end_to_end.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "221yILajn9DK"
      },
      "source": [
        "# Welcome to your end-to-end machine learning project \n",
        "> Try it yourself !!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6VdbvXViouof"
      },
      "source": [
        "!pip install shap # Install SHAP"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R54NSi6m6pnf"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "# Visualization\n",
        "import seaborn as sn\n",
        "import matplotlib.pyplot as plt\n",
        "# To show the plots in notebook.\n",
        "%matplotlib inline\n",
        "\n",
        "from sklearn import preprocessing,model_selection,metrics\n",
        "from sklearn import ensemble\n",
        "\n",
        "from imblearn import over_sampling\n",
        "import shap\n",
        "\n",
        "RANDOM_STATE = 1234"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qlwfEJM07iVb"
      },
      "source": [
        "## Frame the problem"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AOcxI_88Cc7t"
      },
      "source": [
        "**Source**\n",
        "> https://www.kaggle.com/rikdifos/credit-card-approval-prediction \n",
        "\n",
        "**Goal**\n",
        "> To predict if a new application should be approved or not using the past history data of the company's credit-card customers.\n",
        "\n",
        "**Features**\n",
        ">Information in the application form.\n",
        "\n",
        "**Target**\n",
        "> `Good client` if they cleared their credit card outstanding balance witin 60 days after the latest due, otherwise `Bad client`.\n",
        "\n",
        "*Surely there should be other better definitions to label good and bad clients.* "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2TKsPCfv7lA1"
      },
      "source": [
        "## Select performance measures"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rhVC2g-CwcaP"
      },
      "source": [
        "> F1, ROC curve + AUC, Precision-Recall curve + AUC"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HokLbQPV7R4G"
      },
      "source": [
        "## Get the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "35E14QIp33Uy",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "2a5c685b-e9eb-4514-e9f8-ef326811c165"
      },
      "source": [
        "from google.colab import files # Colab library to upload files to notebook\n",
        "\n",
        "# Install kaggle package so that we can use the kaggle api e.g. for downloading data.  \n",
        "!pip install kaggle\n",
        "\n",
        "# Upload kaggle API key file\n",
        "uploaded = files.upload()\n",
        "\n",
        "!mkdir '/root/.kaggle'\n",
        "!cp 'kaggle.json' '/root/.kaggle/.'\n",
        "!chmod 600 '/root/.kaggle/kaggle.json'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.7/dist-packages (1.5.12)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from kaggle) (4.41.1)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.7/dist-packages (from kaggle) (5.0.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from kaggle) (2021.5.30)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.23.0)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.8.1)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.24.3)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.15.0)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.7/dist-packages (from python-slugify->kaggle) (1.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (2.10)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-640d8aeb-1d71-456d-8687-6ab905bddc44\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-640d8aeb-1d71-456d-8687-6ab905bddc44\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving kaggle.json to kaggle.json\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ad24k5DM4JCW",
        "outputId": "450822d0-1d62-4f01-fba5-4bc8ce18916e"
      },
      "source": [
        "!kaggle datasets list -s credit"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.4)\n",
            "ref                                                          title                                                 size  lastUpdated          downloadCount  \n",
            "-----------------------------------------------------------  --------------------------------------------------  ------  -------------------  -------------  \n",
            "sakshigoyal7/credit-card-customers                           Credit Card customers                                379KB  2020-11-19 07:38:44          40000  \n",
            "mlg-ulb/creditcardfraud                                      Credit Card Fraud Detection                           66MB  2018-03-23 01:17:27         335123  \n",
            "uciml/german-credit                                          German Credit Risk                                    11KB  2016-12-14 21:25:02          25543  \n",
            "rikdifos/credit-card-approval-prediction                     Credit Card Approval Prediction                        5MB  2020-03-24 10:04:48          15239  \n",
            "upadorprofzs/credit-risk                                     Credit risk                                           58KB  2020-06-28 00:32:27            902  \n",
            "mishra5001/credit-card                                       Credit Card Fraud Detection                          112MB  2019-07-15 06:36:02           4436  \n",
            "uciml/default-of-credit-card-clients-dataset                 Default of Credit Card Clients Dataset              1001KB  2016-11-03 03:39:18          41656  \n",
            "arjunbhasin2013/ccdata                                       Credit Card Dataset for Clustering                   340KB  2018-03-02 08:35:03          22423  \n",
            "praveengovi/credit-risk-classification-dataset               Credit Risk Classification Dataset                   148KB  2020-05-05 14:34:44           1878  \n",
            "kartik2112/fraud-detection                                   Credit Card Transactions Fraud Detection Dataset     202MB  2020-08-05 15:20:55           4396  \n",
            "kabure/german-credit-data-with-risk                          German Credit Risk - With Target                      11KB  2018-01-09 15:09:46           7185  \n",
            "dansbecker/aer-credit-card-data                              Credit Card Data from book \"Econometric Analysis\"     24KB  2017-10-13 17:17:33           5031  \n",
            "laotse/credit-risk-dataset                                   Credit Risk Dataset                                  368KB  2020-06-02 06:10:39           1778  \n",
            "isaikumar/creditcardfraud                                    Credit Card Fraud Detection Dataset                   66MB  2018-05-05 09:38:01           3792  \n",
            "sajidhussain3/jobathon-may-2021-credit-card-lead-prediction  JOB-A-THON - May 2021- Credit Card Lead Prediction     7MB  2021-05-27 18:32:05            431  \n",
            "shayannaveed/credit-card-fraud-detection                     Credit Card Fraud Detection                           66MB  2019-12-24 08:07:24            569  \n",
            "ananta/credit-card-data                                      Credit Card Data                                       1MB  2018-08-24 02:22:36           2462  \n",
            "brycecf/give-me-some-credit-dataset                          Give Me Some Credit :: 2011 Competition Data           5MB  2017-08-09 20:08:21           6117  \n",
            "ranadeep/credit-risk-dataset                                 Credit Risk Analysis                                 101MB  2021-06-12 14:52:07            339  \n",
            "aryashah2k/credit-card-customer-data                         Credit Card Customer Data                              6KB  2021-05-15 08:59:46            620  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "odX2Smcl4FtS",
        "outputId": "ef3a914d-c2f0-47aa-c060-bb0b87ecdb09"
      },
      "source": [
        "!kaggle datasets download -d rikdifos/credit-card-approval-prediction # Download the data from the competition."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading credit-card-approval-prediction.zip to /content\n",
            " 94% 5.00M/5.32M [00:00<00:00, 35.6MB/s]\n",
            "100% 5.32M/5.32M [00:00<00:00, 33.9MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Vi0EnyG57ND",
        "outputId": "5186cd63-7f87-4677-a92d-dfa36d6d777f"
      },
      "source": [
        "!unzip \"/content/credit-card-approval-prediction.zip\" -d \"/content/credit-card-approval-prediction\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/credit-card-approval-prediction.zip\n",
            "  inflating: /content/credit-card-approval-prediction/application_record.csv  \n",
            "  inflating: /content/credit-card-approval-prediction/credit_record.csv  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D0Ln0XbP8f3N"
      },
      "source": [
        "df_app_rec = pd.read_csv('/content/credit-card-approval-prediction/application_record.csv') \n",
        "df_credit_rec = pd.read_csv('/content/credit-card-approval-prediction/credit_record.csv') # Time-series data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DII291BK78ul"
      },
      "source": [
        "### Creating labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 684
        },
        "id": "A86TDw0k8B1v",
        "outputId": "8e40f754-1bd1-4544-cfb9-649d7a0b4b81"
      },
      "source": [
        "'''\n",
        "MONTHS_BALANCE\n",
        "The month of the extracted data is the starting point, backwards, \n",
        "0 is the current month, -1 is the previous month, and so on\n",
        "\n",
        "STATUS\n",
        "0: 1-29 days past due \n",
        "1: 30-59 days past due \n",
        "2: 60-89 days overdue \n",
        "3: 90-119 days overdue \n",
        "4: 120-149 days overdue \n",
        "5: Overdue or bad debts, write-offs for more than 150 days \n",
        "C: paid off that month \n",
        "X: No loan for the month\n",
        "'''\n",
        "\n",
        "df_credit_rec[:20]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>MONTHS_BALANCE</th>\n",
              "      <th>STATUS</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5001711</td>\n",
              "      <td>0</td>\n",
              "      <td>X</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>5001711</td>\n",
              "      <td>-1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>5001711</td>\n",
              "      <td>-2</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5001711</td>\n",
              "      <td>-3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5001712</td>\n",
              "      <td>0</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-1</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-2</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-3</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-4</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-5</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-6</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-7</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-8</td>\n",
              "      <td>C</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-9</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-10</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-11</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-12</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-13</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-14</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>5001712</td>\n",
              "      <td>-15</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         ID  MONTHS_BALANCE STATUS\n",
              "0   5001711               0      X\n",
              "1   5001711              -1      0\n",
              "2   5001711              -2      0\n",
              "3   5001711              -3      0\n",
              "4   5001712               0      C\n",
              "5   5001712              -1      C\n",
              "6   5001712              -2      C\n",
              "7   5001712              -3      C\n",
              "8   5001712              -4      C\n",
              "9   5001712              -5      C\n",
              "10  5001712              -6      C\n",
              "11  5001712              -7      C\n",
              "12  5001712              -8      C\n",
              "13  5001712              -9      0\n",
              "14  5001712             -10      0\n",
              "15  5001712             -11      0\n",
              "16  5001712             -12      0\n",
              "17  5001712             -13      0\n",
              "18  5001712             -14      0\n",
              "19  5001712             -15      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vBhkTULsIPbB"
      },
      "source": [
        "df_credit_rec['Target'] = df_credit_rec['STATUS'].isin(['X','C','0'])\n",
        "df_credit_rec_uniqueID = df_credit_rec.sort_values(['ID','MONTHS_BALANCE']).drop_duplicates(subset='ID',keep='last')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 620
        },
        "id": "4sITbn4iIqYe",
        "outputId": "adfa9e41-f074-4298-a778-56a12e8f6c7d"
      },
      "source": [
        "df_credit_rec[df_credit_rec['ID'] == 5150482]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>MONTHS_BALANCE</th>\n",
              "      <th>STATUS</th>\n",
              "      <th>Target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1048494</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-11</td>\n",
              "      <td>C</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048495</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-12</td>\n",
              "      <td>C</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048496</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-13</td>\n",
              "      <td>C</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048497</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-14</td>\n",
              "      <td>C</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048498</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-15</td>\n",
              "      <td>C</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048499</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-16</td>\n",
              "      <td>C</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048500</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-17</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048501</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-18</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048502</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-19</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048503</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-20</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048504</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-21</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048505</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-22</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048506</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-23</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048507</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-24</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048508</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-25</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048509</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-26</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048510</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-27</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048511</th>\n",
              "      <td>5150482</td>\n",
              "      <td>-28</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              ID  MONTHS_BALANCE STATUS  Target\n",
              "1048494  5150482             -11      C    True\n",
              "1048495  5150482             -12      C    True\n",
              "1048496  5150482             -13      C    True\n",
              "1048497  5150482             -14      C    True\n",
              "1048498  5150482             -15      C    True\n",
              "1048499  5150482             -16      C    True\n",
              "1048500  5150482             -17      0    True\n",
              "1048501  5150482             -18      0    True\n",
              "1048502  5150482             -19      0    True\n",
              "1048503  5150482             -20      0    True\n",
              "1048504  5150482             -21      0    True\n",
              "1048505  5150482             -22      0    True\n",
              "1048506  5150482             -23      0    True\n",
              "1048507  5150482             -24      0    True\n",
              "1048508  5150482             -25      0    True\n",
              "1048509  5150482             -26      0    True\n",
              "1048510  5150482             -27      0    True\n",
              "1048511  5150482             -28      0    True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xpcMj480-yGY"
      },
      "source": [
        "# Merge the application records and the defined targets.\n",
        "df_data = pd.merge(df_app_rec,df_credit_rec_uniqueID,how='inner',on='ID').drop(\n",
        "    ['ID','MONTHS_BALANCE','STATUS'],axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6XQfOHnmJmW_",
        "outputId": "71768f55-bd3f-4d31-fab5-8643f5f7d04e"
      },
      "source": [
        "df_data['Target'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True     36075\n",
              "False      382\n",
              "Name: Target, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JMIL9nPsMsw_"
      },
      "source": [
        "df_data['Target'] = 1-df_data['Target']*1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 572
        },
        "id": "ezOz_p_XNKEy",
        "outputId": "6978f7fa-a51c-4b7b-ed8f-16d6d333ac97"
      },
      "source": [
        "df_data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CODE_GENDER</th>\n",
              "      <th>FLAG_OWN_CAR</th>\n",
              "      <th>FLAG_OWN_REALTY</th>\n",
              "      <th>CNT_CHILDREN</th>\n",
              "      <th>AMT_INCOME_TOTAL</th>\n",
              "      <th>NAME_INCOME_TYPE</th>\n",
              "      <th>NAME_EDUCATION_TYPE</th>\n",
              "      <th>NAME_FAMILY_STATUS</th>\n",
              "      <th>NAME_HOUSING_TYPE</th>\n",
              "      <th>DAYS_BIRTH</th>\n",
              "      <th>DAYS_EMPLOYED</th>\n",
              "      <th>FLAG_MOBIL</th>\n",
              "      <th>FLAG_WORK_PHONE</th>\n",
              "      <th>FLAG_PHONE</th>\n",
              "      <th>FLAG_EMAIL</th>\n",
              "      <th>OCCUPATION_TYPE</th>\n",
              "      <th>CNT_FAM_MEMBERS</th>\n",
              "      <th>Target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>M</td>\n",
              "      <td>Y</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>427500.0</td>\n",
              "      <td>Working</td>\n",
              "      <td>Higher education</td>\n",
              "      <td>Civil marriage</td>\n",
              "      <td>Rented apartment</td>\n",
              "      <td>-12005</td>\n",
              "      <td>-4542</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>M</td>\n",
              "      <td>Y</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>427500.0</td>\n",
              "      <td>Working</td>\n",
              "      <td>Higher education</td>\n",
              "      <td>Civil marriage</td>\n",
              "      <td>Rented apartment</td>\n",
              "      <td>-12005</td>\n",
              "      <td>-4542</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>M</td>\n",
              "      <td>Y</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>112500.0</td>\n",
              "      <td>Working</td>\n",
              "      <td>Secondary / secondary special</td>\n",
              "      <td>Married</td>\n",
              "      <td>House / apartment</td>\n",
              "      <td>-21474</td>\n",
              "      <td>-1134</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Security staff</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>F</td>\n",
              "      <td>N</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>270000.0</td>\n",
              "      <td>Commercial associate</td>\n",
              "      <td>Secondary / secondary special</td>\n",
              "      <td>Single / not married</td>\n",
              "      <td>House / apartment</td>\n",
              "      <td>-19110</td>\n",
              "      <td>-3051</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Sales staff</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>F</td>\n",
              "      <td>N</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>270000.0</td>\n",
              "      <td>Commercial associate</td>\n",
              "      <td>Secondary / secondary special</td>\n",
              "      <td>Single / not married</td>\n",
              "      <td>House / apartment</td>\n",
              "      <td>-19110</td>\n",
              "      <td>-3051</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Sales staff</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36452</th>\n",
              "      <td>M</td>\n",
              "      <td>Y</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>315000.0</td>\n",
              "      <td>Working</td>\n",
              "      <td>Secondary / secondary special</td>\n",
              "      <td>Married</td>\n",
              "      <td>House / apartment</td>\n",
              "      <td>-17348</td>\n",
              "      <td>-2420</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Managers</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36453</th>\n",
              "      <td>F</td>\n",
              "      <td>N</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>157500.0</td>\n",
              "      <td>Commercial associate</td>\n",
              "      <td>Higher education</td>\n",
              "      <td>Married</td>\n",
              "      <td>House / apartment</td>\n",
              "      <td>-12387</td>\n",
              "      <td>-1325</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Medicine staff</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36454</th>\n",
              "      <td>F</td>\n",
              "      <td>N</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>157500.0</td>\n",
              "      <td>Pensioner</td>\n",
              "      <td>Higher education</td>\n",
              "      <td>Married</td>\n",
              "      <td>House / apartment</td>\n",
              "      <td>-12387</td>\n",
              "      <td>-1325</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Medicine staff</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36455</th>\n",
              "      <td>F</td>\n",
              "      <td>N</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>283500.0</td>\n",
              "      <td>Working</td>\n",
              "      <td>Secondary / secondary special</td>\n",
              "      <td>Married</td>\n",
              "      <td>House / apartment</td>\n",
              "      <td>-17958</td>\n",
              "      <td>-655</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Sales staff</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36456</th>\n",
              "      <td>M</td>\n",
              "      <td>N</td>\n",
              "      <td>Y</td>\n",
              "      <td>0</td>\n",
              "      <td>112500.0</td>\n",
              "      <td>Working</td>\n",
              "      <td>Secondary / secondary special</td>\n",
              "      <td>Single / not married</td>\n",
              "      <td>Rented apartment</td>\n",
              "      <td>-9188</td>\n",
              "      <td>-1193</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Laborers</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>36457 rows × 18 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      CODE_GENDER FLAG_OWN_CAR  ... CNT_FAM_MEMBERS  Target\n",
              "0               M            Y  ...             2.0       0\n",
              "1               M            Y  ...             2.0       0\n",
              "2               M            Y  ...             2.0       0\n",
              "3               F            N  ...             1.0       0\n",
              "4               F            N  ...             1.0       0\n",
              "...           ...          ...  ...             ...     ...\n",
              "36452           M            Y  ...             2.0       1\n",
              "36453           F            N  ...             2.0       0\n",
              "36454           F            N  ...             2.0       0\n",
              "36455           F            N  ...             2.0       1\n",
              "36456           M            N  ...             1.0       0\n",
              "\n",
              "[36457 rows x 18 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J-GUNhDdwPRn"
      },
      "source": [
        "### Clean the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Cl_bMiGwTzU"
      },
      "source": [
        "# Todo: Check duplicated IDs, etc.scห"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y5UOCoTG8SaL"
      },
      "source": [
        "## Split the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I6ZTDQFlNODg"
      },
      "source": [
        "train,test = model_selection.train_test_split(df_data,\n",
        "                      test_size=0.2,\n",
        "                      random_state=RANDOM_STATE,shuffle=True,\n",
        "                      stratify=df_data['Target'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GJ_fQIMexZF1"
      },
      "source": [
        "train,test = train.reset_index(drop=True),test.reset_index(drop=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8nn4HhYKSVSN"
      },
      "source": [
        "X_train,y_train = train.iloc[:,:-1],train.iloc[:,-1]\n",
        "X_test,y_test = test.iloc[:,:-1],test.iloc[:,-1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lc01560EStZZ",
        "outputId": "7c889ab8-d940-4b8b-ee78-5381656129c6"
      },
      "source": [
        "y_train.sum()/len(y_train),y_test.sum()/len(y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.010492028115892336, 0.010422380691168404)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0cnyuPOW7fys"
      },
      "source": [
        "## Understand and preprocess the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PW1YzY8Wi48W",
        "outputId": "777b0730-7247-486e-d397-43a2e71fb68f"
      },
      "source": [
        "X_train.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 29165 entries, 0 to 29164\n",
            "Data columns (total 17 columns):\n",
            " #   Column               Non-Null Count  Dtype  \n",
            "---  ------               --------------  -----  \n",
            " 0   CODE_GENDER          29165 non-null  object \n",
            " 1   FLAG_OWN_CAR         29165 non-null  object \n",
            " 2   FLAG_OWN_REALTY      29165 non-null  object \n",
            " 3   CNT_CHILDREN         29165 non-null  int64  \n",
            " 4   AMT_INCOME_TOTAL     29165 non-null  float64\n",
            " 5   NAME_INCOME_TYPE     29165 non-null  object \n",
            " 6   NAME_EDUCATION_TYPE  29165 non-null  object \n",
            " 7   NAME_FAMILY_STATUS   29165 non-null  object \n",
            " 8   NAME_HOUSING_TYPE    29165 non-null  object \n",
            " 9   DAYS_BIRTH           29165 non-null  int64  \n",
            " 10  DAYS_EMPLOYED        29165 non-null  int64  \n",
            " 11  FLAG_MOBIL           29165 non-null  int64  \n",
            " 12  FLAG_WORK_PHONE      29165 non-null  int64  \n",
            " 13  FLAG_PHONE           29165 non-null  int64  \n",
            " 14  FLAG_EMAIL           29165 non-null  int64  \n",
            " 15  OCCUPATION_TYPE      20116 non-null  object \n",
            " 16  CNT_FAM_MEMBERS      29165 non-null  float64\n",
            "dtypes: float64(2), int64(7), object(8)\n",
            "memory usage: 3.8+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J7xQ1ZN2rYth"
      },
      "source": [
        "col_names_2drop = ['OCCUPATION_TYPE'] # Drop it due to missing\n",
        "# Todo: Try other methods"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303
        },
        "id": "WUXFojJbNnJu",
        "outputId": "190e535b-b837-474c-f477-c738a31f1689"
      },
      "source": [
        "X_train.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CNT_CHILDREN</th>\n",
              "      <th>AMT_INCOME_TOTAL</th>\n",
              "      <th>DAYS_BIRTH</th>\n",
              "      <th>DAYS_EMPLOYED</th>\n",
              "      <th>FLAG_MOBIL</th>\n",
              "      <th>FLAG_WORK_PHONE</th>\n",
              "      <th>FLAG_PHONE</th>\n",
              "      <th>FLAG_EMAIL</th>\n",
              "      <th>CNT_FAM_MEMBERS</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>29165.000000</td>\n",
              "      <td>2.916500e+04</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.0</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.432745</td>\n",
              "      <td>1.866744e+05</td>\n",
              "      <td>-15976.598971</td>\n",
              "      <td>59542.030859</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.224104</td>\n",
              "      <td>0.294634</td>\n",
              "      <td>0.089525</td>\n",
              "      <td>2.201680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.740998</td>\n",
              "      <td>1.019835e+05</td>\n",
              "      <td>4204.237563</td>\n",
              "      <td>137898.369167</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.416998</td>\n",
              "      <td>0.455886</td>\n",
              "      <td>0.285505</td>\n",
              "      <td>0.910885</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.700000e+04</td>\n",
              "      <td>-25152.000000</td>\n",
              "      <td>-15661.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.215000e+05</td>\n",
              "      <td>-19435.000000</td>\n",
              "      <td>-3153.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.575000e+05</td>\n",
              "      <td>-15570.000000</td>\n",
              "      <td>-1542.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>2.250000e+05</td>\n",
              "      <td>-12454.000000</td>\n",
              "      <td>-407.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>3.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>14.000000</td>\n",
              "      <td>1.575000e+06</td>\n",
              "      <td>-7705.000000</td>\n",
              "      <td>365243.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>15.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       CNT_CHILDREN  AMT_INCOME_TOTAL  ...    FLAG_EMAIL  CNT_FAM_MEMBERS\n",
              "count  29165.000000      2.916500e+04  ...  29165.000000     29165.000000\n",
              "mean       0.432745      1.866744e+05  ...      0.089525         2.201680\n",
              "std        0.740998      1.019835e+05  ...      0.285505         0.910885\n",
              "min        0.000000      2.700000e+04  ...      0.000000         1.000000\n",
              "25%        0.000000      1.215000e+05  ...      0.000000         2.000000\n",
              "50%        0.000000      1.575000e+05  ...      0.000000         2.000000\n",
              "75%        1.000000      2.250000e+05  ...      0.000000         3.000000\n",
              "max       14.000000      1.575000e+06  ...      1.000000        15.000000\n",
              "\n",
              "[8 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mdTHqI1sOM-0",
        "outputId": "e112fc60-c5b9-4fd4-a0d5-49a75c29e123"
      },
      "source": [
        "'''\n",
        "Count backwards from current day(0). \n",
        "If positive, it means the person currently unemployed.\n",
        "'''\n",
        "X_train.loc[X_train['DAYS_EMPLOYED'] >= 0,'DAYS_EMPLOYED'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([365243])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303
        },
        "id": "n1XafgXCQ2cK",
        "outputId": "aac8aa93-266c-40d8-9081-e49e6d83f6c5"
      },
      "source": [
        "# Put 'DAYS_EMPLOYED' = 0 for unemployed people.\n",
        "X_train = X_train.replace(to_replace={'DAYS_EMPLOYED':365243},value=0)\n",
        "X_train.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CNT_CHILDREN</th>\n",
              "      <th>AMT_INCOME_TOTAL</th>\n",
              "      <th>DAYS_BIRTH</th>\n",
              "      <th>DAYS_EMPLOYED</th>\n",
              "      <th>FLAG_MOBIL</th>\n",
              "      <th>FLAG_WORK_PHONE</th>\n",
              "      <th>FLAG_PHONE</th>\n",
              "      <th>FLAG_EMAIL</th>\n",
              "      <th>CNT_FAM_MEMBERS</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>29165.000000</td>\n",
              "      <td>2.916500e+04</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.0</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.000000</td>\n",
              "      <td>29165.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.432745</td>\n",
              "      <td>1.866744e+05</td>\n",
              "      <td>-15976.598971</td>\n",
              "      <td>-2197.999657</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.224104</td>\n",
              "      <td>0.294634</td>\n",
              "      <td>0.089525</td>\n",
              "      <td>2.201680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.740998</td>\n",
              "      <td>1.019835e+05</td>\n",
              "      <td>4204.237563</td>\n",
              "      <td>2369.951242</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.416998</td>\n",
              "      <td>0.455886</td>\n",
              "      <td>0.285505</td>\n",
              "      <td>0.910885</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.700000e+04</td>\n",
              "      <td>-25152.000000</td>\n",
              "      <td>-15661.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.215000e+05</td>\n",
              "      <td>-19435.000000</td>\n",
              "      <td>-3153.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.575000e+05</td>\n",
              "      <td>-15570.000000</td>\n",
              "      <td>-1542.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>2.250000e+05</td>\n",
              "      <td>-12454.000000</td>\n",
              "      <td>-407.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>3.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>14.000000</td>\n",
              "      <td>1.575000e+06</td>\n",
              "      <td>-7705.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>15.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       CNT_CHILDREN  AMT_INCOME_TOTAL  ...    FLAG_EMAIL  CNT_FAM_MEMBERS\n",
              "count  29165.000000      2.916500e+04  ...  29165.000000     29165.000000\n",
              "mean       0.432745      1.866744e+05  ...      0.089525         2.201680\n",
              "std        0.740998      1.019835e+05  ...      0.285505         0.910885\n",
              "min        0.000000      2.700000e+04  ...      0.000000         1.000000\n",
              "25%        0.000000      1.215000e+05  ...      0.000000         2.000000\n",
              "50%        0.000000      1.575000e+05  ...      0.000000         2.000000\n",
              "75%        1.000000      2.250000e+05  ...      0.000000         3.000000\n",
              "max       14.000000      1.575000e+06  ...      1.000000        15.000000\n",
              "\n",
              "[8 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "cQ4m3Y13oUrj",
        "outputId": "32c9f056-8d9c-4687-dc52-d0f1c488dfd2"
      },
      "source": [
        "# Todo: Explore each single feature if there is any suspected values. If so, do something.\n",
        "''' \n",
        "Recommended tool:\n",
        "   https://pandas-profiling.github.io/pandas-profiling/docs/master/rtd/index.html\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' \\nRecommended tool:\\n   https://pandas-profiling.github.io/pandas-profiling/docs/master/rtd/index.html\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j5dt5PYPlfcB",
        "outputId": "38340798-2df6-41b7-a9ce-8cc1886cd38c"
      },
      "source": [
        "# One-hot encoding\n",
        "cat_features = ['CODE_GENDER','FLAG_OWN_CAR','FLAG_OWN_REALTY',\n",
        "                'NAME_INCOME_TYPE','NAME_FAMILY_STATUS',\n",
        "                'NAME_HOUSING_TYPE','NAME_EDUCATION_TYPE']\n",
        "onthot_enc = preprocessing.OneHotEncoder()\n",
        "onthot_enc.fit(X_train[cat_features])\n",
        "\n",
        "X_encoded = onthot_enc.transform(X_train[cat_features]).toarray()\n",
        "print(X_encoded.shape)\n",
        "cat_col_names = []\n",
        "for i in range(len(cat_features)):\n",
        "  cat_col_names = cat_col_names + [cat_features[i]+'_'+cat_enc for cat_enc in onthot_enc.categories_[i]]\n",
        "df_train_cat_encoded = pd.DataFrame(X_encoded,columns=cat_col_names)\n",
        "print(df_train_cat_encoded.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(29165, 27)\n",
            "(29165, 27)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qs5S3NDImQwn",
        "outputId": "b6724a05-1836-4b41-af14-334886572418"
      },
      "source": [
        "# Todo: Ordinal encoding for NAME_EDUCATION_TYPE' instead\n",
        "X_train['NAME_EDUCATION_TYPE'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Higher education', 'Secondary / secondary special',\n",
              "       'Incomplete higher', 'Lower secondary', 'Academic degree'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pT8SAHtQaIjD"
      },
      "source": [
        "# DAYS_BIRTH >> age_year\n",
        "def days_birth_to_year(X):\n",
        "  X.loc[:,'DAYS_BIRTH'] = (X['DAYS_BIRTH']*-1/365).astype(int)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PGW4iLvEaN2p"
      },
      "source": [
        "# Todo: DAYS_EMPLOYED >> ??"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wt44JEIGJBj8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ea6ff9a9-90bb-40f2-9f2c-45637f196947"
      },
      "source": [
        "# Prepapre the final feature matrix\n",
        "col_names_2drop += cat_features\n",
        "X_train = pd.concat([X_train,df_train_cat_encoded],axis=1).drop(col_names_2drop,axis=1)\n",
        "print(X_train.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(29165, 36)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1MYZrQnetIuf"
      },
      "source": [
        "# Todo: Make the code more modular."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 466
        },
        "id": "VRwRRuBLlgdP",
        "outputId": "56572f95-6d7f-4881-e5cd-39ed157e3d77"
      },
      "source": [
        "X_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CNT_CHILDREN</th>\n",
              "      <th>AMT_INCOME_TOTAL</th>\n",
              "      <th>DAYS_BIRTH</th>\n",
              "      <th>DAYS_EMPLOYED</th>\n",
              "      <th>FLAG_MOBIL</th>\n",
              "      <th>FLAG_WORK_PHONE</th>\n",
              "      <th>FLAG_PHONE</th>\n",
              "      <th>FLAG_EMAIL</th>\n",
              "      <th>CNT_FAM_MEMBERS</th>\n",
              "      <th>CODE_GENDER_F</th>\n",
              "      <th>CODE_GENDER_M</th>\n",
              "      <th>FLAG_OWN_CAR_N</th>\n",
              "      <th>FLAG_OWN_CAR_Y</th>\n",
              "      <th>FLAG_OWN_REALTY_N</th>\n",
              "      <th>FLAG_OWN_REALTY_Y</th>\n",
              "      <th>NAME_INCOME_TYPE_Commercial associate</th>\n",
              "      <th>NAME_INCOME_TYPE_Pensioner</th>\n",
              "      <th>NAME_INCOME_TYPE_State servant</th>\n",
              "      <th>NAME_INCOME_TYPE_Student</th>\n",
              "      <th>NAME_INCOME_TYPE_Working</th>\n",
              "      <th>NAME_FAMILY_STATUS_Civil marriage</th>\n",
              "      <th>NAME_FAMILY_STATUS_Married</th>\n",
              "      <th>NAME_FAMILY_STATUS_Separated</th>\n",
              "      <th>NAME_FAMILY_STATUS_Single / not married</th>\n",
              "      <th>NAME_FAMILY_STATUS_Widow</th>\n",
              "      <th>NAME_HOUSING_TYPE_Co-op apartment</th>\n",
              "      <th>NAME_HOUSING_TYPE_House / apartment</th>\n",
              "      <th>NAME_HOUSING_TYPE_Municipal apartment</th>\n",
              "      <th>NAME_HOUSING_TYPE_Office apartment</th>\n",
              "      <th>NAME_HOUSING_TYPE_Rented apartment</th>\n",
              "      <th>NAME_HOUSING_TYPE_With parents</th>\n",
              "      <th>NAME_EDUCATION_TYPE_Academic degree</th>\n",
              "      <th>NAME_EDUCATION_TYPE_Higher education</th>\n",
              "      <th>NAME_EDUCATION_TYPE_Incomplete higher</th>\n",
              "      <th>NAME_EDUCATION_TYPE_Lower secondary</th>\n",
              "      <th>NAME_EDUCATION_TYPE_Secondary / secondary special</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>1350000.0</td>\n",
              "      <td>-12999</td>\n",
              "      <td>-1560</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>270000.0</td>\n",
              "      <td>-11451</td>\n",
              "      <td>-2206</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>135000.0</td>\n",
              "      <td>-15842</td>\n",
              "      <td>-1419</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>225000.0</td>\n",
              "      <td>-13021</td>\n",
              "      <td>-5082</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>135000.0</td>\n",
              "      <td>-9167</td>\n",
              "      <td>-947</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29160</th>\n",
              "      <td>0</td>\n",
              "      <td>180000.0</td>\n",
              "      <td>-12461</td>\n",
              "      <td>-678</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29161</th>\n",
              "      <td>0</td>\n",
              "      <td>126000.0</td>\n",
              "      <td>-23751</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29162</th>\n",
              "      <td>0</td>\n",
              "      <td>292500.0</td>\n",
              "      <td>-12554</td>\n",
              "      <td>-5326</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29163</th>\n",
              "      <td>0</td>\n",
              "      <td>157500.0</td>\n",
              "      <td>-19843</td>\n",
              "      <td>-1922</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29164</th>\n",
              "      <td>1</td>\n",
              "      <td>180000.0</td>\n",
              "      <td>-10754</td>\n",
              "      <td>-65</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>29165 rows × 36 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       CNT_CHILDREN  ...  NAME_EDUCATION_TYPE_Secondary / secondary special\n",
              "0                 0  ...                                                0.0\n",
              "1                 0  ...                                                0.0\n",
              "2                 0  ...                                                1.0\n",
              "3                 0  ...                                                0.0\n",
              "4                 0  ...                                                1.0\n",
              "...             ...  ...                                                ...\n",
              "29160             0  ...                                                1.0\n",
              "29161             0  ...                                                1.0\n",
              "29162             0  ...                                                0.0\n",
              "29163             0  ...                                                1.0\n",
              "29164             1  ...                                                0.0\n",
              "\n",
              "[29165 rows x 36 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SOypJ5CY8vQ4"
      },
      "source": [
        "## Train, evaluate, fine-tune the models\n",
        "> This is your model selection step"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iUZsUVl7r61Q",
        "outputId": "c48eaa7f-3986-49a4-bbf7-72fb4c186582"
      },
      "source": [
        "# OK\n",
        "model_selection.cross_val_score(ensemble.RandomForestClassifier(),X_train,y_train,cv=5,scoring='f1')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.15384615, 0.1686747 , 0.17073171, 0.1038961 , 0.09090909])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gwbn8Cr4seFE",
        "outputId": "8d08a404-8d80-48b0-b52c-d8580d6f54c8"
      },
      "source": [
        "# !!!!! Wrong because there is information leak from the training set to the validation set !!!!!\n",
        "oversampler = over_sampling.SMOTE()\n",
        "X_train_oversampled,y_train_oversampled = oversampler.fit_resample(X_train,y_train)\n",
        "model_selection.cross_val_score(ensemble.RandomForestClassifier(),X_train_oversampled,y_train_oversampled,cv=5,scoring='f1')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.97804039, 0.99749287, 0.99757827, 0.99688904, 0.99654398])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "28Y5V8nhuAQC"
      },
      "source": [
        "def smote(X,y):\n",
        "  oversampler = over_sampling.SMOTE()\n",
        "  X_balanced,y_balanced = oversampler.fit_resample(X,y)\n",
        "  return X_balanced,y_balanced "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "48QGeHf2uRnG"
      },
      "source": [
        "# Todo: Try other data-balancing methods"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HV3cy6QC28fn"
      },
      "source": [
        "def run_model(model,resampler=None,preprocessors=[]):\n",
        "  # Return values\n",
        "  f1_train_list,f1_val_list = [],[]\n",
        "  y_train_val_list,y_train_val_pred_list = [],[]\n",
        "\n",
        "  # Stratified cross-validation\n",
        "  kf = model_selection.KFold(n_splits=5,shuffle=True,random_state=RANDOM_STATE)\n",
        "  for train_index, val_index in kf.split(X_train):\n",
        "    # Split the data\n",
        "    X_train_train, X_train_val = X_train.iloc[train_index], X_train.iloc[val_index]\n",
        "    y_train_train, y_train_val = y_train.iloc[train_index], y_train.iloc[val_index]\n",
        "\n",
        "    # Feature engineering\n",
        "    for preprocessor in preprocessors:\n",
        "      preprocessor(X_train_train)\n",
        "      preprocessor(X_train_val)\n",
        "\n",
        "    # Balance the data\n",
        "    if resampler is not None:\n",
        "      X_train_train,y_train_train = resampler(X_train_train,y_train_train)\n",
        "\n",
        "    # Train the model\n",
        "    model.fit(X_train_train,y_train_train)\n",
        "\n",
        "    # Evaluate on the training set\n",
        "    y_train_train_pred = model.predict(X_train_train)\n",
        "    f1 = metrics.f1_score(y_train_train,y_train_train_pred)\n",
        "    f1_train_list.append(f1)\n",
        "\n",
        "    # Evaluate on the validation set\n",
        "    y_train_val_pred = model.predict(X_train_val)\n",
        "    f1 = metrics.f1_score(y_train_val,y_train_val_pred)\n",
        "    f1_val_list.append(f1)\n",
        "\n",
        "    y_train_val_list += list(y_train_val)\n",
        "    y_train_val_pred_list += list(model.predict_proba(X_train_val)[:,1])\n",
        "\n",
        "  return f1_train_list,f1_val_list,y_train_val_list,y_train_val_pred_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z0AHVnOK8UpP"
      },
      "source": [
        "f1_train,f1_val = {},{}\n",
        "y_train_val,y_train_val_pred = {},{}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RMvasNtSIYVp"
      },
      "source": [
        "model_name = 'rf'\n",
        "f1_train[model_name],f1_val[model_name],y_train_val[model_name],y_train_val_pred[model_name] = run_model(\n",
        "    ensemble.RandomForestClassifier(),resampler=None,preprocessors = [days_birth_to_year])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uMTnsg59IW6W"
      },
      "source": [
        "model_name = 'rf_smote'\n",
        "f1_train[model_name],f1_val[model_name],y_train_val[model_name],y_train_val_pred[model_name] = run_model(\n",
        "    ensemble.RandomForestClassifier(),resampler=smote,preprocessors = [days_birth_to_year])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M1M0xVvmups7"
      },
      "source": [
        "# Todo: Try other models."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GyhMnDjNuofV"
      },
      "source": [
        "# Todo: Fine-tune hyper-parameters. You need to implement your own grid / randomized search, due to the resampling scheme."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "EqqAyLx4B-ZP",
        "outputId": "da5e5f8c-906c-41e8-bc8b-7cc4eb1a14a9"
      },
      "source": [
        "# Model comparison\n",
        "bp = plt.boxplot([f1_val['rf'],f1_val['rf_smote']])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANaUlEQVR4nO3df6jd9X3H8eerNwn+4Wxz8TLEZI1jbiQLpd2OGV3VoqxdukHcQDfdr2YEHBSlY/iHEJjWEhiVjQ7mH2ZL/3CsEee6kTG7TEpGCejIibXtkuCWio3XlXmLKU6Km9H3/rgncry9yf3e3Hvuufnc5wMu5Hx/3feBy/N+8z3ne26qCklSu9437gEkSaNl6CWpcYZekhpn6CWpcYZekhq3btwDzHX11VfXli1bxj2GJF1Wjh8//v2qmppv3aoL/ZYtW+j3++MeQ5IuK0m+e6F1XrqRpMYZeklqnKGXpMYZeklqnKGXpMYZeklqnKGXpMYZeklq3Kq7YUpSW5Jc0n7+rYzlY+gljdTFgp3EoK8AL91IUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuM6hT7JziQvJDmd5P551v9RkpNJvpXka0k+OLTu00n+c/D16eUcXpK0sAVDn2QCeAT4FLANuCvJtjmbfQPoVdWHgCeBLwz2nQQeAH4B2AE8kGTj8o0vSVpIlzP6HcDpqnqxqv4PeBy4bXiDqjpSVT8cPHwW2DT49y8DT1fVa1V1Fnga2Lk8o0uSuugS+muBl4ceTw+WXcge4KuL2TfJ3Un6SfozMzMdRpIkdbWsL8Ym+R2gBzy8mP2qan9V9aqqNzU1tZwjSdKa1yX0rwCbhx5vGix7jyS/BOwFdlXV/y5mX0nS6HQJ/THg+iTXJdkA3AkcGt4gyUeAR5mN/KtDqw4Dn0yycfAi7CcHyyRJK2TdQhtU1bkk9zAb6AngS1V1IslDQL+qDjF7qeZK4G+TAJypql1V9VqSzzP7ywLgoap6bSTPRJI0r1TVuGd4j16vV/1+f9xjSFoBSVhtDbpcJTleVb351nlnrCQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLUuMMvSQ1ztBLWhaTk5MkWdQXsKjtJycnx/wsL08L3hkrSV2cPXt25Dc/nf/loMXxjF6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalxhl6SGmfoJalx67pslGQn8OfABPBXVfUnc9bfDHwR+BBwZ1U9ObTuC8CvMvtL5Wngs1VVyzO+pNWiHrgKHnz/6L+HFm3B0CeZAB4BPgFMA8eSHKqqk0ObnQF2A/fN2fcXgY8x+wsA4CjwceBflzq4pNUln3udUZ/DJaEeHOm3aFKXM/odwOmqehEgyePAbcC7oa+qlwbr3pmzbwFXABuAAOuB/17y1JKkzrpco78WeHno8fRg2YKq6hngCPC9wdfhqjo1d7skdyfpJ+nPzMx0ObQkqaORvhib5KeArcAmZn853JrkprnbVdX+qupVVW9qamqUI0nSmtMl9K8Am4cebxos6+LXgWer6o2qegP4KvDRxY0oSVqKLqE/Blyf5LokG4A7gUMdj38G+HiSdUnWM/tC7I9cupEkjc6Coa+qc8A9wGFmI/1EVZ1I8lCSXQBJbkgyDdwBPJrkxGD3J4HvAN8Gvgl8s6r+cQTPQ5J0AZ3eR19VTwFPzVn2x0P/PsbsJZ25+70N/MESZ5R0mUgy0uNv3LhxpMdvVafQS9JCvA9y9fIjECSpcYZekhpn6CWpcYZekhpn6CWpcYZekhpn6CWpcYZekhrnDVONuJQ7Er3BRVobDH0jLhTtJAZdWuO8dCNJjTP0ktQ4Qy9JjTP0ktQ4Qy9JjTP0ktQ4Qy9JjTP0ktQ4Qy9JjTP0ktQ4Qy9JjTP0ktQ4Qy9JjTP0ktQ4Q3+ZmZycJEnnL2BR2ydhcnJyzM9S0nLy8+gvM2fPnh3558tfyh8xkbR6eUYvSY0z9JLUOEMvSY0z9JLUOEMvSY3rFPokO5O8kOR0kvvnWX9zkueSnEty+5x1P5HkX5KcSnIyyZblGV2S1MWCoU8yATwCfArYBtyVZNuczc4Au4Evz3OIx4CHq2orsAN4dSkDS5IWp8v76HcAp6vqRYAkjwO3ASfPb1BVLw3WvTO84+AXwrqqenqw3RvLM7Ykqasul26uBV4eejw9WNbFTwM/SPKVJN9I8vDgfwjvkeTuJP0k/ZmZmY6HliR1MeoXY9cBNwH3ATcAP8nsJZ73qKr9VdWrqt7U1NSIR5KktaVL6F8BNg893jRY1sU08HxVvVhV54B/AH5ucSNKkpaiS+iPAdcnuS7JBuBO4FDH4x8DPpDk/Gn6rQxd25ckjd6CoR+cid8DHAZOAU9U1YkkDyXZBZDkhiTTwB3Ao0lODPZ9m9nLNl9L8m0gwF+O5qlIkuaTUX8S4mL1er3q9/vjHmPVSrIin1652n4uJF1ckuNV1ZtvnXfGSlLjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjDL0kNc7QS1LjOoU+yc4kLyQ5neT+edbfnOS5JOeS3D7P+quSTCf5i+UYWpLU3bqFNkgyATwCfAKYBo4lOVRVJ4c2OwPsBu67wGE+D3x9aaMKoB64Ch58/+i/h6RmLBh6YAdwuqpeBEjyOHAb8G7oq+qlwbp35u6c5OeBHwf+GegtfeS1LZ97naoa7fdIqAdH+i0kraAul26uBV4eejw9WLagJO8D/pQLn+mf3+7uJP0k/ZmZmS6HliR1NOoXYz8DPFVV0xfbqKr2V1WvqnpTU1MjHkmS1pYul25eATYPPd40WNbFR4GbknwGuBLYkOSNqvqRF3QlSaPRJfTHgOuTXMds4O8EfqvLwavqt8//O8luoGfkJWllLXjppqrOAfcAh4FTwBNVdSLJQ0l2ASS5Ick0cAfwaJIToxxaktRdRv0OjsXq9XrV7/fHPcaqlWRl3nWzyn4uJF1ckuNVNe87G70zVpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIaZ+glqXGGXpIa1+UvTGmVSTLS42/cuHGkx5e0sgz9ZWaxfxDEPyIiyUs3ktQ4Qy9JjTP0ktQ4Qy9JjTP0ktQ4Qy9JjTP0ktQ4Qy9pxR08eJDt27czMTHB9u3bOXjw4LhHapo3TElaUQcPHmTv3r0cOHCAG2+8kaNHj7Jnzx4A7rrrrjFP1ybP6CWtqH379nHgwAFuueUW1q9fzy233MKBAwfYt2/fuEdrVlbb7fG9Xq/6/f64x2iGH4Gg1WZiYoI333yT9evXv7vsrbfe4oorruDtt98e42SXtyTHq6o33zrP6CWtqK1bt3L06NH3LDt69Chbt24d00TtM/SSVtTevXvZs2cPR44c4a233uLIkSPs2bOHvXv3jnu0ZvlirKQVdf4F13vvvZdTp06xdetW9u3b5wuxI+Q1+sZ5jV5aG5Z8jT7JziQvJDmd5P551t+c5Lkk55LcPrT8w0meSXIiybeS/OalPw1J0qVYMPRJJoBHgE8B24C7kmybs9kZYDfw5TnLfwj8XlX9LLAT+GKSDyx1aElSd12u0e8ATlfViwBJHgduA06e36CqXhqse2d4x6r6j6F//1eSV4Ep4AdLnlyS1EmXSzfXAi8PPZ4eLFuUJDuADcB35ll3d5J+kv7MzMxiDy1JuogVeXtlkmuAvwZ+v6rembu+qvZXVa+qelNTUysxkiStGV1C/wqweejxpsGyTpJcBfwTsLeqnl3ceJKkpeoS+mPA9UmuS7IBuBM41OXgg+3/Hnisqp689DElSZdqwdBX1TngHuAwcAp4oqpOJHkoyS6AJDckmQbuAB5NcmKw+28ANwO7kzw/+PrwSJ6JJGle3jDVOG+YktYGP9RMktYwQy9JjTP0ktQ4Qy9JjfNjihuRZNHrfJFWWhsMfSOMtqQL8dKNJDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS4wy9JDXO0EtS41bdxxQnmQG+O+45GnI18P1xDyFdgD+fy+eDVTXv32JddaHX8krSv9BnVEvj5s/nyvDSjSQ1ztBLUuMMffv2j3sA6SL8+VwBXqOXpMZ5Ri9JjTP0ktQ4Q9+oJF9K8mqSfx/3LNKwJJuTHElyMsmJJJ8d90yt8xp9o5LcDLwBPFZV28c9j3RekmuAa6rquSQ/BhwHfq2qTo55tGZ5Rt+oqvo68Nq455DmqqrvVdVzg3//D3AKuHa8U7XN0EsamyRbgI8A/zbeSdpm6CWNRZIrgb8D/rCqXh/3PC0z9JJWXJL1zEb+b6rqK+Oep3WGXtKKShLgAHCqqv5s3POsBYa+UUkOAs8AP5NkOsmecc8kDXwM+F3g1iTPD75+ZdxDtcy3V0pS4zyjl6TGGXpJapyhl6TGGXpJapyhl6TGGXpJapyhl6TG/T9ppz6sYfkzjgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "aPath1lHHhtv",
        "outputId": "ff217f3f-da30-4229-96c0-457d5bd320b1"
      },
      "source": [
        "# Check if the model is overfitting or underfitting\n",
        "bp = plt.boxplot([f1_train['rf'],f1_train['rf_smote']])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAALvUlEQVR4nO3df6jd913H8efLZEVwW23MVUqSNvkjcxYdVA9VKLiiFtL+kagDSf9QOubyj5k/UKED2WpA/EcUhKgELbOijWWKRChEcZWCdJKTrStLQsolMnOzQe/6Y1MEa8bbP+7pOLs5ueckPTcn993nAw7c7/f7yfm+/7g88+V7zrknVYUkaev7rkUPIEmaD4MuSU0YdElqwqBLUhMGXZKa2L6oE+/cubP27t27qNNL0pZ09uzZr1fV0qRjCwv63r17GQ6Hizq9JG1JSb5yvWPecpGkJgy6JDVh0CWpCYMuSU1MDXqSp5K8muTL1zmeJH+cZDnJy0l+dP5jSpKmmeUK/TPAgQ2OPwLsHz2OAH/6zseSJN2oqUGvqheA1zdYcgh4utZ8HvjeJHfPa0BJ0mzmcQ99F3B5bHtltO8aSY4kGSYZrq6uzuHUkqS33dIPFlXVCeAEwGAw8A+xS1vRk3feovN849acp5F5BP0KsGdse/don6SODO1tax63XE4BvzR6t8tPAN+oqq/N4XklSTdg6hV6kmeAh4CdSVaATwPvAaiqPwOeAx4FloH/AT66WcNKkq5vatCr6rEpxwv4lblNJEm6KX5SVJKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWpipqAnOZDkYpLlJE9MOH5vkn9J8nKSf02ye/6jSpI2MjXoSbYBx4FHgPuAx5Lct27ZHwBPV9WHgGPA7897UEnSxma5Qn8AWK6qS1X1FnASOLRuzX3A50Y/Pz/huCRpk80S9F3A5bHtldG+cV8Cfn70888B70vyfeufKMmRJMMkw9XV1ZuZV5J0HfN6UfS3gA8n+SLwYeAK8K31i6rqRFUNqmqwtLQ0p1NLkgC2z7DmCrBnbHv3aN+3VdVXGV2hJ3kv8JGqenNeQ0qSppvlCv0MsD/JviR3AIeBU+MLkuxM8vZzfRJ4ar5jSpKmmRr0qroKHAVOAxeAZ6vqXJJjSQ6Olj0EXEzyCvADwO9t0rySpOtIVS3kxIPBoIbD4ULOLUlbVZKzVTWYdMxPikpSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqYmZgp7kQJKLSZaTPDHh+D1Jnk/yxSQvJ3l0/qNKkjYyNehJtgHHgUeA+4DHkty3btnvAM9W1f3AYeBP5j2oJGljs1yhPwAsV9WlqnoLOAkcWremgPePfr4T+Or8RpQkzWL7DGt2AZfHtleAH1+35kngn5J8Avge4GfmMp0kaWbzelH0MeAzVbUbeBT4qyTXPHeSI0mGSYarq6tzOrUkCWYL+hVgz9j27tG+cR8DngWoqheB7wZ2rn+iqjpRVYOqGiwtLd3cxJKkiWYJ+hlgf5J9Se5g7UXPU+vW/Cfw0wBJfoi1oHsJLkm30NSgV9VV4ChwGrjA2rtZziU5luTgaNlvAh9P8iXgGeDxqqrNGlqSdK1ZXhSlqp4Dnlu371NjP58HHpzvaJKkG+EnRSWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmZgp6kgNJLiZZTvLEhON/lOSl0eOVJG/Of1RJ0ka2T1uQZBtwHHgYWAHOJDlVVeffXlNVvzG2/hPA/ZswqyRpA7NcoT8ALFfVpap6CzgJHNpg/WPAM/MYTpI0u1mCvgu4PLa9Mtp3jST3AvuAz73z0SRJN2LeL4oeBj5bVd+adDDJkSTDJMPV1dU5n1qS3t1mCfoVYM/Y9u7RvkkOs8Htlqo6UVWDqhosLS3NPqUkaapZgn4G2J9kX5I7WIv2qfWLknwQuAt4cb4jSpJmMTXoVXUVOAqcBi4Az1bVuSTHkhwcW3oYOFlVtTmjSpI2MvVtiwBV9Rzw3Lp9n1q3/eT8xpIk3Sg/KSpJTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1MVPQkxxIcjHJcpInrrPmF5KcT3Iuyd/Md0xJ0jTbpy1Isg04DjwMrABnkpyqqvNja/YDnwQerKo3knz/Zg0sSZpsliv0B4DlqrpUVW8BJ4FD69Z8HDheVW8AVNWr8x1TkjTNLEHfBVwe214Z7Rv3AeADSf4tyeeTHJjXgJKk2Uy95XIDz7MfeAjYDbyQ5Eeq6s3xRUmOAEcA7rnnnjmdWpIEs12hXwH2jG3vHu0btwKcqqr/q6r/AF5hLfDfoapOVNWgqgZLS0s3O7MkaYJZgn4G2J9kX5I7gMPAqXVr/oG1q3OS7GTtFsylOc4pSZpiatCr6ipwFDgNXACerapzSY4lOThadhp4Lcl54Hngt6vqtc0aWpJ0rVTVQk48GAxqOBwu5NyStFUlOVtVg0nH/KSoJDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJamL7ogfQ7JLc1L9b1BeBS7q1vEK/De3YsYMk1zxu1qTn2rFjxxwnlnQ78Ar9NvTGG29s+lX1O/kPQtLtySt0SWrCoEtSEwZdkprwHvptqD79fnjyzs0/h6RWDPptKL/7zU0/x1133cXrT276aSTdQgb9NnQz73BJ4vvNpXc576FLUhMGXZKamCnoSQ4kuZhkOckTE44/nmQ1yUujxy/Pf1RJ0kam3kNPsg04DjwMrABnkpyqqvPrlv5tVR3dhBklSTOY5Qr9AWC5qi5V1VvASeDQ5o4lSbpRswR9F3B5bHtltG+9jyR5Oclnk+yZy3T6DpP+yNb4H+7a6Jik/ub1oug/Anur6kPAPwN/OWlRkiNJhkmGq6urczr1u0dV3dRD0rvDLEG/Aoxfce8e7fu2qnqtqv53tPnnwI9NeqKqOlFVg6oaLC0t3cy8kqTrmCXoZ4D9SfYluQM4DJwaX5Dk7rHNg8CF+Y0oSZrF1He5VNXVJEeB08A24KmqOpfkGDCsqlPAryY5CFwFXgce38SZJUkTZFH3WAeDQQ2Hw4WcW5K2qiRnq2ow6ZifFJWkJgy6JDVh0CWpCYMuSU0s7EXRJKvAVxZy8p52Al9f9BDSBP5uzte9VTXxgzwLC7rmK8nweq98S4vk7+at4y0XSWrCoEtSEwa9jxOLHkC6Dn83bxHvoUtSE16hS1ITBl2SmjDoW1ySp5K8muTLi55FGpdkT5Lnk5xPci7Jry16pu68h77FJflJ4L+Bp6vqhxc9j/S20fck3F1VX0jyPuAs8LMTvmBec+IV+hZXVS+w9jfopdtKVX2tqr4w+vm/WPvim0nfR6w5MeiSNl2SvcD9wL8vdpLeDLqkTZXkvcDfAb9eVd9c9DydGXRJmybJe1iL+V9X1d8vep7uDLqkTZEkwF8AF6rqDxc9z7uBQd/ikjwDvAj8YJKVJB9b9EzSyIPALwI/leSl0ePRRQ/VmW9blKQmvEKXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2Smvh/fCCxUJLdRTYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EdkY0g-080XY"
      },
      "source": [
        "## Evaluate more and interpret the results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JdqDHjRu4fBh"
      },
      "source": [
        "def plot_confusion_matrix(model_name,norm=True):\n",
        "  confusion_matrix = metrics.confusion_matrix(y_train_val[model_name],np.array(y_train_val_pred[model_name])>0.5)\n",
        "  fmt = 'd'\n",
        "  if norm:\n",
        "    confusion_matrix = confusion_matrix / confusion_matrix.sum(axis=1)[:, np.newaxis]\n",
        "    fmt = '.4f'\n",
        "\n",
        "  ax = sn.heatmap(confusion_matrix, cmap=\"Blues\", annot=True,fmt=fmt)\n",
        "  ax.set_xlabel('Predicted')\n",
        "  ax.set_ylabel('Actual')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "id": "3BAw8ON3W94t",
        "outputId": "c45f604c-ea29-428e-c8c6-fa68be860749"
      },
      "source": [
        "plot_confusion_matrix(model_name='rf',norm=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEGCAYAAACjLLT8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcm0lEQVR4nO3dfZxPdd7H8ddnZty0hsEwgxmJsIXalHRjQ3LPculu1bZ7tdWqdqtVW9Fq7eaqLbHdXNG16da2uk/bFKIk0o1ILRFWlGYwgxDCmJnP9cf8jBlhfjTz+50z3s/H4zz6nXO+53u+p4fevn3P+Z5j7o6IiARbQrwbICIi5VNYi4iEgMJaRCQEFNYiIiGgsBYRCYGkeDfgYI5pf50eU5Hv2Tx/XLybIAFUMwn7oXUcTubs/GTcDz7f4VLPWkQkBALbsxYRiSkLdt9VYS0iApCQGO8WHJLCWkQEwGI+DH1YFNYiIqBhEBGRUFDPWkQkBNSzFhEJAfWsRURCQE+DiIiEgIZBRERCQMMgIiIhoJ61iEgIKKxFREIgUTcYRUSCT2PWIiIhoGEQEZEQUM9aRCQE1LMWEQkB9axFREJA081FREJAwyAiIiGgYRARkRBQz1pEJAQU1iIiIaAbjCIiIaAxaxGRENAwiIhICKhnLSISfKawFhEJPoW1iEgIWILCWkQk8NSzFhEJAYW1iEgIKKxFRMIg2FmtsBYRAfWsRURCISFBMxhFRAJPPWsRkTAIdlYT7H6/iEiMmFnUSxR19Taz5Wa20syGH2D/sWY2y8w+MbNFZta3vDoV1iIiVFxYm1kiMB7oA7QBLjGzNvsVux14wd3bA4OBh8trn4ZBRESo0OnmHYGV7r4KwMyeAwYCS0uVcaBO5HcKsLa8ShXWIiIc3g1GMxsCDCm1aYK7T4j8zgC+LrUvGzhjvyr+Aswws+uBWkD38s6psBYR4fDCOhLME8oteHCXAE+5+9/M7CzgaTNr5+5FBztAYS0iQoU+upcDNC21nhnZVtqVQG8Ad//AzGoCDYC8g1WqG4wiIlTo0yDzgVZm1tzMqlN8AzFrvzJrgPMi5z0RqAlsOFSl6lmLiECFPWft7gVmdh0wHUgEnnD3JWY2Cljg7lnAH4BHzexGim82Xu7ufqh6FdYiIlTsdHN3nwpM3W/byFK/lwKdDqdOhbWICJpuLiISDsHOaoV1LPQ4+0TG3nIhiQkJPPWv9xn75Jtl9h/buB5///NlNKiXzOZvv+OKERPJydsCwJ03DKT3OW0BuOfRN3hpxkIA3np8KMm1agKQVr82Cz77kotvepT+XU9i5LX9KXKnoLCIW8e8xPufrorh1Uq03nt3DqPvuYuiwiIGXXARV/5mSJn9+fn5jLjtVj5fsoSUunW592/3k5GRCcDjjz7CKy+/REJiAsNuu51OPz2H9evWMeK2W/lm0yYw48KLLuYXv/xvAG75w1C+Wr0agG3btlG7dm1emPxqbC844NSzPsolJBgPDL+YfteOIyd3C3Mn3cLrsxezbNX6kjJ33ziISVM+YtJr8+hyemtGXT+AK//0D3r/tC2nnNiUMwbfQ41qScx47PdMf28p23bsovuVD5Qc/+zYq3jtnUUAzJq3nNffWQxAu1ZN+OfoKzjl/Dtje9FSrsLCQv561ygeefRJ0tPTufTnF9L13G4c37JlSZlXXn6ROnXq8PobbzJt6hQeuG8sY/72AF+sXMkbU6cwOWsKeXm5XH3Vr8maMp3EpERuvnU4J7Zpy44d2xl80QWceVYnjm/ZkjF/2/fnZey995CcnByPyw60oId1pT26Z2YnmNkwM/vfyDIs8ojKUeX0dsfxxdcb+TJnE3sKCnlx+kL6dz25TJkTWjRm9kfLAZg9fwX9u54EwIktGjF34UoKC4v4blc+i/+TQ8+zy/4rrF2rJl1Ob81rs4rDesfO/JJ9tY6pwaHvL0u8fLZ4EU2bNiOzaVOqVa9O7779eGfWzDJlZr39NgMGDgKgR89efPThB7g778yaSe++/ahevTqZmU1p2rQZny1eRMOGaZzYpvj/wmrVSqZFixbk5eWWqdPdmTF9Gn369Y/NhYZIRb7IqTJUSlib2TDgOYpHgT6KLAY8e6A3UFVlTdJSyM7dXLKek7uZjIYpZcosXpHDwG6nADCw20+ok3wM9VNqsWhFcTgfU7MaqXVr0aVDazIb1Stz7M/OPZl3PlrOth27SrYNOPdkPp18O5P/9xquuWNSJV6dHKm83FwaNW5Usp6Wnk5ubtlgzcvLpVGjxgAkJSWRXLs2W7ZsJjc3l/RG+45Nb5RO3n7H5uRks+zzzznp5J+U2b7w4wWkpqbSrNlxFXxF4WcJFvUSD5U1DHIl0Nbd95TeaGb3AUuAew50UOn59kmZXUlq0LaSmhcst93/CvcPu4jLBpzBewtXkpO7mcLCImZ+uIzT2jZj1lN/YOPm7cxbtJrCwrKzUS/ufRpPvfJBmW1ZsxaRNWsRnU49npG/7Ue/a8bF8nIkzr7bsYM/DL2BW4b/8XvDHdOmvk7vvupVH8jROgxSBDQ5wPbGkX0H5O4T3L2Du3eoKkG9Nm8rmen7esMZ6fXI2bC1TJl1G7Yy+ObHOOuS0fx53GsAbN2+E4B7H5/OmYPvof+14zAz/rNm32zU1Lq16ND2OKa9+9kBz/3ewi9ontGA1Lq1Kvqy5AdKS09n/bp99y3ycnNJT08vWyYtnfXr1wFQUFDA9m3bqFu3Hunp6eSu33ds7vpc0iLH7tmzh5uG3kDffj+je4+eZeorKChg5ltv0rt3ua9OPiodlcMgwFBgpplNM7MJkeUNYCbw+0o6ZyAtWPIVLY9tSLMmqVRLSuSiXqcyJXIzcK/UurVK/gDcckUvJr76IVB8c7J+SnHQtmvVhHatmvDWB8tKjhvUvT3T3v2M3fkFJdtaNG1Q8vuUEzKpUT2JTVt2VNr1yZFp2+4k1qz5kuzsr9mTn88bU6fQ5dxuZcp0PbcbWa++AsCbM6bT8YwzMTO6nNuNN6ZOIT8/n+zsr1mz5kvanXQy7s5fRo6gRYsW/OryX3/vnPM+eJ/mzVuUGUKRfcyiX+KhUoZB3P0NM2tN8XtdMyKbc4D57l5YGecMqsLCIm4c/QKvPfw7EhOMia9+yOer1vOna/uxcOkapsxeTOcOrRh1/QDcYe7ClQy9+wUAqiUl8tYTQwHYtn0XV4yYWGYY5KJepzH2yRllzjfovFO4tP8Z7CkoZNfuPfxy2BOxu1iJWlJSEreNGMm1Q66iqKiQ/xp0AS1btmL8Qw/Stm07unY7j0EXXMiI4bfQv3cP6qSkcO/Y+wFo2bIVPXv3YdCAviQmJvLH20eSmJjIwo8X8HrWq7Rq3ZqLzx8IwPVDb+Kczl0AeGPaVHr37Re3aw66oA+DWDnT0ePmmPbXBbNhEleb52v8Xb6vZtIPn9Ly42HTo86c5aN7xTzZ9Zy1iAjxG96IlsJaRITie0RBprAWEUE9axGRUAj6DUaFtYgI6lmLiIRCRX58oDIorEVEUM9aRCQUNGYtIhICAc9qhbWICKhnLSISCgHPaoW1iAhoBqOISChoGEREJAQCntUKaxERUM9aRCQUAp7VCmsREdANRhGRUNAwiIhICCisRURCIOBZrbAWEQH1rEVEQiHgWa2wFhEBPQ0iIhIKCQHvWgf7OzYiIjFiFv1Sfl3W28yWm9lKMxt+kDIXm9lSM1tiZs+UV6d61iIiVNwNRjNLBMYDPYBsYL6ZZbn70lJlWgG3AZ3cfbOZpZVXr3rWIiJAgkW/lKMjsNLdV7l7PvAcMHC/Mr8Bxrv7ZgB3zyu3fYd/SSIiVU9CgkW9mNkQM1tQahlSqqoM4OtS69mRbaW1Blqb2Xtm9qGZ9S6vfRoGEREBjOiHQdx9AjDhB5wuCWgFdAUygTlmdpK7bznYAepZi4hQocMgOUDTUuuZkW2lZQNZ7r7H3VcDKygO74O37/AuR0SkajKzqJdyzAdamVlzM6sODAay9ivzL4p71ZhZA4qHRVYdqlINg4iIUHEzGN29wMyuA6YDicAT7r7EzEYBC9w9K7Kvp5ktBQqBW9x906HqVViLiFCxk2LcfSowdb9tI0v9duCmyBIVhbWICJpuLiISCgGfba6wFhGB4L8bRGEtIgKH8ZR1fBw0rM3sIcAPtt/db6iUFomIxEGYPz6wIGatEBGJs4DfXzx4WLv7xFg2REQknkL/NIiZNQSGAW2Amnu3u3u3SmyXiEhMBX0YJJrp5pOAz4HmwB3AlxRPpxQRqTIq8N0gldO+KMqkuvvjwB53n+3uVwDqVYtIlVKB7wapFNE8urcn8s91ZtYPWAvUr7wmiYjEXrAHQaIL6zvNLAX4A/AQUAe4sVJbJSISY4lhv8Ho7q9Hfm4Fzq3c5oiIxEfQbzBG8zTIkxxgckxk7FpEpEoIeFZHNQzyeqnfNYFBFI9bi4hUGaF/N4i7v1x63cyeBeZWWotEROIg4Fl9RC9yagWkVXRDvqfxIT9HJkepTdvz490ECaCMutV/cB1VYcx6G2XHrNdTPKNRRKTKSAx7WLt77Vg0REQkngL+5F75MxjNbGY020REwizo080P9T7rmsCPgAZmVo99E3zqABkxaJuISMyEecz6amAo0AT4mH1h/S0wrpLbJSISU0EfBjnU+6wfBB40s+vd/aEYtklEJOYC3rGO6q17RWZWd++KmdUzs99WYptERGIuySzqJR6iCevfuPuWvSvuvhn4TeU1SUQk9syiX+IhmkkxiWZm7u4AZpYI/PAn0EVEAiT0082BN4DnzeyRyPrVwLTKa5KISOwFPKujCuthwBDgmsj6IqBRpbVIRCQOQvs0yF7uXmRm84DjgYuBBsDLhz5KRCRcQvvxATNrDVwSWTYCzwO4uz5AICJVTsCz+pA962XAu0B/d18JYGb6nJeIVEkW8K8wHurRvfOBdcAsM3vUzM4j+N+UFBE5IkF/N8hBw9rd/+Xug4ETgFkUTz1PM7P/M7OesWqgiEgshDas93L3He7+jLv/DMgEPkHvsxaRKsbMol7i4bC+FBOZvTghsoiIVBmJ0cznjqOAN09EJDYSzKJeymNmvc1suZmtNLPhhyh3gZm5mXUor84j+QajiEiVU1Fj0ZFXcowHegDZwHwzy3L3pfuVqw38HpgXVfsqpnkiIuFWgS9y6gisdPdV7p4PPAcMPEC5/wFGA7uiaZ/CWkQESMCiXsxsiJktKLUMKVVVBvB1qfVs9vu6lpmdCjR19ynRtk/DICIiHN6LnNz9iB+0MLME4D7g8sM5TmEtIgIkVdwD1DlA01LrmZFte9UG2gHvRB4DbARkmdkAd19w0PZVVOtERMKsAh+fng+0MrPmFIf0YODSvTvdfSvFL8SLnNfeAW4+VFCDwlpEBKi4jw+4e4GZXQdMBxKBJ9x9iZmNAha4e9aR1KuwFhGhYj8+4O5Tgan7bRt5kLJdo6lTYS0iQvAfjVNYi4hQNb7BKCJS5SmsRURCINhRrbAWEQGqxtfNRUSqvHi9pzpaCmsREfQ0iIhIKOgGo4hICGgYREQkBDQMIiISAupZi4iEQLCjWmEtIgJAonrWIiLBF/CsVliLiABYwAdCFNYiIqhnLSISCgnqWYuIBJ961iIiIaDp5iIiIZAQ7KxWWIuIgJ4GEREJhYCPgiisY6HHac0Ye3UXEhOMp6YvYeyLC8rsPzatNn8f2oMGKcewedsurhgznZxN2wH4xXknMnxwRwDuee4jJs38nGNqJDHptr60aJxCYZEzdd5q/vTUewBUT0rk8Zt70r5lGt9s28Vld09lTd622F6wROWjD+Yy7r7RFBUV0nfA+Vz631eV2Z+fn889d/yRFcuWUielLiPvHEOjJhns2bOH++6+gxXLlmCWwHU3DeeU004HYOb0qTwz8VHMjNQGafzxjrtJqVuPb7du5X9uv5n1a9fSqEkTRt41ltp1UuJx2YEV9J510F80FXoJCcYDv+3KwJH/ov01T3NRl9ac0LR+mTJ3X3kOk2Z+TsffTeKvz85j1K/PBqBecg1GXHoGnW98jnNufI4Rl55B3eQaADwweSGnXP00Z17/DGe1aUzPDs0AuLxXWzZv3027qyby0CufcNcVP43tBUtUCgsLeXDMXdzzwMM8+dyrvD1jGl+u+qJMmWlZk6lduw7/fHkqFw7+JRPG3w/AlH+9BMDjz7zCmIcm8H8PjqGoqIjCggLG3z+a+x5+gscmTaZFy9a88uKzADz7j8dp3+EMnn55Cu07nMGz/3g8thccAgkW/RKX9sXntEeP01un88XarXy5/lv2FBTx4pwV9D+rRZkyJxxbn9n//hqA2f/Opv+Zxft7nNaMmZ+sYfP23WzZvpuZn6yh52nN2Lm7gDmLsgHYU1DEp1/kkZGaDED/M1sw6a2lAEye+x+6/qRprC5VDsOypYvJyDyWJhlNqVatGt169OH9ObPKlHlvzix69hsAQJduPVg4fx7uzlerv6B9hzMAqFc/leTadVj++RIcx93ZuXMn7s53O7bToEHDkrp69RsIQK9+A5k7u+y5pPhpkGiXuLQvLmc9ijRJTSZ7475hiJyN20uCda/FqzcysFNLAAaefTx1flSD+rVrfv/YTdtpst+xKbWq07djC2ZFwr5Jai2yNxQPoRQWOd9+t5vUOjUr5drkyG3MyyMtvVHJeoO0dDZsyC1bZkMeaWnFZRKTkqiVnMy3W7dwfKsf8/67sygsKGDd2mxWLFvKhtz1JCVVY+itt3PVpedzUb9ufLn6C/oMOB+Azd9sIjUS3PVTG7D5m00xutLwsMNY4iHmYW1mvz7EviFmtsDMFhSseT+WzYqr2x57l3PaZfDBQ5dwzkkZ5GzcRmFRUbnHJSYYE4f14eGsT/ly/bcxaKkEQZ+fDaJhWjrXXD6Y8feNpu1JPyEhMYGCgj1kTX6BR55+kRenvE2Llq15ZuJj3zvezAJ/My0egt6zjscNxjuAJw+0w90nABMAjun7oMeyUZVl7abtZDaoXbKe0SC55ObhXuu+2cHgu6YAUKtmNf6rU0u27shn7abtnHNS5r5jU5N5d3F2yfr4G87ji5wtjHv101Ln20Fmw+JzJCYYdX5Ug03f7qqsy5Mj1CAtjbzc9SXrG/NyadgwvWyZhmnk5a2nYXojCgsK2LF9O3VS6mJm/O7GYSXlrrvqMjKbHsfKFcsByMgsHvrq2r0Xz04sHpuuVz+VTRs3kNqgIZs2bqBuvdTKvsTQCfrfX5XSszazRQdZFgPp5VZQhSxYkUvLJnVpll6HakkJXNS5NVM+XFWmTGqdmiU9nVsu7sDEGcVjzm9+/BXdTz2Wusk1qJtcg+6nHsubH38FwJ9/dRYptWpw84TZZeqaMm8Vv+jeBoDzf9qK2Yu+ruQrlCNxwontyPn6K9atzWbPnj28/eY0zurctUyZs8/pyowpWQDMfvtN2nfoiJmxa9dOdu78DoAF894nMTGR41ocT4OGaXy1+gu2bP4GgI/nfUCz5i1K6po+5VUApk95lU6dz43RlYZIwMdBzL3iO7Bmlgv0Ajbvvwt4392blFdHVelZA/TqcBxjru5cPGwxYyn3Pj+fP112Jgv/k8uUeasZ1Kkloy7vhOPM/SyHoePfIb+gEIBf9WjDrT8vfixr9PPzefrNpWSkJrPy6StZtuYbdu8pLvf31//NU9OXUKNaIk/c3IufHN+Qzdt28cvR06rUEMnKZ66NdxMqzIfvzeHh+++lsKiQPj8bxGW/HsKTj4yj9Ylt6dT5XPJ37+avf7mNlSuWUbtOCn+6816aZDRl/docbv39NSQkGA0apnHziFE0alz8n1TW5BeY/Pw/SUpKIq1RE4aNvJOUlLps3bqFUX+8mbz160hv3JiRd/2NOilV59G9jLrVf3CEfrRqa9SZ07FFSswju7LC+nHgSXefe4B9z7j7peXVUZXCWipOVQprqTgVEdbzDyOsT49DWFfKmLW7X3mIfeUGtYhIzAV80FozGEVECP4MRoW1iAh6N4iISCgEPKs1g1FEBPZOFopuiaKu3ma23MxWmtnwA+y/ycyWRh5pnmlmzcqrU2EtIkLxMEi0y6HrsURgPNAHaANcYmZt9iv2CdDB3U8GXgLuLa99CmsRESp0TkxHYKW7r3L3fOA5YGDpAu4+y92/i6x+CGRSDoW1iAgcVlqXfo9RZBlSqqYMoPTU4ezItoO5EphWXvN0g1FEhMN7dK/0e4x+0DnNLgM6AF3KK6uwFhGhQh/dywFKv0g+M7Jtv/NZd2AE0MXdd5dXqYZBRESouBuMwHyglZk1N7PqwGAgq+y5rD3wCDDA3fOiaZ961iIiVNwMRncvMLPrgOlAIvCEuy8xs1HAAnfPAsYAycCLkUcB17j7gEPVq7AWEaFiZzC6+1Rg6n7bRpb63f1w61RYi4gQ/BmMCmsREQh8WiusRUQgbt9WjJbCWkSEwHesFdYiIkDg01phLSKCPj4gIhIKAR+yVliLiEDgR0EU1iIiQFQfFYgnhbWICBoGEREJhYBntcJaRAQIfForrEVE0KN7IiKhoDFrEZEQSFBYi4iEQbDTWmEtIoKGQUREQiHgWa2wFhEB9axFREJB081FREIg2FGtsBYRATQMIiISCprBKCISBsHOaoW1iAgEPqsV1iIiAAkBH7RWWIuIEPwbjAnxboCIiJRPPWsREYLfs1ZYi4igR/dEREJBPWsRkRBQWIuIhICGQUREQkA9axGREAh4ViusRUSAwKe1wlpEhOBPNzd3j3cbpBxmNsTdJ8S7HRIs+nNxdNF083AYEu8GSCDpz8VRRGEtIhICCmsRkRBQWIeDxiXlQPTn4iiiG4wiIiGgnrWISAgorEVEQkBhHXBm1tvMlpvZSjMbHu/2SPyZ2RNmlmdmn8W7LRI7CusAM7NEYDzQB2gDXGJmbeLbKgmAp4De8W6ExJbCOtg6AivdfZW75wPPAQPj3CaJM3efA3wT73ZIbCmsgy0D+LrUenZkm4gcZRTWIiIhoLAOthygaan1zMg2ETnKKKyDbT7Qysyam1l1YDCQFec2iUgcKKwDzN0LgOuA6cDnwAvuviS+rZJ4M7NngQ+AH5tZtpldGe82SeXTdHMRkRBQz1pEJAQU1iIiIaCwFhEJAYW1iEgIKKxFREJAYS2VwswKzexTM/vMzF40sx/9gLqeMrMLI78fO9TLrMysq5mdfQTn+NLMGhxpG0Uqm8JaKstOdz/F3dsB+cA1pXeaWdKRVOruV7n70kMU6QocdliLBJ3CWmLhXaBlpNf7rpllAUvNLNHMxpjZfDNbZGZXA1ixcZH3eL8FpO2tyMzeMbMOkd+9zWyhmf3bzGaa2XEU/6VwY6RXf46ZNTSzlyPnmG9mnSLHpprZDDNbYmaPARbbfyUih+eIejci0Yr0oPsAb0Q2nQq0c/fVZjYE2Orup5tZDeA9M5sBtAd+TPE7vNOBpcAT+9XbEHgU6Bypq767f2Nmfwe2u/vYSLlngPvdfa6ZHUvxbNATgT8Dc919lJn1AzQLUAJNYS2V5Rgz+zTy+13gcYqHJz5y99WR7T2Bk/eORwMpQCugM/CsuxcCa83s7QPUfyYwZ29d7n6w9zt3B9qYlXSc65hZcuQc50eOnWJmm4/wOkViQmEtlWWnu59SekMkMHeU3gRc7+7T9yvXtwLbkQCc6e67DtAWkdDQmLXE03TgWjOrBmBmrc2sFjAH+HlkTLsxcO4Bjv0Q6GxmzSPH1o9s3wbULlVuBnD93hUz2/sXyBzg0si2PkC9CrsqkUqgsJZ4eozi8eiFkY+/PkLx/+29Avwnsu8fFL9hrgx33wAMASab2b+B5yO7XgMG7b3BCNwAdIjcwFzKvqdS7qA47JdQPByyppKuUaRC6K17IiIhoJ61iEgIKKxFREJAYS0iEgIKaxGREFBYi4iEgMJaRCQEFNYiIiHw/8mvpxMfFXLbAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "id": "lKoRMGFwXF87",
        "outputId": "852f1d73-d75e-40d8-8bf0-c1a43790801d"
      },
      "source": [
        "plot_confusion_matrix(model_name='rf_smote',norm=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEGCAYAAACjLLT8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcOElEQVR4nO3deXwW1b3H8c8vCRFEdkgCBClIqAJatIi2IiAKROVCKejF1u1qi9WLC1rFpaWKltalWlvxKoLVat0ViYKgIqsVgWJlc4uIJMEEkB2VQPK7f+QhJhSSJ5jneWbC9/16zcvMzJkzZ3yFL4czc2bM3RERkWBLSnQDRESkegprEZEQUFiLiISAwlpEJAQU1iIiIZCS6AYcSIPjR+kxFfkPmxc/kOgmSADVT8G+ax01yZyv33vgO5+vptSzFhEJgcD2rEVE4sqC3XdVWIuIACQlJ7oFVVJYi4gAWNyHoWtEYS0iAhoGEREJBfWsRURCQD1rEZEQUM9aRCQE9DSIiEgIaBhERCQENAwiIhIC6lmLiISAwlpEJASSdYNRRCT4NGYtIhICGgYREQkB9axFREJAPWsRkRBQz1pEJAQ03VxEJAQ0DCIiEgIaBhERCQH1rEVEQkBhLSISArrBKCISAhqzFhEJAQ2DiIiEgHrWIiLBZwprEZHgU1iLiISAJSmsRUQCTz1rEZEQUFiLiISAwlpEJAyCndUKaxERUM9aRCQUkpI0g1FEJPDUsxYRCYNgZzXB7veLiMSJmUW9RFFXtpl9ZGa5ZnbjfvYfaWazzew9M1tmZmdVV6fCWkSE2gtrM0sGJgBnAl2A88ysyz7FfgM85+7HAyOAB6trn4ZBRESo1enmPYFcd18NYGbPAEOAVRXKONA48nMTYF11lSqsRUSo2Q1GMxsJjKywaaK7T4z83BbIq7AvHzhpnypuBV43syuBhsAZ1Z1TYS0iQs3COhLME6steGDnAY+5+5/M7EfAE2bWzd1LD3SAwlpEhFp9dK8AaFdhPTOyraJLgWwAd3/HzOoDLYH1B6pUNxhFRKjVp0EWA1lm1sHMUim7gZizT5m1wOmR8x4D1Ac2VFWpetYiIlBrz1m7+x4zGwXMBJKBR919pZmNA5a4ew5wHfCImY2m7Gbjxe7uVdWrsBYRoXanm7v7dGD6PtvGVvh5FXBKTepUWIuIoOnmIiLhEOys1g3GeOj/42N4f8pvWTH1d/z6f/r/x/4jWzdj+kNXsujZm5j5yNW0TWtavu+Oq4aw5PmbWfL8zQwfcEKl42793/9i2ctjee/F33DFeX3Kt//phuGsmPo7Fj17E92Pzozdhcl38vb8eQw+eyCDsvsz+ZH/fAqsuLiY66+7hkHZ/fn5iHMoKMgv3zf5kYcZlN2fwWcP5O0F8ysdV1JSwrnDfsKoKy4r35afn8fPR5zDoOz+XH/dNewuLo7dhYVUbU43jwWFdYwlJRl/vvFchox6kOOH3cE52T/k6I4Zlcr8YfRQ/jFtET3/+w+Mn/ga464cDEB2r650P6YdJ434I70vuIdrLjydRg3rA3DB4JPJzGjKD4bezvHD7uD5Gf8CYGCvLhx1ZCu6DbmNUXc8zV9uHhHfC5aolJSUMP7343jwoUlMyZnGjOmv8mlubqUyU158nsaNG/PqjDc4/8KL+fO99wDwaW4uM6ZP46WcaTz48CTG33EbJSUl5cf944m/07HjUZXquv/eezj/wot5dcYbNG7cmCkvvRD7iwyZQzaszexoMxtjZn+JLGMij6gcUk7s9j0+zdvImoIv2b2nhOdnLmVQ3+MqlTm6Y2vmLvoIgLmLP2ZQ32MBOKZjBguW5lJSUspX3xSz/JMCBvy47H/hyHN6MX7ia+y9gbxh8w4ABvU5jqdeXQTAouVraNKoARktGyPBsmL5Mtq1a09mu3bUS00l+6yzmTN7VqUys996i8FDhgLQf8BAFi18B3dnzuxZZJ91NqmpqWRmtqNdu/asWL4MgKLCQubPm8PQYcPL63F3Fr27kP4DBgIweMhQ3ppV+VxyiIa1mY0BnqFsFGhRZDHg6f29gaoua5PWhPyizeXrBUWbaduqSaUyyz8uYEi/7gAM6fcDGh/RgOZNGrLs47JwblC/Hi2aNqRPj85kZjQDoENmK4YP+CEL/nEDLz9wOUcd2SpyvqbkF1Y83xbaVBhWkWBYX1RERutv/4WVlp5OUVFR5TLri8jIaA1ASkoKRzRqxJYtmykqKiI949tj0zPSWR859q4/jmf0dddXerJhy5bNNGrUmJSUsltU6ekZrF9f+VxS9m6QaJdEiNUNxkuBru6+u+JGM7sXWAn8cX8HVZxvn5LZl5SWXWPUvGC56b4p3DfmHM4ffBJvL82loGgzJSWlzFr4IT/s2p7Zj13Hxs07eHfZZ5SUlM1GPSw1hV3Fu+n187sY0u8HPPy7n3PGpX9O8JVIIs2dM5vmzZvTpWs3Fi96N9HNCZ1D9WmQUqAN8Pk+21tH9u1Xxfn2DY4fVeUD4mGxbv1WMtObla+3TW9GwYatlcp8sWErI349CYCGDVL5yend2brjawDumjyTuybPBOCx8Rfzydqy2agFRZt5edb7AEx9630evvX8yPm2lPe+y87XlHXrt8To6uRgpaWnU/hFYfn6+qIi0tPTK5dJS6ew8AvSMzLYs2cPO7Zvp2nTZqSnp1NU+O2xRYVFpKWnM2f2W8yZ8xYL5s9j165d7Ny5g5vG/Jrxf7yb7du3sWfPHlJSUigqKiQtrfK5JPhhHasx62uAWWb2mplNjCwzgFnA1TE6ZyAtWfk5nY5sRfs2LaiXksw5A09g2pxllcq0aNqw/Bfl+ksG8vjUhUDZzcnmTRoC0C2rDd2y2vDmOx8C8MqcZfQ5MQuAU3+YRW4kxKfNXc7PBvUEoOex32Pbjq8p3Lgt9hcqNdK127GsXbuG/Pw8dhcXM2P6NPqc1q9Smb6n9SNn6hQA3nh9Jj1POhkzo89p/ZgxfRrFxcXk5+exdu0auh17HFePvo433prHa2+8xZ333MuJJ53MH+68BzPjxJ4n8cbrZX/p50ydwmn9+v1Hmw51ZtEviRCTnrW7zzCzzpS917VtZHMBsNjdSw58ZN1TUlLK6Duf45UH/5fkJOPxqQv5YHUhv738bJauWsu0ucvp3SOLcVcOxh0WLM3lmj88B0C9lGTefPQaALbv+IZLbnm8fBjknkff4G/jL+LKn/dj59e7uHzcUwDMWLCSgb26sjLnd3z1zW4uu/XJxFy4VCklJYWbbhnL5SN/QWlpCT8ZOoxOnbKY8Nf76dq1G337nc7QYcO55cbrGZTdn8ZNmnDXPfcB0KlTFgOyz2To4LNITk7m5t+MJTk5ucrzXXPt9dzw69FM+MufOfqYYxg67Jx4XGaoBL1nbdVMR0+YujIMIrVr8+IHEt0ECaD6Kd99Ssv3x8yMOnM+unNg3JNdMxhFREjc8Ea0FNYiIpTdIwoyhbWICOpZi4iEQtBvMCqsRURQz1pEJBRq8+MDsaCwFhFBPWsRkVDQmLWISAgEPKsV1iIioJ61iEgoBDyrFdYiIqAZjCIioaBhEBGREAh4ViusRURAPWsRkVAIeFYrrEVEQDcYRURCQcMgIiIhoLAWEQmBgGe1wlpEBNSzFhEJhYBntcJaRAT0NIiISCgkBbxrHezv2IiIxIlZ9Ev1dVm2mX1kZrlmduMBypxrZqvMbKWZPVVdnepZi4hQezcYzSwZmAD0B/KBxWaW4+6rKpTJAm4CTnH3zWaWVl296lmLiABJFv1SjZ5Arruvdvdi4BlgyD5lfglMcPfNAO6+vtr21fySRETqnqQki3oxs5FmtqTCMrJCVW2BvArr+ZFtFXUGOpvZ22a20Myyq2ufhkFERAAj+mEQd58ITPwOp0sBsoC+QCYwz8yOdfctBzpAPWsREWp1GKQAaFdhPTOyraJ8IMfdd7v7Z8DHlIX3gdtXs8sREambzCzqpRqLgSwz62BmqcAIIGefMi9T1qvGzFpSNiyyuqpKNQwiIkLtzWB09z1mNgqYCSQDj7r7SjMbByxx95zIvgFmtgooAa539y+rqldhLSJC7U6KcffpwPR9to2t8LMD10aWqCisRUTQdHMRkVAI+GxzhbWICAT/3SAKaxERqMFT1olxwLA2s78CfqD97n5VTFokIpIAYf74wJK4tUJEJMECfn/xwGHt7o/HsyEiIokU+qdBzKwVMAboAtTfu93d+8WwXSIicRX0YZBoppv/A/gA6ADcBqyhbDqliEidUYvvBolN+6Io08LdJwO73X2uu18CqFctInVKLb4bJCaieXRvd+S/X5jZ2cA6oHnsmiQiEn/BHgSJLqzvMLMmwHXAX4HGwOiYtkpEJM6Sw36D0d1fjfy4FTgtts0REUmMoN9gjOZpkL+xn8kxkbFrEZE6IeBZHdUwyKsVfq4PDKVs3FpEpM4I/btB3P3Fiutm9jSwIGYtEhFJgIBn9UG9yCkLSKvthuzLOp4Q61NICK3b/E2imyAB1LFV/eoLVaMujFlvp/KYdSFlMxpFROqM5LCHtbs3ikdDREQSKeBP7lU/g9HMZkWzTUQkzII+3byq91nXBw4HWppZM76d4NMYaBuHtomIxE2Yx6wvA64B2gD/4tuw3gY8EON2iYjEVdCHQap6n/X9wP1mdqW7/zWObRIRibuAd6yjeuteqZk13btiZs3M7IoYtklEJO5SzKJeEiGasP6lu2/Zu+Lum4Ffxq5JIiLxZxb9kgjRTIpJNjNzdwcws2QgNbbNEhGJr9BPNwdmAM+a2cOR9cuA12LXJBGR+At4VkcV1mOAkcCvIuvLgIyYtUhEJAFC+zTIXu5eambvAkcB5wItgRerPkpEJFxC+/EBM+sMnBdZNgLPAri7PkAgInVOwLO6yp71h8B8YJC75wKYmT7nJSJ1kgX8K4xVPbr3U+ALYLaZPWJmpxP8b0qKiByUoL8b5IBh7e4vu/sI4GhgNmVTz9PM7P/MbEC8GigiEg+hDeu93H2nuz/l7v8FZALvofdZi0gdY2ZRL4lQoy/FRGYvTowsIiJ1RnI087kTKODNExGJjySzqJfqmFm2mX1kZrlmdmMV5YaZmZtZj+rqPJhvMIqI1Dm1NRYdeSXHBKA/kA8sNrMcd1+1T7lGwNXAu1G1r3aaJyISbrX4IqeeQK67r3b3YuAZYMh+yt0O3AlE9RVohbWICJCERb2Y2UgzW1JhGVmhqrZAXoX1fPb5upaZnQC0c/dp0bZPwyAiItTsRU7uftAPWphZEnAvcHFNjlNYi4gAKbX3AHUB0K7CemZk216NgG7AnMhjgBlAjpkNdvclB2xfbbVORCTMavHx6cVAlpl1oCykRwA/27vT3bdS9kK8yHltDvDrqoIaFNYiIkDtfXzA3feY2ShgJpAMPOruK81sHLDE3XMOpl6FtYgItfvxAXefDkzfZ9vYA5TtG02dCmsREYL/aJzCWkSEuvENRhGROk9hLSISAsGOaoW1iAhQN75uLiJS5yXqPdXRUliLiKCnQUREQkE3GEVEQkDDICIiIaBhEBGREFDPWkQkBIId1QprEREAktWzFhEJvoBntcJaRATAAj4QorAWEUE9axGRUEhSz1pEJPjUsxYRCQFNNxcRCYGkYGe1wlpEBPQ0iIhIKAR8FERhHQ/9u7fl7ktOJjnJeGzWx/xpyrJK+zNbNuSRK3vT9PBUkpKNsU8uYebSfAC6tW/GXy87hUaH16O01Dl1zCvs2l1CvZQk7vvFjzi1awal7tz61L+YuvBzUlOSmHRVb47v2JJN23dxwb2zWbthRyIuW6qxZOHbPHT/nZSWlpI9aCjnXnBppf3L//0vHv7LXXz26SfceOudnHpafwCKCtdx+82j8VJnz57dDB5+Hmf/5NxKx9465ioK1+Xz0BMvAbB921b+MPYGigrXkZ7RhpvG3U2jxo3jc6EhoZ71IS4pybjvlz9i0LiZFHy5k/l3Dmba4rV8mL+lvMyNw7vz0j8/45GZH3J0ZlOm3NKfYy5/nuQkY/LVffjF/fNY/vkmmh9xGLtLSgEYM+wHbNj6NT+48kXMoPkRhwFw8emd2bKjmGNHvcDwUzpwxwU9uPDeOYm4dKlCSUkJE+4dz/j7HqZlWjpX/+JnnNSrL+07HFVeJi09g+tuvp0Xn3680rHNW7Ti3oeeIDU1la+/+opfXTiMk3v1pUXLNADenvsmDRocXumY5558lO4/7Mm5F1zKc09M5rknJ3PpFaNjf6EhEvQx66C/FTD0enRqyaeF21hTtJ3de0p5YcFqBp14ZKUy7k6jBvUAaHx4Pb7Y9BUAZ3Rvy4o1m1j++SYANu3YRWmpA3BhvyzufmlZ5Hj4cvsuAM7ueSRPzvkEgCnvrKHvsW1ifo1Scx9/sII2me1o3TaTevXq0eeMbBYumFOpTHrrtnTo1BlLqvzHtF69eqSmpgKwe3cxXlpavu/rr77ipWeeYMRFv6x0zDvzZ3PGmYMBOOPMwbwzf3YMrircksyiXhJBPesYa9O8IQUbd5avF2zayYlZrSqV+f2z75EzdiCXn9WFww9LYdBtMwDo1LoxDkz97QBaNa7P8ws+476py2lyeNkf1LHnnUDvrhmsLtzOtZPeYf3Wbyqdr6TU2fZVMS0aHVYe5hIMGzesp1VaRvl6y1ZpfLRqedTHbygqZOwNo/giP49Lrxhd3qv++6QJ/HTEhdSvX79S+S2bN9G8ZdnvXbMWLdmyeVMtXEXdEvCOdfx71mb2P1XsG2lmS8xsyZ7P5sazWQl1zqkdeXJ2Llkjn2Xo719n0lW9MYOU5CR+fHQ6l/x5LqffMo3BJ7Wn77GtSUk2MlsewcIP1/Pj63N49+P1jL+oZ6IvQ+KoVXoG//f4C0x+9hXenJHD5k1f8uknH/JFQR6n9Dm9ymPNgj46mxhB71knYhjktgPtcPeJ7t7D3XukdOgTzzbFzLpNO2nbsmH5etvmDVn35VeVylx0emde/OdnACz6eAP1U1No2ag+BV/uZMGqQr7cvouvi0uYuTSP7h1b8OX2Xez8ZjdT310DwEv/XEP3ji3+43zJSUbjw1PVqw6glq3S2LC+sHx944b1tGiVXuN6WrRMo32HTqx4fykfrFjGJx+u4qLhZ3LdFRdTkPc5N4wqu2nZtFlzNm3cAMCmjRto0qx57VxIHWI1WBIhJmFtZssOsCwHav4bGWL/yt1Ip9ZNaJ92BPVSkhjeqyPTlqytVCZ/w05OO641AN9v24T69ZLZsO0b3vx3Ad3aN6NBajLJSUavrq35MK/sxuT0JXn07lp2zGnHVdi+OI/z+2YBMPRH32Puii/idalSA52P7sq6vLUUrstn9+7dzH1zBiefEl0HZcP6Inbt+gaA7du2sWrZe2Qe+T0GDT2Xf0x9k8dfeI0/PfgYbdu1564HJgNwcq++vPlaDgBvvpbDj049LTYXFmYBT+tYjVmnAwOBzftsN+CfMTpnIJWUOtdOeoec3w4kOcn4+1uf8EHeFn474niW5m5k2pI8bnx8ERMuP4VRg7qBOyMfmAfAlp3F/OWVlcy/azDuMHNpHjMij/T95snFTL6qD3ddchIbt37DZRPmA/DYrI+ZfFVvlj8wnM07dnHhfXMSdelSheSUFC6/9iZ+c+3llJSWMuDsn9C+Yyf+PmkCnY/uysm9+vLRByu4/ebR7Ni+jXffnsuTkx/k4SenkPf5ah554E8YhuP89LyL6HBUVpXnO/f8Sxg/9npmTnuZtPTW3Hz73XG60vAI+nRzc/far9RsMvA3d1+wn31PufvPqqvj8GGP1n7DJPRWPFTtr44cgjq2qv+dk3bx6q1RZ86JHZvEPdlj0rN290ur2Kc/bSISPMHuWOvRPRER0AxGEZFQCPiQtcJaRAQCPwqi6eYiIhCZLBTlEkVd2Wb2kZnlmtmN+9l/rZmtijzSPMvM2ldXp8JaRISyYZBol6rrsWRgAnAm0AU4z8y67FPsPaCHux8HvADcVV37FNYiItTqnJieQK67r3b3YuAZYEjFAu4+2933TmVeCGRWV6nCWkQEapTWFd9jFFlGVqipLZBXYT0/su1ALgVeq655usEoIkLNHt1z94nAxO98TrPzgR5Ate8aUFiLiFCrj+4VAO0qrGdGtu1zPjsDuAXo4+7Vvm1NwyAiItTeDUZgMZBlZh3MLBUYAeRUPpcdDzwMDHb39dG0Tz1rERFqbwaju+8xs1HATCAZeNTdV5rZOGCJu+cAdwNHAM9HHgVc6+6Dq6pXYS0iQu3OYHT36cD0fbaNrfDzGTWtU2EtIkLwZzAqrEVEIPBprbAWESH4Hx9QWIuIEPiOtcJaRAQIfForrEVE0McHRERCIeBD1gprEREI/CiIwlpEBIjqowKJpLAWEUHDICIioRDwrFZYi4gAgU9rhbWICHp0T0QkFDRmLSISAkkKaxGRMAh2WiusRUTQMIiISCgEPKsV1iIioJ61iEgoaLq5iEgIBDuqFdYiIoCGQUREQkEzGEVEwiDYWa2wFhGBwGe1wlpEBCAp4IPWCmsREYJ/gzEp0Q0QEZHqqWctIkLwe9YKaxER9OieiEgoqGctIhICCmsRkRDQMIiISAioZy0iEgIBz2qFtYgIEPi0VliLiBD86ebm7olug1TDzEa6+8REt0OCRb8XhxZNNw+HkYlugASSfi8OIQprEZEQUFiLiISAwjocNC4p+6Pfi0OIbjCKiISAetYiIiGgsBYRCQGFdcCZWbaZfWRmuWZ2Y6LbI4lnZo+a2XozW5Hotkj8KKwDzMySgQnAmUAX4Dwz65LYVkkAPAZkJ7oREl8K62DrCeS6+2p3LwaeAYYkuE2SYO4+D9iU6HZIfCmsg60tkFdhPT+yTUQOMQprEZEQUFgHWwHQrsJ6ZmSbiBxiFNbBthjIMrMOZpYKjAByEtwmEUkAhXWAufseYBQwE/gAeM7dVya2VZJoZvY08A7wfTPLN7NLE90miT1NNxcRCQH1rEVEQkBhLSISAgprEZEQUFiLiISAwlpEJAQU1hITZlZiZv82sxVm9ryZHf4d6nrMzIZHfp5U1cuszKyvmf34IM6xxsxaHmwbRWJNYS2x8rW7d3f3bkAx8KuKO80s5WAqdfdfuPuqKor0BWoc1iJBp7CWeJgPdIr0euebWQ6wysySzexuM1tsZsvM7DIAK/NA5D3ebwJpeysyszlm1iPyc7aZLTWz981slpl9j7K/FEZHevWnmlkrM3sxco7FZnZK5NgWZva6ma00s0mAxfd/iUjNHFTvRiRakR70mcCMyKYTgG7u/pmZjQS2uvuJZnYY8LaZvQ4cD3yfsnd4pwOrgEf3qbcV8AjQO1JXc3ffZGYPATvc/Z5IuaeA+9x9gZkdSdls0GOA3wEL3H2cmZ0NaBagBJrCWmKlgZn9O/LzfGAyZcMTi9z9s8j2AcBxe8ejgSZAFtAbeNrdS4B1ZvbWfuo/GZi3ty53P9D7nc8AupiVd5wbm9kRkXP8NHLsNDPbfJDXKRIXCmuJla/dvXvFDZHA3FlxE3Clu8/cp9xZtdiOJOBkd/9mP20RCQ2NWUsizQQuN7N6AGbW2cwaAvOA/46MabcGTtvPsQuB3mbWIXJs88j27UCjCuVeB67cu2Jme/8CmQf8LLLtTKBZrV2VSAworCWRJlE2Hr008vHXhyn7194U4JPIvr9T9oa5Stx9AzASeMnM3geejex6BRi69wYjcBXQI3IDcxXfPpVyG2Vhv5Ky4ZC1MbpGkVqht+6JiISAetYiIiGgsBYRCQGFtYhICCisRURCQGEtIhICCmsRkRBQWIuIhMD/A9qekJr43QqVAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "id": "LEtk8Xt-YubQ",
        "outputId": "69f184af-f009-4005-c8ab-65f0cd63a157"
      },
      "source": [
        "# Precision-recall curve\n",
        "for model_name,color in zip(['rf','rf_smote'],['b','g']):\n",
        "  precisions, recalls, thresholds = metrics.precision_recall_curve(y_train_val[model_name],y_train_val_pred[model_name])\n",
        "  plt.plot(recalls,precisions,color=color)\n",
        "  plt.xlabel('Recall')\n",
        "  plt.ylabel('Precision')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV5Z3H8c8vuSHsYUmEsImyKCoKJO6ouFUUi1u1MGLrUq1Wnc7YztTW1hbttLaOtrWDo7ZatbXi0o6DQqsOtbUuUKBuLBUQAYNRokAEWZKb/OaPc0MuISE3kHNPkvN9v173dc99zsm9vyPIN8957nMec3dERCS+cqIuQEREoqUgEBGJOQWBiEjMKQhERGJOQSAiEnOJqAtoqcLCQh86dGjUZYiItCuLFi36yN2LGtvX7oJg6NChLFy4MOoyRETaFTNb09Q+XRoSEYk5BYGISMwpCEREYk5BICIScwoCEZGYCy0IzOwBM1tvZoub2G9mdpeZrTSzN81sXFi1iIhI08LsETwITNzD/jOBEanHVcB/h1iLiIg0IbR5BO7+opkN3cMh5wAPe3Af7Hlm1svMit29PIx67n7mZR5Z8Ayn+A8wbLf9I0bAJZeE8ckiIm1blBPKBgLvpb0uS7XtFgRmdhVBr4EhQ4bs1Yc99+bfeSXnNl6581ps86Bd9tUtyXDBBdC16169vYhIu9UuBovd/T53L3X30qKiRmdIN+vfLykB4Kn5i6itZZfHr34VHFMeSl9ERKRtizII1gGD014PSrWFYkz/MeRYDovKF+22b8CA4FlBICJxFGUQzAK+kPr20DFAZVjjAwBd87oyqnBUo0FQXBw8v/9+WJ8uItJ2hTZGYGaPAhOAQjMrA74L5AG4+z3AHOAsYCWwFbgsrFrqlAwo4dmVz+LumNUPGKtHICJxFua3hqY2s9+Ba8P6/MaUFJfw8BsP8/7m9xnYc+DO9j59oFMn9QhEJJ7axWBxaykpDgaMG14eMgsuD6lHICJxFKsg2Dlg/H7j4wTqEYhIHMUqCLp16sbBhQc3+c0h9QhEJI5iFQQQXB5q6ptD6hGISBzFMgg+2PIB72/e9V/9AQNg0ybYti2iwkREIhK/IBiQGjBuME5QN5fggw+yXZGISLRiFwRj+o/BsN0uD9XNJdDlIRGJm9gFQfdO3RsdMK7rEWjAWETiJnZBAMHloYaXhtQjEJG4imcQFJdQvqWc8s31v/737Qt5eeoRiEj8xDYIYNcZxmbQv796BCISP7EMgrHFY4MB40YuD6lHICJxE8sg6N6pOwcVHtTogLF6BCISN7EMAmh8hrF6BCISR7EOgvc3v88HW+pnkPXtCxs2RFiUiEgE4hsEjcwwTlurRkQkNmIbBGP7j210hrGISNzENgh65PdgZN+RCgIRib3YBgE0PsNYRCRu4h0ExSWs27yOD7d8GHUpIiKRiX0QwO5rGIuIxEmsg2Bs8Vhg97UJRETiJNZB0DO/pwaMRST2Yh0E0PQaxiIicaEgKC6h7JMy1n+6PupSREQioSBoYg1jEZG4iH0QjO0fDBj/vfzvO9vWq3MgIjES+yAo6FzAiD4jdhkn6NcvwoJERLIs9kEAqRnG5Ytwj7oSEZHsUxAQDBivrVzLvDc/iroUEZGsUxBQP8PYBmrAWETiJ9QgMLOJZva2ma00sxsb2T/EzF4ws9fM7E0zOyvMepoyrngcAMecv4gLL4QcxaOIxEho/+SZWS4wAzgTOASYamaHNDjs28Dj7j4WmALcHVY9e1LQuYDhfYaz+ONFHHRQFBWIiEQnzN99jwJWuvsqd68CZgLnNDjGgZ6p7QIgsqXjS4p1S2oRiacwg2Ag8F7a67JUW7rvAdPMrAyYA1zf2BuZ2VVmttDMFlZUVIRRKyXFJaypXMNWPg7l/UVE2qqor4ZPBR5090HAWcCvzWy3mtz9PncvdffSoqKiUAqpm2FcbuoViEi8hBkE64DBaa8HpdrSXQE8DuDurwKdgcIQa2pS3YDxBygIRCRewgyCBcAIMzvAzDoRDAbPanDMWuBUADMbRRAE4Vz7aUavzr0Y1nuYegQiEjuhBYG7J4HrgGeBZQTfDlpiZreY2eTUYV8DrjSzN4BHgUvdo5vfWzKgREEgIrGTCPPN3X0OwSBwetvNadtLgePDrKElSopLeHzJ41iXj4G+UZcjIpIVUQ8Wtyl1M4wp/vueDxQR6UAUBGnqBowp1uUhEYkPBUGa3l1608sPxBUEIhIjCoIGir1EPQIRiRUFQQPFXgK932XDtg1RlyIikhUKggb6EwwYpy9dKSLSkSkIGij2YMBYN6ATkbhQEDTQhT6w8YBd1jAWEenIFASNKS9REIhIbCgIGmHlJazauIqN2zZGXYqISOgUBI0p14CxiMSHgqAx5akBY10eEpEYCPWmc+2VbevL/r2GKghEJBbUI2iC1jAWkbhQEDShpLiEdza+w6btm6IuRUQkVAqCJtStYawBYxHp6BQETahbm0CXh0Sko1MQNKFv177sX7C/BoxFpMNTEOxByQDNMBaRjk9BsAclxSWs3LCSyu2VUZciIhIaBUEj3INH3TiBBoxFpCNTEDTQs2cQAlu21H9zSJeHRKQjUxA00K9f8Pzhh1DYtZAhBUMUBCLSoSkIGkgPAtAMYxHp+BQEDTQWBCs2rNCAsYh0WAqCBnYLgtQ4wWsfvBZRRSIi4VIQNFBUBGa79ghAM4xFpONSEDSQSEDfvjBjBpSWwmO/KmJQj8EsKl/Eqo2reOofT0VdoohIq9J6BI3o1w+WLIENG2DRIkhMK+H5zYvYnvw6c9+dS+WNGi8QkY5DPYJGFBYGz1/6Erz6KhTacD6qfYc5K+ZQU1sTbXEiIq1MQdCIuvGBE0+EY46BIUNyIKeGHTU7oi1MRCQEoQaBmU00s7fNbKWZ3djEMReZ2VIzW2Jmvw2znkxt3x48n3hitHWIiGRDaGMEZpYLzABOB8qABWY2y92Xph0zAvgmcLy7bzSz/cKqpyV+/3uYMwcGD466EhGR8IXZIzgKWOnuq9y9CpgJnNPgmCuBGe6+EcDd14dYT8bGjoWbbtq9/bjBx2W/GBGRkGUUBGZ2vJk9b2bLzWyVmb1rZqua+bGBwHtpr8tSbelGAiPN7GUzm2dmE5v4/KvMbKGZLayoqMik5FY1oPZoclZ8lqMHHp31zxYRCVumPYL7gTuB8cCRQGnqeV8lgBHABGAq8Asz69XwIHe/z91L3b20qKioFT62ZUbWnE+n380ixzS2LiIdT6ZjBJXu/ocWvvc6IP0q+6BUW7oyYL67VwPvmtlygmBY0MLPEhGRvZTpr7gvmNntZnasmY2rezTzMwuAEWZ2gJl1AqYAsxoc8xRBbwAzKyS4VNTcJScREWlFmfYI6i6Ol6a1OXBKUz/g7kkzuw54FsgFHnD3JWZ2C7DQ3Wel9n3GzJYCNcC/ufvHLT0JERHZexkFgbufvDdv7u5zgDkN2m5O23bghtRDREQikOm3hgrM7M66b+6Y2R1mVhB2cW1RrdeSrE1GXYaISKvJdIzgAWAzcFHq8Qnwq7CKasu2JbeRd2te1GWIiLSaTMcIhrn7BWmvp5vZ62EU1JbphnMi0hFl2iPYZmbj616Y2fHAtnBKarteXPti1CWIiLS6TIPgGmCGma02szXAfwFXh1dW21RaXP+lqU3bN0VYiYhI68koCNz9dXc/AjgcGO3uY939jXBLa3t+ftbPueTwSwDo/aPefGnWl7SEpYi0e3scIzCzae7+GzO7oUE7AO5+Z4i1tTmdcjvRo1OPna/vf+1+Zq+YTfnXyiOsSkRk3zQ3WNwt9dxjj0fFyNPLn965nWM57EhqsRoRad/2GATufm/qeXp2ymn7vjn+m3xlzlcAuHj0xTyz/JmIKxIR2TeZTij7sZn1NLM8M5trZhVmNi3s4tqia468ho3f2Miya5fRq/NuN0oVEWl3Mv3W0Gfc/RPgbGA1MBz4t7CKaut6de7FwYUHR12GiEiryDQI6i4hTQKecPfKkOppt2Yvn830P+sKmoi0P5kGwTNm9g+gBJhrZkXA9vDKal+StUmunXMtMxbMiLoUEZEWy3QewY3AcUBpahGZT9l9/eFY2rh9Iyc/dDJrKtdEXYqIyF7ZYxCY2Smp5/MJFpA5J7U9kSAYYq1uMfuX1r4UcSUiInuvuR7BSannzzbyODvEutqFKYdN2Tlo3Ltz74irERHZO3sMAnf/bur5skYel2enxLbtc6M+B8B5B58XcSUiInsn03kEPzCzXmmve5vZ98Mrq/249ZRbqfp2FfmJfDZu38ghMw5hypNToi5LRCRjmX5r6Ex333m7TXffCJwVTkntT15uHicMOYFDiw7lkx2f8Jc1f4m6JBGRjGUaBLlmll/3wsy6APl7OD52po6eyutXv86kEZOiLkVEpEUyXaHsEYL5A3XLU14GPBROSSIikk2ZziP4EfB9YFTqcau7/zjMwtoad5g7F8aPh4kTmz6ud5fefLz1Yy1cIyLtRqaXhgCWAX90968DfzWzWN2aescOOO00ePllePbZpo87f9T5VNdW8z/L/id7xYmI7INMvzV0JfAkcG+qaSDwVFhFtTVjx8Jhh0FOBv+1jhxwJMN6D+PRxY+GX5iISCvItEdwLXA88AmAu68A9gurqLZmyhR46y0444zmjzUzTtr/JJ5f9TxPv/108z8gIhKxTINgh7tX1b0wswTg4ZTUdj30EJxwAvRo5qJYjdcAMHnmZJ5757ksVCYisvcyDYK/mNm3gC5mdjrwBBC7X3eLiqC0tPnjXnnvlZ3b5z92fogViYjsu0yD4BtABfAW8GVgDvDtsIpq735yxk/44hFfZHif4fTM7xl1OSIie9TsPAIzywWWuPvBwC/CL6n9mzRyEpNGTuKK/72C51bp0pCItG3N9gjcvQZ428yGZKEeERHJskxnFvcGlpjZ3wgWpQHA3SeHUpWIiGRNpkHwnb15czObCPwMyAV+6e63NXHcBQTzFI5094V781kiIrJ39hgEZtYZuBoYTjBQfL+7JzN549TYwgzgdKAMWGBms9x9aYPjegBfBea3vPxolZXBnDmQnw+HHx5MPNvtmE/K+NFLP+Ib47+R/QJFRDLQ3BjBQ0ApQQicCdzRgvc+Cljp7qtScxBm0vg6x7cCPwK2t+C924Srr4YvfxkuvRQ++9nd95dvKQfgxrk3ZrcwEZEWaC4IDnH3ae5+L/A54IQWvPdA4L2012Wptp3MbBww2N1n7+mNzOwqM1toZgsrKipaUEJ43n036A18/etwySWwvZEYq6qp2r1RRKSNaS4Iqus2Mr0klCkzywHuBL7W3LHufp+7l7p7aVFRUWuWsVeSSZgxI7j30L/8C/RsYqrAg+c+yGH7HcbgnoOzW6CISAs0FwRHmNknqcdm4PC6bTP7pJmfXQek/ws4KNVWpwdwGPBnM1sNHAPMMrMM5u5G59hjYds2+OlP4dxzYeDAxo9bvx42rB7EkQOOzG6BIiIt1Nzi9bnu3jP16OHuibTt5qbMLgBGmNkBZtYJmALMSnvvSncvdPeh7j4UmAdMbuvfGjrvPBg8GGpq4Nprg7YuXWDLFti6NXjtDuecA+fr7hIi0g60ZD2CFkldSroOeJZgLYPH3X2Jmd1iZu12/kEiAbfdBlOnwoQJQdukScF6BU+n7r40cybMmxf0HERE2rpM5xHsFXefQ3BfovS2m5s4dkKYtbSmf/qn4FHnhBOCS0SPPAKTJ8M3Ut8UdQ8eIiJtWWg9gjjJzQ3WKpg/H+68E957D0aNgvJymDWr+Z8XEYmSgqCV5OcHA8Q//GEwjlDXE9iwIdq6RESaoyBoJZtSa9VXVcGPfxzMLYBgTKGmJrq6RESaE+oYQZwMGBA8P/ggDB8O3/oWbN4Mty0LegoiIm2VegStZPp0WL1610Hk//gPGDpUA8Yi0rapR9BKunULHulyciAvL5p6REQypR6BiEjMKQhERGJOQSAiEnMKAhGRmFMQhKzatlDT/T2m/m5q1KWIiDRKQRCyvNrgJq0zF8+kfHM5S9YvibgiEZFdKQhCVrLpB+RVlFCQX8CRvziSC5+4MOqSRER2oSAIWZfa/cjbOJrKHZWs27yO7cl2tzSziHRwCoIsyKkqoCC/gHHF46itDWYgi4i0FQqCLOi18DZWXL+C3tWHsmYNjB4ddUUiIvUUBFlgNZ3Z9H4Rc+cGr7dsibYeEZF0CoIs2L4dLrgAyNsKvd+lx9RrcHdqauvvT71lC1RURFejiMSXgiALKipg8WI49cQuAGw+6B4O+NmB9PhhD55d8X/88pfQowecckrEhYpILCkIsmT6dHjoktsoqBoFwJrK1WxLbmPib0/nyp8+Dr1Xsfj9laxbt+vPzSubx0trX4qgYhGJC92GOmSTJ0NBAdx0E+TkDOSgDTfwt42z4Z0z4MTvQ891cOHnoSYPNgxjwqQ/s+L1frg79792P9fMvoZxxeOY/6X5UZ+KiHRQ5u1s1ZTS0lJfuHBh1GXsteeeg6eegrFj4Te/gTcPuohNA5+oP+DD0cz+19uY9NtJO5tKiktYeFX7PWcRiZ6ZLXL30kb3KQiitbZyLeWby/n5/Lt5ZPHDu+w75YBTSOQk+HjrxwoCEdknewoCjRFEbEjBEI4edDT3Tf5vem2aEDR+PIKz1v2NuV+YSyInwaLyRfzqtV9FWqeIdFwaI2gjuuZ1pSTvn5i7aDj84S7mF3Th1sHwj87B6PEVs65gWJ9hVNVUMa54HH269Im4YhHpKHRpqA2prQ0Wuj/ySHjttaCtz1F/ZMNZZ+5yXF75cfzusy/y2Um5EVQpIu2RLg21Ezk5kJsLEybUtw3YOpGTe12+y3HVxa8w+eGL+NOfslufiHRMCoI26I47gt7BmDHBRLQXbrgXbqnm7v1quXHwM8FBvdZw5ZXR1ikiHYOCoA0yCx7jxgWvRwxLsHRxgmuuMX54+SRGdzsNBiyiy/C/RVuoiHQICoI27J57gvsULV8Oo0bVt9d2LQfgnSMuAWDrVtixI4oKRaQjUBC0YXl5kJ+/e/vtp98OwPaNvZhw9nq6TfwPxlz0DB99lOUCRaRDCDUIzGyimb1tZivN7MZG9t9gZkvN7E0zm2tm+4dZT0dx5ogzKfj4VChaxl+O7Aenfpt/HHA9Bx26naVLo65ORNqb0ILAzHKBGcCZwCHAVDM7pMFhrwGl7n448CTw47Dq6WhKD+1LIr+KI/sdHzT0Xs2Gk77ApLNr2bQp2tpEpH0Js0dwFLDS3Ve5exUwEzgn/QB3f8Hdt6ZezgMGhVhPh/LUZfez/t/L+dvVLzF9wvSg8dAnWD3mUn2bSERaJMwgGAi8l/a6LNXWlCuAPzS2w8yuMrOFZrawQqu3ANC9U3d6d+kNwM0n3czPz/x5sKPvcubPDyakfec7cPDBcP/9ERYqIm1em7jFhJlNA0qBkxrb7+73AfdBMLM4i6W1G9cddR0zF8/kZV7mvao3GTfu8J37fvYzOPlkKCoKFsAREUkXZo9gHTA47fWgVNsuzOw04CZgsrvrS5D7oHJHJQCdL/gK48fDr38d/OP/1lswbBhcd13EBYpImxRmECwARpjZAWbWCZgCzEo/wMzGAvcShMD6EGuJhbvPuhuAY47K469/hWnT4Nvfhn79gv3l5REWJyJtVmiXhtw9aWbXAc8CucAD7r7EzG4BFrr7LOB2oDvwhJkBrHX3yWHV1NGdsP8JHD/4eCo+reDmF26mpraG8z93Psu/PIKTj+vZ6JwEEZFQxwjcfQ4wp0HbzWnbp4X5+XG0cftGllYsZUnFEgB+8NIPuHj0xcBvoi1MRNoszSzuYK4/6nquLrmah859iNMPPB2AR956hE396q/KbanaQrI2GVWJItLGaD2CDu7s357N7BWzyd8yksGbP8fK4h8A8OWSL3PP2fdEXJ2IZIvWI4ixJy96kuLuxezovpyV/W/b2X7vons5/NL7taaBiCgIOrrOic7cfvrtFC75Dvx0FXzPYf71ALx1wJc49bK/YkXLsOP/k+/+Qre1FomjNjGhTMJ18eEXs3YszK+CzZvhuJK7eKpgIYsrX4XLT9x53P1LL2Q6j0dYqYhEQWMEMbXuk3Xc+eqdvLj2Ra4YewXXzL5mt2MePOdBvjjmixFUJyKtTWMEspuBPQdyxxl3sODKBVxdejUHvn3Xbsc8vfT5CCoTkWxTj0CAYCW0/feH9XXzu79WDD0+oDBnOH1659C/e39uOuEmPjPsM5HWKSJ7Rz0CaVbnzvD443DWWTB9OvTzcZDM56PalSz/eDkvrnmRM35zBk8seYL29suDiOyZegTSKHfnrrvgX765AfK2MfrGa3mrqn5S2r1n38tVJVdFWKGItIR6BNJiZsZXv2qMOagvfDKIt771v/DETLrXBktK3Ph/N7Luk91uJisi7ZCCQPbo+edhzhy4/npgyee5qKyMA3odwMbtG/nxy1pZVKQjUBDIHhUWwplnwl13wcCBYAZ/nPZHALYnt0dcnYi0BgWBtNjIviPplteN+/5+H88sfybqckRkHykIpEW2bgV3OG/UeQDMXj474opEZF/pFhOSsW3b4NFHYeRI+PX3fs1z7zwXdUki0grUI5CMzZgRPE+fDrfeGqxrcM+ie/j9st9HW5iI7BMFgWRsyhS4JnVLoptvhq2vXgLABY9fwO+X/Z5lFcu04I1IO6QJZdJiL7wAP/kJPPssVF11MBS+vcv+8UPGM6bfGMb0H8N5o86jT5c+EVUqInX2NKFMQSD75IFHKrni1r/A4Ffg0Meg9+rdjrl8zOX88OT/ZL+evbNfoIgACgIJ2WOPwfe/D4sXA7k7ICcJhf+Anutg6jk7jzs15xb+9xs30K1Tt+iKFYkpBYFkxcyZsGIFrFkThMLq1XDEyStYOPw8Nmxw2G8pAIW5B/LIBQ/zmVHHR1uwSIwoCCRylZXQ96pp1BzyyM62bjm9GdJrIH2792L/gv3Zr9t+HD3waM49+FzyE/kRVivS8SgIpE349FNYvhzGXfAnGDgfepZROKyMj/rO2u3YicMncszAY5g4fCJH9D+CzonOEVQs0nEoCKRNWbUKNm2CZ56B734XyNvKuefXMLT0H9yz9VS2++Zdjk/kJBi932hG9B3BYUWHcf3R19Orc69oihdppxQE0ia5w9SpwWBzI3uhZxkMWMgZly9gW995vPjeCzv3XjbmMo4ddCw983viOFuqtrClagsfbvmQQT0Hcdzg4zi83+Hk5uRm7XxE2jIFgbRptbXBUplVVVBdHTxXVcErr8C0afXH9R+6iZzTvsP7g/4ro/ft0akHxww6hvFDxjN+yHiOHni0vrEksaUgkHZr9mz48EP44IPgG0krVsDbZR/y0acboOvHkOwMVd3p37cbRQXd6dKrkuIhW6nq8wZlOS/xnr3Epk5vgTm5lsvY4rGM7DuSLokuHNHvCEoGlFDYtZCe+T0pyC+gc6IzZhb1aYu0OgWBdDiVlbByZX04LFlSf4kpkYAuXWDHjqBnQedNMGgeDHmJnGF/onbgq02+r2H0SPQhYZ04sPto8nPzyc/tTEGnPuTlJuiUyCMvN0FeTrDdKZF6zk2Ql0iQX9eWl7adm0ciJ0EiJ0FeTrCd10hbentjbXXtOZajsJIWUxBIbG3YEMxn+OlPYe1a6NkTnp6dpNfg98nrv5zc/O18WlPJ5qpK6FoBAxZB3+XQ+x3Y3gvyN8O2PmC1kFMdTJbLTXuOiNXmkUMC8wQ5BNu55GEkyCVoyyVBjgXtuZYgYXnkWIKEJUjkpNpygvbcJsKnLsTycoPtvNwEidwg5OqCcZcQzEuQnwrH/ESC/E55qdcJOu/czjwEFXitZ09BoNtQS4fWp0/wePjh9NYEMCT1CJSXQ1lZsJ1MBo+amvrtptqqkzXsSCapSibZUV1NdU2SqppqqpJJqpNJqmtT2zVJqmuqqa6tf07WJEnWprZrq0nWJoOHV1NTmyTpSWq8mhpPtXmSWk9SQzU11G87wXMtSWpJUmXV1FoSt6DNLVkfYjlVkPtparuuLS3c0tt3aavJ4p9amtpczBOY56WeE+SktnMItnOoD8O6QMyx1LYFgdhoCKYCcLdAyskjkRsEU15OEHx1IVgXiHU9weB1gvy8+rb8VI+wc6otPy/Y3zkvCMMgFBN06ZRHfl7wnlF/qSHUIDCzicDPgFzgl+5+W4P9+cDDQAnwMfB5d18dZk0ijSkuDh4tl5t6tO0JcO67hljDQGvudXWylqrqGrZXV7OjOgi2HdXVu4dgspqqmiAEq2rqg7G6Jgi56tR2evhV1wTPNZ5q82QqCIPwq9n5XB94NR6EYXoIJkmF384Q3IFbEs+pDp5T21hTIVgNuRHdPdcNahNQmwe1QeBZbd6uz57g6lHf42dXfr7VPz60IDCzXGAGcDpQBiwws1nuvjTtsCuAje4+3MymAD8CWv8sRWLOLBg7Sez1//E5qUde6xUVofRgTA+96mqnOlnLtqog8IJHsL29Oujd1QVfXQhWJavrn2uSqR5ggxBMbQfBlxaCdT3BVAim9wR39ghJph7VFPcK506+YfYIjgJWuvsqADObCZwDpAfBOcD3UttPAv9lZubtbeBCRNqVpoPRqO/lxUeYC9MMBN5Le12Wamv0GHdPApVA34ZvZGZXmdlCM1tYUVERUrkiIvHULlYoc/f73L3U3UuLioqiLkdEpEMJMwjWAYPTXg9KtTV6jJklgAKCQWMREcmSMINgATDCzA4ws07AFKDhbSZnAV9MbX8O+JPGB0REsiu0wWJ3T5rZdcCzBCMvD7j7EjO7BVjo7rOA+4Ffm9lKYANBWIiISBaFOo/A3ecAcxq03Zy2vR24MMwaRERkz9rFYLGIiIRHQSAiEnPt7qZzZlYBrNnLHy8EPmrFctoDnXM86JzjYV/OeX93b/T79+0uCPaFmS1s6u57HZXOOR50zvEQ1jnr0pCISMwpCEREYi5uQXBf1AVEQOccDzrneAjlnGM1RiAiIruLW49AREQaUBCIiMRchwwCM5toZm+b2Uozu7GR/flm9lhq/3wzG5r9KltXBud8g5ktNbM3zWyume0fRSt7F4cAAARzSURBVJ2tqblzTjvuAjNzM2v3XzXM5JzN7KLUn/USM/tttmtsbRn83R5iZi+Y2Wupv99nRVFnazGzB8xsvZktbmK/mdldqf8eb5rZuH3+UHfvUA+CG9y9AxwIdALeAA5pcMxXgHtS21OAx6KuOwvnfDLQNbV9TRzOOXVcD+BFYB5QGnXdWfhzHgG8BvROvd4v6rqzcM73Adektg8BVkdd9z6e84nAOGBxE/vPAv5AsJzaMcD8ff3Mjtgj2LlEprtXAXVLZKY7B3gotf0kcKqZWRZrbG3NnrO7v+DuW1Mv5xGsD9GeZfLnDHArwVrY27NZXEgyOecrgRnuvhHA3ddnucbWlsk5O9AztV0AvJ/F+lqdu79IcDfmppwDPOyBeUAvMyvel8/siEHQaktktiOZnHO6Kwh+o2jPmj3nVJd5sLvPzmZhIcrkz3kkMNLMXjazeWY2MWvVhSOTc/4eMM3Mygjudnx9dkqLTEv/f29WqLehlrbHzKYBpcBJUdcSJjPLAe4ELo24lGxLEFwemkDQ63vRzEa7+6ZIqwrXVOBBd7/DzI4lWOPkMHevjbqw9qIj9gjiuERmJueMmZ0G3ARMdvcdWaotLM2dcw/gMODPZraa4FrqrHY+YJzJn3MZMMvdq939XWA5QTC0V5mc8xXA4wDu/irQmeDmbB1VRv+/t0RHDII4LpHZ7Dmb2VjgXoIQaO/XjaGZc3b3SncvdPeh7j6UYFxksrsvjKbcVpHJ3+2nCHoDmFkhwaWiVdksspVlcs5rgVMBzGwUQRBUZLXK7JoFfCH17aFjgEp3L9+XN+xwl4Y8hktkZnjOtwPdgSdS4+Jr3X1yZEXvowzPuUPJ8JyfBT5jZkuBGuDf3L3d9nYzPOevAb8ws38lGDi+tD3/YmdmjxKEeWFq3OO7QB6Au99DMA5yFrAS2Apcts+f2Y7/e4mISCvoiJeGRESkBRQEIiIxpyAQEYk5BYGISMwpCEREYk5BINIIM6sxs9fNbLGZPW1mvVr5/VenvuePmW1pzfcWaSkFgUjjtrn7GHc/jGCuybVRFyQSFgWBSPNeJXVTLzMbZmZ/NLNFZvZXMzs41d7PzP7HzN5IPY5LtT+VOnaJmV0V4TmINKnDzSwWaU1mlktw+4L7U033AVe7+wozOxq4GzgFuAv4i7ufl/qZ7qnjL3f3DWbWBVhgZr9rzzN9pWNSEIg0rouZvU7QE1gGPG9m3YHjqL9NB0B+6vkU4AsA7l5DcGtzgH82s/NS24MJbgCnIJA2RUEg0rht7j7GzLoS3OfmWuBBYJO7j8nkDcxsAnAacKy7bzWzPxPcEE2kTdEYgcgepFZ1+2eCG5ttBd41swth59qxR6QOnUuwBChmlmtmBQS3N9+YCoGDCW6FLdLmKAhEmuHurwFvEiyAcjFwhZm9ASyhftnErwInm9lbwCKCtXP/CCTMbBlwG8GtsEXaHN19VEQk5tQjEBGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTm/h/jdh/pC2G28gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7nBiehmgKXVT"
      },
      "source": [
        "# Todo: Analyze your model candidates with shap values; you might find something useful. https://www.kaggle.com/dansbecker/shap-values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gpb6VJ9ypj3r"
      },
      "source": [
        "## Evaluate your best model on the test set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "91QbaoT3p01V"
      },
      "source": [
        "# Todo: What is your best model? Try it on the test set and this is the unbiased generalization performance of your model."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "reUiEbmI85N6"
      },
      "source": [
        "## Launch and maintain your system"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K_AgYCv-9A2A"
      },
      "source": [
        "# Retrain your final model using all dataset."
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}