code/2.DataProcessing.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DNWMn_ox_f2B"
      },
      "source": [
        "# Load the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[33mWARNING: Ignoring invalid distribution -ip (/Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages)\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -ip (/Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages)\u001b[0m\u001b[33m\n",
            "\u001b[0mRequirement already satisfied: appnope==0.1.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 1)) (0.1.2)\n",
            "Requirement already satisfied: asttokens==2.0.5 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 2)) (2.0.5)\n",
            "Requirement already satisfied: backcall==0.2.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 3)) (0.2.0)\n",
            "Requirement already satisfied: category-encoders==2.4.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 4)) (2.4.0)\n",
            "Requirement already satisfied: cloudpickle==2.0.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 5)) (2.0.0)\n",
            "Requirement already satisfied: cycler==0.11.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 6)) (0.11.0)\n",
            "Requirement already satisfied: debugpy==1.5.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 7)) (1.5.1)\n",
            "Requirement already satisfied: decorator==5.1.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 8)) (5.1.1)\n",
            "Requirement already satisfied: entrypoints==0.4 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 9)) (0.4)\n",
            "Requirement already satisfied: executing==0.8.3 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 10)) (0.8.3)\n",
            "Requirement already satisfied: fonttools==4.29.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 11)) (4.29.1)\n",
            "Requirement already satisfied: future==0.18.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 12)) (0.18.2)\n",
            "Requirement already satisfied: hyperopt==0.2.7 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 13)) (0.2.7)\n",
            "Requirement already satisfied: ipykernel==6.9.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 14)) (6.9.1)\n",
            "Requirement already satisfied: ipython==8.1.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 15)) (8.1.1)\n",
            "Requirement already satisfied: jedi==0.18.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 16)) (0.18.1)\n",
            "Requirement already satisfied: joblib==1.1.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 17)) (1.1.0)\n",
            "Requirement already satisfied: jupyter-client==7.1.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 18)) (7.1.2)\n",
            "Requirement already satisfied: jupyter-core==4.9.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 19)) (4.9.2)\n",
            "Requirement already satisfied: kiwisolver==1.3.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 20)) (1.3.2)\n",
            "Requirement already satisfied: lxml==4.8.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 21)) (4.8.0)\n",
            "Requirement already satisfied: matplotlib==3.5.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 22)) (3.5.1)\n",
            "Requirement already satisfied: matplotlib-inline==0.1.3 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 23)) (0.1.3)\n",
            "Requirement already satisfied: MiniSom==2.3.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 24)) (2.3.0)\n",
            "Requirement already satisfied: nest-asyncio==1.5.4 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 25)) (1.5.4)\n",
            "Requirement already satisfied: networkx==2.7.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 26)) (2.7.1)\n",
            "Requirement already satisfied: numpy==1.22.3 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 27)) (1.22.3)\n",
            "Requirement already satisfied: packaging==21.3 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 28)) (21.3)\n",
            "Requirement already satisfied: pandas==1.4.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 29)) (1.4.1)\n",
            "Requirement already satisfied: parso==0.8.3 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 30)) (0.8.3)\n",
            "Requirement already satisfied: patsy==0.5.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 31)) (0.5.2)\n",
            "Requirement already satisfied: pexpect==4.8.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 32)) (4.8.0)\n",
            "Requirement already satisfied: pickle-mixin==1.0.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 33)) (1.0.2)\n",
            "Requirement already satisfied: pickleshare==0.7.5 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 34)) (0.7.5)\n",
            "Requirement already satisfied: Pillow==9.0.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 35)) (9.0.1)\n",
            "Requirement already satisfied: prompt-toolkit==3.0.28 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 36)) (3.0.28)\n",
            "Requirement already satisfied: ptyprocess==0.7.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 37)) (0.7.0)\n",
            "Requirement already satisfied: pure-eval==0.2.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 38)) (0.2.2)\n",
            "Requirement already satisfied: py4j==0.10.9.5 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 39)) (0.10.9.5)\n",
            "Requirement already satisfied: Pygments==2.11.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 40)) (2.11.2)\n",
            "Requirement already satisfied: PyMySQL==1.0.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 41)) (1.0.2)\n",
            "Requirement already satisfied: pyparsing==3.0.7 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 42)) (3.0.7)\n",
            "Requirement already satisfied: python-dateutil==2.8.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 43)) (2.8.2)\n",
            "Requirement already satisfied: pytz==2021.3 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 44)) (2021.3)\n",
            "Requirement already satisfied: pyzmq==22.3.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 45)) (22.3.0)\n",
            "Requirement already satisfied: scikit-learn==1.0.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 46)) (1.0.2)\n",
            "Requirement already satisfied: scipy==1.8.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 47)) (1.8.0)\n",
            "Requirement already satisfied: seaborn==0.11.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 48)) (0.11.2)\n",
            "Requirement already satisfied: six==1.16.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 49)) (1.16.0)\n",
            "Requirement already satisfied: sklearn==0.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 50)) (0.0)\n",
            "Requirement already satisfied: stack-data==0.2.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 51)) (0.2.0)\n",
            "Requirement already satisfied: statsmodels==0.13.2 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 52)) (0.13.2)\n",
            "Requirement already satisfied: threadpoolctl==3.1.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 53)) (3.1.0)\n",
            "Requirement already satisfied: tornado==6.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 54)) (6.1)\n",
            "Requirement already satisfied: tqdm==4.63.0 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 55)) (4.63.0)\n",
            "Requirement already satisfied: traitlets==5.1.1 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 56)) (5.1.1)\n",
            "Requirement already satisfied: wcwidth==0.2.5 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from -r requirement.txt (line 57)) (0.2.5)\n",
            "Requirement already satisfied: setuptools>=18.5 in /Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages (from ipython==8.1.1->-r requirement.txt (line 15)) (60.5.0)\n",
            "\u001b[33mWARNING: Ignoring invalid distribution -ip (/Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages)\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -ip (/Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages)\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -ip (/Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages)\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "import warnings\n",
        "!pip install -r requirement.txt\n",
        "warnings.filterwarnings('ignore')\n",
        "from FeatureEngineering import *"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "69QDAEp1Cm5l",
        "outputId": "df32de0f-b4d3-4bc7-85a9-13b59dd9c768"
      },
      "outputs": [],
      "source": [
        "from FeatureEngineering import *\n",
        "data_dir = './data/'\n",
        "data = my_read_csv(data_dir+'data_v1.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G38vliXZC0oS",
        "outputId": "875aa664-fef3-42cf-dc69-9475ef793464"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 32902 entries, 0 to 32901\n",
            "Data columns (total 218 columns):\n",
            " #    Column                        Dtype  \n",
            "---   ------                        -----  \n",
            " 0    SK_ID_CURR                    int64  \n",
            " 1    TARGET                        float64\n",
            " 2    NAME_CONTRACT_TYPE_x          object \n",
            " 3    CODE_GENDER                   int64  \n",
            " 4    FLAG_OWN_CAR                  int64  \n",
            " 5    FLAG_OWN_REALTY               int64  \n",
            " 6    CNT_CHILDREN                  int64  \n",
            " 7    AMT_INCOME_TOTAL              float64\n",
            " 8    AMT_CREDIT_x                  float64\n",
            " 9    AMT_ANNUITY_x                 float64\n",
            " 10   AMT_GOODS_PRICE_x             float64\n",
            " 11   NAME_TYPE_SUITE_x             object \n",
            " 12   NAME_INCOME_TYPE              object \n",
            " 13   NAME_EDUCATION_TYPE           object \n",
            " 14   NAME_FAMILY_STATUS            object \n",
            " 15   NAME_HOUSING_TYPE             object \n",
            " 16   REGION_POPULATION_RELATIVE    float64\n",
            " 17   DAYS_BIRTH                    float64\n",
            " 18   DAYS_EMPLOYED                 float64\n",
            " 19   DAYS_REGISTRATION             float64\n",
            " 20   DAYS_ID_PUBLISH               float64\n",
            " 21   OWN_CAR_AGE                   float64\n",
            " 22   FLAG_MOBIL                    float64\n",
            " 23   FLAG_EMP_PHONE                float64\n",
            " 24   FLAG_WORK_PHONE               float64\n",
            " 25   FLAG_CONT_MOBILE              float64\n",
            " 26   FLAG_PHONE                    float64\n",
            " 27   FLAG_EMAIL                    float64\n",
            " 28   OCCUPATION_TYPE               object \n",
            " 29   CNT_FAM_MEMBERS               float64\n",
            " 30   REGION_RATING_CLIENT          float64\n",
            " 31   REGION_RATING_CLIENT_W_CITY   float64\n",
            " 32   WEEKDAY_APPR_PROCESS_START_x  object \n",
            " 33   HOUR_APPR_PROCESS_START_x     float64\n",
            " 34   REG_REGION_NOT_LIVE_REGION    float64\n",
            " 35   REG_REGION_NOT_WORK_REGION    float64\n",
            " 36   LIVE_REGION_NOT_WORK_REGION   float64\n",
            " 37   REG_CITY_NOT_LIVE_CITY        float64\n",
            " 38   REG_CITY_NOT_WORK_CITY        float64\n",
            " 39   LIVE_CITY_NOT_WORK_CITY       float64\n",
            " 40   ORGANIZATION_TYPE             object \n",
            " 41   EXT_SOURCE_1                  float64\n",
            " 42   EXT_SOURCE_2                  float64\n",
            " 43   EXT_SOURCE_3                  float64\n",
            " 44   APARTMENTS_AVG                float64\n",
            " 45   BASEMENTAREA_AVG              float64\n",
            " 46   YEARS_BEGINEXPLUATATION_AVG   float64\n",
            " 47   YEARS_BUILD_AVG               float64\n",
            " 48   COMMONAREA_AVG                float64\n",
            " 49   ELEVATORS_AVG                 float64\n",
            " 50   ENTRANCES_AVG                 float64\n",
            " 51   FLOORSMAX_AVG                 float64\n",
            " 52   FLOORSMIN_AVG                 float64\n",
            " 53   LANDAREA_AVG                  float64\n",
            " 54   LIVINGAPARTMENTS_AVG          float64\n",
            " 55   LIVINGAREA_AVG                float64\n",
            " 56   NONLIVINGAPARTMENTS_AVG       float64\n",
            " 57   NONLIVINGAREA_AVG             float64\n",
            " 58   APARTMENTS_MODE               float64\n",
            " 59   BASEMENTAREA_MODE             float64\n",
            " 60   YEARS_BEGINEXPLUATATION_MODE  float64\n",
            " 61   YEARS_BUILD_MODE              float64\n",
            " 62   COMMONAREA_MODE               float64\n",
            " 63   ELEVATORS_MODE                float64\n",
            " 64   ENTRANCES_MODE                float64\n",
            " 65   FLOORSMAX_MODE                float64\n",
            " 66   FLOORSMIN_MODE                float64\n",
            " 67   LANDAREA_MODE                 float64\n",
            " 68   LIVINGAPARTMENTS_MODE         float64\n",
            " 69   LIVINGAREA_MODE               float64\n",
            " 70   NONLIVINGAPARTMENTS_MODE      float64\n",
            " 71   NONLIVINGAREA_MODE            float64\n",
            " 72   APARTMENTS_MEDI               float64\n",
            " 73   BASEMENTAREA_MEDI             float64\n",
            " 74   YEARS_BEGINEXPLUATATION_MEDI  float64\n",
            " 75   YEARS_BUILD_MEDI              float64\n",
            " 76   COMMONAREA_MEDI               float64\n",
            " 77   ELEVATORS_MEDI                float64\n",
            " 78   ENTRANCES_MEDI                float64\n",
            " 79   FLOORSMAX_MEDI                float64\n",
            " 80   FLOORSMIN_MEDI                float64\n",
            " 81   LANDAREA_MEDI                 float64\n",
            " 82   LIVINGAPARTMENTS_MEDI         float64\n",
            " 83   LIVINGAREA_MEDI               float64\n",
            " 84   NONLIVINGAPARTMENTS_MEDI      float64\n",
            " 85   NONLIVINGAREA_MEDI            float64\n",
            " 86   FONDKAPREMONT_MODE            object \n",
            " 87   HOUSETYPE_MODE                object \n",
            " 88   TOTALAREA_MODE                float64\n",
            " 89   WALLSMATERIAL_MODE            object \n",
            " 90   EMERGENCYSTATE_MODE           object \n",
            " 91   OBS_30_CNT_SOCIAL_CIRCLE      float64\n",
            " 92   DEF_30_CNT_SOCIAL_CIRCLE      float64\n",
            " 93   OBS_60_CNT_SOCIAL_CIRCLE      float64\n",
            " 94   DEF_60_CNT_SOCIAL_CIRCLE      float64\n",
            " 95   DAYS_LAST_PHONE_CHANGE        float64\n",
            " 96   FLAG_DOCUMENT_2               float64\n",
            " 97   FLAG_DOCUMENT_3               float64\n",
            " 98   FLAG_DOCUMENT_4               float64\n",
            " 99   FLAG_DOCUMENT_5               float64\n",
            " 100  FLAG_DOCUMENT_6               float64\n",
            " 101  FLAG_DOCUMENT_7               float64\n",
            " 102  FLAG_DOCUMENT_8               float64\n",
            " 103  FLAG_DOCUMENT_9               float64\n",
            " 104  FLAG_DOCUMENT_10              float64\n",
            " 105  FLAG_DOCUMENT_11              float64\n",
            " 106  FLAG_DOCUMENT_12              float64\n",
            " 107  FLAG_DOCUMENT_13              float64\n",
            " 108  FLAG_DOCUMENT_14              float64\n",
            " 109  FLAG_DOCUMENT_15              float64\n",
            " 110  FLAG_DOCUMENT_16              float64\n",
            " 111  FLAG_DOCUMENT_17              float64\n",
            " 112  FLAG_DOCUMENT_18              float64\n",
            " 113  FLAG_DOCUMENT_19              float64\n",
            " 114  FLAG_DOCUMENT_20              float64\n",
            " 115  FLAG_DOCUMENT_21              float64\n",
            " 116  AMT_REQ_CREDIT_BUREAU_HOUR    float64\n",
            " 117  AMT_REQ_CREDIT_BUREAU_DAY     float64\n",
            " 118  AMT_REQ_CREDIT_BUREAU_WEEK    float64\n",
            " 119  AMT_REQ_CREDIT_BUREAU_MON     float64\n",
            " 120  AMT_REQ_CREDIT_BUREAU_QRT     float64\n",
            " 121  AMT_REQ_CREDIT_BUREAU_YEAR    float64\n",
            " 122  DAYS_EMPLOYED_PERC            float64\n",
            " 123  INCOME_CREDIT_PERC            float64\n",
            " 124  INCOME_PER_PERSON             float64\n",
            " 125  ANNUITY_INCOME_PERC           float64\n",
            " 126  PAYMENT_RATE                  float64\n",
            " 127  CREDIT_ACTIVE                 object \n",
            " 128  CREDIT_CURRENCY               object \n",
            " 129  DAYS_CREDIT                   float64\n",
            " 130  CREDIT_DAY_OVERDUE            float64\n",
            " 131  DAYS_CREDIT_ENDDATE           float64\n",
            " 132  DAYS_ENDDATE_FACT             float64\n",
            " 133  AMT_CREDIT_MAX_OVERDUE        float64\n",
            " 134  CNT_CREDIT_PROLONG            float64\n",
            " 135  AMT_CREDIT_SUM                float64\n",
            " 136  AMT_CREDIT_SUM_DEBT           float64\n",
            " 137  AMT_CREDIT_SUM_LIMIT          float64\n",
            " 138  AMT_CREDIT_SUM_OVERDUE        float64\n",
            " 139  CREDIT_TYPE                   object \n",
            " 140  DAYS_CREDIT_UPDATE            float64\n",
            " 141  AMT_ANNUITY_y                 float64\n",
            " 142  MONTHS_BALANCE_MIN            float64\n",
            " 143  MONTHS_BALANCE_MAX            float64\n",
            " 144  MONTHS_BALANCE_SIZE           float64\n",
            " 145  SK_ID_PREV_x                  float64\n",
            " 146  NAME_CONTRACT_TYPE_y          object \n",
            " 147  AMT_ANNUITY                   float64\n",
            " 148  AMT_APPLICATION               float64\n",
            " 149  AMT_CREDIT_y                  float64\n",
            " 150  AMT_DOWN_PAYMENT              float64\n",
            " 151  AMT_GOODS_PRICE_y             float64\n",
            " 152  WEEKDAY_APPR_PROCESS_START_y  object \n",
            " 153  HOUR_APPR_PROCESS_START_y     float64\n",
            " 154  FLAG_LAST_APPL_PER_CONTRACT   object \n",
            " 155  NFLAG_LAST_APPL_IN_DAY        float64\n",
            " 156  RATE_DOWN_PAYMENT             float64\n",
            " 157  RATE_INTEREST_PRIMARY         float64\n",
            " 158  RATE_INTEREST_PRIVILEGED      float64\n",
            " 159  NAME_CASH_LOAN_PURPOSE        object \n",
            " 160  NAME_CONTRACT_STATUS_x        object \n",
            " 161  DAYS_DECISION                 float64\n",
            " 162  NAME_PAYMENT_TYPE             object \n",
            " 163  CODE_REJECT_REASON            object \n",
            " 164  NAME_TYPE_SUITE_y             object \n",
            " 165  NAME_CLIENT_TYPE              object \n",
            " 166  NAME_GOODS_CATEGORY           object \n",
            " 167  NAME_PORTFOLIO                object \n",
            " 168  NAME_PRODUCT_TYPE             object \n",
            " 169  CHANNEL_TYPE                  object \n",
            " 170  SELLERPLACE_AREA              float64\n",
            " 171  NAME_SELLER_INDUSTRY          object \n",
            " 172  CNT_PAYMENT                   float64\n",
            " 173  NAME_YIELD_GROUP              object \n",
            " 174  PRODUCT_COMBINATION           object \n",
            " 175  DAYS_FIRST_DRAWING            float64\n",
            " 176  DAYS_FIRST_DUE                float64\n",
            " 177  DAYS_LAST_DUE_1ST_VERSION     float64\n",
            " 178  DAYS_LAST_DUE                 float64\n",
            " 179  DAYS_TERMINATION              float64\n",
            " 180  NFLAG_INSURED_ON_APPROVAL     float64\n",
            " 181  APP_CREDIT_PERC               float64\n",
            " 182  SK_ID_PREV_y                  float64\n",
            " 183  MONTHS_BALANCE_x              float64\n",
            " 184  CNT_INSTALMENT                float64\n",
            " 185  CNT_INSTALMENT_FUTURE         float64\n",
            " 186  NAME_CONTRACT_STATUS_y        object \n",
            " 187  SK_DPD_x                      float64\n",
            " 188  SK_DPD_DEF_x                  float64\n",
            " 189  SK_ID_PREV_x.1                float64\n",
            " 190  NUM_INSTALMENT_VERSION        float64\n",
            " 191  NUM_INSTALMENT_NUMBER         float64\n",
            " 192  DAYS_INSTALMENT               float64\n",
            " 193  DAYS_ENTRY_PAYMENT            float64\n",
            " 194  AMT_INSTALMENT                float64\n",
            " 195  AMT_PAYMENT                   float64\n",
            " 196  SK_ID_PREV_y.1                float64\n",
            " 197  MONTHS_BALANCE_y              float64\n",
            " 198  AMT_BALANCE                   float64\n",
            " 199  AMT_CREDIT_LIMIT_ACTUAL       float64\n",
            " 200  AMT_DRAWINGS_ATM_CURRENT      float64\n",
            " 201  AMT_DRAWINGS_CURRENT          float64\n",
            " 202  AMT_DRAWINGS_OTHER_CURRENT    float64\n",
            " 203  AMT_DRAWINGS_POS_CURRENT      float64\n",
            " 204  AMT_INST_MIN_REGULARITY       float64\n",
            " 205  AMT_PAYMENT_CURRENT           float64\n",
            " 206  AMT_PAYMENT_TOTAL_CURRENT     float64\n",
            " 207  AMT_RECEIVABLE_PRINCIPAL      float64\n",
            " 208  AMT_RECIVABLE                 float64\n",
            " 209  AMT_TOTAL_RECEIVABLE          float64\n",
            " 210  CNT_DRAWINGS_ATM_CURRENT      float64\n",
            " 211  CNT_DRAWINGS_CURRENT          float64\n",
            " 212  CNT_DRAWINGS_OTHER_CURRENT    float64\n",
            " 213  CNT_DRAWINGS_POS_CURRENT      float64\n",
            " 214  CNT_INSTALMENT_MATURE_CUM     float64\n",
            " 215  NAME_CONTRACT_STATUS          object \n",
            " 216  SK_DPD_y                      float64\n",
            " 217  SK_DPD_DEF_y                  float64\n",
            "dtypes: float64(179), int64(5), object(34)\n",
            "memory usage: 54.7+ MB\n"
          ]
        }
      ],
      "source": [
        "data.info(verbose=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "The data description for each columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Table</th>\n",
              "      <th>Row</th>\n",
              "      <th>Description</th>\n",
              "      <th>Special</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>209</th>\n",
              "      <td>previous_application.csv</td>\n",
              "      <td>DAYS_TERMINATION</td>\n",
              "      <td>Relative to application date of current applic...</td>\n",
              "      <td>time only relative to the application</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>210</th>\n",
              "      <td>previous_application.csv</td>\n",
              "      <td>NFLAG_INSURED_ON_APPROVAL</td>\n",
              "      <td>Did the client requested insurance during the ...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>211</th>\n",
              "      <td>installments_payments.csv</td>\n",
              "      <td>SK_ID_PREV</td>\n",
              "      <td>ID of previous credit in Home credit related t...</td>\n",
              "      <td>hashed</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>212</th>\n",
              "      <td>installments_payments.csv</td>\n",
              "      <td>SK_ID_CURR</td>\n",
              "      <td>ID of loan in our sample</td>\n",
              "      <td>hashed</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>installments_payments.csv</td>\n",
              "      <td>NUM_INSTALMENT_VERSION</td>\n",
              "      <td>Version of installment calendar (0 is for cred...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>214</th>\n",
              "      <td>installments_payments.csv</td>\n",
              "      <td>NUM_INSTALMENT_NUMBER</td>\n",
              "      <td>On which installment we observe payment</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>215</th>\n",
              "      <td>installments_payments.csv</td>\n",
              "      <td>DAYS_INSTALMENT</td>\n",
              "      <td>When the installment of previous credit was su...</td>\n",
              "      <td>time only relative to the application</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>216</th>\n",
              "      <td>installments_payments.csv</td>\n",
              "      <td>DAYS_ENTRY_PAYMENT</td>\n",
              "      <td>When was the installments of previous credit p...</td>\n",
              "      <td>time only relative to the application</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>installments_payments.csv</td>\n",
              "      <td>AMT_INSTALMENT</td>\n",
              "      <td>What was the prescribed installment amount of ...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>218</th>\n",
              "      <td>installments_payments.csv</td>\n",
              "      <td>AMT_PAYMENT</td>\n",
              "      <td>What the client actually paid on previous cred...</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         Table                        Row  \\\n",
              "209   previous_application.csv           DAYS_TERMINATION   \n",
              "210   previous_application.csv  NFLAG_INSURED_ON_APPROVAL   \n",
              "211  installments_payments.csv                SK_ID_PREV    \n",
              "212  installments_payments.csv                 SK_ID_CURR   \n",
              "213  installments_payments.csv     NUM_INSTALMENT_VERSION   \n",
              "214  installments_payments.csv      NUM_INSTALMENT_NUMBER   \n",
              "215  installments_payments.csv            DAYS_INSTALMENT   \n",
              "216  installments_payments.csv         DAYS_ENTRY_PAYMENT   \n",
              "217  installments_payments.csv             AMT_INSTALMENT   \n",
              "218  installments_payments.csv                AMT_PAYMENT   \n",
              "\n",
              "                                           Description  \\\n",
              "209  Relative to application date of current applic...   \n",
              "210  Did the client requested insurance during the ...   \n",
              "211  ID of previous credit in Home credit related t...   \n",
              "212                           ID of loan in our sample   \n",
              "213  Version of installment calendar (0 is for cred...   \n",
              "214            On which installment we observe payment   \n",
              "215  When the installment of previous credit was su...   \n",
              "216  When was the installments of previous credit p...   \n",
              "217  What was the prescribed installment amount of ...   \n",
              "218  What the client actually paid on previous cred...   \n",
              "\n",
              "                                   Special  \n",
              "209  time only relative to the application  \n",
              "210                                    NaN  \n",
              "211                                 hashed  \n",
              "212                                 hashed  \n",
              "213                                    NaN  \n",
              "214                                    NaN  \n",
              "215  time only relative to the application  \n",
              "216  time only relative to the application  \n",
              "217                                    NaN  \n",
              "218                                    NaN  "
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data_desc = my_read_csv(data_dir+'HomeCredit_columns_description.csv')\n",
        "data_desc.tail(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "For the duplicated columns during join, like 'AMT_GOODS_PRICE_x' and 'AMT_GOODS_PRICE_y', only keep one of them"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Removed 12 duplicated columns\n",
            "Keeped 12 columns for each orginal column\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>SK_ID_CURR</th>\n",
              "      <th>TARGET</th>\n",
              "      <th>NAME_CONTRACT_TYPE</th>\n",
              "      <th>CODE_GENDER</th>\n",
              "      <th>FLAG_OWN_CAR</th>\n",
              "      <th>FLAG_OWN_REALTY</th>\n",
              "      <th>CNT_CHILDREN</th>\n",
              "      <th>AMT_INCOME_TOTAL</th>\n",
              "      <th>AMT_CREDIT</th>\n",
              "      <th>AMT_ANNUITY</th>\n",
              "      <th>...</th>\n",
              "      <th>AMT_PAYMENT_TOTAL_CURRENT</th>\n",
              "      <th>AMT_RECEIVABLE_PRINCIPAL</th>\n",
              "      <th>AMT_RECIVABLE</th>\n",
              "      <th>AMT_TOTAL_RECEIVABLE</th>\n",
              "      <th>CNT_DRAWINGS_ATM_CURRENT</th>\n",
              "      <th>CNT_DRAWINGS_CURRENT</th>\n",
              "      <th>CNT_DRAWINGS_OTHER_CURRENT</th>\n",
              "      <th>CNT_DRAWINGS_POS_CURRENT</th>\n",
              "      <th>CNT_INSTALMENT_MATURE_CUM</th>\n",
              "      <th>NAME_CONTRACT_STATUS</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>100002</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Cash loans</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>202500.0</td>\n",
              "      <td>406597.5</td>\n",
              "      <td>24700.5</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>100003</td>\n",
              "      <td>0.0</td>\n",
              "      <td>Cash loans</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>270000.0</td>\n",
              "      <td>1293502.5</td>\n",
              "      <td>35698.5</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>100004</td>\n",
              "      <td>0.0</td>\n",
              "      <td>Revolving loans</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>67500.0</td>\n",
              "      <td>135000.0</td>\n",
              "      <td>6750.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>100006</td>\n",
              "      <td>0.0</td>\n",
              "      <td>Cash loans</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>135000.0</td>\n",
              "      <td>312682.5</td>\n",
              "      <td>29686.5</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>100007</td>\n",
              "      <td>0.0</td>\n",
              "      <td>Cash loans</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>121500.0</td>\n",
              "      <td>513000.0</td>\n",
              "      <td>21865.5</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 206 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE  CODE_GENDER  FLAG_OWN_CAR  \\\n",
              "0      100002     1.0         Cash loans            0             0   \n",
              "1      100003     0.0         Cash loans            1             0   \n",
              "2      100004     0.0    Revolving loans            0             1   \n",
              "3      100006     0.0         Cash loans            1             0   \n",
              "4      100007     0.0         Cash loans            0             0   \n",
              "\n",
              "   FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
              "0                0             0          202500.0    406597.5      24700.5   \n",
              "1                1             0          270000.0   1293502.5      35698.5   \n",
              "2                0             0           67500.0    135000.0       6750.0   \n",
              "3                0             0          135000.0    312682.5      29686.5   \n",
              "4                0             0          121500.0    513000.0      21865.5   \n",
              "\n",
              "   ...  AMT_PAYMENT_TOTAL_CURRENT AMT_RECEIVABLE_PRINCIPAL AMT_RECIVABLE  \\\n",
              "0  ...                        NaN                      NaN           NaN   \n",
              "1  ...                        NaN                      NaN           NaN   \n",
              "2  ...                        NaN                      NaN           NaN   \n",
              "3  ...                        NaN                      NaN           NaN   \n",
              "4  ...                        NaN                      NaN           NaN   \n",
              "\n",
              "  AMT_TOTAL_RECEIVABLE CNT_DRAWINGS_ATM_CURRENT CNT_DRAWINGS_CURRENT  \\\n",
              "0                  NaN                      NaN                  NaN   \n",
              "1                  NaN                      NaN                  NaN   \n",
              "2                  NaN                      NaN                  NaN   \n",
              "3                  NaN                      NaN                  NaN   \n",
              "4                  NaN                      NaN                  NaN   \n",
              "\n",
              "   CNT_DRAWINGS_OTHER_CURRENT  CNT_DRAWINGS_POS_CURRENT  \\\n",
              "0                         NaN                       NaN   \n",
              "1                         NaN                       NaN   \n",
              "2                         NaN                       NaN   \n",
              "3                         NaN                       NaN   \n",
              "4                         NaN                       NaN   \n",
              "\n",
              "   CNT_INSTALMENT_MATURE_CUM  NAME_CONTRACT_STATUS  \n",
              "0                        NaN                   NaN  \n",
              "1                        NaN                   NaN  \n",
              "2                        NaN                   NaN  \n",
              "3                        NaN                   NaN  \n",
              "4                        NaN                   NaN  \n",
              "\n",
              "[5 rows x 206 columns]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data = remove_duplicated_cols(data, list(data_desc.Row))\n",
        "data.head()\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Column</th>\n",
              "      <th>Desc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SK_ID_CURR</td>\n",
              "      <td>ID of loan in our sample</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NAME_CONTRACT_TYPE</td>\n",
              "      <td>Identification if loan is cash or revolving</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CODE_GENDER</td>\n",
              "      <td>Gender of the client</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>FLAG_OWN_CAR</td>\n",
              "      <td>Flag if the client owns a car</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>FLAG_OWN_REALTY</td>\n",
              "      <td>Flag if client owns a house or flat</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>200</th>\n",
              "      <td>CNT_DRAWINGS_CURRENT</td>\n",
              "      <td>Number of drawings during this month on the pr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>201</th>\n",
              "      <td>CNT_DRAWINGS_OTHER_CURRENT</td>\n",
              "      <td>Number of other drawings during this month on ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202</th>\n",
              "      <td>CNT_DRAWINGS_POS_CURRENT</td>\n",
              "      <td>Number of drawings for goods during this month...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>203</th>\n",
              "      <td>CNT_INSTALMENT_MATURE_CUM</td>\n",
              "      <td>Number of paid installments on the previous cr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>204</th>\n",
              "      <td>NAME_CONTRACT_STATUS</td>\n",
              "      <td>Contract status during the month</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>205 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                         Column  \\\n",
              "0                    SK_ID_CURR   \n",
              "1            NAME_CONTRACT_TYPE   \n",
              "2                   CODE_GENDER   \n",
              "3                  FLAG_OWN_CAR   \n",
              "4               FLAG_OWN_REALTY   \n",
              "..                          ...   \n",
              "200        CNT_DRAWINGS_CURRENT   \n",
              "201  CNT_DRAWINGS_OTHER_CURRENT   \n",
              "202    CNT_DRAWINGS_POS_CURRENT   \n",
              "203   CNT_INSTALMENT_MATURE_CUM   \n",
              "204        NAME_CONTRACT_STATUS   \n",
              "\n",
              "                                                  Desc  \n",
              "0                             ID of loan in our sample  \n",
              "1          Identification if loan is cash or revolving  \n",
              "2                                 Gender of the client  \n",
              "3                        Flag if the client owns a car  \n",
              "4                  Flag if client owns a house or flat  \n",
              "..                                                 ...  \n",
              "200  Number of drawings during this month on the pr...  \n",
              "201  Number of other drawings during this month on ...  \n",
              "202  Number of drawings for goods during this month...  \n",
              "203  Number of paid installments on the previous cr...  \n",
              "204                   Contract status during the month  \n",
              "\n",
              "[205 rows x 2 columns]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "x_vars, y_var = list(data.loc[:,data.columns !='TARGET'].columns),'TARGET'\n",
        "desc_df = show_description(x_vars, data_desc)\n",
        "desc_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 12 columns without description in all the columns\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Column</th>\n",
              "      <th>Desc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>121</th>\n",
              "      <td>DAYS_EMPLOYED_PERC</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>122</th>\n",
              "      <td>INCOME_CREDIT_PERC</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>123</th>\n",
              "      <td>INCOME_PER_PERSON</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>124</th>\n",
              "      <td>ANNUITY_INCOME_PERC</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>125</th>\n",
              "      <td>PAYMENT_RATE</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>140</th>\n",
              "      <td>MONTHS_BALANCE_MIN</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>141</th>\n",
              "      <td>MONTHS_BALANCE_MAX</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>142</th>\n",
              "      <td>MONTHS_BALANCE_SIZE</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>143</th>\n",
              "      <td>SK_ID_PREV</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173</th>\n",
              "      <td>APP_CREDIT_PERC</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>179</th>\n",
              "      <td>SK_ID_PREV_x.1</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>186</th>\n",
              "      <td>SK_ID_PREV_y.1</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  Column  Desc\n",
              "121   DAYS_EMPLOYED_PERC  None\n",
              "122   INCOME_CREDIT_PERC  None\n",
              "123    INCOME_PER_PERSON  None\n",
              "124  ANNUITY_INCOME_PERC  None\n",
              "125         PAYMENT_RATE  None\n",
              "140   MONTHS_BALANCE_MIN  None\n",
              "141   MONTHS_BALANCE_MAX  None\n",
              "142  MONTHS_BALANCE_SIZE  None\n",
              "143           SK_ID_PREV  None\n",
              "173      APP_CREDIT_PERC  None\n",
              "179       SK_ID_PREV_x.1  None\n",
              "186       SK_ID_PREV_y.1  None"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "non_desc_df = desc_df[desc_df.Desc.isnull()]\n",
        "print('There are {} columns without description in all the columns'.format(len(non_desc_df)))\n",
        "non_desc_df\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "I droped all the columns without description"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [],
      "source": [
        "non_desc_list = list(non_desc_df.Column)\n",
        "data.drop(columns = non_desc_list,inplace=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P6zPIxskEiLx"
      },
      "source": [
        "# Select catergorical and numerical columns\n",
        "## Numerical features:\n",
        "- 1. Select those columns whose datatype is float or int64\n",
        "- 2. Removes all low-variance features whose variance lower than $1*10^{-8}$\n",
        "- 3. Check columns such as 'SK_DPD_*', 'FLAG_*' to see if it's really numerical features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "0vT9qjlJDviX"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 164 columns whose datatype is ['float64', 'int64']\n"
          ]
        }
      ],
      "source": [
        "# 1. select those columns whose datatype is float or int\n",
        "potential_num_df = select_col_by_dtype(data, ['float64','int64'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [],
      "source": [
        "potential_num_list = list(potential_num_df.columns)\n",
        "potential_num_list.remove('TARGET')\n",
        "potential_num_df = potential_num_df.loc[:,potential_num_list]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Column</th>\n",
              "      <th>Desc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SK_ID_CURR</td>\n",
              "      <td>ID of loan in our sample</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CODE_GENDER</td>\n",
              "      <td>Gender of the client</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>FLAG_OWN_CAR</td>\n",
              "      <td>Flag if the client owns a car</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>FLAG_OWN_REALTY</td>\n",
              "      <td>Flag if client owns a house or flat</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CNT_CHILDREN</td>\n",
              "      <td>Number of children the client has</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>160</th>\n",
              "      <td>CNT_DRAWINGS_ATM_CURRENT</td>\n",
              "      <td>Number of drawings at ATM during this month on...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>161</th>\n",
              "      <td>CNT_DRAWINGS_CURRENT</td>\n",
              "      <td>Number of drawings during this month on the pr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>162</th>\n",
              "      <td>CNT_DRAWINGS_OTHER_CURRENT</td>\n",
              "      <td>Number of other drawings during this month on ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>163</th>\n",
              "      <td>CNT_DRAWINGS_POS_CURRENT</td>\n",
              "      <td>Number of drawings for goods during this month...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>164</th>\n",
              "      <td>CNT_INSTALMENT_MATURE_CUM</td>\n",
              "      <td>Number of paid installments on the previous cr...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>165 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                         Column  \\\n",
              "0                    SK_ID_CURR   \n",
              "1                   CODE_GENDER   \n",
              "2                  FLAG_OWN_CAR   \n",
              "3               FLAG_OWN_REALTY   \n",
              "4                  CNT_CHILDREN   \n",
              "..                          ...   \n",
              "160    CNT_DRAWINGS_ATM_CURRENT   \n",
              "161        CNT_DRAWINGS_CURRENT   \n",
              "162  CNT_DRAWINGS_OTHER_CURRENT   \n",
              "163    CNT_DRAWINGS_POS_CURRENT   \n",
              "164   CNT_INSTALMENT_MATURE_CUM   \n",
              "\n",
              "                                                  Desc  \n",
              "0                             ID of loan in our sample  \n",
              "1                                 Gender of the client  \n",
              "2                        Flag if the client owns a car  \n",
              "3                  Flag if client owns a house or flat  \n",
              "4                    Number of children the client has  \n",
              "..                                                 ...  \n",
              "160  Number of drawings at ATM during this month on...  \n",
              "161  Number of drawings during this month on the pr...  \n",
              "162  Number of other drawings during this month on ...  \n",
              "163  Number of drawings for goods during this month...  \n",
              "164  Number of paid installments on the previous cr...  \n",
              "\n",
              "[165 rows x 2 columns]"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "potential_num_df_desc = show_description(potential_num_list, data_desc)\n",
        "potential_num_df_desc"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "I find that all the columns containing 'FLAG' is catergorical"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Column</th>\n",
              "      <th>Desc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>FLAG_OWN_CAR</td>\n",
              "      <td>Flag if the client owns a car</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>FLAG_OWN_REALTY</td>\n",
              "      <td>Flag if client owns a house or flat</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>FLAG_MOBIL</td>\n",
              "      <td>Did client provide mobile phone (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>FLAG_EMP_PHONE</td>\n",
              "      <td>Did client provide work phone (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>FLAG_WORK_PHONE</td>\n",
              "      <td>Did client provide home phone (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>FLAG_CONT_MOBILE</td>\n",
              "      <td>Was mobile phone reachable (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>FLAG_PHONE</td>\n",
              "      <td>Did client provide home phone (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>FLAG_EMAIL</td>\n",
              "      <td>Did client provide email (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>FLAG_DOCUMENT_2</td>\n",
              "      <td>Did client provide document 2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>FLAG_DOCUMENT_3</td>\n",
              "      <td>Did client provide document 3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>FLAG_DOCUMENT_4</td>\n",
              "      <td>Did client provide document 4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>FLAG_DOCUMENT_5</td>\n",
              "      <td>Did client provide document 5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>FLAG_DOCUMENT_6</td>\n",
              "      <td>Did client provide document 6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>FLAG_DOCUMENT_7</td>\n",
              "      <td>Did client provide document 7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>FLAG_DOCUMENT_8</td>\n",
              "      <td>Did client provide document 8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>FLAG_DOCUMENT_9</td>\n",
              "      <td>Did client provide document 9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>FLAG_DOCUMENT_10</td>\n",
              "      <td>Did client provide document 10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>FLAG_DOCUMENT_11</td>\n",
              "      <td>Did client provide document 11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>FLAG_DOCUMENT_12</td>\n",
              "      <td>Did client provide document 12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>FLAG_DOCUMENT_13</td>\n",
              "      <td>Did client provide document 13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>FLAG_DOCUMENT_14</td>\n",
              "      <td>Did client provide document 14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>FLAG_DOCUMENT_15</td>\n",
              "      <td>Did client provide document 15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>FLAG_DOCUMENT_16</td>\n",
              "      <td>Did client provide document 16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>FLAG_DOCUMENT_17</td>\n",
              "      <td>Did client provide document 17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>FLAG_DOCUMENT_18</td>\n",
              "      <td>Did client provide document 18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>FLAG_DOCUMENT_19</td>\n",
              "      <td>Did client provide document 19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>FLAG_DOCUMENT_20</td>\n",
              "      <td>Did client provide document 20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>FLAG_DOCUMENT_21</td>\n",
              "      <td>Did client provide document 21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>NFLAG_LAST_APPL_IN_DAY</td>\n",
              "      <td>Flag if the application was the last applicati...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>NFLAG_INSURED_ON_APPROVAL</td>\n",
              "      <td>Did the client requested insurance during the ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                       Column  \\\n",
              "0                FLAG_OWN_CAR   \n",
              "1             FLAG_OWN_REALTY   \n",
              "2                  FLAG_MOBIL   \n",
              "3              FLAG_EMP_PHONE   \n",
              "4             FLAG_WORK_PHONE   \n",
              "5            FLAG_CONT_MOBILE   \n",
              "6                  FLAG_PHONE   \n",
              "7                  FLAG_EMAIL   \n",
              "8             FLAG_DOCUMENT_2   \n",
              "9             FLAG_DOCUMENT_3   \n",
              "10            FLAG_DOCUMENT_4   \n",
              "11            FLAG_DOCUMENT_5   \n",
              "12            FLAG_DOCUMENT_6   \n",
              "13            FLAG_DOCUMENT_7   \n",
              "14            FLAG_DOCUMENT_8   \n",
              "15            FLAG_DOCUMENT_9   \n",
              "16           FLAG_DOCUMENT_10   \n",
              "17           FLAG_DOCUMENT_11   \n",
              "18           FLAG_DOCUMENT_12   \n",
              "19           FLAG_DOCUMENT_13   \n",
              "20           FLAG_DOCUMENT_14   \n",
              "21           FLAG_DOCUMENT_15   \n",
              "22           FLAG_DOCUMENT_16   \n",
              "23           FLAG_DOCUMENT_17   \n",
              "24           FLAG_DOCUMENT_18   \n",
              "25           FLAG_DOCUMENT_19   \n",
              "26           FLAG_DOCUMENT_20   \n",
              "27           FLAG_DOCUMENT_21   \n",
              "28     NFLAG_LAST_APPL_IN_DAY   \n",
              "29  NFLAG_INSURED_ON_APPROVAL   \n",
              "\n",
              "                                                 Desc  \n",
              "0                       Flag if the client owns a car  \n",
              "1                 Flag if client owns a house or flat  \n",
              "2       Did client provide mobile phone (1=YES, 0=NO)  \n",
              "3         Did client provide work phone (1=YES, 0=NO)  \n",
              "4         Did client provide home phone (1=YES, 0=NO)  \n",
              "5            Was mobile phone reachable (1=YES, 0=NO)  \n",
              "6         Did client provide home phone (1=YES, 0=NO)  \n",
              "7              Did client provide email (1=YES, 0=NO)  \n",
              "8                       Did client provide document 2  \n",
              "9                       Did client provide document 3  \n",
              "10                      Did client provide document 4  \n",
              "11                      Did client provide document 5  \n",
              "12                      Did client provide document 6  \n",
              "13                      Did client provide document 7  \n",
              "14                      Did client provide document 8  \n",
              "15                      Did client provide document 9  \n",
              "16                     Did client provide document 10  \n",
              "17                     Did client provide document 11  \n",
              "18                     Did client provide document 12  \n",
              "19                     Did client provide document 13  \n",
              "20                     Did client provide document 14  \n",
              "21                     Did client provide document 15  \n",
              "22                     Did client provide document 16  \n",
              "23                     Did client provide document 17  \n",
              "24                     Did client provide document 18  \n",
              "25                     Did client provide document 19  \n",
              "26                     Did client provide document 20  \n",
              "27                     Did client provide document 21  \n",
              "28  Flag if the application was the last applicati...  \n",
              "29  Did the client requested insurance during the ...  "
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "flag_col_list = []\n",
        "for col in potential_num_df.columns:\n",
        "    if 'FLAG' in col:\n",
        "        flag_col_list.append(col)\n",
        "show_description(flag_col_list,data_desc)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [],
      "source": [
        "potential_num_df.drop(columns = flag_col_list, inplace=True)\n",
        "# update potential_num_list\n",
        "potential_num_list = list(potential_num_df.columns)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 constant features are removed, namely []\n",
            "There are 147 columns after removing constants\n"
          ]
        }
      ],
      "source": [
        "# 2. Remove low-variance features whose variance lower than 1*10^(-8) - similar to constant\n",
        "variance_threshhold = 0.00000001\n",
        "potential_num_df = remove_lowvariance(potential_num_df, variance_threshhold)\n",
        "potential_num_df\n",
        "potential_num_list = list(potential_num_df.columns)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Catergorical columns:\n",
        "- If the datatype is 'object' or contains 'FLAG', it must be catergorical feature\n",
        "- Elif the datatype is 'int64', it may be catergorical feature\n",
        "- If the datatype is 'float64', it may be catergorical feature"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "qarL0vtnDA4N"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 30 columns whose datatype is ['object']\n",
            "After adding in 30 columns contain FLAG\n",
            "There are 62 columns that I am sure they are catergorical data\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "['NAME_CONTRACT_TYPE',\n",
              " 'NAME_TYPE_SUITE',\n",
              " 'NAME_INCOME_TYPE',\n",
              " 'NAME_EDUCATION_TYPE',\n",
              " 'NAME_FAMILY_STATUS',\n",
              " 'NAME_HOUSING_TYPE',\n",
              " 'OCCUPATION_TYPE',\n",
              " 'WEEKDAY_APPR_PROCESS_START',\n",
              " 'ORGANIZATION_TYPE',\n",
              " 'FONDKAPREMONT_MODE',\n",
              " 'HOUSETYPE_MODE',\n",
              " 'WALLSMATERIAL_MODE',\n",
              " 'EMERGENCYSTATE_MODE',\n",
              " 'CREDIT_ACTIVE',\n",
              " 'CREDIT_CURRENCY',\n",
              " 'CREDIT_TYPE',\n",
              " 'FLAG_LAST_APPL_PER_CONTRACT',\n",
              " 'NAME_CASH_LOAN_PURPOSE',\n",
              " 'NAME_CONTRACT_STATUS',\n",
              " 'NAME_CONTRACT_STATUS',\n",
              " 'NAME_PAYMENT_TYPE',\n",
              " 'CODE_REJECT_REASON',\n",
              " 'NAME_CLIENT_TYPE',\n",
              " 'NAME_GOODS_CATEGORY',\n",
              " 'NAME_PORTFOLIO',\n",
              " 'NAME_PRODUCT_TYPE',\n",
              " 'CHANNEL_TYPE',\n",
              " 'NAME_SELLER_INDUSTRY',\n",
              " 'NAME_YIELD_GROUP',\n",
              " 'PRODUCT_COMBINATION',\n",
              " 'NAME_CONTRACT_STATUS',\n",
              " 'NAME_CONTRACT_STATUS',\n",
              " 'FLAG_OWN_CAR',\n",
              " 'FLAG_OWN_REALTY',\n",
              " 'FLAG_MOBIL',\n",
              " 'FLAG_EMP_PHONE',\n",
              " 'FLAG_WORK_PHONE',\n",
              " 'FLAG_CONT_MOBILE',\n",
              " 'FLAG_PHONE',\n",
              " 'FLAG_EMAIL',\n",
              " 'FLAG_DOCUMENT_2',\n",
              " 'FLAG_DOCUMENT_3',\n",
              " 'FLAG_DOCUMENT_4',\n",
              " 'FLAG_DOCUMENT_5',\n",
              " 'FLAG_DOCUMENT_6',\n",
              " 'FLAG_DOCUMENT_7',\n",
              " 'FLAG_DOCUMENT_8',\n",
              " 'FLAG_DOCUMENT_9',\n",
              " 'FLAG_DOCUMENT_10',\n",
              " 'FLAG_DOCUMENT_11',\n",
              " 'FLAG_DOCUMENT_12',\n",
              " 'FLAG_DOCUMENT_13',\n",
              " 'FLAG_DOCUMENT_14',\n",
              " 'FLAG_DOCUMENT_15',\n",
              " 'FLAG_DOCUMENT_16',\n",
              " 'FLAG_DOCUMENT_17',\n",
              " 'FLAG_DOCUMENT_18',\n",
              " 'FLAG_DOCUMENT_19',\n",
              " 'FLAG_DOCUMENT_20',\n",
              " 'FLAG_DOCUMENT_21',\n",
              " 'NFLAG_LAST_APPL_IN_DAY',\n",
              " 'NFLAG_INSURED_ON_APPROVAL']"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Firstly, the columns of 'object' dtype must be catergorical data \n",
        "cat_df = select_col_by_dtype(data, ['object'])\n",
        "cat_columns = list(cat_df.columns)\n",
        "print('After adding in {} columns contain FLAG'.format(len(flag_col_list)))\n",
        "# Add the columns containing FLAG into cat_columns\n",
        "cat_columns = cat_columns + flag_col_list\n",
        "cat_df = data.loc[:,cat_columns]\n",
        "print('There are {} columns that I am sure they are catergorical data'.format(len(cat_columns)))\n",
        "cat_columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Column</th>\n",
              "      <th>Desc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NAME_CONTRACT_TYPE</td>\n",
              "      <td>Identification if loan is cash or revolving</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NAME_TYPE_SUITE</td>\n",
              "      <td>Who was accompanying client when he was applyi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NAME_INCOME_TYPE</td>\n",
              "      <td>Clients income type (businessman, working, mat...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NAME_EDUCATION_TYPE</td>\n",
              "      <td>Level of highest education the client achieved</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NAME_FAMILY_STATUS</td>\n",
              "      <td>Family status of the client</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>FLAG_DOCUMENT_19</td>\n",
              "      <td>Did client provide document 19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58</th>\n",
              "      <td>FLAG_DOCUMENT_20</td>\n",
              "      <td>Did client provide document 20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>FLAG_DOCUMENT_21</td>\n",
              "      <td>Did client provide document 21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60</th>\n",
              "      <td>NFLAG_LAST_APPL_IN_DAY</td>\n",
              "      <td>Flag if the application was the last applicati...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>NFLAG_INSURED_ON_APPROVAL</td>\n",
              "      <td>Did the client requested insurance during the ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>62 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                       Column  \\\n",
              "0          NAME_CONTRACT_TYPE   \n",
              "1             NAME_TYPE_SUITE   \n",
              "2            NAME_INCOME_TYPE   \n",
              "3         NAME_EDUCATION_TYPE   \n",
              "4          NAME_FAMILY_STATUS   \n",
              "..                        ...   \n",
              "57           FLAG_DOCUMENT_19   \n",
              "58           FLAG_DOCUMENT_20   \n",
              "59           FLAG_DOCUMENT_21   \n",
              "60     NFLAG_LAST_APPL_IN_DAY   \n",
              "61  NFLAG_INSURED_ON_APPROVAL   \n",
              "\n",
              "                                                 Desc  \n",
              "0         Identification if loan is cash or revolving  \n",
              "1   Who was accompanying client when he was applyi...  \n",
              "2   Clients income type (businessman, working, mat...  \n",
              "3      Level of highest education the client achieved  \n",
              "4                         Family status of the client  \n",
              "..                                                ...  \n",
              "57                     Did client provide document 19  \n",
              "58                     Did client provide document 20  \n",
              "59                     Did client provide document 21  \n",
              "60  Flag if the application was the last applicati...  \n",
              "61  Did the client requested insurance during the ...  \n",
              "\n",
              "[62 rows x 2 columns]"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cat_df_desc = show_description(cat_columns, data_desc)\n",
        "cat_df_desc"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "All the catergorical data has meaning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Column    0\n",
              "Desc      0\n",
              "dtype: int64"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cat_df_desc.isnull().sum()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mxIjFSfMhRtt"
      },
      "source": [
        "# Check missing values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 55.258039% of data whose target is none\n"
          ]
        }
      ],
      "source": [
        "print('There are {:%} of data whose target is none'.format(len(data[data.TARGET.isnull()])/len(data)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Delete rows whose target is none"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "After deleting those whose target is none, 18181 rows will be deleted, and there are only 14721 rows left\n"
          ]
        }
      ],
      "source": [
        "new_data = data.dropna(subset=['TARGET'])\n",
        "print('After deleting those whose target is none, {} rows will be deleted, and there are only {} rows left'.format( len(data)-len(new_data),len(new_data)))\n",
        "data = new_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(14721, 194)"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "new_data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(14721, 194)"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Check the number of missing features for each row"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([ 73,  68, 120, ..., 125, 125, 125])"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "missing_num = data.isnull().sum(axis=1).values\n",
        "missing_num"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t90wEIr3oZDM",
        "outputId": "74efb179-ce1d-450a-a380-b8ade498eb0a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "If we dropna for our current 14721 rows of data, there will be 0 rows left\n"
          ]
        }
      ],
      "source": [
        "print(\"If we dropna for our current {} rows of data, there will be {} rows left\".format(len(data),len(data.dropna())))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "As we can see, the highly missing values are majorly some processed data such as `RATE_INTEREST_PRIMARY\t`:'Interest rate normalized on previous credi'. But I haven't figured out the reason for that."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Column</th>\n",
              "      <th>Desc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>RATE_INTEREST_PRIVILEGED</td>\n",
              "      <td>Interest rate normalized on previous credit</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>RATE_INTEREST_PRIMARY</td>\n",
              "      <td>Interest rate normalized on previous credit</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DAYS_FIRST_DRAWING</td>\n",
              "      <td>Relative to application date of current applic...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AMT_CREDIT_MAX_OVERDUE</td>\n",
              "      <td>Maximal amount overdue on the Credit Bureau cr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>DAYS_ENDDATE_FACT</td>\n",
              "      <td>Days since CB credit ended at the time of appl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73</th>\n",
              "      <td>NONLIVINGAREA_AVG</td>\n",
              "      <td>Normalized information about building where th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>NONLIVINGAREA_MODE</td>\n",
              "      <td>Normalized information about building where th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75</th>\n",
              "      <td>ELEVATORS_MEDI</td>\n",
              "      <td>Normalized information about building where th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>76</th>\n",
              "      <td>ELEVATORS_AVG</td>\n",
              "      <td>Normalized information about building where th...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77</th>\n",
              "      <td>ELEVATORS_MODE</td>\n",
              "      <td>Normalized information about building where th...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>78 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                      Column  \\\n",
              "0   RATE_INTEREST_PRIVILEGED   \n",
              "1      RATE_INTEREST_PRIMARY   \n",
              "2         DAYS_FIRST_DRAWING   \n",
              "3     AMT_CREDIT_MAX_OVERDUE   \n",
              "4          DAYS_ENDDATE_FACT   \n",
              "..                       ...   \n",
              "73         NONLIVINGAREA_AVG   \n",
              "74        NONLIVINGAREA_MODE   \n",
              "75            ELEVATORS_MEDI   \n",
              "76             ELEVATORS_AVG   \n",
              "77            ELEVATORS_MODE   \n",
              "\n",
              "                                                 Desc  \n",
              "0         Interest rate normalized on previous credit  \n",
              "1         Interest rate normalized on previous credit  \n",
              "2   Relative to application date of current applic...  \n",
              "3   Maximal amount overdue on the Credit Bureau cr...  \n",
              "4   Days since CB credit ended at the time of appl...  \n",
              "..                                                ...  \n",
              "73  Normalized information about building where th...  \n",
              "74  Normalized information about building where th...  \n",
              "75  Normalized information about building where th...  \n",
              "76  Normalized information about building where th...  \n",
              "77  Normalized information about building where th...  \n",
              "\n",
              "[78 rows x 2 columns]"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Let's look at the columns whose missing ratio is larger than 0.5\n",
        "highly_missing_cols = filter_columns_by_missing_ratio(data, 0.5)\n",
        "\n",
        "desc_df = show_description(highly_missing_cols, data_desc)\n",
        "# list_1 = [i for i in desc_df['Desc'].values if 'normalized' in i.lower()]\n",
        "# print('There are {} columns in the missing cols that is normalized, which is {:%} out of all the missing values'.format(len(list_1),len(list_1)/len(highly_missing_cols)))\n",
        "desc_df\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['RATE_INTEREST_PRIMARY', 'RATE_INTEREST_PRIVILEGED']"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# I want to check if there are non-normalized data for 'interest' but I failed.\n",
        "[i for i in data.columns if 'INTEREST' in i]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>MissingRatio</th>\n",
              "      <th>ColNumb</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.1</td>\n",
              "      <td>64.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.2</td>\n",
              "      <td>78.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.3</td>\n",
              "      <td>87.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.4</td>\n",
              "      <td>97.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.5</td>\n",
              "      <td>116.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.6</td>\n",
              "      <td>129.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0.7</td>\n",
              "      <td>146.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0.8</td>\n",
              "      <td>151.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>0.9</td>\n",
              "      <td>151.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   MissingRatio  ColNumb\n",
              "0           0.1     64.0\n",
              "1           0.2     78.0\n",
              "2           0.3     87.0\n",
              "3           0.4     97.0\n",
              "4           0.5    116.0\n",
              "5           0.6    129.0\n",
              "6           0.7    146.0\n",
              "7           0.8    151.0\n",
              "8           0.9    151.0"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEGCAYAAACevtWaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAnS0lEQVR4nO3deXhU5d3G8e8vC0lYA4QtCRBEdsIaQKxVxAW0KqBWxQ0tSq1dxCpa69tq27d1wbbaV22LVoW6QFVKaaul1KJWK6vsSwQFIWELSwiQffK8f8yAEQJZJsmZ5f5cF5czZ87M3Ixw8+Q55zxjzjlERCSyxHgdQERE6p/KXUQkAqncRUQikMpdRCQCqdxFRCJQnNcBAFJSUlxGRobXMUREwsqKFSv2OefaVfVYSJR7RkYGy5cv9zqGiEhYMbPPT/WYpmVERCKQyl1EJAKp3EVEIlBIzLlXpaysjJycHIqLi72OErYSExNJT08nPj7e6ygi0shCttxzcnJo0aIFGRkZmJnXccKOc479+/eTk5NDt27dvI4jIo0sZMu9uLhYxR4EM6Nt27bk5eV5HUXEU/NW5jJ9QTY784tITU5i2phejB+c5nWsBs8VsuUOqNiDpM9Pot28lbk8MHctRWU+AHLzi3hg7loATwu+MXKFdLmLiARj+oJNxwv0mKIyH9MXZHNez3a8+OHWk54zuk8HBnVOZm9BMS8vPvk08rH9O9E3tSU7DhTy+vIdJz1+xaBUzmzfgs/yjjBvZe5Jj189tDPTF2SfMpfKvZHs3r2bqVOnsmzZMpKTk+nQoQNPPvkkPXv2PGnfbdu2cdlll7Fu3Treffddzj//fObPn8/ll18OwGWXXca9997LqFGjgs517MKvlJSUoF9LJBIdKiwjN7/qEzJ25hdxqKiM/1u05aTH2rVM9Jf74ZIqH+/Wrhl9U1uyM7+oysf7p7XizPYt+Hx/YZWPn3VGW3bmF50yV32JmHJviPkr5xwTJkxg0qRJzJ49G4DVq1ezZ8+eKsv9ROnp6fz85z8/Xu4i0jDyDpewaNNe3tm0h+SkJjx29QBaNY2naZNYCkt9J+2fmpxERkoztj7ytVO+Zv+0Vqd9fMQZbU/7+Pm925/y8dTkJHKrKPLU5KRTvl5tRcR57sfmr3Lzi3B8MX9V1Y9EtbFo0SLi4+O54447jm8bOHAg55xzDtOmTaN///5kZmYyZ86cKp8/cOBAWrVqxcKFC096LCMjg3379gGwfPny46P5hx9+mEmTJvHVr36Vrl27MnfuXO677z4yMzMZO3YsZWVlx1/j8ccfJzMzk+HDh7Nly8kjBJFIN3vpdsY98yHDfv4v7ntzDWtyDpHc7ItTf38xIZOk+NgvPScpPpZpY3o1dtQvmTamV4PnCpuR+7W//+ikbZcN6MRNIzN4/B9Vz6s9/Nf1jB+cxoGjpXzr5RVfenzON0dW+57r1q1j6NChJ22fO3cuq1atYvXq1ezbt49hw4Zx7rnnVvkaDz74ID/60Y+46KKLqn2/Yz799FMWLVrEhg0bGDlyJG+++SaPP/44EyZM4O9//zvjx48HoFWrVqxdu5ZZs2YxdepU/va3v9X4PUTCTXGZj48+3c97n+Tx4Nf6EB8bw9b9RwG456KejO7Tnr6dWn7pRIJjP72H2tkyjZErbMr9dHYdqnpeLb+wrMrtwfrggw+YOHEisbGxdOjQgfPOO49ly5YxYMCAk/Y9VvoffPBBjV//kksuIT4+nszMTHw+H2PHjgUgMzOTbdu2Hd9v4sSJx/979913B/E7EglNB46WsmD9bt7ZuIcPtuyjuKyCpk1imTi8C706tuD+Mb2JiTn9WWHjB6d5XuZVaehcYVPupxtpn2r+Ki0wf9WmWZMajdRP1K9fP954441aP+9EDz74IP/7v/9LXNwXH3dcXBwVFRUAJ12Fm5CQAEBMTAzx8fHHRyIxMTGUl5cf36/yCEWnPUokqKhwrN9ZQMukOLq2bcbmPYd5YO5a0pKTuDarM6P7dGBEtzYkBqY0qiv2aBYRc+4NNX81evRoSkpKmDFjxvFta9asITk5mTlz5uDz+cjLy+P9999n+PDhp3ydiy++mIMHD7JmzZrj2zIyMlixwj9V9Oabb9Yp37G5/jlz5jByZO3/8RIJBYWl5SzcsIcfvLmGsx55h8uf/oBZH/lPQRzatTULpp7LB/efz0/G9ee8nu2OF7ucXtiM3E+noeavzIw///nPTJ06lccee4zExEQyMjJ48sknOXLkCAMHDsTMePzxx+nYseOXpkxO9OCDDzJu3Ljj9x966CEmT57Mj370ozqfGnnw4EEGDBhAQkICr732Wp1eQ8QLR0vKaZYQh3OO0U+8x+6CYponxHFez3aM7t2eUb383z8RFxtDr44tPE4bnsw553UGsrKy3Ilf1rFx40b69OnjUaLIoc9RQkFFhWN1Tj7/3rSXf23cS3GZj3/fc55/ALUyh/YtEhmW0YYmcRExmdBozGyFcy6rqsciYuQuIt471bUms5du54l/ZrPvSCkxBlkZbRg/KBVfhSMu1pgwON3r6BFJ5S4iQatqrZT73/QfY2rbPIGR3VO4IDDdkty0iZdRo0ZIl7tzTmeBBCEUptwkOlS1VkpJeQXTF2Tz4Q9Gc1HfDh4li14hO8GVmJjI/v37VVB1dGw998TERK+jSBSo6lRkqN+1UqR2Qnbknp6eTk5OjtYjD8Kxb2ISaWgtE+MoKC4/aXt9rpUitROy5R4fH69vEBIJccVlPhLjY/npuP78YO4aissqjj8WCmu4RLOQnZYRkdC2+LP9jJr+LutyDzF+cBqPXjmAtOQkDP/V4Y9cmRmSl/1Hi5AduYtI6Fq69QDfeGkZqclJdGjpP64Tqmu4RCuN3EWkVlZ8foBbX1xKx1aJvHr7CNq1SPA6klRB5S4iNfbJnsNMemEZ7Vsm8trtZ9G+hc7GClWalhGRGuvatilXDUnjjlHdj0/HSGhSuYtItTbuKqBDy0TaNGvCT8b19zqO1ICmZUTktNbvPMR1MxZz3xurvY4itVBtuZvZC2a218zWVfHYPWbmzCwlcN/M7DdmtsXM1pjZkIYILSKNY9PuAm58fgnNmsTy0OX9vI4jtVCTkftLwNgTN5pZZ+BiYHulzZcAPQK/pgC/DT6iiHjhkz2HueG5JSTExfLalLPo3Kap15GkFqotd+fc+8CBKh76NXAfUHnxl3HALOe3GEg2s071klREGo1zjh/OXUtsjPHalLPo2raZ15Gklup0QNXMxgG5zrnVJ6zamAbsqHQ/J7BtVxWvMQX/6J4uXbrUJYaINBAz4+nrh3C0tJxuKSr2cFTrA6pm1hT4IfDjYN7YOTfDOZflnMtq165dMC8lIvVk676j/OxvG/BVODq2SqR7u+ZeR5I6qsvZMt2BbsBqM9sGpAMfm1lHIBfoXGnf9MA2EQlxn+8/ysQZi/nzylwt1RsBal3uzrm1zrn2zrkM51wG/qmXIc653cB84ObAWTNnAYeccydNyYhIaNlxoJCJMxZTUu7jldtG6OBpBKjJqZCvAR8Bvcwsx8wmn2b3t4DPgC3Ac8Cd9ZJSRBpMzsFCJj63mKOlPl6+bQR9OrX0OpLUg2oPqDrnJlbzeEal2w74dvCxRKSx7MwvptzneOW2EfRLbeV1HKknWn5AJEod+6KN4d3a8O60USTGx3odSeqRlh8QiUJ7C4q59Df/4dUl/msQVeyRR+UuEmX2Hi5m4nOL2XOomF4ddapjpNK0jEgU2XekhBueW8LO/GJmfmM4Q7u28TqSNBCN3EWiRHGZjxufX8KOg4W8cMswhndTsUcyjdxFokRifCzXZHWmd8cWjOze1us40sBU7iIRLr+wlJyDRfRPa8U3zunmdRxpJJqWEYlghwrLuPEPS5j0wlKOlpR7HUcakUbuIhGqoLiMm19YQvbuw/z+pqE0S9Bf92iikbtIBDpcXMakF5ayfmcBz94wlNG9O3gdSRqZ/ikXiUDP/Wcra3IO8cz1Q7ior4o9GqncRSLQd0efybk9UsjK0OmO0UrTMiIRoqjUxw//vJa8wyXEx8ao2KOcyl0kAhSX+bht1jJeW7qdj7cf9DqOhACVu0iYKy7zcfus5fz30/08cfVAxvTr6HUkCQEqd5EwVlLu45t/XMF/Nu/jsasGcNXQdK8jSYjQAVWRMDNvZS7TF2SzM7+IDq0ScRWOR67M5JqsztU/WaKGyl0kjMxbmcsDc9dSVOYDYPehYhLjY0jSeuxyAk3LiISR6Quyjxf7McVlFUxfkO1RIglVKneRMLIzv6hW2yV6qdxFwsSiTXtP+VhqclIjJpFwoHIXCQNHSsq55/XVdGqVSELcl//aJsXHMm1ML4+SSajSAVWREHaoqIwWCXE0T4jjj5OH071dc/6xbvfxs2VSk5OYNqYX4weneR1VQozKXSRErd6Rz52vfMwNZ3XhzlFn0i+1FQDjB6epzKVampYRCTHOOV5dsp2v/+4jAM45M8XjRBKONHIXCSFFpT7+Z9463vw4h3N7tuOpawfRulkTr2NJGFK5i4SQjbsLmL86l7su6MH3LuhBbIx5HUnClMpdJAR8lneEM9o1Z0iX1rw77XzSdGqjBElz7iIe8lU4pi/YxIW/eo8Pt+wDULFLvdDIXcQj+4+UcNfsVXywZR/XZnVmaNfWXkeSCKJyF/HAyu0HufOVj9l/tJTHrsrk2mFdvI4kEUblLuKBdTsLiI0x5n7rbPqntfI6jkQglbtIIykq9bFhVwFDu7bmxhFduHJwGs0S9FdQGoYOqIo0gq37jjLh2Q+55YWlHCosw8xU7NKg9KdLpIEtWL+be/+0mthY4+kbhtCqabzXkSQKqNxFGohzjsf+kc3v3vuUAemtePaGIaS3bup1LIkSKneRBmJmFBSXcf2ILjx0eV8S4vRVeNJ4qp1zN7MXzGyvma2rtG26mW0yszVm9mczS6702ANmtsXMss1sTAPlFglZKz4/yIadBQD8bFx/fjEhU8Uuja4mB1RfAsaesG0h0N85NwD4BHgAwMz6AtcB/QLPedbM9KdaooJzjpc+3Mq1v/+IR97eCKC1YcQz1Za7c+594MAJ2/7pnCsP3F0MpAdujwNmO+dKnHNbgS3A8HrMKxKSCkvLuWv2Kh7+6wZG9WrH09cP8TqSRLn6mHP/BjAncDsNf9kfkxPYdhIzmwJMAejSRVfnSfjaU1DMTX9Ywpa9R5g2phffOq87MRqxi8eCKnczexAoB16p7XOdczOAGQBZWVkumBwiXmrTrAlnpDTnx5f145we+mINCQ11vojJzG4BLgNucM4dK+dcoHOl3dID20QiSrmvgt+8s5n9R0qIj43hdzcNVbFLSKlTuZvZWOA+4ArnXGGlh+YD15lZgpl1A3oAS4OPKRI69h4u5vrnl/CrhZ/w1tpdXscRqVK10zJm9howCkgxsxzgIfxnxyQAC80MYLFz7g7n3Hoz+xOwAf90zbedc76GCi/S2JZtO8Cdr3zM4eIyfn3tQCYMTq/+SSIesC9mVLyTlZXlli9f7nUMkdP6x7rdfOfVj0lvncTvbhpK744tvY4kUc7MVjjnsqp6TFeoipzCvJW5TF+Qzc78IlKTk/jWqO5cM6wzP7ikNy0TtT6MhDatCilShXkrc3lg7lpy84twQG5+ET//+0aGZ7RRsUtYULmLVGH6gmyKyr58uKiozMf0BdkeJRKpHZW7SBV25hfVartIqFG5i5zgwNHSU15hmpqc1MhpROpG5S5ygr2Hi2mZGEdC3Jf/eiTFxzJtTC+PUonUjspdJODYacG9O7ZkyQ8v5LGrBpCWnIQBaclJPHJlJuMHV7lUkkjI0amQIviL/aH562mWEMd9Y3rRJC6G8YPTVOYStjRyFwFmvP8Zsz76nHJfBYGrrkXCmspdot781Tt55O1NfG1AJx64pI/XcUTqhcpdotriz/Zz759WM7xbG3759YFah10ihspdolre4RK6t2/OjJuGkhivb4SUyKEDqhKVKiocMTHG5QNTuaR/R+JiNc6RyKI/0RJ1DheXceVv/8vf1uwEULFLRNKfaokqZb4K7nzlY9bmHqKFFgCTCKZyl6jhnOOBuWv5z+Z9PHJlJuf1bOd1JJEGo3KXqPHrf23mjRU5TL2wB9dkda7+CSJhTOUuUcE5R0m5j2uy0rnrgh5exxFpcDpbRiJeua+CuNgYHrikDxUVTlegSlTQyF0i2tqcQ1z4q/fYsLMAQBcpSdRQuUvE2nGgkFtfWkaZz5HSoonXcUQalaZlJCLlF5Yy6cWllPkqmD1lBO1bJHodSaRRqdwl4hSX+bh91nJyDhbx8uQRnNm+hdeRRBqdpmUk4vgqHC0S4/n1NYMY3q2N13FEPKGRu0SUknIfzRLi+MOkLJ0VI1FNI3eJGM//5zOu+u1/OVRYpmKXqKdyl4jw1tpd/PytjaQnN6V5on4gFVG5S9hbtu0AU+esYkiX1jx53SBidS67iMpdwtuWvUe4beZy0pOTeP7mLH3hhkiAyl3CWkJcDL07tmDmN4bTupkuVBI5RpOTEpaKy3w0iY2hc5umzPnmSK/jiIQcjdwl7JT7Krjj5RXc+/pqnHNexxEJSSp3CSvOOf5n3jrezc4jK6ONTnkUOQWVu4SV//v3FmYv28F3zj+T60d08TqOSMhSuUvYeH35Dn618BOuHJLGPRf39DqOSEhTuUvYSGudxKWZHXn0ygGajhGpRrXlbmYvmNleM1tXaVsbM1toZpsD/20d2G5m9hsz22Jma8xsSEOGl+hwuLgMgLO7p/DsDUNpEqcxiUh1avK35CVg7AnbfgC845zrAbwTuA9wCdAj8GsK8Nv6iSnRKje/iAt/9R4vL/7c6ygiYaXacnfOvQ8cOGHzOGBm4PZMYHyl7bOc32Ig2cw61VNWiTKHCsu45YWlFJb6GJahpXtFaqOuP992cM7tCtzeDXQI3E4DdlTaLyew7SRmNsXMlpvZ8ry8vDrGkEhVUu5jyh+Xs23/UX5/01B6ddQXbojURtCTl85/FUmtryRxzs1wzmU557LatWsXbAyJIM457n19DUu2HuCJrw/k7O4pXkcSCTt1XX5gj5l1cs7tCky77A1szwU6V9ovPbBNpMbMjOEZremf2pJxg6r8wU9EqlHXkft8YFLg9iTgL5W23xw4a+Ys4FCl6RuRauUXlgJw08gMvnled4/TiISvakfuZvYaMApIMbMc4CHgUeBPZjYZ+By4JrD7W8ClwBagELi1ATJLhJm3MpfpC7LJzS/CgLsu7MHUC3WRkkgwqi1359zEUzx0QRX7OuDbwYaS6DFvZS4PzF1LUZkP8B+8+d17n5LRthnjB2tKRqSudDWIeGr6gk3Hi/2Y4rIKpi/I9iiRSGRQuYuncvOLq9y+M7+okZOIRBaVuzS6wtJyVnx+EIDU5MQq90lNTmrMSCIRR+UujcY5x9trd3HhL9/j1heXcqSknPvG9CbphO89TYqPZdqYXh6lFIkM+po9aRRb9h7hJ39dz38276N3xxY8NXEwzRPijh80nb4gm535RaQmJzFtTC8dTBUJkspdGtyOA4Vc8tT7JMbH8vDlfbnxrK7ExX7xQ+P4wWkqc5F6pnKXBuGcY+Ouw/RNbUnnNk15+Ip+XNy3I+1aJHgdTSQqaM5d6t3mPYe54fklXP70B2zZewSAG0Z0VbGLNCKN3KXeHCkp56l/fcKLH26jaRP/FEy3lGZexxKJSip3qRcl5T7GPvk+uflFXJvVmWljetG2uUbqIl5RuUtQcg4Wkt66KQlxsdw56kz6prZkUOdkr2OJRD3NuUudFBSX8ZO/rue86e/y3if+L1u5fkQXFbtIiNDIXWqlosIxd2Uuj769kf1HS7l+eBcGpLXyOpaInEDlLrXyzZdXsHDDHgZ1TubFW4aTma5iFwlFKnep1qGiMponxBEbY3wtsxMX9enA1UPTiYkxr6OJyClozl1OqaLC8adlOxj9xLu8unQ74L+a9JphnVXsIiFOI3ep0rrcQ/zoL+tYuT2foV1bM6RLsteRRKQWVO5ykmcWbeGJf2bTtlkCv/z6QK4ckoaZRuoi4UTlLoB/CqasooKEuFgGpidzy9kZ3H1RT1omxnsdTUTqQOUurN6Rz4//so6zzmjLA5f24ZweKZzTI8XrWCISBJV7FDtwtJTpCzYxe9kOUpon0E/nq4tEDJV7FJm3Mvf4l2K0adaEwtJySn2OyV/pxl0X9qCFpmBEIobKPUrMW5nLA3PXUlTmA2D/0VJiDKaN6cW3Rp3pcToRqW86zz1KPPL2xuPFfkyFg5cXb/cokYg0JI3cI1ze4RL+79+b2VNQUuXjO/OLGjmRiDQGlXsEezd7L3e+8jEl5RU0bRJLYanvpH1Sk5M8SCYiDU3TMhGmpNxHbmA0PiA9mbH9OrLw7nP5xYRMkuJjv7RvUnws08b08iKmiDQwjdwjREWFY/7qnfxyYTZtmjZh3re/QptmTfjVtYMAOKNdc4DjZ8ukJicxbUwvxg9O8zC1iDQUlXuYc87x3id5PPaPbDbuKqBvp5bcc3HVo/Hxg9NU5iJRQuUe5v6+dhffeXUlndsk8dR1g7h8QKpWbBQRlXs4+izvCLsPFXP2mSlc1LcDv5iQydVD02kSp0MoIuKncg8jewuKefKdzcxZtoNuKc1YePe5JMTFcv2ILl5HE5EQo3IPAwXFZcx47zP+8MFWyisquHFEF74zuoeW4RWRU1K5h4HFn+7n6UVbuGJgKvdc3JOubZt5HUlEQpzKPQT5KhzzVuZypKScSWdncFHfDiy8+1x6dGjhdTQRCRMq9xDinGNR9l4eezub7D2HGdGtDTeP7IqZqdhFpFaCKnczuxu4DXDAWuBWoBMwG2gLrABucs6VBpkz4m3cVcBD89ezdOsBurZtytPXD+bS/p00ry4idVLnc+fMLA34HpDlnOsPxALXAY8Bv3bOnQkcBCbXR9BI5ZwD/FMxW/cd5Wfj+vGv75/HZTpfXUSCEOy0TByQZGZlQFNgFzAauD7w+EzgYeC3Qb5PxNl1qIgnF27G4Xj86oH0T2vFh/eP1rnqIlIv6lzuzrlcM3sC2A4UAf/EPw2T75wrD+yWA+h690oOFZbx2/c+5cUPt1LhHDePzMA5h5mp2EWk3tS53M2sNTAO6AbkA68DY2vx/CnAFIAuXaLjIpz3P8nju6+tpKC4jPGD0vj+RT3p3Kap17FEJAIFMy1zIbDVOZcHYGZzga8AyWYWFxi9pwO5VT3ZOTcDmAGQlZXlgsgRUip/T2lqchL3XNSTr/ZsR7sWCfTq2IIR3dow9cKe9E1t6XVUEYlgwcwDbAfOMrOm5j+l4wJgA7AIuDqwzyTgL8FFDB/Hvqc0N78IB+TmF3HP66u59vcf4ZyjQ8tEZtycpWIXkQZX53J3zi0B3gA+xn8aZAz+kfj9wPfNbAv+0yH/UA85w8L0BdknfU+pA/ILdSaoiDSuoM6Wcc49BDx0wubPgOHBvG64OtX3kR4sLNP56iLSqHR6Rj061feR6ntKRaSxqdzrga/C8bv3PuW7o8/U95SKSEhQuQepoLiMyTOX8ejbm4gx45ErM0lLTsKAtOQkHrkyU19tJyKNTguHBWH7/kImz1zG1n1H+cWETK4Z1hlAZS4inlO519HK7Qf5xkvLqHAwa/Jwzu6e4nUkEZHjVO511L5lIj07tODRqwbQLUVfniEioUVz7rXgq3C8vnwHFRWOtOQk5nxzpIpdREKSRu41dKSknLteW8k7m/aS3LQJF/Xt4HUkEZFTUrnXwI4Dhdw2czlb8o7w03H9VOwiEvJU7tVY8fkBpsxaQamvgpduHcZXe7TzOpKISLVU7tWocNC2eROevWEoZ7Zv7nUcEZEa0QHVKlRUOD7YvA+AYRltePuuc1XsIhJWVO4nOFpSzh0vr+DGPyxh1Y58AGL1XaYiEmY0LVNJbn4Rt81cTvbuAh6+vC8D01t5HUlEpE5U7gEfbz/IlFkrKCnz8cItwxjVq73XkURE6kzlHrB5z2GaNonltdtH0KNDC6/jiIgEJarLvaLC8cnew/Tu2JJrh3XhioFpJDWJrf6JIiIhLmoPqBaWlvPtVz9m/DMfsuNAIYCKXUQiRlSO3HcfKua2WctYv7OABy/tQ3prfVOSiESWqCv3NTn53DZzOUdLynn+5iwu6KOlBEQk8kRduf919U6axMUwa/LZ9O7Y0us4IiINIirK3TnH3sMldGiZyP1je3PnqDNp3ayJ17FERBpMxB9QLS7z8b3Zqxj/zIccKiwjLjZGxS4iES+iR+57C4q5fdZy1uQe4v6xvWmZFNG/XRGR4yK27dblHuK2mcspKC7j9zcO5eJ+Hb2OJCLSaCK23J/812ZiDN6442z6purAqYhEl4gqd+cchaU+miXE8cuvD6TE56N9i0SvY4mINLqIOaBaXObj7jmruPmFpZSWV9CqabyKXUSiVtiO3OetzGX6gmx25hfRoWUiCXHG5weKuPfinsTHav11EYluYVnu81bm8sDctRSV+QDYXVAMwK1nd+U7o3t4GU1EJCSE5bTM9AXZx4u9sn9u2OtBGhGR0BOW5b4zv6hW20VEok1YlntqctWrOJ5qu4hItAnLcp82phdJ8V9eez0pPpZpY3p5lEhEJLSE5QHV8YPTAI6fLZOanMS0Mb2ObxcRiXZhWe7gL3iVuYhI1cJyWkZERE4vqHI3s2Qze8PMNpnZRjMbaWZtzGyhmW0O/Ld1fYUVEZGaCXbk/hTwD+dcb2AgsBH4AfCOc64H8E7gvoiINKI6l7uZtQLOBf4A4Jwrdc7lA+OAmYHdZgLjg4soIiK1FczIvRuQB7xoZivN7HkzawZ0cM7tCuyzG6jyG6jNbIqZLTez5Xl5eUHEEBGRE5lzrm5PNMsCFgNfcc4tMbOngALgu8655Er7HXTOnXbe3czygM/rFARSgH11fG5DCtVcELrZlKt2lKt2IjFXV+dcu6oeCOZUyBwgxzm3JHD/Dfzz63vMrJNzbpeZdQKqXfDlVOFqwsyWO+ey6vr8hhKquSB0sylX7ShX7URbrjpPyzjndgM7zOzYZaEXABuA+cCkwLZJwF+CSigiIrUW7EVM3wVeMbMmwGfArfj/wfiTmU3GP9VyTZDvISIitRRUuTvnVgFV/ThxQTCvW0szGvG9aiNUc0HoZlOu2lGu2omqXHU+oCoiIqFLyw+IiEQglbuISAQKm3I3s7Fmlm1mW8zspCUNzOxcM/vYzMrN7OoQyvV9M9tgZmvM7B0z6xoiue4ws7VmtsrMPjCzvqGQq9J+V5mZC1xP4XkuM7vFzPICn9cqM7stFHIF9rkm8GdsvZm9Ggq5zOzXlT6rT8wsP0RydTGzRYELL9eY2aUhkqtroB/WmNm7ZpYe9Js650L+FxALfAqcATQBVgN9T9gnAxgAzAKuDqFc5wNNA7e/BcwJkVwtK92+Av8aQZ7nCuzXAngf/0VyWaGQC7gFeLox/lzVMlcPYCXQOnC/fSjkOmH/7wIvhEIu/AcvvxW43RfYFiK5XgcmBW6PBv4Y7PuGy8h9OLDFOfeZc64UmI1/DZvjnHPbnHNrgIoQy7XIOVcYuLsYCP5f5PrJVVDpbjOgMY6sV5sr4GfAY0BxI2SqTa7GVpNctwPPOOcOAjjnGuNb4mv7eU0EXguRXA5oGbjdCtgZIrn6Av8O3F5UxeO1Fi7lngbsqHQ/J7DNa7XNNRl4u0ET+dUol5l928w+BR4HvhcKucxsCNDZOff3RshT41wBVwV+bH7DzDqHSK6eQE8z+9DMFpvZ2BDJBfinG/CvQ/Xvqh73INfDwI1mlgO8hf+nilDItRq4MnB7AtDCzNoG86bhUu5hz8xuxH9NwHSvsxzjnHvGOdcduB/4H6/zmFkM8CvgHq+zVOGvQIZzbgCwkC9WPvVaHP6pmVH4R8jPmVmyl4FOcB3whnPO53WQgInAS865dOBS4I+BP3deuxc4z8xWAucBuUBQn1ko/KZqIheoPFJKD2zzWo1ymdmFwIPAFc65klDJVclsGmdp5upytQD6A++a2TbgLGB+IxxUrfbzcs7tr/T/7nlgaANnqlEu/KPA+c65MufcVuAT/GXvda5jrqNxpmSgZrkmA38CcM59BCTiX7jL01zOuZ3OuSudc4PxdwXOv4R63TX0wYR6OiARh395g258cUCi3yn2fYnGO6BabS5gMP6DKT1C6fOqnAe4HFgeCrlO2P9dGueAak0+r06Vbk8AFodIrrHAzMDtFPw//rf1Oldgv97ANgIXS4bI5/U2cEvgdh/8c+4Nmq+GuVKAmMDtnwM/Dfp9G+NDr6cP6FL8o5JPgQcD236KfzQMMAz/KOYosB9YHyK5/gXsAVYFfs0PkVxPAesDmRadrmQbM9cJ+zZKudfw83ok8HmtDnxevUMkl+GfytoArAWuC4VcgfsPA482Rp5afF59gQ8D/x9XAReHSK6rgc2BfZ4HEoJ9Ty0/ICISgcJlzl1ERGpB5S4iEoFU7iIiEUjlLiISgVTuIiIRSOUuYSGwQuTLle7HBVZp/Fvg/hWnW2XyNK/73yAyVV4pcpOZ3V3D56RWuv98Y63IKdEl2O9QFWksR4H+ZpbknCsCLqLSVX7Oufn4v5y9VpxzZweZa45z7juBdUCyzewN59yO0+x/C7COwIJVzrlGWTpYoo9G7hJO3gK+Frj9pZUGAyPipwO3v25m68xstZm9H9jWz8yWBkbZa8ysR2D7kcB/RwXW0X4jMAp/xcws8NilgW0rzOw3x35aqMw5tx/YAnQKPOfHZrYskGOG+V2Nf32hVwI5kgLvmRV4zkTzr7G/zswea5BPUKKGyl3CyWzgOjNLxL92/5JT7PdjYIxzbiD+teoB7gCecs4Nwl+wOVU8bzAwFf9VjGcAXwm81++BS5xzQ4F2Vb2hmXXBv07JmsCmp51zw5xz/YEk4DLn3BvAcuAG59ygwE8gx56fin+Z49HAIGCYmY0/7achchoqdwkbzr9efwb+Uftbp9n1Q+AlM7sd/xclAHwE/NDM7ge6Vi7WSpY653KccxX4L03PwL8+ymfOvygXnLwI1rVmtgb/qP1Z59yxNejPN7MlZrYWf2H3q+a3Nwx41zmX55wrB14Bzq3mOSKnpHKXcDMfeILTrDTonLsD/xLGnYEVZtbWOfcq/lF8EfCWmY2u4qmVV+z0UbNjUnOcfxngs4FHzaxjYLT/LP4F7DKB5/CP6kUajcpdws0LwE+cc2tPtYOZdXfOLXHO/RjIAzqb2Rn4R+C/Af6Cf1qnJrKBM8wsI3D/2qp2cs4tB/4I3MUXRb7PzJrjXxTqmMP4lzY+0VL863mnmFks/p9O3qthRpGT6GwZCSvOuRzgN9XsNj1wwNSAd/CvAHg/cJOZlQG7gV/U8P2KzOxO4B9mdhRYdprdHwM+Drz2c/jPitl9wnNeAn5nZkXAyErvsytwKueiQO6/O+f+UpOMIlXRqpAi1TCz5s65I4GzZ54BNjvnfu11LpHT0bSMSPVuN7NV+Ndzb4X/7BmRkKaRu4hIBNLIXUQkAqncRUQikMpdRCQCqdxFRCKQyl1EJAL9P2AUHPXvGYaOAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "plot_col_num_for_dif_missing_ratio(data)  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "After we delete columns whose missing ratio is larger than 0.1, the missing value for this data shrink from 32902 to 60"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "SK_ID_CURR            0\n",
              "TARGET                0\n",
              "NAME_CONTRACT_TYPE    0\n",
              "CODE_GENDER           0\n",
              "FLAG_OWN_CAR          0\n",
              "                     ..\n",
              "FLAG_DOCUMENT_17      4\n",
              "FLAG_DOCUMENT_18      4\n",
              "FLAG_DOCUMENT_19      4\n",
              "FLAG_DOCUMENT_20      4\n",
              "FLAG_DOCUMENT_21      4\n",
              "Length: 63, dtype: int64"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "missing_cols = filter_columns_by_missing_ratio(data, 0.1)\n",
        "new_df = data.drop(columns=missing_cols)\n",
        "new_df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "If I drop none, the original 14721 data will be reduced to 14661 rows, which mean 60(0.407581%) get removed\n"
          ]
        }
      ],
      "source": [
        "print('If I drop none, the original {} data will be reduced to {} rows, which mean {}({:%}) get removed'.format(len(new_df),len(new_df.dropna()),len(new_df)-len(new_df.dropna()),(len(new_df)-len(new_df.dropna()))/len(new_df)))\n",
        "new_df = new_df.dropna()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "'TARGET' in potential_num_df.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['AMT_GOODS_PRICE',\n",
              " 'EXT_SOURCE_2',\n",
              " 'OBS_30_CNT_SOCIAL_CIRCLE',\n",
              " 'DAYS_REGISTRATION',\n",
              " 'AMT_CREDIT',\n",
              " 'REG_REGION_NOT_LIVE_REGION',\n",
              " 'LIVE_REGION_NOT_WORK_REGION',\n",
              " 'CODE_GENDER',\n",
              " 'AMT_INCOME_TOTAL',\n",
              " 'REG_CITY_NOT_LIVE_CITY',\n",
              " 'CNT_FAM_MEMBERS',\n",
              " 'DEF_60_CNT_SOCIAL_CIRCLE',\n",
              " 'REGION_RATING_CLIENT',\n",
              " 'REG_REGION_NOT_WORK_REGION',\n",
              " 'REGION_POPULATION_RELATIVE',\n",
              " 'REGION_RATING_CLIENT_W_CITY',\n",
              " 'LIVE_CITY_NOT_WORK_CITY',\n",
              " 'DAYS_BIRTH',\n",
              " 'DEF_30_CNT_SOCIAL_CIRCLE',\n",
              " 'HOUR_APPR_PROCESS_START',\n",
              " 'DAYS_LAST_PHONE_CHANGE',\n",
              " 'SK_ID_CURR',\n",
              " 'DAYS_ID_PUBLISH',\n",
              " 'CNT_CHILDREN',\n",
              " 'OBS_60_CNT_SOCIAL_CIRCLE',\n",
              " 'REG_CITY_NOT_WORK_CITY']"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "num_col = list(set(potential_num_list) & set(new_df.columns))\n",
        "num_df = new_df.loc[:,num_col]\n",
        "num_col"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Column</th>\n",
              "      <th>Desc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>AMT_GOODS_PRICE</td>\n",
              "      <td>For consumer loans it is the price of the good...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>EXT_SOURCE_2</td>\n",
              "      <td>Normalized score from external data source</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>OBS_30_CNT_SOCIAL_CIRCLE</td>\n",
              "      <td>How many observation of client's social surrou...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>DAYS_REGISTRATION</td>\n",
              "      <td>How many days before the application did clien...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>AMT_CREDIT</td>\n",
              "      <td>Credit amount of the loan</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>REG_REGION_NOT_LIVE_REGION</td>\n",
              "      <td>Flag if client's permanent address does not ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>LIVE_REGION_NOT_WORK_REGION</td>\n",
              "      <td>Flag if client's contact address does not matc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>CODE_GENDER</td>\n",
              "      <td>Gender of the client</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>AMT_INCOME_TOTAL</td>\n",
              "      <td>Income of the client</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>REG_CITY_NOT_LIVE_CITY</td>\n",
              "      <td>Flag if client's permanent address does not ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>CNT_FAM_MEMBERS</td>\n",
              "      <td>How many family members does client have</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>DEF_60_CNT_SOCIAL_CIRCLE</td>\n",
              "      <td>How many observation of client's social surrou...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>REGION_RATING_CLIENT</td>\n",
              "      <td>Our rating of the region where client lives (1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>REG_REGION_NOT_WORK_REGION</td>\n",
              "      <td>Flag if client's permanent address does not ma...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>REGION_POPULATION_RELATIVE</td>\n",
              "      <td>Normalized population of region where client l...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>REGION_RATING_CLIENT_W_CITY</td>\n",
              "      <td>Our rating of the region where client lives wi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>LIVE_CITY_NOT_WORK_CITY</td>\n",
              "      <td>Flag if client's contact address does not matc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>DAYS_BIRTH</td>\n",
              "      <td>Client's age in days at the time of application</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>DEF_30_CNT_SOCIAL_CIRCLE</td>\n",
              "      <td>How many observation of client's social surrou...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>HOUR_APPR_PROCESS_START</td>\n",
              "      <td>Approximately at what hour did the client appl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>DAYS_LAST_PHONE_CHANGE</td>\n",
              "      <td>How many days before application did client ch...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>SK_ID_CURR</td>\n",
              "      <td>ID of loan in our sample</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>DAYS_ID_PUBLISH</td>\n",
              "      <td>How many days before the application did clien...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>CNT_CHILDREN</td>\n",
              "      <td>Number of children the client has</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>OBS_60_CNT_SOCIAL_CIRCLE</td>\n",
              "      <td>How many observation of client's social surrou...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>REG_CITY_NOT_WORK_CITY</td>\n",
              "      <td>Flag if client's permanent address does not ma...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         Column  \\\n",
              "0               AMT_GOODS_PRICE   \n",
              "1                  EXT_SOURCE_2   \n",
              "2      OBS_30_CNT_SOCIAL_CIRCLE   \n",
              "3             DAYS_REGISTRATION   \n",
              "4                    AMT_CREDIT   \n",
              "5    REG_REGION_NOT_LIVE_REGION   \n",
              "6   LIVE_REGION_NOT_WORK_REGION   \n",
              "7                   CODE_GENDER   \n",
              "8              AMT_INCOME_TOTAL   \n",
              "9        REG_CITY_NOT_LIVE_CITY   \n",
              "10              CNT_FAM_MEMBERS   \n",
              "11     DEF_60_CNT_SOCIAL_CIRCLE   \n",
              "12         REGION_RATING_CLIENT   \n",
              "13   REG_REGION_NOT_WORK_REGION   \n",
              "14   REGION_POPULATION_RELATIVE   \n",
              "15  REGION_RATING_CLIENT_W_CITY   \n",
              "16      LIVE_CITY_NOT_WORK_CITY   \n",
              "17                   DAYS_BIRTH   \n",
              "18     DEF_30_CNT_SOCIAL_CIRCLE   \n",
              "19      HOUR_APPR_PROCESS_START   \n",
              "20       DAYS_LAST_PHONE_CHANGE   \n",
              "21                   SK_ID_CURR   \n",
              "22              DAYS_ID_PUBLISH   \n",
              "23                 CNT_CHILDREN   \n",
              "24     OBS_60_CNT_SOCIAL_CIRCLE   \n",
              "25       REG_CITY_NOT_WORK_CITY   \n",
              "\n",
              "                                                 Desc  \n",
              "0   For consumer loans it is the price of the good...  \n",
              "1          Normalized score from external data source  \n",
              "2   How many observation of client's social surrou...  \n",
              "3   How many days before the application did clien...  \n",
              "4                           Credit amount of the loan  \n",
              "5   Flag if client's permanent address does not ma...  \n",
              "6   Flag if client's contact address does not matc...  \n",
              "7                                Gender of the client  \n",
              "8                                Income of the client  \n",
              "9   Flag if client's permanent address does not ma...  \n",
              "10           How many family members does client have  \n",
              "11  How many observation of client's social surrou...  \n",
              "12  Our rating of the region where client lives (1...  \n",
              "13  Flag if client's permanent address does not ma...  \n",
              "14  Normalized population of region where client l...  \n",
              "15  Our rating of the region where client lives wi...  \n",
              "16  Flag if client's contact address does not matc...  \n",
              "17    Client's age in days at the time of application  \n",
              "18  How many observation of client's social surrou...  \n",
              "19  Approximately at what hour did the client appl...  \n",
              "20  How many days before application did client ch...  \n",
              "21                           ID of loan in our sample  \n",
              "22  How many days before the application did clien...  \n",
              "23                  Number of children the client has  \n",
              "24  How many observation of client's social surrou...  \n",
              "25  Flag if client's permanent address does not ma...  "
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "num_desc = show_description(num_col, data_desc)\n",
        "num_desc.to_csv('num_desc.csv')\n",
        "num_desc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['NAME_EDUCATION_TYPE',\n",
              " 'NAME_FAMILY_STATUS',\n",
              " 'FLAG_DOCUMENT_17',\n",
              " 'ORGANIZATION_TYPE',\n",
              " 'FLAG_DOCUMENT_21',\n",
              " 'NAME_TYPE_SUITE',\n",
              " 'FLAG_MOBIL',\n",
              " 'FLAG_DOCUMENT_7',\n",
              " 'FLAG_DOCUMENT_9',\n",
              " 'NAME_INCOME_TYPE',\n",
              " 'FLAG_CONT_MOBILE',\n",
              " 'FLAG_DOCUMENT_10',\n",
              " 'FLAG_DOCUMENT_11',\n",
              " 'FLAG_DOCUMENT_4',\n",
              " 'FLAG_DOCUMENT_15',\n",
              " 'FLAG_DOCUMENT_20',\n",
              " 'NAME_HOUSING_TYPE',\n",
              " 'FLAG_PHONE',\n",
              " 'FLAG_DOCUMENT_19',\n",
              " 'FLAG_EMAIL',\n",
              " 'FLAG_OWN_REALTY',\n",
              " 'FLAG_EMP_PHONE',\n",
              " 'FLAG_DOCUMENT_12',\n",
              " 'FLAG_DOCUMENT_5',\n",
              " 'FLAG_DOCUMENT_16',\n",
              " 'FLAG_OWN_CAR',\n",
              " 'FLAG_DOCUMENT_14',\n",
              " 'FLAG_DOCUMENT_6',\n",
              " 'FLAG_DOCUMENT_13',\n",
              " 'FLAG_DOCUMENT_3',\n",
              " 'FLAG_DOCUMENT_18',\n",
              " 'FLAG_WORK_PHONE',\n",
              " 'FLAG_DOCUMENT_2',\n",
              " 'WEEKDAY_APPR_PROCESS_START',\n",
              " 'FLAG_DOCUMENT_8',\n",
              " 'NAME_CONTRACT_TYPE']"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cat_col = list(set(cat_columns) & set(new_df.columns))\n",
        "cat_df = new_df.loc[:,cat_col]\n",
        "cat_col"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Column</th>\n",
              "      <th>Desc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NAME_EDUCATION_TYPE</td>\n",
              "      <td>Level of highest education the client achieved</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NAME_FAMILY_STATUS</td>\n",
              "      <td>Family status of the client</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>FLAG_DOCUMENT_17</td>\n",
              "      <td>Did client provide document 17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ORGANIZATION_TYPE</td>\n",
              "      <td>Type of organization where client works</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>FLAG_DOCUMENT_21</td>\n",
              "      <td>Did client provide document 21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>NAME_TYPE_SUITE</td>\n",
              "      <td>Who was accompanying client when he was applyi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>FLAG_MOBIL</td>\n",
              "      <td>Did client provide mobile phone (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>FLAG_DOCUMENT_7</td>\n",
              "      <td>Did client provide document 7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>FLAG_DOCUMENT_9</td>\n",
              "      <td>Did client provide document 9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>NAME_INCOME_TYPE</td>\n",
              "      <td>Clients income type (businessman, working, mat...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>FLAG_CONT_MOBILE</td>\n",
              "      <td>Was mobile phone reachable (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>FLAG_DOCUMENT_10</td>\n",
              "      <td>Did client provide document 10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>FLAG_DOCUMENT_11</td>\n",
              "      <td>Did client provide document 11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>FLAG_DOCUMENT_4</td>\n",
              "      <td>Did client provide document 4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>FLAG_DOCUMENT_15</td>\n",
              "      <td>Did client provide document 15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>FLAG_DOCUMENT_20</td>\n",
              "      <td>Did client provide document 20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>NAME_HOUSING_TYPE</td>\n",
              "      <td>What is the housing situation of the client (r...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>FLAG_PHONE</td>\n",
              "      <td>Did client provide home phone (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>FLAG_DOCUMENT_19</td>\n",
              "      <td>Did client provide document 19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>FLAG_EMAIL</td>\n",
              "      <td>Did client provide email (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>FLAG_OWN_REALTY</td>\n",
              "      <td>Flag if client owns a house or flat</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>FLAG_EMP_PHONE</td>\n",
              "      <td>Did client provide work phone (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>FLAG_DOCUMENT_12</td>\n",
              "      <td>Did client provide document 12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>FLAG_DOCUMENT_5</td>\n",
              "      <td>Did client provide document 5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>FLAG_DOCUMENT_16</td>\n",
              "      <td>Did client provide document 16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>FLAG_OWN_CAR</td>\n",
              "      <td>Flag if the client owns a car</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>FLAG_DOCUMENT_14</td>\n",
              "      <td>Did client provide document 14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>FLAG_DOCUMENT_6</td>\n",
              "      <td>Did client provide document 6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>FLAG_DOCUMENT_13</td>\n",
              "      <td>Did client provide document 13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>FLAG_DOCUMENT_3</td>\n",
              "      <td>Did client provide document 3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>FLAG_DOCUMENT_18</td>\n",
              "      <td>Did client provide document 18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>FLAG_WORK_PHONE</td>\n",
              "      <td>Did client provide home phone (1=YES, 0=NO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>FLAG_DOCUMENT_2</td>\n",
              "      <td>Did client provide document 2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>WEEKDAY_APPR_PROCESS_START</td>\n",
              "      <td>On which day of the week did the client apply ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>FLAG_DOCUMENT_8</td>\n",
              "      <td>Did client provide document 8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>NAME_CONTRACT_TYPE</td>\n",
              "      <td>Identification if loan is cash or revolving</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                        Column  \\\n",
              "0          NAME_EDUCATION_TYPE   \n",
              "1           NAME_FAMILY_STATUS   \n",
              "2             FLAG_DOCUMENT_17   \n",
              "3            ORGANIZATION_TYPE   \n",
              "4             FLAG_DOCUMENT_21   \n",
              "5              NAME_TYPE_SUITE   \n",
              "6                   FLAG_MOBIL   \n",
              "7              FLAG_DOCUMENT_7   \n",
              "8              FLAG_DOCUMENT_9   \n",
              "9             NAME_INCOME_TYPE   \n",
              "10            FLAG_CONT_MOBILE   \n",
              "11            FLAG_DOCUMENT_10   \n",
              "12            FLAG_DOCUMENT_11   \n",
              "13             FLAG_DOCUMENT_4   \n",
              "14            FLAG_DOCUMENT_15   \n",
              "15            FLAG_DOCUMENT_20   \n",
              "16           NAME_HOUSING_TYPE   \n",
              "17                  FLAG_PHONE   \n",
              "18            FLAG_DOCUMENT_19   \n",
              "19                  FLAG_EMAIL   \n",
              "20             FLAG_OWN_REALTY   \n",
              "21              FLAG_EMP_PHONE   \n",
              "22            FLAG_DOCUMENT_12   \n",
              "23             FLAG_DOCUMENT_5   \n",
              "24            FLAG_DOCUMENT_16   \n",
              "25                FLAG_OWN_CAR   \n",
              "26            FLAG_DOCUMENT_14   \n",
              "27             FLAG_DOCUMENT_6   \n",
              "28            FLAG_DOCUMENT_13   \n",
              "29             FLAG_DOCUMENT_3   \n",
              "30            FLAG_DOCUMENT_18   \n",
              "31             FLAG_WORK_PHONE   \n",
              "32             FLAG_DOCUMENT_2   \n",
              "33  WEEKDAY_APPR_PROCESS_START   \n",
              "34             FLAG_DOCUMENT_8   \n",
              "35          NAME_CONTRACT_TYPE   \n",
              "\n",
              "                                                 Desc  \n",
              "0      Level of highest education the client achieved  \n",
              "1                         Family status of the client  \n",
              "2                      Did client provide document 17  \n",
              "3             Type of organization where client works  \n",
              "4                      Did client provide document 21  \n",
              "5   Who was accompanying client when he was applyi...  \n",
              "6       Did client provide mobile phone (1=YES, 0=NO)  \n",
              "7                       Did client provide document 7  \n",
              "8                       Did client provide document 9  \n",
              "9   Clients income type (businessman, working, mat...  \n",
              "10           Was mobile phone reachable (1=YES, 0=NO)  \n",
              "11                     Did client provide document 10  \n",
              "12                     Did client provide document 11  \n",
              "13                      Did client provide document 4  \n",
              "14                     Did client provide document 15  \n",
              "15                     Did client provide document 20  \n",
              "16  What is the housing situation of the client (r...  \n",
              "17        Did client provide home phone (1=YES, 0=NO)  \n",
              "18                     Did client provide document 19  \n",
              "19             Did client provide email (1=YES, 0=NO)  \n",
              "20                Flag if client owns a house or flat  \n",
              "21        Did client provide work phone (1=YES, 0=NO)  \n",
              "22                     Did client provide document 12  \n",
              "23                      Did client provide document 5  \n",
              "24                     Did client provide document 16  \n",
              "25                      Flag if the client owns a car  \n",
              "26                     Did client provide document 14  \n",
              "27                      Did client provide document 6  \n",
              "28                     Did client provide document 13  \n",
              "29                      Did client provide document 3  \n",
              "30                     Did client provide document 18  \n",
              "31        Did client provide home phone (1=YES, 0=NO)  \n",
              "32                      Did client provide document 2  \n",
              "33  On which day of the week did the client apply ...  \n",
              "34                      Did client provide document 8  \n",
              "35        Identification if loan is cash or revolving  "
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cat_desc = show_description(cat_col, data_desc)\n",
        "cat_desc.to_csv('cat_desc.csv')\n",
        "cat_desc"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Handling missing values"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Check nan for all the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 34,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "new_df.isnull().sum().sum()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Train test split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(14661, 63)"
            ]
          },
          "execution_count": 35,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data = new_df.loc[:,cat_col+num_col+['TARGET']]\n",
        "data.to_csv('data_without_missing_val.csv')\n",
        "data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {},
      "outputs": [],
      "source": [
        "# for col in cat_col:\n",
        "#     print(col,data.loc[:,col].unique())\n",
        "df = cat_df.copy(deep=True)\n",
        "for column in cat_col:\n",
        "    tempdf = pd.get_dummies(df[column], prefix=column)\n",
        "    df = pd.merge(\n",
        "        left=df,\n",
        "        right=tempdf,\n",
        "        left_index=True,\n",
        "        right_index=True,\n",
        "    )\n",
        "    df = df.drop(columns=column)\n",
        "cat_df = df\n",
        "cat_col = [i for i in cat_df.columns if (i != 'TARGET') and (i not in num_col)]\n",
        "cat_df = cat_df.loc[:,cat_col]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['NAME_EDUCATION_TYPE_Academic degree',\n",
              " 'NAME_EDUCATION_TYPE_Higher education',\n",
              " 'NAME_EDUCATION_TYPE_Incomplete higher',\n",
              " 'NAME_EDUCATION_TYPE_Lower secondary',\n",
              " 'NAME_EDUCATION_TYPE_Secondary / secondary special',\n",
              " 'NAME_FAMILY_STATUS_Civil marriage',\n",
              " 'NAME_FAMILY_STATUS_Married',\n",
              " 'NAME_FAMILY_STATUS_Separated',\n",
              " 'NAME_FAMILY_STATUS_Single / not married',\n",
              " 'NAME_FAMILY_STATUS_Widow',\n",
              " 'FLAG_DOCUMENT_17_0.0',\n",
              " 'FLAG_DOCUMENT_17_1.0',\n",
              " 'ORGANIZATION_TYPE_Advertising',\n",
              " 'ORGANIZATION_TYPE_Agriculture',\n",
              " 'ORGANIZATION_TYPE_Bank',\n",
              " 'ORGANIZATION_TYPE_Business Entity Type 1',\n",
              " 'ORGANIZATION_TYPE_Business Entity Type 2',\n",
              " 'ORGANIZATION_TYPE_Business Entity Type 3',\n",
              " 'ORGANIZATION_TYPE_Cleaning',\n",
              " 'ORGANIZATION_TYPE_Construction',\n",
              " 'ORGANIZATION_TYPE_Culture',\n",
              " 'ORGANIZATION_TYPE_Electricity',\n",
              " 'ORGANIZATION_TYPE_Emergency',\n",
              " 'ORGANIZATION_TYPE_Government',\n",
              " 'ORGANIZATION_TYPE_Hotel',\n",
              " 'ORGANIZATION_TYPE_Housing',\n",
              " 'ORGANIZATION_TYPE_Industry: type 1',\n",
              " 'ORGANIZATION_TYPE_Industry: type 10',\n",
              " 'ORGANIZATION_TYPE_Industry: type 11',\n",
              " 'ORGANIZATION_TYPE_Industry: type 12',\n",
              " 'ORGANIZATION_TYPE_Industry: type 13',\n",
              " 'ORGANIZATION_TYPE_Industry: type 2',\n",
              " 'ORGANIZATION_TYPE_Industry: type 3',\n",
              " 'ORGANIZATION_TYPE_Industry: type 4',\n",
              " 'ORGANIZATION_TYPE_Industry: type 5',\n",
              " 'ORGANIZATION_TYPE_Industry: type 7',\n",
              " 'ORGANIZATION_TYPE_Industry: type 8',\n",
              " 'ORGANIZATION_TYPE_Industry: type 9',\n",
              " 'ORGANIZATION_TYPE_Insurance',\n",
              " 'ORGANIZATION_TYPE_Kindergarten',\n",
              " 'ORGANIZATION_TYPE_Legal Services',\n",
              " 'ORGANIZATION_TYPE_Medicine',\n",
              " 'ORGANIZATION_TYPE_Military',\n",
              " 'ORGANIZATION_TYPE_Mobile',\n",
              " 'ORGANIZATION_TYPE_Other',\n",
              " 'ORGANIZATION_TYPE_Police',\n",
              " 'ORGANIZATION_TYPE_Postal',\n",
              " 'ORGANIZATION_TYPE_Realtor',\n",
              " 'ORGANIZATION_TYPE_Religion',\n",
              " 'ORGANIZATION_TYPE_Restaurant',\n",
              " 'ORGANIZATION_TYPE_School',\n",
              " 'ORGANIZATION_TYPE_Security',\n",
              " 'ORGANIZATION_TYPE_Security Ministries',\n",
              " 'ORGANIZATION_TYPE_Self-employed',\n",
              " 'ORGANIZATION_TYPE_Services',\n",
              " 'ORGANIZATION_TYPE_Telecom',\n",
              " 'ORGANIZATION_TYPE_Trade: type 1',\n",
              " 'ORGANIZATION_TYPE_Trade: type 2',\n",
              " 'ORGANIZATION_TYPE_Trade: type 3',\n",
              " 'ORGANIZATION_TYPE_Trade: type 4',\n",
              " 'ORGANIZATION_TYPE_Trade: type 5',\n",
              " 'ORGANIZATION_TYPE_Trade: type 6',\n",
              " 'ORGANIZATION_TYPE_Trade: type 7',\n",
              " 'ORGANIZATION_TYPE_Transport: type 1',\n",
              " 'ORGANIZATION_TYPE_Transport: type 2',\n",
              " 'ORGANIZATION_TYPE_Transport: type 3',\n",
              " 'ORGANIZATION_TYPE_Transport: type 4',\n",
              " 'ORGANIZATION_TYPE_University',\n",
              " 'ORGANIZATION_TYPE_XNA',\n",
              " 'FLAG_DOCUMENT_21_0.0',\n",
              " 'FLAG_DOCUMENT_21_1.0',\n",
              " 'NAME_TYPE_SUITE_Children',\n",
              " 'NAME_TYPE_SUITE_Family',\n",
              " 'NAME_TYPE_SUITE_Group of people',\n",
              " 'NAME_TYPE_SUITE_Other_A',\n",
              " 'NAME_TYPE_SUITE_Other_B',\n",
              " 'NAME_TYPE_SUITE_Spouse, partner',\n",
              " 'NAME_TYPE_SUITE_Unaccompanied',\n",
              " 'FLAG_MOBIL_1.0',\n",
              " 'FLAG_DOCUMENT_7_0.0',\n",
              " 'FLAG_DOCUMENT_9_0.0',\n",
              " 'FLAG_DOCUMENT_9_1.0',\n",
              " 'NAME_INCOME_TYPE_Commercial associate',\n",
              " 'NAME_INCOME_TYPE_Pensioner',\n",
              " 'NAME_INCOME_TYPE_State servant',\n",
              " 'NAME_INCOME_TYPE_Working',\n",
              " 'FLAG_CONT_MOBILE_0.0',\n",
              " 'FLAG_CONT_MOBILE_1.0',\n",
              " 'FLAG_DOCUMENT_10_0.0',\n",
              " 'FLAG_DOCUMENT_11_0.0',\n",
              " 'FLAG_DOCUMENT_11_1.0',\n",
              " 'FLAG_DOCUMENT_4_0.0',\n",
              " 'FLAG_DOCUMENT_15_0.0',\n",
              " 'FLAG_DOCUMENT_15_1.0',\n",
              " 'FLAG_DOCUMENT_20_0.0',\n",
              " 'FLAG_DOCUMENT_20_1.0',\n",
              " 'NAME_HOUSING_TYPE_Co-op apartment',\n",
              " 'NAME_HOUSING_TYPE_House / apartment',\n",
              " 'NAME_HOUSING_TYPE_Municipal apartment',\n",
              " 'NAME_HOUSING_TYPE_Office apartment',\n",
              " 'NAME_HOUSING_TYPE_Rented apartment',\n",
              " 'NAME_HOUSING_TYPE_With parents',\n",
              " 'FLAG_PHONE_0.0',\n",
              " 'FLAG_PHONE_1.0',\n",
              " 'FLAG_DOCUMENT_19_0.0',\n",
              " 'FLAG_DOCUMENT_19_1.0',\n",
              " 'FLAG_EMAIL_0.0',\n",
              " 'FLAG_EMAIL_1.0',\n",
              " 'FLAG_OWN_REALTY_0',\n",
              " 'FLAG_OWN_REALTY_1',\n",
              " 'FLAG_EMP_PHONE_0.0',\n",
              " 'FLAG_EMP_PHONE_1.0',\n",
              " 'FLAG_DOCUMENT_12_0.0',\n",
              " 'FLAG_DOCUMENT_5_0.0',\n",
              " 'FLAG_DOCUMENT_5_1.0',\n",
              " 'FLAG_DOCUMENT_16_0.0',\n",
              " 'FLAG_DOCUMENT_16_1.0',\n",
              " 'FLAG_OWN_CAR_0',\n",
              " 'FLAG_OWN_CAR_1',\n",
              " 'FLAG_DOCUMENT_14_0.0',\n",
              " 'FLAG_DOCUMENT_14_1.0',\n",
              " 'FLAG_DOCUMENT_6_0.0',\n",
              " 'FLAG_DOCUMENT_6_1.0',\n",
              " 'FLAG_DOCUMENT_13_0.0',\n",
              " 'FLAG_DOCUMENT_13_1.0',\n",
              " 'FLAG_DOCUMENT_3_0.0',\n",
              " 'FLAG_DOCUMENT_3_1.0',\n",
              " 'FLAG_DOCUMENT_18_0.0',\n",
              " 'FLAG_DOCUMENT_18_1.0',\n",
              " 'FLAG_WORK_PHONE_0.0',\n",
              " 'FLAG_WORK_PHONE_1.0',\n",
              " 'FLAG_DOCUMENT_2_0.0',\n",
              " 'WEEKDAY_APPR_PROCESS_START_FRIDAY',\n",
              " 'WEEKDAY_APPR_PROCESS_START_MONDAY',\n",
              " 'WEEKDAY_APPR_PROCESS_START_SATURDAY',\n",
              " 'WEEKDAY_APPR_PROCESS_START_SUNDAY',\n",
              " 'WEEKDAY_APPR_PROCESS_START_THURSDAY',\n",
              " 'WEEKDAY_APPR_PROCESS_START_TUESDAY',\n",
              " 'WEEKDAY_APPR_PROCESS_START_WEDNESDAY',\n",
              " 'FLAG_DOCUMENT_8_0.0',\n",
              " 'FLAG_DOCUMENT_8_1.0',\n",
              " 'NAME_CONTRACT_TYPE_Cash loans',\n",
              " 'NAME_CONTRACT_TYPE_Revolving loans']"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cat_col"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Data shape (14661, 26)\n",
            "Finished Standarization\n"
          ]
        }
      ],
      "source": [
        "num_df = my_standarization(\"RobustScaler\", x_vars = num_col, data = num_df)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Finished Normalization\n"
          ]
        }
      ],
      "source": [
        "num_df = my_normalization(\"l2\", x_vars = num_col, data = num_df)\n",
        "data = pd.concat([num_df,cat_df,data['TARGET']],axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 40,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "'SK_ID_CURR' in data.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [],
      "source": [
        "x_vars = [i for i in data.columns if (i != 'TARGET')]\n",
        "y_var = 'TARGET'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 14661 entries, 0 to 14716\n",
            "Columns: 143 entries, NAME_EDUCATION_TYPE_Academic degree to NAME_CONTRACT_TYPE_Revolving loans\n",
            "dtypes: uint8(143)\n",
            "memory usage: 2.6 MB\n"
          ]
        }
      ],
      "source": [
        "data.loc[:,cat_col].info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {},
      "outputs": [],
      "source": [
        "X, y = data.loc[:,data.columns != 'TARGET'], data.loc[:,['TARGET']]\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0,stratify = y,test_size=0.2)\n",
        "train_data = pd.concat([X_train,y_train],axis=1)\n",
        "test_data = pd.concat([X_test,y_test],axis=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Save train and test data "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {},
      "outputs": [],
      "source": [
        "train_data = pd.concat([X_train,y_train],axis=1)\n",
        "test_data = pd.concat([X_test,y_test],axis=1)\n",
        "train_data.to_csv('train_data.csv')\n",
        "test_data.to_csv('test_data.csv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Select features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Choose by  RandomForest\n",
            "Choose by  ExtraTree\n",
            "Choose by  LogisticRegression\n"
          ]
        }
      ],
      "source": [
        "feature_num = 20\n",
        "feature_dict = select_features(data, x_vars, y_var, feature_num)      "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Modeling"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## RandomForest\n",
        "1. Use the features selected by RandomForest to train, and test using cross_val_score\n",
        "2. Use all the features to train, and test using cross_val_score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RandomForest classifer achieved recall 0.7907\n",
            "RandomForest classifer achieved roc_auc 0.9824\n"
          ]
        }
      ],
      "source": [
        "make_classification(model_name='RandomForest', x_vars = feature_dict['RandomForest'], y_var='TARGET', train_data = train_data, test_data = test_data)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RandomForest classifer achieved recall 0.7907\n",
            "RandomForest classifer achieved roc_auc 0.983\n"
          ]
        }
      ],
      "source": [
        "make_classification(model_name='RandomForest', x_vars = x_vars, y_var='TARGET', train_data = train_data, test_data = test_data)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "LogisticRegression classifer achieved mean: 0.07813953488372093 recall (std: 0.025029998227113878)<br>\n",
        "LogisticRegression classifer achieved mean: 0.7853682058905498 roc_auc (std: 0.009058205642588855)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "LogisticRegression classifer achieved recall 0.06512\n",
            "LogisticRegression classifer achieved roc_auc 0.7799\n"
          ]
        }
      ],
      "source": [
        "make_classification(model_name='LogisticRegression', x_vars = feature_dict['RandomForest'], y_var='TARGET', train_data = train_data, test_data = test_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ExtraTree classifer achieved recall 0.7907\n",
            "ExtraTree classifer achieved roc_auc 0.9856\n"
          ]
        }
      ],
      "source": [
        "make_classification(model_name='ExtraTree', x_vars = feature_dict['RandomForest'], y_var='TARGET', train_data = train_data, test_data = test_data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# please install the packages using 'pip install -r requirement.txt'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[33mWARNING: Ignoring invalid distribution -ip (/Users/huanganni/Library/CloudStorage/OneDrive-SingaporeManagementUniversity/Huang Anni/Term2/CS606-AI Planning-G1/Assignments/A2 release/code_template/.venv/lib/python3.9/site-packages)\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip freeze > requirement.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'RandomForest': ['AMT_GOODS_PRICE',\n",
              "  'EXT_SOURCE_2',\n",
              "  'OBS_30_CNT_SOCIAL_CIRCLE',\n",
              "  'DAYS_REGISTRATION',\n",
              "  'AMT_CREDIT',\n",
              "  'LIVE_REGION_NOT_WORK_REGION',\n",
              "  'CODE_GENDER',\n",
              "  'AMT_INCOME_TOTAL',\n",
              "  'CNT_FAM_MEMBERS',\n",
              "  'REGION_RATING_CLIENT',\n",
              "  'REGION_POPULATION_RELATIVE',\n",
              "  'REGION_RATING_CLIENT_W_CITY',\n",
              "  'DAYS_BIRTH',\n",
              "  'DEF_30_CNT_SOCIAL_CIRCLE',\n",
              "  'HOUR_APPR_PROCESS_START',\n",
              "  'DAYS_LAST_PHONE_CHANGE',\n",
              "  'SK_ID_CURR',\n",
              "  'DAYS_ID_PUBLISH',\n",
              "  'CNT_CHILDREN',\n",
              "  'OBS_60_CNT_SOCIAL_CIRCLE'],\n",
              " 'ExtraTree': ['AMT_GOODS_PRICE',\n",
              "  'EXT_SOURCE_2',\n",
              "  'OBS_30_CNT_SOCIAL_CIRCLE',\n",
              "  'DAYS_REGISTRATION',\n",
              "  'AMT_CREDIT',\n",
              "  'CODE_GENDER',\n",
              "  'AMT_INCOME_TOTAL',\n",
              "  'CNT_FAM_MEMBERS',\n",
              "  'REGION_POPULATION_RELATIVE',\n",
              "  'DAYS_BIRTH',\n",
              "  'DEF_30_CNT_SOCIAL_CIRCLE',\n",
              "  'HOUR_APPR_PROCESS_START',\n",
              "  'DAYS_LAST_PHONE_CHANGE',\n",
              "  'SK_ID_CURR',\n",
              "  'DAYS_ID_PUBLISH',\n",
              "  'CNT_CHILDREN',\n",
              "  'OBS_60_CNT_SOCIAL_CIRCLE',\n",
              "  'ORGANIZATION_TYPE_Medicine',\n",
              "  'ORGANIZATION_TYPE_Other',\n",
              "  'WEEKDAY_APPR_PROCESS_START_FRIDAY'],\n",
              " 'LogisticRegression': ['EXT_SOURCE_2',\n",
              "  'DAYS_REGISTRATION',\n",
              "  'LIVE_REGION_NOT_WORK_REGION',\n",
              "  'CODE_GENDER',\n",
              "  'REG_REGION_NOT_WORK_REGION',\n",
              "  'DEF_30_CNT_SOCIAL_CIRCLE',\n",
              "  'CNT_CHILDREN',\n",
              "  'ORGANIZATION_TYPE_Advertising',\n",
              "  'ORGANIZATION_TYPE_Business Entity Type 2',\n",
              "  'ORGANIZATION_TYPE_Hotel',\n",
              "  'ORGANIZATION_TYPE_Housing',\n",
              "  'ORGANIZATION_TYPE_Industry: type 1',\n",
              "  'ORGANIZATION_TYPE_Industry: type 3',\n",
              "  'ORGANIZATION_TYPE_Industry: type 9',\n",
              "  'ORGANIZATION_TYPE_Transport: type 1',\n",
              "  'NAME_TYPE_SUITE_Children',\n",
              "  'NAME_INCOME_TYPE_State servant',\n",
              "  'FLAG_DOCUMENT_11_0.0',\n",
              "  'FLAG_DOCUMENT_11_1.0',\n",
              "  'NAME_CONTRACT_TYPE_Revolving loans']}"
            ]
          },
          "execution_count": 51,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "feature_dict"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "FeatureEngineering.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}