exchange_pairs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1: Exchange pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.optimize import differential_evolution\n",
    "import time\n",
    "np.set_printoptions(suppress = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_matched = pd.read_stata('sample_matched_demean.dta')\n",
    "df_unmatched = pd.read_stata('sample_unmatched_demean.dta').iloc[:,: 7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lender_id</th>\n",
       "      <th>loan_id</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "      <th>var3</th>\n",
       "      <th>var4</th>\n",
       "      <th>var5</th>\n",
       "      <th>USPS_ZIP_PREF_CITY</th>\n",
       "      <th>USPS_ZIP_PREF_STATE</th>\n",
       "      <th>yearapproved</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5051.0</td>\n",
       "      <td>97252.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-550.811646</td>\n",
       "      <td>3.875915e+05</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4081.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-334.724243</td>\n",
       "      <td>-8.013543e+05</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2605.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-560.586060</td>\n",
       "      <td>-1.799459e+05</td>\n",
       "      <td>0.184932</td>\n",
       "      <td>-0.347326</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>31464.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-544.871094</td>\n",
       "      <td>1.192297e+05</td>\n",
       "      <td>-0.078142</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AMHERST</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2129.0</td>\n",
       "      <td>40976.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-547.824036</td>\n",
       "      <td>2.413198e+05</td>\n",
       "      <td>0.014364</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AMHERST</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88834</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34115.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-560.586060</td>\n",
       "      <td>1.404330e+05</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>KETCHIKAN</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88835</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34117.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-560.586060</td>\n",
       "      <td>-1.448854e+06</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>KETCHIKAN</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88836</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-494.896057</td>\n",
       "      <td>1.634662e+05</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88837</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34127.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-498.872070</td>\n",
       "      <td>3.707646e+05</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88838</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34131.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-498.872070</td>\n",
       "      <td>4.398640e+05</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>88839 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       lender_id  loan_id      var1        var2          var3      var4  \\\n",
       "0         5051.0  97252.0 -0.001122 -550.811646  3.875915e+05  0.091452   \n",
       "1         4081.0  78177.0 -0.001122 -334.724243 -8.013543e+05  0.091452   \n",
       "2         2605.0  27917.0 -0.001122 -560.586060 -1.799459e+05  0.184932   \n",
       "3         1339.0  31464.0 -0.001122 -544.871094  1.192297e+05 -0.078142   \n",
       "4         2129.0  40976.0 -0.001122 -547.824036  2.413198e+05  0.014364   \n",
       "...          ...      ...       ...         ...           ...       ...   \n",
       "88834     1631.0  34115.0 -0.001122 -560.586060  1.404330e+05 -0.024180   \n",
       "88835     1631.0  34117.0 -0.001122 -560.586060 -1.448854e+06 -0.024180   \n",
       "88836     1631.0  34122.0 -0.001122 -494.896057  1.634662e+05 -0.139812   \n",
       "88837     1631.0  34127.0 -0.001122 -498.872070  3.707646e+05 -0.139812   \n",
       "88838     1631.0  34131.0 -0.001122 -498.872070  4.398640e+05 -0.139812   \n",
       "\n",
       "           var5 USPS_ZIP_PREF_CITY USPS_ZIP_PREF_STATE  yearapproved  \n",
       "0      0.045413             AGAWAM                  MA        2020.0  \n",
       "1     -0.070219             AGAWAM                  MA        2020.0  \n",
       "2     -0.347326             AGAWAM                  MA        2020.0  \n",
       "3     -0.070219            AMHERST                  MA        2020.0  \n",
       "4     -0.070219            AMHERST                  MA        2020.0  \n",
       "...         ...                ...                 ...           ...  \n",
       "88834  0.045413          KETCHIKAN                  AK        2020.0  \n",
       "88835  0.045413          KETCHIKAN                  AK        2020.0  \n",
       "88836  0.045413              CRAIG                  AK        2020.0  \n",
       "88837  0.045413           WRANGELL                  AK        2020.0  \n",
       "88838 -0.070219           WRANGELL                  AK        2020.0  \n",
       "\n",
       "[88839 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_matched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_matched = pd.read_stata('sample_matched_NY_demean1.dta').iloc[:,: 7]\n",
    "df_unmatched = pd.read_stata('sample_unmatched_NY_demean1.dta').iloc[:,: 7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value1</th>\n",
       "      <th>value2</th>\n",
       "      <th>value3</th>\n",
       "      <th>value4</th>\n",
       "      <th>value5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-196.802643</td>\n",
       "      <td>-6.432140e+06</td>\n",
       "      <td>1.862645e-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-199.432587</td>\n",
       "      <td>-6.432144e+05</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-195.945038</td>\n",
       "      <td>1.286428e+06</td>\n",
       "      <td>-5.587935e-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-190.481293</td>\n",
       "      <td>-3.216070e+06</td>\n",
       "      <td>1.862645e-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-188.347870</td>\n",
       "      <td>-4.180891e+06</td>\n",
       "      <td>1.862645e-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11463945</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-163.896790</td>\n",
       "      <td>-2.731600e+04</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11463946</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-163.896790</td>\n",
       "      <td>1.092640e+05</td>\n",
       "      <td>2.600000e+00</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11463947</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-152.651550</td>\n",
       "      <td>-5.463202e+04</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11463948</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-31.092957</td>\n",
       "      <td>-1.092640e+05</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11463949</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-24.015137</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.600000e+00</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11463950 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          value1      value2        value3        value4  value5\n",
       "0            1.0 -196.802643 -6.432140e+06  1.862645e-09     0.0\n",
       "1            1.0 -199.432587 -6.432144e+05  0.000000e+00     0.0\n",
       "2            1.0 -195.945038  1.286428e+06 -5.587935e-09     0.0\n",
       "3            0.0 -190.481293 -3.216070e+06  1.862645e-09     0.0\n",
       "4            0.0 -188.347870 -4.180891e+06  1.862645e-09     0.0\n",
       "...          ...         ...           ...           ...     ...\n",
       "11463945     0.0 -163.896790 -2.731600e+04  2.000000e+00    -1.0\n",
       "11463946     0.0 -163.896790  1.092640e+05  2.600000e+00    -1.0\n",
       "11463947     0.0 -152.651550 -5.463202e+04  3.000000e+00     0.0\n",
       "11463948     0.0  -31.092957 -1.092640e+05  4.000000e+00    -1.0\n",
       "11463949     0.0  -24.015137  0.000000e+00  3.600000e+00    -1.0\n",
       "\n",
       "[11463950 rows x 5 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_matched = pd.read_stata('sample_matched_NY_demean1.dta').iloc[:,: 7]\n",
    "df_unmatched = pd.read_stata('sample_unmatched_NY_demean1.dta').iloc[:,: 7]\n",
    "\n",
    "'''\n",
    "df_matched = pd.read_stata('test_sample_matched.dta')\n",
    "df_unmatched = pd.read_stata('sample_unmatched.dta')\n",
    "'''\n",
    "def Exchange_pairs(df_matched, df_unmatched):\n",
    "    # t1 = time.time()\n",
    "    \n",
    "    df_matched_column = df_matched.columns\n",
    "    df_matched.columns = df_matched_column + '1m'\n",
    "    df_unmatched.columns = df_unmatched.columns.str.replace('lender_id', 'lender_id1m')\n",
    "    df1 = pd.merge(df_matched, df_unmatched, on = 'lender_id1m', how = 'inner')\n",
    "    \n",
    "    l = df1.columns[:-6].append([df1.columns[-6:] + '1um'])\n",
    "    df1.columns = l\n",
    "    df_matched.columns = df_matched_column\n",
    "    df2 = pd.merge(df_matched, df1, left_on = 'loan_id', right_on = 'loan_id1um', how = 'inner') \n",
    "    \n",
    "    ll = (df2.columns[:7]+'2m').append(df2.columns[7:])\n",
    "    df2.columns = ll\n",
    "    df_unmatched.columns = df_unmatched.columns.str.replace('lender_id1m', 'lender_id')\n",
    "    df3 = pd.merge(df_unmatched, df2, left_on = ['lender_id','loan_id'], right_on = ['lender_id2m','loan_id1m'], how = 'inner')\n",
    "    lll = (df3.columns[:7]+'2um').append(df3.columns[7:])\n",
    "    df3.columns = lll\n",
    "    \n",
    "    df_keep = pd.DataFrame()\n",
    "    for i in range(1, 6):\n",
    "        name = \"value\" + str(i)\n",
    "        df_keep[name] = df3[\"var\"+str(i)+\"1m\"] + df3[\"var\"+str(i)+\"2m\"] - df3[\"var\"+str(i)+\"1um\"] - df3[\"var\"+str(i)+\"2um\"]\n",
    "    # t2 = time.time()\n",
    "    # print(\"Running time: \", t2-t1)\n",
    "    return df_keep\n",
    "df_keep = Exchange_pairs(df_matched, df_unmatched)\n",
    "df_keep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2: Define objective function "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Step 2: Define objective function -- (P15 equation 7 in Fox (2018) or P818 equation 4 in Schwert (2018) )\n",
    "define a function that calculates the score of the matching model for given parameters\n",
    "i.e. count how many inequalities are satisfied under given parameters \n",
    "objective function (score) =sum of 1(inequalities satisfied), where\n",
    "$$Inequalities = beta \\times value \\geqslant 0$$\n",
    "$\\beta$ is the parameter vector\n",
    "Value vector for a given data point (i.e. value 1-6 for a given data point)\n",
    "Hint: apply (multiply) the parameter vector to the data matrix (transposed) and then count the non-negative values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The objective function $$Q(\\beta) = \\Sigma \\mathbb{1}[X_{N\\times 6}\\beta_{6\\times 1}]$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3: Differential Evolution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use differential_evolution in scipy.optimize to find the parameters than maximize the score calculated in step 2\n",
    "# Reference\n",
    "# https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.differential_evolution.html\n",
    "# https://pablormier.github.io/2017/09/05/a-tutorial-on-differential-evolution-with-python/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value1</th>\n",
       "      <th>value2</th>\n",
       "      <th>value3</th>\n",
       "      <th>value4</th>\n",
       "      <th>value5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-128.862122</td>\n",
       "      <td>95195504.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-123.937958</td>\n",
       "      <td>96773328.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>94.940582</td>\n",
       "      <td>97036304.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-97.312332</td>\n",
       "      <td>23914.0</td>\n",
       "      <td>3.413793</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-97.312332</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   value1      value2      value3    value4  value5\n",
       "0     1.0 -128.862122  95195504.0  0.000000     0.0\n",
       "1     1.0 -123.937958  96773328.0  0.000000     0.0\n",
       "2     0.0   94.940582  97036304.0  0.000000     0.0\n",
       "3     0.0  -97.312332     23914.0  3.413793     0.0\n",
       "4     0.0  -97.312332         0.0  0.413793     1.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = df_keep.iloc[: 5, :]\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. When input, make sure df contains all the values you need\n",
    "# 2. specify df = X in def objectfunc(beta, df = df_test) before you run differential_evolution\n",
    "def objectfunc(beta, df = df_keep):\n",
    "    '''\n",
    "    num = df.shape[1] # This is the num of Parameters( dimension of beta)\n",
    "    obj = 0\n",
    "    for i in range(0, num):\n",
    "        obj = obj + np.array(df.iloc[:, i]) * beta[i]\n",
    "        # print(obj)\n",
    "    return -sum(obj >= 0)\n",
    "    '''\n",
    "    return -sum(df.dot(beta) >=0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-5908089"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is a test for debugging\n",
    "beta = np.array([1, 2, 3, 4, 5])\n",
    "obj = objectfunc(beta, df_keep)\n",
    "obj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-4"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is a test for debugging\n",
    "beta = np.array([1, 2, 3, 4, 5])\n",
    "obj = objectfunc(beta, df_test)\n",
    "obj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     fun: -5989014.0\n",
      " message: 'Optimization terminated successfully.'\n",
      "    nfev: 306\n",
      "     nit: 3\n",
      " success: True\n",
      "       x: array([  1.        ,  -4.47262905,  36.31679535, -41.54346339,\n",
      "        88.40841153])\n",
      "Running time:  364.816956281662\n"
     ]
    }
   ],
   "source": [
    "# specify bounds of every parameters\n",
    "'''\n",
    "num = df_test.shape[0]\n",
    "num = df_test.shape[0]\n",
    "bounds = [(-100, 100)] * num\n",
    "'''\n",
    "t1 = time.time()\n",
    "bounds = [(1, 1.000000001), (-100, 100), (-100, 100), (-100, 100), (-100, 100)]\n",
    "result = differential_evolution(objectfunc, bounds)\n",
    "print(result)\n",
    "t2 = time.time()\n",
    "print(\"Running time: \", t2-t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The bumber of inequalities satisfied is\n",
      "5989014\n"
     ]
    }
   ],
   "source": [
    "print(\"The bumber of inequalities satisfied is\")\n",
    "print(round(-result.fun))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 4: Use subsampling to construct confidence intervals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The idea is to select some random subsamples (100-200) that mimics the original sample and re-do the step1-3 and get another estimated parameter. Then calculate the 5% and 95% percentile of the 100 (200) parameters.\n",
    "To mimic the original sample, we choose the subsample for fixed share for each state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n# The random sampling ratio = 10% here\\ndef Subsample(df, ratio):\\n    # t1 = time.time()\\n    df_sample = df.loc[np.random.choice(df.index, round(len(df) * ratio), replace = False)]\\n    # t2 = time.time()\\n    # print(\"Running time: \", t2-t1)\\n    return df_sample\\ndf_matched_sample = Subsample(df_matched, 0.1)\\ndf_unmatched_sample = Subsample(df_unmatched, 0.1)\\n'"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# The random sampling ratio = 10% here\n",
    "def Subsample(df, ratio):\n",
    "    # t1 = time.time()\n",
    "    df_sample = df.loc[np.random.choice(df.index, round(len(df) * ratio), replace = False)]\n",
    "    # t2 = time.time()\n",
    "    # print(\"Running time: \", t2-t1)\n",
    "    return df_sample\n",
    "df_matched_sample = Subsample(df_matched, 0.1)\n",
    "df_unmatched_sample = Subsample(df_unmatched, 0.1)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lender_id</th>\n",
       "      <th>loan_id</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "      <th>var3</th>\n",
       "      <th>var4</th>\n",
       "      <th>var5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>814.0</td>\n",
       "      <td>19445.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.481559</td>\n",
       "      <td>2150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>275.0</td>\n",
       "      <td>3498.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>169.902786</td>\n",
       "      <td>11088.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3096.0</td>\n",
       "      <td>60530.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.044008</td>\n",
       "      <td>2823.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3813.0</td>\n",
       "      <td>71810.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.056799</td>\n",
       "      <td>28242.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60.0</td>\n",
       "      <td>456.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.011763</td>\n",
       "      <td>1764.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6000</th>\n",
       "      <td>1085.0</td>\n",
       "      <td>23929.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>222.663376</td>\n",
       "      <td>38430.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6001</th>\n",
       "      <td>1085.0</td>\n",
       "      <td>23938.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>222.663376</td>\n",
       "      <td>96075.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6002</th>\n",
       "      <td>1795.0</td>\n",
       "      <td>36141.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.144510</td>\n",
       "      <td>5346.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6003</th>\n",
       "      <td>1795.0</td>\n",
       "      <td>36138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.873976</td>\n",
       "      <td>5346.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6004</th>\n",
       "      <td>821.0</td>\n",
       "      <td>19525.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.001764</td>\n",
       "      <td>8004.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6005 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      lender_id  loan_id  var1        var2     var3  var4  var5\n",
       "0         814.0  19445.0   0.0   11.481559   2150.0   0.0   0.0\n",
       "1         275.0   3498.0   1.0  169.902786  11088.0   0.0   0.0\n",
       "2        3096.0  60530.0   1.0    7.044008   2823.0   0.0   0.0\n",
       "3        3813.0  71810.0   0.0    1.056799  28242.0   0.0   0.0\n",
       "4          60.0    456.0   0.0    3.011763   1764.0   0.0   0.0\n",
       "...         ...      ...   ...         ...      ...   ...   ...\n",
       "6000     1085.0  23929.0   0.0  222.663376  38430.0   0.0   0.0\n",
       "6001     1085.0  23938.0   0.0  222.663376  96075.0   0.0   0.0\n",
       "6002     1795.0  36141.0   0.0   20.144510   5346.0   0.0   0.0\n",
       "6003     1795.0  36138.0   0.0   17.873976   5346.0   0.0   0.0\n",
       "6004      821.0  19525.0   0.0    3.001764   8004.0   0.0   0.0\n",
       "\n",
       "[6005 rows x 7 columns]"
      ]
     },
     "execution_count": 349,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_matched"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 5% quantile of parameters are\n",
      "0     1.000000\n",
      "1   -99.151884\n",
      "2     0.010407\n",
      "3   -95.174940\n",
      "4   -82.695684\n",
      "Name: 0.05, dtype: float64\n",
      "The 95% quantile of parameters are\n",
      "0     1.000000\n",
      "1   -36.245882\n",
      "2     0.493030\n",
      "3    98.743652\n",
      "4    89.493345\n",
      "Name: 0.95, dtype: float64\n",
      "Simulation time:  8354.007978200912\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-99.231250</td>\n",
       "      <td>0.004879</td>\n",
       "      <td>-12.488045</td>\n",
       "      <td>-54.667913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-78.910727</td>\n",
       "      <td>0.507259</td>\n",
       "      <td>39.188229</td>\n",
       "      <td>25.413048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-64.231139</td>\n",
       "      <td>0.086076</td>\n",
       "      <td>-95.673616</td>\n",
       "      <td>-64.225877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-70.381379</td>\n",
       "      <td>0.264324</td>\n",
       "      <td>-30.725268</td>\n",
       "      <td>-75.176655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-87.567491</td>\n",
       "      <td>0.026632</td>\n",
       "      <td>-7.489319</td>\n",
       "      <td>84.613679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-71.167678</td>\n",
       "      <td>0.025789</td>\n",
       "      <td>99.757212</td>\n",
       "      <td>76.561598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-86.993004</td>\n",
       "      <td>0.310994</td>\n",
       "      <td>97.504857</td>\n",
       "      <td>-88.847617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-86.512356</td>\n",
       "      <td>0.023114</td>\n",
       "      <td>-62.291873</td>\n",
       "      <td>52.593936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-13.348854</td>\n",
       "      <td>0.475639</td>\n",
       "      <td>-86.376915</td>\n",
       "      <td>23.807160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-99.054880</td>\n",
       "      <td>0.017162</td>\n",
       "      <td>-94.565448</td>\n",
       "      <td>93.485799</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0          1         2          3          4\n",
       "0  1.0 -99.231250  0.004879 -12.488045 -54.667913\n",
       "1  1.0 -78.910727  0.507259  39.188229  25.413048\n",
       "2  1.0 -64.231139  0.086076 -95.673616 -64.225877\n",
       "3  1.0 -70.381379  0.264324 -30.725268 -75.176655\n",
       "4  1.0 -87.567491  0.026632  -7.489319  84.613679\n",
       "5  1.0 -71.167678  0.025789  99.757212  76.561598\n",
       "6  1.0 -86.993004  0.310994  97.504857 -88.847617\n",
       "7  1.0 -86.512356  0.023114 -62.291873  52.593936\n",
       "8  1.0 -13.348854  0.475639 -86.376915  23.807160\n",
       "9  1.0 -99.054880  0.017162 -94.565448  93.485799"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# num is the # of random samples(or the # of simulations)\n",
    "# ratio is the random sampling ratio, which is 10% here\n",
    "def Simulation(num, ratio, df_matched, df_unmatched):\n",
    "    df_result = pd.DataFrame()\n",
    "    for i in range(0, num):\n",
    "        # df_matched_sample = Subsample(df_matched, ratio)\n",
    "        # df_unmatched_sample = Subsample(df_unmatched, ratio)\n",
    "        \n",
    "        loan_id = df_matched[\"loan_id\"].unique() # loan_id = df_unmatched[\"loan_id\"].unique() -- 应该是从matched里取~\n",
    "        sample_loan_id = np.random.choice(loan_id, round(loan_id.shape[0] * ratio), replace = False)\n",
    "        df_unmatched_sample = df_unmatched.loc[df_unmatched[\"loan_id\"].isin(sample_loan_id)]\n",
    "        df_matched_sample = df_matched.loc[df_matched[\"loan_id\"].isin(sample_loan_id)]\n",
    "       \n",
    "        df_keep = Exchange_pairs(df_matched_sample, df_unmatched_sample)\n",
    "        bounds = [(1, 1.000000001), (-100, 100), (-100, 100), (-100, 100), (-100, 100)] # fix beta_1 = 1\n",
    "        result = differential_evolution(objectfunc, bounds)\n",
    "        df_result = df_result.append(pd.Series(result.x), ignore_index = True)\n",
    "        # print(i)\n",
    "    print(\"The 5% quantile of parameters are\")\n",
    "    print(df_result.quantile(0.05))\n",
    "    print(\"The 95% quantile of parameters are\")\n",
    "    print(df_result.quantile(0.95))\n",
    "    return df_result\n",
    "\n",
    "\n",
    "t1 = time.time()\n",
    "df_result = Simulation(10, 0.1, df_matched, df_unmatched)\n",
    "t2 = time.time()\n",
    "print(\"Simulation time: \", t2-t1)\n",
    "df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 5% quantile of parameters are\n",
      "0     1.000000\n",
      "1   -92.747828\n",
      "2     0.054753\n",
      "3   -31.420180\n",
      "4   -82.814493\n",
      "Name: 0.05, dtype: float64\n",
      "The 95% quantile of parameters are\n",
      "0     1.000000\n",
      "1    -0.477537\n",
      "2    65.198585\n",
      "3    80.092220\n",
      "4    97.542895\n",
      "Name: 0.95, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(\"The 5% quantile of parameters are\")\n",
    "print(df_result.quantile(0.05))\n",
    "print(\"The 95% quantile of parameters are\")\n",
    "print(df_result.quantile(0.95))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 5: Model Fitness"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1) Number of inequalities satisfied: count how many inequalities are satisfied under the optimized parameter choice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Fox_func(path1, path2, num, ratio):\n",
    "\n",
    "    df_matched = pd.read_stata(path1).iloc[:,: 7]\n",
    "    df_unmatched = pd.read_stata(path2).iloc[:,: 7]\n",
    "    df_keep = Exchange_pairs(df_matched, df_unmatched)\n",
    "    def objectfunc(beta, df = df_keep):\n",
    "        return -sum(df.dot(beta) >=0 )\n",
    "    t1 = time.time()\n",
    "    bounds = [(1, 1.000000001), (-100, 100), (-100, 100), (-100, 100), (-100, 100)]\n",
    "    result = differential_evolution(objectfunc, bounds)\n",
    "    print(result)\n",
    "    t2 = time.time()\n",
    "    print(\"Differential Evolution time: \", t2-t1)\n",
    "    print(\"The bumber of inequalities satisfied is\")\n",
    "    print(round(-result.fun))\n",
    "    # Simulation(num, ratio, df_matched, df_unmatched)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2) In sample prediction: assign firms to lenders according to the estimated value function and count how many of the observed (P820, footnote 51 – Schwert (2018)) Target Output – Table IX X in Schwert (2018)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Calculate the quota for each lender.\n",
    "- Using the estimated value function, matching values are calculated for all pairs in matched sample and unmatched sample.\n",
    "- For every lender, rank the pairs by matching values in descending order and pick the top \"quota\" pairs.\n",
    "- Calculate how many pairs are in matched sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Ratio_Calculate(path1, path2, beta):\n",
    "    df_matched = pd.read_stata(path1)\n",
    "    df_unmatched = pd.read_stata(path2)\n",
    "    values, counts = np.unique(df_matched[\"lender_id\"], return_counts = True)\n",
    "    df_quota = pd.DataFrame(counts, index = values, columns = [\"quota\"])\n",
    "    df_total = df_matched.append(df_unmatched)\n",
    "    df_total[\"matching_value\"] = df_total.iloc[:,2:].dot(beta)\n",
    "    s = 0\n",
    "    for i in df_quota.index:\n",
    "        quota = int(df_quota.loc[i])\n",
    "        id_sorted = df_total[df_total[\"lender_id\"] == i].sort_values(by = \"matching_value\")[0:quota][\"loan_id\"]\n",
    "        s = s + sum(df_matched[df_matched[\"lender_id\"] == i][\"loan_id\"].isin(id_sorted))\n",
    "    print(s/df_matched.shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.07139395897270231\n"
     ]
    }
   ],
   "source": [
    "Ratio_Calculate(\"sample_matched_NY_demean1.dta\", \"sample_unmatched_NY_demean1.dta\", r11.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.07607817800032304\n"
     ]
    }
   ],
   "source": [
    "Ratio_Calculate(\"sample_matched_NY_org.dta\", \"sample_unmatched_NY_org.dta\", r2.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.06251009529962849\n"
     ]
    }
   ],
   "source": [
    "Ratio_Calculate(\"sample_matched_NY_std.dta\", \"sample_unmatched_NY_std.dta\", r3.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quota</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14.0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49.0</th>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57.0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60.0</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67.0</th>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5001.0</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5007.0</th>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5009.0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5084.0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090.0</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>267 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        quota\n",
       "14.0        1\n",
       "49.0       13\n",
       "57.0        1\n",
       "60.0        5\n",
       "67.0       28\n",
       "...       ...\n",
       "5001.0      8\n",
       "5007.0     34\n",
       "5009.0      1\n",
       "5084.0      1\n",
       "5090.0      9\n",
       "\n",
       "[267 rows x 1 columns]"
      ]
     },
     "execution_count": 356,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values, counts = np.unique(df_matched[\"lender_id\"], return_counts = True)\n",
    "df_quota = pd.DataFrame(counts, index = values, columns = [\"quota\"])\n",
    "df_quota\n",
    "beta = result.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lender_id</th>\n",
       "      <th>loan_id</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "      <th>var3</th>\n",
       "      <th>var4</th>\n",
       "      <th>var5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>814.0</td>\n",
       "      <td>19445.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.481559</td>\n",
       "      <td>2150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>275.0</td>\n",
       "      <td>3498.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>169.902786</td>\n",
       "      <td>11088.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3096.0</td>\n",
       "      <td>60530.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.044008</td>\n",
       "      <td>2823.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3813.0</td>\n",
       "      <td>71810.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.056799</td>\n",
       "      <td>28242.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60.0</td>\n",
       "      <td>456.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.011763</td>\n",
       "      <td>1764.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430180</th>\n",
       "      <td>3278.0</td>\n",
       "      <td>36138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>174.711655</td>\n",
       "      <td>1337778.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430181</th>\n",
       "      <td>697.0</td>\n",
       "      <td>19525.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1279.236694</td>\n",
       "      <td>772416.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430182</th>\n",
       "      <td>697.0</td>\n",
       "      <td>36138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1279.236694</td>\n",
       "      <td>2317248.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430183</th>\n",
       "      <td>3923.0</td>\n",
       "      <td>36138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>515700.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430184</th>\n",
       "      <td>3923.0</td>\n",
       "      <td>19525.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>171900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>436190 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        lender_id  loan_id  var1         var2       var3  var4  var5\n",
       "0           814.0  19445.0   0.0    11.481559     2150.0   0.0   0.0\n",
       "1           275.0   3498.0   1.0   169.902786    11088.0   0.0   0.0\n",
       "2          3096.0  60530.0   1.0     7.044008     2823.0   0.0   0.0\n",
       "3          3813.0  71810.0   0.0     1.056799    28242.0   0.0   0.0\n",
       "4            60.0    456.0   0.0     3.011763     1764.0   0.0   0.0\n",
       "...           ...      ...   ...          ...        ...   ...   ...\n",
       "430180     3278.0  36138.0   0.0   174.711655  1337778.0   0.0   0.0\n",
       "430181      697.0  19525.0   0.0  1279.236694   772416.0   0.0   0.0\n",
       "430182      697.0  36138.0   0.0  1279.236694  2317248.0   0.0   0.0\n",
       "430183     3923.0  36138.0   0.0     0.000000   515700.0   4.5   0.0\n",
       "430184     3923.0  19525.0   0.0     0.000000   171900.0   1.0   1.0\n",
       "\n",
       "[436190 rows x 7 columns]"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_total = df_matched.append(df_unmatched)\n",
    "df_total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1.        ,  -4.47262905,  36.31679535, -41.54346339,\n",
       "        88.40841153])"
      ]
     },
     "execution_count": 354,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beta = result.x\n",
    "beta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lender_id</th>\n",
       "      <th>loan_id</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "      <th>var3</th>\n",
       "      <th>var4</th>\n",
       "      <th>var5</th>\n",
       "      <th>matching_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>814.0</td>\n",
       "      <td>19445.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.481559</td>\n",
       "      <td>2150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.802976e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>275.0</td>\n",
       "      <td>3498.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>169.902786</td>\n",
       "      <td>11088.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.019217e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3096.0</td>\n",
       "      <td>60530.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.044008</td>\n",
       "      <td>2823.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.024918e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3813.0</td>\n",
       "      <td>71810.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.056799</td>\n",
       "      <td>28242.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.025654e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60.0</td>\n",
       "      <td>456.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.011763</td>\n",
       "      <td>1764.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.404936e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430180</th>\n",
       "      <td>3278.0</td>\n",
       "      <td>36138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>174.711655</td>\n",
       "      <td>1337778.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.858303e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430181</th>\n",
       "      <td>697.0</td>\n",
       "      <td>19525.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1279.236694</td>\n",
       "      <td>772416.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.804595e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430182</th>\n",
       "      <td>697.0</td>\n",
       "      <td>36138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1279.236694</td>\n",
       "      <td>2317248.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.414930e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430183</th>\n",
       "      <td>3923.0</td>\n",
       "      <td>36138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>515700.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.872838e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>430184</th>\n",
       "      <td>3923.0</td>\n",
       "      <td>19525.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>171900.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.242904e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>436190 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        lender_id  loan_id  var1         var2       var3  var4  var5  \\\n",
       "0           814.0  19445.0   0.0    11.481559     2150.0   0.0   0.0   \n",
       "1           275.0   3498.0   1.0   169.902786    11088.0   0.0   0.0   \n",
       "2          3096.0  60530.0   1.0     7.044008     2823.0   0.0   0.0   \n",
       "3          3813.0  71810.0   0.0     1.056799    28242.0   0.0   0.0   \n",
       "4            60.0    456.0   0.0     3.011763     1764.0   0.0   0.0   \n",
       "...           ...      ...   ...          ...        ...   ...   ...   \n",
       "430180     3278.0  36138.0   0.0   174.711655  1337778.0   0.0   0.0   \n",
       "430181      697.0  19525.0   0.0  1279.236694   772416.0   0.0   0.0   \n",
       "430182      697.0  36138.0   0.0  1279.236694  2317248.0   0.0   0.0   \n",
       "430183     3923.0  36138.0   0.0     0.000000   515700.0   4.5   0.0   \n",
       "430184     3923.0  19525.0   0.0     0.000000   171900.0   1.0   1.0   \n",
       "\n",
       "        matching_value  \n",
       "0         7.802976e+04  \n",
       "1         4.019217e+05  \n",
       "2         1.024918e+05  \n",
       "3         1.025654e+06  \n",
       "4         6.404936e+04  \n",
       "...                ...  \n",
       "430180    4.858303e+07  \n",
       "430181    2.804595e+07  \n",
       "430182    8.414930e+07  \n",
       "430183    1.872838e+07  \n",
       "430184    6.242904e+06  \n",
       "\n",
       "[436190 rows x 8 columns]"
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_total[\"matching_value\"] = df_total.iloc[:,2:].dot(beta)\n",
    "df_total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.07610324729392173\n"
     ]
    }
   ],
   "source": [
    "s = 0\n",
    "for i in df_quota.index:\n",
    "    quota = int(df_quota.loc[i])\n",
    "    id_sorted = df_total[df_total[\"lender_id\"] == i].sort_values(by = \"matching_value\")[0:quota][\"loan_id\"]\n",
    "    s = s + sum(df_matched[df_matched[\"lender_id\"] == i][\"loan_id\"].isin(id_sorted))\n",
    "print(s/df_matched.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12/29"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num is the # of random samples(or the # of simulations)\n",
    "# ratio is the random sampling ratio, which is 10% here\n",
    "def Simulation(num, ratio, df_matched, df_unmatched):\n",
    "    df_result = pd.DataFrame()\n",
    "    for i in range(0, num):\n",
    "        # df_matched_sample = Subsample(df_matched, ratio)\n",
    "        # df_unmatched_sample = Subsample(df_unmatched, ratio)\n",
    "        \n",
    "        loan_id = df_unmatched[\"loan_id\"].unique()\n",
    "        sample_loan_id = np.random.choice(loan_id, round(loan_id.shape[0] * ratio), replace = False)\n",
    "        df_unmatched_sample = df_unmatched.loc[df_unmatched[\"loan_id\"].isin(sample_loan_id)]\n",
    "        df_matched_sample = df_matched.loc[df_matched[\"loan_id\"].isin(sample_loan_id)]\n",
    "       \n",
    "        df_keep = Exchange_pairs(df_matched_sample, df_unmatched_sample)\n",
    "        bounds = [(1, 1.000000001), (-100, 100), (-100, 100), (-100, 100), (-100, 100)] # fix beta_1 = 1\n",
    "        result = differential_evolution(objectfunc, bounds)\n",
    "        df_result = df_result.append(pd.Series(result.x), ignore_index = True)\n",
    "        # print(i)\n",
    "    print(\"The 5% quantile of parameters are\")\n",
    "    print(df_result.quantile(0.05))\n",
    "    print(\"The 95% quantile of parameters are\")\n",
    "    print(df_result.quantile(0.95))\n",
    "    return df_result\n",
    "\n",
    "\n",
    "t1 = time.time()\n",
    "df_result = Simulation(10, 0.1, df_matched, df_unmatched)\n",
    "t2 = time.time()\n",
    "print(\"Simulation time: \", t2-t1)\n",
    "df_result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1/3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    }
   ],
   "source": [
    "df_sample = pd.read_stata('sample.dta')\n",
    "df_matched = df_sample[df_sample[\"match\"] == 1]\n",
    "df_unmatched = df_sample[df_sample[\"match\"] == 0]\n",
    "\n",
    "df_matched.drop([\"match\"], axis = 1, inplace = True) \n",
    "df_unmatched.drop([\"match\"], axis = 1, inplace = True) \n",
    "df_matched = df_matched[df_matched[\"USPS_ZIP_PREF_STATE\"] == \"NY\"]\n",
    "df_matched = df_matched[df_matched[\"yearapproved\"] == 2021].iloc[:,: 7]\n",
    "\n",
    "df_unmatched = df_unmatched[df_unmatched[\"USPS_ZIP_PREF_STATE\"] == \"NY\"]\n",
    "df_unmatched = df_unmatched[df_unmatched[\"yearapproved\"] == 2021].iloc[:,: 7]\n",
    "\n",
    "df_exchange_pairs = Exchange_pairs(df_matched, df_unmatched)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lender_id</th>\n",
       "      <th>loan_id</th>\n",
       "      <th>match</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "      <th>var3</th>\n",
       "      <th>var4</th>\n",
       "      <th>var5</th>\n",
       "      <th>USPS_ZIP_PREF_CITY</th>\n",
       "      <th>USPS_ZIP_PREF_STATE</th>\n",
       "      <th>yearapproved</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322744</td>\n",
       "      <td>376543.375000</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>97252.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322744</td>\n",
       "      <td>399935.531250</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322744</td>\n",
       "      <td>376543.375000</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3402.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.350215</td>\n",
       "      <td>379006.218750</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3402.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.350215</td>\n",
       "      <td>379006.218750</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117067</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>52924.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>255598.812500</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>KETCHIKAN</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117068</th>\n",
       "      <td>3923.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-32325.736328</td>\n",
       "      <td>1.069300</td>\n",
       "      <td>-0.347326</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117069</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>65.689995</td>\n",
       "      <td>163466.203125</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117070</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34127.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.713989</td>\n",
       "      <td>370764.562500</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117071</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34131.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.713989</td>\n",
       "      <td>439864.031250</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4117072 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         lender_id  loan_id  match  var1       var2           var3      var4  \\\n",
       "0           1339.0  27917.0    0.0   0.0  17.322744  376543.375000 -0.024180   \n",
       "1           1339.0  97252.0    0.0   0.0  17.322744  399935.531250  0.091452   \n",
       "2           1339.0  78177.0    0.0   0.0  17.322744  376543.375000  0.091452   \n",
       "3           3402.0  78177.0    0.0   0.0  10.350215  379006.218750  0.091452   \n",
       "4           3402.0  27917.0    0.0   0.0  10.350215  379006.218750 -0.024180   \n",
       "...            ...      ...    ...   ...        ...            ...       ...   \n",
       "4117067     1631.0  52924.0    0.0   0.0   0.000000  255598.812500 -0.139812   \n",
       "4117068     3923.0  34122.0    0.0   0.0   0.000000  -32325.736328  1.069300   \n",
       "4117069     1631.0  34122.0    1.0   0.0  65.689995  163466.203125 -0.139812   \n",
       "4117070     1631.0  34127.0    1.0   0.0  61.713989  370764.562500 -0.139812   \n",
       "4117071     1631.0  34131.0    1.0   0.0  61.713989  439864.031250 -0.139812   \n",
       "\n",
       "             var5 USPS_ZIP_PREF_CITY USPS_ZIP_PREF_STATE  yearapproved  \n",
       "0        0.045413             AGAWAM                  MA        2020.0  \n",
       "1        0.045413             AGAWAM                  MA        2020.0  \n",
       "2       -0.070219             AGAWAM                  MA        2020.0  \n",
       "3       -0.070219             AGAWAM                  MA        2020.0  \n",
       "4        0.045413             AGAWAM                  MA        2020.0  \n",
       "...           ...                ...                 ...           ...  \n",
       "4117067  0.045413          KETCHIKAN                  AK        2020.0  \n",
       "4117068 -0.347326              CRAIG                  AK        2020.0  \n",
       "4117069  0.045413              CRAIG                  AK        2020.0  \n",
       "4117070  0.045413           WRANGELL                  AK        2020.0  \n",
       "4117071 -0.070219           WRANGELL                  AK        2020.0  \n",
       "\n",
       "[4117072 rows x 11 columns]"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value1</th>\n",
       "      <th>value2</th>\n",
       "      <th>value3</th>\n",
       "      <th>value4</th>\n",
       "      <th>value5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.062592</td>\n",
       "      <td>282140.000000</td>\n",
       "      <td>-7.450581e-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-134.978073</td>\n",
       "      <td>-112856.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-57.051155</td>\n",
       "      <td>14934.007812</td>\n",
       "      <td>7.450581e-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-11.757530</td>\n",
       "      <td>13364.000000</td>\n",
       "      <td>7.450581e-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>6.960756</td>\n",
       "      <td>739.999023</td>\n",
       "      <td>-7.450581e-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101681</th>\n",
       "      <td>0.0</td>\n",
       "      <td>6.236197</td>\n",
       "      <td>12312.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101682</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14364.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101683</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-162.729996</td>\n",
       "      <td>20072.031250</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101684</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.490269</td>\n",
       "      <td>-3750.031250</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101685</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.490270</td>\n",
       "      <td>-3750.031250</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>101686 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        value1      value2         value3        value4  value5\n",
       "0          0.0   -0.062592  282140.000000 -7.450581e-09     0.0\n",
       "1          0.0 -134.978073 -112856.000000  0.000000e+00     0.0\n",
       "2          0.0  -57.051155   14934.007812  7.450581e-09     0.0\n",
       "3          0.0  -11.757530   13364.000000  7.450581e-09     0.0\n",
       "4          0.0    6.960756     739.999023 -7.450581e-09     0.0\n",
       "...        ...         ...            ...           ...     ...\n",
       "101681     0.0    6.236197   12312.000000  0.000000e+00     0.0\n",
       "101682     0.0    0.000000   14364.000000  0.000000e+00     0.0\n",
       "101683     0.0 -162.729996   20072.031250  0.000000e+00     0.0\n",
       "101684     0.0   -1.490269   -3750.031250  0.000000e+00     0.0\n",
       "101685     0.0   -1.490270   -3750.031250  0.000000e+00     0.0\n",
       "\n",
       "[101686 rows x 5 columns]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_exchange_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Fox_func(num, ratio):\n",
    "    \n",
    "    df_sample = pd.read_stata('sample.dta')\n",
    "    df_matched = df_sample[df_sample[\"match\"] == 1]\n",
    "    df_unmatched = df_sample[df_sample[\"match\"] == 0]\n",
    "    df_matched.drop([\"match\"], axis = 1, inplace = True) \n",
    "    df_unmatched.drop([\"match\"], axis = 1, inplace = True) \n",
    "    df_matched = df_matched[df_matched[\"USPS_ZIP_PREF_STATE\"] == \"NY\"]\n",
    "    df_matched = df_matched[df_matched[\"yearapproved\"] == 2020].iloc[:,: 7]\n",
    "    df_unmatched = df_unmatched[df_unmatched[\"USPS_ZIP_PREF_STATE\"] == \"NY\"]\n",
    "    df_unmatched = df_unmatched[df_unmatched[\"yearapproved\"] == 2020].iloc[:,: 7]\n",
    "    \n",
    "    df_keep = Exchange_pairs(df_matched, df_unmatched)\n",
    "    \n",
    "    df_keep.drop([\"value3\"], axis = 1, inplace = True) \n",
    "    # df_keep.drop([\"value4\"], axis = 1, inplace = True) \n",
    "    \n",
    "    # df_keep.drop([\"value2\"], axis = 1) \n",
    "    # df_keep.drop([\"value3\"], axis = 1) \n",
    "\n",
    "    def objectfunc(beta, df = df_keep):\n",
    "        return -sum(df.dot(beta) >=0 )\n",
    "    t1 = time.time()\n",
    "    # bounds = [(1, 1.0000000001), (-100, 100), (-100, 100), (-100, 100)]\n",
    "    # bounds = [(-1.000000001, -1), (-500, 500), (-500, 500)]\n",
    "    bounds = [(-500, 500), (-1.000000001, -1), (-500, 500), (-500, 500)]\n",
    "    # bounds = [(-500, 500), (-500, 500), (-500, 500)]\n",
    "    result = differential_evolution(objectfunc, bounds)\n",
    "    # print(result)\n",
    "\n",
    "    r = np.append(result.x, round(-result.fun))\n",
    "    \n",
    "    # t2 = time.time()\n",
    "    # print(\"Differential Evolution time: \", t2 - t1)\n",
    "    # print(\"The number of inequalities satisfied is\")\n",
    "    # print(round(-result.fun))\n",
    "    # Simulation(num, ratio, df_matched, df_unmatched)\n",
    "    return r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "53\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "61\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "67\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "68\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "69\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "89\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "98\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:4312: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  errors=errors,\n"
     ]
    }
   ],
   "source": [
    "df_result = pd.DataFrame()\n",
    "for i in range(0, 100):\n",
    "    print(i)\n",
    "    result = Fox_func(100, 0.1)\n",
    "    df_result = df_result.append([list(result)], ignore_index = True)\n",
    "df_result.columns = [\"Beta_1\", \"Beta_2\", \"Beta_4\", \"Beta_5\", \"Number of of inequalities satisfied\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Beta_1</th>\n",
       "      <th>Beta_2</th>\n",
       "      <th>Beta_4</th>\n",
       "      <th>Beta_5</th>\n",
       "      <th>Number of of inequalities satisfied</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>449.525596</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.406714</td>\n",
       "      <td>265.011253</td>\n",
       "      <td>5936247.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Beta_1  Beta_2    Beta_4      Beta_5  \\\n",
       "82  449.525596    -1.0 -0.406714  265.011253   \n",
       "\n",
       "    Number of of inequalities satisfied  \n",
       "82                            5936247.0  "
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Don't use argmax or idxmax since if the maximum is achieved in multiple locations, only the first is returned.\n",
    "num_max = df_result[\"Number of of inequalities satisfied\"].max()\n",
    "df_result[df_result[\"Number of of inequalities satisfied\"] == num_max]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df_result\n",
    "df_test = df_test.append(df_result[df_result[\"Number of of inequalities satisfied\"] == num_max])\n",
    "df_test = df_test.append(df_result.mean(axis = 0).rename(\"mean\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Beta_1</th>\n",
       "      <th>Beta_2</th>\n",
       "      <th>Beta_4</th>\n",
       "      <th>Beta_5</th>\n",
       "      <th>Number of of inequalities satisfied</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>450.221994</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-2.630644</td>\n",
       "      <td>121.118366</td>\n",
       "      <td>5926412.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>295.447559</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.718904</td>\n",
       "      <td>143.286266</td>\n",
       "      <td>5930628.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>368.195846</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.472396</td>\n",
       "      <td>262.935498</td>\n",
       "      <td>5935122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>353.166882</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-2.383213</td>\n",
       "      <td>333.106743</td>\n",
       "      <td>5929836.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>443.246448</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.538437</td>\n",
       "      <td>461.061858</td>\n",
       "      <td>5923845.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>286.969504</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-3.166268</td>\n",
       "      <td>277.449091</td>\n",
       "      <td>5930284.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>285.237308</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.521603</td>\n",
       "      <td>151.499033</td>\n",
       "      <td>5931376.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>303.845445</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-2.425117</td>\n",
       "      <td>299.887522</td>\n",
       "      <td>5927658.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>449.525596</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.406714</td>\n",
       "      <td>265.011253</td>\n",
       "      <td>5936247.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>381.182240</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-3.194430</td>\n",
       "      <td>262.139217</td>\n",
       "      <td>5928906.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Beta_1  Beta_2    Beta_4      Beta_5  \\\n",
       "0     450.221994    -1.0 -2.630644  121.118366   \n",
       "1     295.447559    -1.0 -1.718904  143.286266   \n",
       "2     368.195846    -1.0 -1.472396  262.935498   \n",
       "3     353.166882    -1.0 -2.383213  333.106743   \n",
       "4     443.246448    -1.0 -0.538437  461.061858   \n",
       "...          ...     ...       ...         ...   \n",
       "97    286.969504    -1.0 -3.166268  277.449091   \n",
       "98    285.237308    -1.0 -0.521603  151.499033   \n",
       "99    303.845445    -1.0 -2.425117  299.887522   \n",
       "82    449.525596    -1.0 -0.406714  265.011253   \n",
       "mean  381.182240    -1.0 -3.194430  262.139217   \n",
       "\n",
       "      Number of of inequalities satisfied  \n",
       "0                               5926412.0  \n",
       "1                               5930628.0  \n",
       "2                               5935122.0  \n",
       "3                               5929836.0  \n",
       "4                               5923845.0  \n",
       "...                                   ...  \n",
       "97                              5930284.0  \n",
       "98                              5931376.0  \n",
       "99                              5927658.0  \n",
       "82                              5936247.0  \n",
       "mean                            5928906.2  \n",
       "\n",
       "[102 rows x 5 columns]"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_csv(\"NY2020;Beta_2=-1;ExcludeVar3;1230.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Not defining Fox_func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result = pd.DataFrame()\n",
    "for i in range(0, 100):\n",
    "    print(i)\n",
    "    df_sample = pd.read_stata('sample.dta')\n",
    "    df_matched = df_sample[df_sample[\"match\"] == 1]\n",
    "    df_unmatched = df_sample[df_sample[\"match\"] == 0]\n",
    "    df_matched.drop([\"match\"], axis = 1, inplace = True) \n",
    "    df_unmatched.drop([\"match\"], axis = 1, inplace = True) \n",
    "    df_matched = df_matched[df_matched[\"USPS_ZIP_PREF_STATE\"] == \"NY\"]\n",
    "    df_matched = df_matched[df_matched[\"yearapproved\"] == 2021].iloc[:,: 7]\n",
    "    df_unmatched = df_unmatched[df_unmatched[\"USPS_ZIP_PREF_STATE\"] == \"NY\"]\n",
    "    df_unmatched = df_unmatched[df_unmatched[\"yearapproved\"] == 2021].iloc[:,: 7]\n",
    "    \n",
    "    df_keep = Exchange_pairs(df_matched, df_unmatched)\n",
    "    \n",
    "    df_keep.drop([\"value1\"], axis = 1, inplace = True) \n",
    "    df_keep.drop([\"value3\"], axis = 1, inplace = True) \n",
    "    \n",
    "    # df_keep.drop([\"value2\"], axis = 1) \n",
    "    # df_keep.drop([\"value3\"], axis = 1) \n",
    "\n",
    "    def objectfunc(beta, df = df_keep):\n",
    "        return -sum(df.dot(beta) >=0 )\n",
    "    t1 = time.time()\n",
    "    # bounds = [(1, 1.0000000001), (-100, 100), (-100, 100), (-100, 100)]\n",
    "    bounds = [(-1.000000001, -1), (-500, 500), (-500, 500)]\n",
    "    # bounds = [(-500, 500), (-500, 500), (-500, 500)]\n",
    "    result = differential_evolution(objectfunc, bounds)\n",
    "    # print(result)\n",
    "\n",
    "    r = np.append(result.x, round(-result.fun))\n",
    "    \n",
    "    \n",
    "    \n",
    "    result = Fox_func(100, 0.1)\n",
    "    df_result = df_result.append([list(r)], ignore_index = True)\n",
    "df_result.columns = [\"Beta_2\", \"Beta_4\", \"Beta_5\", \"Number of of inequalities satisfied\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_csv(\"NY2021;Beta_2=-1;ExcludeVar1Var3;1230.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "df_w1\n",
    "- var1 = Relationship_Dum(demean)\n",
    "- var2 = mi_to_zcta5(demean)\n",
    "- var3 = FinTechIndicator(demean) * rating_avg(demean)\n",
    "- var4 = FinTechIndicator(demean) * minority_yelp(demean)\n",
    "\n",
    "\n",
    "df_w2\n",
    "- var1 = Relationship_Dum\n",
    "- var2 = mi_to_zcta5\n",
    "- var3 = FinTechIndicator(demean) * rating_avg(demean)\n",
    "- var4 = FinTechIndicator(demean) * minority_yelp(demean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_org = pd.read_stata('sample_org.dta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_org['var1'] = df_org['Relationship_Dum'] - df_org['Relationship_Dum'].mean(axis = 0)\n",
    "df_org['var2'] = df_org['mi_to_zcta5'] - df_org['mi_to_zcta5'].mean(axis = 0)\n",
    "df_org['var3'] = (df_org['FinTechIndicator'] - df_org['FinTechIndicator'].mean(axis = 0)) * (df_org['rating_avg'] - df_org['rating_avg'].mean(axis = 0))\n",
    "df_org['var4'] = (df_org['FinTechIndicator'] - df_org['FinTechIndicator'].mean(axis = 0)) * (df_org['minority_yelp'] - df_org['minority_yelp'].mean(axis = 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lender_id</th>\n",
       "      <th>loan_id</th>\n",
       "      <th>rating_avg</th>\n",
       "      <th>minority_yelp</th>\n",
       "      <th>USPS_ZIP_PREF_CITY</th>\n",
       "      <th>USPS_ZIP_PREF_STATE</th>\n",
       "      <th>FinTechIndicator</th>\n",
       "      <th>Relationship_Dum</th>\n",
       "      <th>match</th>\n",
       "      <th>mi_to_zcta5</th>\n",
       "      <th>yearapproved</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "      <th>var3</th>\n",
       "      <th>var4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322745</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-543.263339</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>97252.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322745</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-543.263339</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>0.045413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322745</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-543.263339</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3402.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.350215</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-550.235868</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3402.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.350215</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-550.235868</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117067</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34117.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>KETCHIKAN</td>\n",
       "      <td>AK</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-560.586083</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117068</th>\n",
       "      <td>3923.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-560.586083</td>\n",
       "      <td>1.069299</td>\n",
       "      <td>-0.347326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117069</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>65.689996</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-494.896087</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117070</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34131.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>61.713988</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-498.872095</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>-0.070219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117071</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34127.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>61.713988</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-498.872095</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4117072 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         lender_id  loan_id  rating_avg  minority_yelp USPS_ZIP_PREF_CITY  \\\n",
       "0           1339.0  27917.0         4.0            0.0             AGAWAM   \n",
       "1           1339.0  97252.0         3.0            0.0             AGAWAM   \n",
       "2           1339.0  78177.0         3.0            1.0             AGAWAM   \n",
       "3           3402.0  27917.0         4.0            0.0             AGAWAM   \n",
       "4           3402.0  78177.0         3.0            1.0             AGAWAM   \n",
       "...            ...      ...         ...            ...                ...   \n",
       "4117067     1631.0  34117.0         4.0            0.0          KETCHIKAN   \n",
       "4117068     3923.0  34122.0         5.0            0.0              CRAIG   \n",
       "4117069     1631.0  34122.0         5.0            0.0              CRAIG   \n",
       "4117070     1631.0  34131.0         5.0            1.0           WRANGELL   \n",
       "4117071     1631.0  34127.0         5.0            0.0           WRANGELL   \n",
       "\n",
       "        USPS_ZIP_PREF_STATE  FinTechIndicator  Relationship_Dum  match  \\\n",
       "0                        MA                 0               0.0    0.0   \n",
       "1                        MA                 0               0.0    0.0   \n",
       "2                        MA                 0               0.0    0.0   \n",
       "3                        MA                 0               0.0    0.0   \n",
       "4                        MA                 0               0.0    0.0   \n",
       "...                     ...               ...               ...    ...   \n",
       "4117067                  AK                 0               0.0    1.0   \n",
       "4117068                  AK                 1               0.0    0.0   \n",
       "4117069                  AK                 0               0.0    1.0   \n",
       "4117070                  AK                 0               0.0    1.0   \n",
       "4117071                  AK                 0               0.0    1.0   \n",
       "\n",
       "         mi_to_zcta5  yearapproved      var1        var2      var3      var4  \n",
       "0          17.322745        2020.0 -0.001122 -543.263339 -0.024180  0.045413  \n",
       "1          17.322745        2020.0 -0.001122 -543.263339  0.091452  0.045413  \n",
       "2          17.322745        2020.0 -0.001122 -543.263339  0.091452 -0.070219  \n",
       "3          10.350215        2020.0 -0.001122 -550.235868 -0.024180  0.045413  \n",
       "4          10.350215        2020.0 -0.001122 -550.235868  0.091452 -0.070219  \n",
       "...              ...           ...       ...         ...       ...       ...  \n",
       "4117067     0.000000        2020.0 -0.001122 -560.586083 -0.024180  0.045413  \n",
       "4117068     0.000000        2020.0 -0.001122 -560.586083  1.069299 -0.347326  \n",
       "4117069    65.689996        2020.0 -0.001122 -494.896087 -0.139812  0.045413  \n",
       "4117070    61.713988        2020.0 -0.001122 -498.872095 -0.139812 -0.070219  \n",
       "4117071    61.713988        2020.0 -0.001122 -498.872095 -0.139812  0.045413  \n",
       "\n",
       "[4117072 rows x 15 columns]"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_org"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1597: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[key] = value\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1676: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(ilocs[0], value, pi)\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1738: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value[:, i].tolist(), pi)\n"
     ]
    }
   ],
   "source": [
    "df_w1 = pd.DataFrame()\n",
    "df_w1 = df_org[['lender_id', 'loan_id', 'match']]\n",
    "df_w1.loc[:, 'var1'] = df_org['var1'].values\n",
    "df_w1.loc[:, 'var2'] = df_org['var2'].values\n",
    "df_w1.loc[:, 'var3'] = df_org['var3'].values\n",
    "df_w1.loc[:, 'var4'] = df_org['var4'].values\n",
    "df_w1.loc[:, ('USPS_ZIP_PREF_CITY', 'USPS_ZIP_PREF_STATE', 'yearapproved')] = df_org[['USPS_ZIP_PREF_CITY', 'USPS_ZIP_PREF_STATE', 'yearapproved']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lender_id</th>\n",
       "      <th>loan_id</th>\n",
       "      <th>match</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "      <th>var3</th>\n",
       "      <th>var4</th>\n",
       "      <th>USPS_ZIP_PREF_CITY</th>\n",
       "      <th>USPS_ZIP_PREF_STATE</th>\n",
       "      <th>yearapproved</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-543.263339</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>97252.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-543.263339</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-543.263339</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3402.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-550.235868</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3402.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-550.235868</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117067</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34117.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-560.586083</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>KETCHIKAN</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117068</th>\n",
       "      <td>3923.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-560.586083</td>\n",
       "      <td>1.069299</td>\n",
       "      <td>-0.347326</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117069</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-494.896087</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117070</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34131.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-498.872095</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117071</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34127.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.001122</td>\n",
       "      <td>-498.872095</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4117072 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         lender_id  loan_id  match      var1        var2      var3      var4  \\\n",
       "0           1339.0  27917.0    0.0 -0.001122 -543.263339 -0.024180  0.045413   \n",
       "1           1339.0  97252.0    0.0 -0.001122 -543.263339  0.091452  0.045413   \n",
       "2           1339.0  78177.0    0.0 -0.001122 -543.263339  0.091452 -0.070219   \n",
       "3           3402.0  27917.0    0.0 -0.001122 -550.235868 -0.024180  0.045413   \n",
       "4           3402.0  78177.0    0.0 -0.001122 -550.235868  0.091452 -0.070219   \n",
       "...            ...      ...    ...       ...         ...       ...       ...   \n",
       "4117067     1631.0  34117.0    1.0 -0.001122 -560.586083 -0.024180  0.045413   \n",
       "4117068     3923.0  34122.0    0.0 -0.001122 -560.586083  1.069299 -0.347326   \n",
       "4117069     1631.0  34122.0    1.0 -0.001122 -494.896087 -0.139812  0.045413   \n",
       "4117070     1631.0  34131.0    1.0 -0.001122 -498.872095 -0.139812 -0.070219   \n",
       "4117071     1631.0  34127.0    1.0 -0.001122 -498.872095 -0.139812  0.045413   \n",
       "\n",
       "        USPS_ZIP_PREF_CITY USPS_ZIP_PREF_STATE  yearapproved  \n",
       "0                   AGAWAM                  MA        2020.0  \n",
       "1                   AGAWAM                  MA        2020.0  \n",
       "2                   AGAWAM                  MA        2020.0  \n",
       "3                   AGAWAM                  MA        2020.0  \n",
       "4                   AGAWAM                  MA        2020.0  \n",
       "...                    ...                 ...           ...  \n",
       "4117067          KETCHIKAN                  AK        2020.0  \n",
       "4117068              CRAIG                  AK        2020.0  \n",
       "4117069              CRAIG                  AK        2020.0  \n",
       "4117070           WRANGELL                  AK        2020.0  \n",
       "4117071           WRANGELL                  AK        2020.0  \n",
       "\n",
       "[4117072 rows x 10 columns]"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_w1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1597: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[key] = value\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1676: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(ilocs[0], value, pi)\n",
      "/opt/conda/lib/python3.7/site-packages/pandas/core/indexing.py:1738: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value[:, i].tolist(), pi)\n"
     ]
    }
   ],
   "source": [
    "df_w2 = pd.DataFrame()\n",
    "df_w2 = df_org[['lender_id', 'loan_id', 'match']]\n",
    "df_w2.loc[:, 'var1'] = df_org['Relationship_Dum'].values\n",
    "df_w2.loc[:, 'var2'] = df_org['mi_to_zcta5'].values\n",
    "df_w2.loc[:, 'var3'] = df_org['var3'].values\n",
    "df_w2.loc[:, 'var4'] = df_org['var4'].values\n",
    "df_w2.loc[:, ('USPS_ZIP_PREF_CITY', 'USPS_ZIP_PREF_STATE', 'yearapproved')] = df_org[['USPS_ZIP_PREF_CITY', 'USPS_ZIP_PREF_STATE', 'yearapproved']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lender_id</th>\n",
       "      <th>loan_id</th>\n",
       "      <th>match</th>\n",
       "      <th>var1</th>\n",
       "      <th>var2</th>\n",
       "      <th>var3</th>\n",
       "      <th>var4</th>\n",
       "      <th>USPS_ZIP_PREF_CITY</th>\n",
       "      <th>USPS_ZIP_PREF_STATE</th>\n",
       "      <th>yearapproved</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322745</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>97252.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322745</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1339.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.322745</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3402.0</td>\n",
       "      <td>27917.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.350215</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3402.0</td>\n",
       "      <td>78177.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.350215</td>\n",
       "      <td>0.091452</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>AGAWAM</td>\n",
       "      <td>MA</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117067</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34117.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.024180</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>KETCHIKAN</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117068</th>\n",
       "      <td>3923.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.069299</td>\n",
       "      <td>-0.347326</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117069</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34122.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>65.689996</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>CRAIG</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117070</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34131.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.713988</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>-0.070219</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117071</th>\n",
       "      <td>1631.0</td>\n",
       "      <td>34127.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.713988</td>\n",
       "      <td>-0.139812</td>\n",
       "      <td>0.045413</td>\n",
       "      <td>WRANGELL</td>\n",
       "      <td>AK</td>\n",
       "      <td>2020.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4117072 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         lender_id  loan_id  match  var1       var2      var3      var4  \\\n",
       "0           1339.0  27917.0    0.0   0.0  17.322745 -0.024180  0.045413   \n",
       "1           1339.0  97252.0    0.0   0.0  17.322745  0.091452  0.045413   \n",
       "2           1339.0  78177.0    0.0   0.0  17.322745  0.091452 -0.070219   \n",
       "3           3402.0  27917.0    0.0   0.0  10.350215 -0.024180  0.045413   \n",
       "4           3402.0  78177.0    0.0   0.0  10.350215  0.091452 -0.070219   \n",
       "...            ...      ...    ...   ...        ...       ...       ...   \n",
       "4117067     1631.0  34117.0    1.0   0.0   0.000000 -0.024180  0.045413   \n",
       "4117068     3923.0  34122.0    0.0   0.0   0.000000  1.069299 -0.347326   \n",
       "4117069     1631.0  34122.0    1.0   0.0  65.689996 -0.139812  0.045413   \n",
       "4117070     1631.0  34131.0    1.0   0.0  61.713988 -0.139812 -0.070219   \n",
       "4117071     1631.0  34127.0    1.0   0.0  61.713988 -0.139812  0.045413   \n",
       "\n",
       "        USPS_ZIP_PREF_CITY USPS_ZIP_PREF_STATE  yearapproved  \n",
       "0                   AGAWAM                  MA        2020.0  \n",
       "1                   AGAWAM                  MA        2020.0  \n",
       "2                   AGAWAM                  MA        2020.0  \n",
       "3                   AGAWAM                  MA        2020.0  \n",
       "4                   AGAWAM                  MA        2020.0  \n",
       "...                    ...                 ...           ...  \n",
       "4117067          KETCHIKAN                  AK        2020.0  \n",
       "4117068              CRAIG                  AK        2020.0  \n",
       "4117069              CRAIG                  AK        2020.0  \n",
       "4117070           WRANGELL                  AK        2020.0  \n",
       "4117071           WRANGELL                  AK        2020.0  \n",
       "\n",
       "[4117072 rows x 10 columns]"
      ]
     },
     "execution_count": 356,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}