notebooks/0.1_data_preprocessing_FullName.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ce92ae6b-1cfb-47f9-b947-55f2448b7500",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Full Name Dataset (Train/Validation/Test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a34fa1c0-cf2e-464a-bc56-73a4f7a38a55",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af05aa15-8218-4086-9626-adadd2552183",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15455110, 3)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/fl_reg_name_race_2022.csv.gz')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "05503af7-99a2-4cf2-bdd4-b25d9015aa92",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size after dropping missing first and last names: (15454979, 3)\n",
      "Size after dropping unknown: (15009244, 3)\n",
      "Size after dropping last names less than 2 chars: (14933334, 3)\n"
     ]
    }
   ],
   "source": [
    "# Remove NA first/last\n",
    "df.dropna(subset=['name_first', 'name_last'], inplace=True)\n",
    "print(\"Size after dropping missing first and last names:\", df.shape)\n",
    "\n",
    "# We assume unknown as missing at random\n",
    "sdf = df[df.race.isin(['unknown']) == False]\n",
    "print(\"Size after dropping unknown:\", sdf.shape)\n",
    "del df\n",
    "\n",
    "# Drop cases where last name is less than 2 chars\n",
    "sdf = sdf.drop(sdf[sdf['name_last'].str.len() < 2].index)\n",
    "print(\"Size after dropping last names less than 2 chars:\", sdf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "98a49e48-1dfc-4d1b-ad98-874ce9559d0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Full Name\n",
    "sdf['name_first'] = sdf.name_first.str.strip().str.title()\n",
    "sdf['name_last'] = sdf.name_last.str.strip().str.title()\n",
    "sdf['full_name'] = sdf['name_last'] + ' ' + sdf['name_first']\n",
    "# Remove special chars\n",
    "sdf['full_name'] = sdf['full_name'].str.replace(\"[^a-zA-Z' -]\", '', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9bfb3b34-bdea-4c60-bff9-ffb0a63aa265",
   "metadata": {},
   "outputs": [],
   "source": [
    "# recode race\n",
    "mapping = {'multi_racial': 'other', 'native_indian': 'other'}\n",
    "sdf['race'] = sdf['race'].replace(mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "041729a5-8518-405f-9cd5-5e6e1869cc2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>full_name</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "      <th>total_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A Arup Erik</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A Bitang Ahmad</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A De Feria Graciela</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A F R Stephenson John Alexander</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A Felix Noehmi</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9018613</th>\n",
       "      <td>Zyzdryn Krzysztof</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9018614</th>\n",
       "      <td>Zyznomyrsky John</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9018615</th>\n",
       "      <td>Zzaman Md</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9018616</th>\n",
       "      <td>Zzaman Mohammad</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9018617</th>\n",
       "      <td>Zzie Richard</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9018618 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "race                           full_name  asian  hispanic  nh_black  nh_white  \\\n",
       "0                            A Arup Erik    0.0       0.0       0.0       1.0   \n",
       "1                         A Bitang Ahmad    0.0       0.0       1.0       0.0   \n",
       "2                    A De Feria Graciela    0.0       1.0       0.0       0.0   \n",
       "3        A F R Stephenson John Alexander    0.0       0.0       0.0       1.0   \n",
       "4                         A Felix Noehmi    0.0       1.0       0.0       0.0   \n",
       "...                                  ...    ...       ...       ...       ...   \n",
       "9018613                Zyzdryn Krzysztof    0.0       0.0       0.0       1.0   \n",
       "9018614                 Zyznomyrsky John    0.0       0.0       0.0       1.0   \n",
       "9018615                        Zzaman Md    0.0       0.0       0.0       0.0   \n",
       "9018616                  Zzaman Mohammad    0.0       0.0       0.0       0.0   \n",
       "9018617                     Zzie Richard    0.0       0.0       0.0       1.0   \n",
       "\n",
       "race     other  total_n  \n",
       "0          0.0      1.0  \n",
       "1          0.0      1.0  \n",
       "2          0.0      1.0  \n",
       "3          0.0      1.0  \n",
       "4          0.0      1.0  \n",
       "...        ...      ...  \n",
       "9018613    0.0      1.0  \n",
       "9018614    0.0      1.0  \n",
       "9018615    1.0      1.0  \n",
       "9018616    1.0      1.0  \n",
       "9018617    0.0      1.0  \n",
       "\n",
       "[9018618 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Summing the count of each name & race combination\n",
    "gdf = sdf.groupby(['full_name','race'], as_index=False)['race'].agg(['count'])\n",
    "# creating a pivot table so that each name has a count of the # of races with that last name\n",
    "gdf = gdf.pivot_table(values='count', columns='race', index='full_name')\n",
    "\n",
    "# Converting NaN to zeros since that means there is no one that identifies with that race with that last name\n",
    "gdf = gdf.fillna(0)\n",
    "\n",
    "gdf['total_n'] = gdf.sum(axis=1)\n",
    "gdf.reset_index(inplace=True)\n",
    "gdf.iloc[:, 1:-1] = gdf.iloc[:, 1:-1].div(gdf.total_n, axis=0)\n",
    "\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3e953fb6-2fe8-4d34-be56-10cf6698dc35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['asian', 'hispanic', 'nh_black', 'nh_white', 'other']\n"
     ]
    }
   ],
   "source": [
    "races = sorted(sdf.race.unique().tolist())\n",
    "print(races)\n",
    "\n",
    "def get_race_idx(val, races):\n",
    "    race_idx = races.index(val)\n",
    "    return race_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "db8c31ec-17d4-464a-8693-754a389c9bf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for one set of analysis, we define 'true race/ethincity' = where max prob (so modal race = true race)\n",
    "gdf['race'] = gdf[races].idxmax(axis=1)\n",
    "gdf['race_code'] = gdf['race'].apply(lambda c: get_race_idx(c,races))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "734fc268-8d20-4120-8693-7da140e4c8ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>full_name</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "      <th>total_n</th>\n",
       "      <th>race</th>\n",
       "      <th>race_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A Arup Erik</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A Bitang Ahmad</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>nh_black</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A De Feria Graciela</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>hispanic</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A F R Stephenson John Alexander</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>nh_white</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A Felix Noehmi</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>hispanic</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "race                        full_name  asian  hispanic  nh_black  nh_white  \\\n",
       "0                         A Arup Erik    0.0       0.0       0.0       1.0   \n",
       "1                      A Bitang Ahmad    0.0       0.0       1.0       0.0   \n",
       "2                 A De Feria Graciela    0.0       1.0       0.0       0.0   \n",
       "3     A F R Stephenson John Alexander    0.0       0.0       0.0       1.0   \n",
       "4                      A Felix Noehmi    0.0       1.0       0.0       0.0   \n",
       "\n",
       "race  other  total_n      race  race_code  \n",
       "0       0.0      1.0  nh_white          3  \n",
       "1       0.0      1.0  nh_black          2  \n",
       "2       0.0      1.0  hispanic          1  \n",
       "3       0.0      1.0  nh_white          3  \n",
       "4       0.0      1.0  hispanic          1  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "036bbb11-0d02-45db-801c-03a2873291c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf.to_csv(\"train_validation_test/gdf_fullname.csv.gz\", index = False, compression=\"gzip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3378db9-94f4-4268-94a0-435a448d0d77",
   "metadata": {},
   "source": [
    "## Split dataset in train, validation and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3a11b216-fde5-48c7-b5ec-904dbde4bb29",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, rest_df = train_test_split(gdf, test_size=0.2, random_state=42, stratify=gdf['race_code'])\n",
    "val_df, test_df = train_test_split(rest_df, test_size=0.5, random_state=42, stratify=rest_df['race_code'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e47fa000-d58f-4360-9487-bd33c149433e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df = train_df.reset_index(drop=True)\n",
    "val_df = val_df.reset_index(drop=True)\n",
    "test_df = test_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fce465b6-0a17-437d-ad38-7d8036c74d0e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7214894, 9)\n",
      "(901862, 9)\n",
      "(901862, 9)\n"
     ]
    }
   ],
   "source": [
    "print(train_df.shape)\n",
    "print(val_df.shape)\n",
    "print(test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "66f97f4e-4fda-44bc-8d3c-65fbe6eea9b6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>asian</th>\n",
       "      <td>206042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>1308198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_black</th>\n",
       "      <td>1067770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_white</th>\n",
       "      <td>4421898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>210986</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "race      full_name\n",
       "race               \n",
       "asian        206042\n",
       "hispanic    1308198\n",
       "nh_black    1067770\n",
       "nh_white    4421898\n",
       "other        210986"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.groupby('race').agg({'full_name':'nunique'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7d9a625b-fd10-4374-b744-e809620c86d5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>asian</th>\n",
       "      <td>25755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>163525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_black</th>\n",
       "      <td>133471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_white</th>\n",
       "      <td>552738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>26373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "race      full_name\n",
       "race               \n",
       "asian         25755\n",
       "hispanic     163525\n",
       "nh_black     133471\n",
       "nh_white     552738\n",
       "other         26373"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.groupby('race').agg({'full_name':'nunique'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e61fc0f4-1b6c-42a0-a1d5-cf7cbff8a290",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>asian</th>\n",
       "      <td>25756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>163525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_black</th>\n",
       "      <td>133471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_white</th>\n",
       "      <td>552737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>26373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "race      full_name\n",
       "race               \n",
       "asian         25756\n",
       "hispanic     163525\n",
       "nh_black     133471\n",
       "nh_white     552737\n",
       "other         26373"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.groupby('race').agg({'full_name':'nunique'})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c82ede6e-5c68-4786-b3c3-68b346de2a2e",
   "metadata": {},
   "source": [
    "## Save datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ecc12a2c-bee0-49bd-b42d-ab8cb5589a15",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df.to_csv(\"data/fl_2022_FullName_train.csv.gz\", index=False, compression=\"gzip\")\n",
    "val_df.to_csv(\"data/fl_2022_FullName_val.csv.gz\", index=False, compression=\"gzip\")\n",
    "test_df.to_csv(\"data/fl_2022_FullName_test.csv.gz\", index=False, compression=\"gzip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41d34d62-afb2-479e-a21b-040f1eb962ff",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}