add community transmission and svi rank

notebooks/COMMUNITY_TRANSMISSION.ipynb

@@ -0,0 +1,231 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "26b0e81b",
+   "metadata": {},
+   "source": [
+    "# Level Of Community Transmission\n",
+    "\n",
+    "### https://covid.cdc.gov/covid-data-tracker/#county-view"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d9f6944f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import pycountry\n",
+    "import json\n",
+    "from datetime import datetime\n",
+    "import requests\n",
+    "import io"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6ba5bf0",
+   "metadata": {
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# papermill parameters\n",
+    "output_folder = \"../output/\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7144c483",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s = requests.Session()\n",
+    "response = s.get(\"https://covid.cdc.gov/covid-data-tracker/content/CoronaViewJson_01/fips-codes-covid-tracker.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d28c2536",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fips = pd.read_csv(io.StringIO(response.text))\n",
+    "subdivisions = pycountry.subdivisions.get(country_code='US')\n",
+    "subdivision_dict = {subdivision.name: subdivision.code for subdivision in subdivisions}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ccd0f85",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = s.get(\"https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_latest_external_data\")\n",
+    "transition_df = pd.DataFrame(json.loads(response.text)[\"integrated_county_latest_external_data\"])\n",
+    "transition_df[\"fips_code\"] = transition_df[\"fips_code\"].astype(str)\n",
+    "transition_df[\"fips_code\"] = transition_df[\"fips_code\"].str.zfill(5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5d429cd5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.DataFrame()\n",
+    "\n",
+    "for i in range(len(fips)):\n",
+    "    area = fips.iloc[i]\n",
+    "    fips_code = str(area[\"fips_code\"]).zfill(5)\n",
+    "    \n",
+    "    # get response\n",
+    "    url = \"https://covid.cdc.gov/covid-data-tracker/COVIDData/getAjaxData?id=integrated_county_timeseries_fips_{fips_code}_external\".format(fips_code=fips_code)\n",
+    "    response = s.get(url)\n",
+    "    \n",
+    "    # create df for area\n",
+    "    try:\n",
+    "        tmp_df = pd.read_json(io.StringIO(json.dumps(json.loads(response.text)[\"integrated_county_timeseries_external_data\"])))\n",
+    "    except:\n",
+    "        continue\n",
+    "    tmp_df[\"state_name\"] = area[\"state_name\"]\n",
+    "    tmp_df[\"county\"] = area[\"county\"]\n",
+    "    tmp_df[\"fips_code\"] = fips_code\n",
+    "    tmp_df[\"svi\"] = transition_df.loc[transition_df[\"fips_code\"] == fips_code][\"svi_rank\"]\n",
+    "    tmp_df[\"ISO3166_2\"] = subdivision_dict[area[\"state_name\"]][-2:]\n",
+    "    tmp_df = tmp_df.drop(columns=[\"state\"])\n",
+    "    tmp_df[\"new_cases_7_day_rolling_average\"] = tmp_df.replace({\"suppressed\": np.nan})\n",
+    "    tmp_df[\"new_deaths_7_day_rolling_average\"] = tmp_df.replace({\"suppressed\": np.nan})\n",
+    "    df = df.append(tmp_df, ignore_index=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9e10fc77",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[\"Last_Updated_Date\"] = datetime.utcnow()\n",
+    "df['Last_Reported_Flag'] = df[\"date\"].max() == df[\"date\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a7051e6",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "df = df.sort_values(by=['state_name', 'date'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "213085d3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[\"date\"] = pd.to_datetime(df[\"date\"], format=\"%Y-%m-%d\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1b1a0ef9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df.rename(columns={\n",
+    "    \"fips_code\": \"FIPS\",\n",
+    "    \"state_name\": \"PROVINCE_STATE\",\n",
+    "    \"county\": \"COUNTY\"\n",
+    "})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8d79ff76",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.dtypes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ac3d56eb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[\"new_cases_7_day_rolling_average\"].unique()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d816fbc7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.to_csv(output_folder + \"COMMUNITY_TRANSMISSION.csv\", index=False, columns=[\n",
+    "    \"FIPS\",\n",
+    "    \"PROVINCE_STATE\",\n",
+    "    \"COUNTY\",\n",
+    "    \"ISO3166_2\",\n",
+    "    \"new_cases_7_day_rolling_average\",\n",
+    "    \"new_deaths_7_day_rolling_average\",\n",
+    "    \"new_test_results_reported_7_day_rolling_average\",\n",
+    "    \"admissions_covid_confirmed_7_day_rolling_average\",\n",
+    "    \"percent_adult_inpatient_beds_used_confirmed_covid\",\n",
+    "    \"percent_adult_icu_beds_used_confirmed_covid\",\n",
+    "    \"report_date_window_start\",\n",
+    "    \"report_date_window_end\",\n",
+    "    \"community_transmission_level\",\n",
+    "    \"svi\",\n",
+    "    \"date\",\n",
+    "    \"Last_Updated_Date\",\n",
+    "    \"Last_Reported_Flag\"\n",
+    "])"
+   ]
+  }
+ ],
+ "metadata": {
+  "celltoolbar": "Tags",
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}