From d2f1236f7adc4397f46fe6666b4d01fe3acc2f5d Mon Sep 17 00:00:00 2001 From: Ashutosh Date: Wed, 7 Feb 2024 16:23:24 -0500 Subject: [PATCH 1/3] Update process.ipynb --- process.ipynb | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/process.ipynb b/process.ipynb index 69ea1e8..d895d38 100644 --- a/process.ipynb +++ b/process.ipynb @@ -684,22 +684,28 @@ "symbol_list = pd.DataFrame([i.split('|') for i in fred_time_series], columns=['Id', 'Name', 'Unit'])\n", "session = requests.Session()\n", "\n", - "for row in symbol_list.iterrows():\n", - " symbol = row[1].Id\n", - " name = row[1].Name\n", + "for retries in range(5):\n", + " try:\n", + " for row in symbol_list.iterrows():\n", + " symbol = row[1].Id\n", + " name = row[1].Name\n", "\n", - " print(f'Downloading {name} ... ')\n", + " print(f'Downloading {name} ... ')\n", "\n", - " df = get_dataframe_from_request(symbol, API_KEY, session)\n", - " \n", - " if df is None:\n", - " print(f'Download failed for {name} - observations were null')\n", - " continue\n", + " df = get_dataframe_from_request(symbol, API_KEY, session)\n", + " \n", + " if df is None:\n", + " print(f'Download failed for {name} - observations were null')\n", + " continue\n", "\n", - " destination_file = destination_folder / f'{symbol.lower()}.csv'\n", - " df.to_csv(destination_file.resolve(), header=None, date_format='%Y%m%d')\n", + " destination_file = destination_folder / f'{symbol.lower()}.csv'\n", + " df.to_csv(destination_file.resolve(), header=None, date_format='%Y%m%d')\n", "\n", - " sleep(1)" + " sleep(1)\n", + " except Exception as e:\n", + " print(f'Failed to Download data - retrying')\n", + " sleep(10)\n", + "raise Exception(f'Failed to Download data')" ] }, { @@ -730,6 +736,11 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" + }, + "vscode": { + "interpreter": { + "hash": "5c7b89af1651d0b8571dde13640ecdccf7d5a6204171d6ab33e7c296e100e08a" + } } }, "nbformat": 4, From cc524745b5b62c8016b5c96884a66d2442c49941 Mon Sep 17 00:00:00 2001 From: Ashutosh Date: Wed, 7 Feb 2024 16:30:21 -0500 Subject: [PATCH 2/3] Update process.ipynb --- process.ipynb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/process.ipynb b/process.ipynb index d895d38..dd1c5a1 100644 --- a/process.ipynb +++ b/process.ipynb @@ -736,12 +736,7 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" - }, - "vscode": { - "interpreter": { - "hash": "5c7b89af1651d0b8571dde13640ecdccf7d5a6204171d6ab33e7c296e100e08a" - } - } + } }, "nbformat": 4, "nbformat_minor": 5 From 90c753448c3c301d0188234ecf07bcef78e43b0e Mon Sep 17 00:00:00 2001 From: Ashutosh Date: Thu, 8 Feb 2024 13:11:47 -0500 Subject: [PATCH 3/3] Update process.ipynb --- process.ipynb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/process.ipynb b/process.ipynb index dd1c5a1..3cd4b6a 100644 --- a/process.ipynb +++ b/process.ipynb @@ -649,6 +649,7 @@ " From FRED request's response, get the data as pandas DataFrame.\n", " :return: pandas DataFrame with the values correctly parsed.\n", " \"\"\"\n", + " url = BASE_URL + f'&api_key={api_key}&series_id={symbol}'\n", " for retries in range(5):\n", " try:\n", " response = download_fred_symbol(symbol, api_key, session).json()\n", @@ -702,10 +703,13 @@ " df.to_csv(destination_file.resolve(), header=None, date_format='%Y%m%d')\n", "\n", " sleep(1)\n", + " break\n", " except Exception as e:\n", - " print(f'Failed to Download data - retrying')\n", - " sleep(10)\n", - "raise Exception(f'Failed to Download data')" + " if retries == 4:\n", + " raise Exception(f'Failed to download data')\n", + " \n", + " print(f'Failed to download data - retrying')\n", + " sleep(10)" ] }, { @@ -736,7 +740,7 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" - } + } }, "nbformat": 4, "nbformat_minor": 5