[fix] replace NaN values in nyiso df with None

calblueprint · Oct 20, 2024 · 9b1f6a9 · 9b1f6a9
1 parent 6a58223
commit 9b1f6a9
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/api/webscraper/database.py b/api/webscraper/database.py
@@ -26,7 +26,7 @@
 
 def nyserda_large_to_database():
   database = []
-  database.extend(query_nyserda_large())
+  database.extend(query_nyserda_large()[0:10])
   for project in database:
     if project.get('proposed_cod', None) is not None:
       ymd = datetime.strptime(project.get('proposed_cod'), '%Y').strftime('%Y-%m-%d')
@@ -120,6 +120,6 @@ def nyiso_to_database():
 '''
 For testing
 '''
-# nyserda_large_to_database()
+nyserda_large_to_database()
 # nyserda_solar_to_database()
 # nyiso_to_database()
diff --git a/api/webscraper/nyiso_scraper.py b/api/webscraper/nyiso_scraper.py
@@ -8,7 +8,7 @@ def query_nyiso():
   nyiso_data = nyiso.content
   nyiso_df = pd.read_excel(BytesIO(nyiso_data))
   nyiso_df.dropna(subset=['Project Name'], inplace=True) # drops rows of xlsx that don't correspond to project data
-  nyiso_df.fillna('', inplace=True) # replaces NaN values with empty strings
+  nyiso_df.fillna(None, inplace=True) # replaces NaN values with empty strings
   nyiso_df.replace(to_replace=['', 'N/A', 'n/a', 'NAN', 'n/a'], value=None, inplace=True)
   nyiso_list = nyiso_df.to_dict(orient='records')