Added support for neighbourhoods in db

backend/app/database.py

@@ -7,7 +7,11 @@
 
 SQLALCHEMY_DATABASE_URL = os.getenv("DATABASE_URL")
 
-engine = create_engine(SQLALCHEMY_DATABASE_URL)
+engine = create_engine(
+    SQLALCHEMY_DATABASE_URL, 
+    pool_pre_ping=True, 
+    pool_recycle=300
+)
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 Base = declarative_base()

backend/app/db_models.py

@@ -30,6 +30,7 @@ class Building(Base):
                 autoincrement=True, unique=True)
     name = Column(String, index=True)
     address = Column(String, index=True)
+    neighbourhood = Column(String, index=True)
     city = Column(String, index=True)
     lat = Column(Float, primary_key=True)
     lon = Column(Float, primary_key=True)

backend/app/pydantic_schemas/Analysis.py

@@ -11,7 +11,7 @@ class AddListing(BaseModel):
     bath: float
     sqft: float
     price: float
-    pets: float
+    pets: bool
     latitude: float
     longitude: float
     date: datetime

backend/app/scripts/seed.py

@@ -9,7 +9,8 @@
 session.query(Building).delete()
 
 # Recreate tables after dropping them (if needed)
-Base.metadata.create_all(engine)
+# Base.metadata.drop_all(bind=engine)
+Base.metadata.create_all(bind=engine)
 
 # Commit changes and close the session
 session.commit()

backend/app/services/create.py

@@ -14,6 +14,7 @@ def row_to_building(row, db: Session) -> Building:
     return Building(
         name=row[TableHeaders.BUILDING.value],
         address=row[TableHeaders.ADDRESS.value],
+        neighbourhood = row[TableHeaders.NEIGHBOURHOOD.value],
         city=row[TableHeaders.CITY.value],
         lat=row[TableHeaders.LAT.value],
         lon=row[TableHeaders.LON.value],

data/configs.py

@@ -22,7 +22,7 @@ def create_chrome_driver(*, debugging_port):
     chrome_options.add_argument(f"user-agent={user_agent}")
     chrome_options.add_argument(f"--remote-debugging-port={debugging_port}")
     chrome_options.add_argument('--window-size=1920x1080')
-    # chrome_options.add_argument("--headless")  # Enable headless mode (does not open browser GUI)
+    chrome_options.add_argument("--headless")  # Enable headless mode (does not open browser GUI)
 
     # First ChromeService instance
     chrome_service = ChromeService(executable_path=chrome_driver_path)

data/main.py

@@ -39,15 +39,13 @@ def extract_raw_data(filepath: str, listing_urls: list[str]) -> pd.DataFrame:
             file.write('\n'.join(padmapper_scraper.urls))
 
         current_100_units = []
-        current_city_units = []
 
         # Scrape page content of collected URLs to get rental listing data 
         for url in padmapper_scraper.urls:
             try:
                 # on every 100 listings read, write them to the excel sheet (in case of crash)
                 if len(current_100_units) >= 100:
-                    current_city_units += current_100_units
-                    extracted_listing_data += current_city_units
+                    extracted_listing_data += current_100_units
                     extracted_listing_data_df = pd.DataFrame(extracted_listing_data, columns=table_columns)
                     extracted_listing_data_df.to_excel(filepath, index=False)
                     current_100_units.clear()