diff --git a/lib/shared/layers/python-sdk/python/genai_core/websites/crawler.py b/lib/shared/layers/python-sdk/python/genai_core/websites/crawler.py
index 9e32539bd..e426b7383 100644
--- a/lib/shared/layers/python-sdk/python/genai_core/websites/crawler.py
+++ b/lib/shared/layers/python-sdk/python/genai_core/websites/crawler.py
@@ -101,7 +101,7 @@ def parse_url(url: str):
     base_url = f"{root_url_parse.scheme}://{root_url_parse.netloc}"
 
     response = requests.get(url, timeout=20)
-    if response.headers["Content-Type"] != "text/html":
+    if "text/html" not in response.headers["Content-Type"]:
         raise Exception(f"Invalid content type {response.headers['Content-Type']}")
     soup = BeautifulSoup(response.content, "html.parser")
     content = soup.text