Merge pull request #100 from reworkd/s3-scheme

✨ Don't normalize S3 urls

core/harambe_core/normalize_url.py

@@ -11,7 +11,12 @@ def normalize_url(path: str, base_path: str | None) -> str:
     """
     path = sanitize_scheme(path)
     validate_allowed_scheme(path)
-    path = _normalize(path)
+    if not is_s3_url(path):
+        # We append actual URLs at the end of S3 urls occasionally
+        # Normalization will turn https:// into http:/
+        # TODO: When we handle dynamic downloads in our worker, we need to remove this logic
+        #  we should also remove s3 as an allowed scheme all together
+        path = _normalize(path)
     escaped_path = path.replace(" ", "%20")
 
     if base_path is None:
@@ -45,7 +50,12 @@ def sanitize_scheme(url: str) -> str:
     return base + url[last_scheme_index + 1 :] if last_scheme_index > 0 else url
 
 
-allowed_url_schemes = ["http", "https", "s3", "file"]
+s3_scheme = "s3"
+allowed_url_schemes = ["http", "https", s3_scheme, "file"]
+
+
+def is_s3_url(url: str) -> bool:
+    return urlparse(url).scheme == s3_scheme
 
 
 def validate_allowed_scheme(url: str, scheme_required: bool = False) -> None:

core/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "harambe-core"
-version = "0.55.0"
+version = "0.56.0"
 description = "Core types for harambe SDK 🐒🍌"
 authors = [
     { name = "Adam Watkins", email = "[email protected]" }

core/test/parser/test_type_url.py

@@ -61,6 +61,11 @@
             "https://example.com",
             "https://example.com/doc1",
         ),
+        (
+            "s3://deworkd-local-files/5a9a3ac5-a572-494d-a888-8a065e6e3878/579c900dbfb97dae2fbf329cf6b3411a.pdf;https://example.com/",
+            "https://www.example.com",
+            "s3://deworkd-local-files/5a9a3ac5-a572-494d-a888-8a065e6e3878/579c900dbfb97dae2fbf329cf6b3411a.pdf;https://example.com/",
+        ),
     ],
 )
 def test_pydantic_type_url_validate_type_success(url, base_url_, expected):

core/test/test_normalize_url.py

@@ -112,6 +112,11 @@
             "//primoliquors.com/cdn/shop/files/oxcuimdzbczobdeo248f.png?v=1727810919&width=1946",
             "https://primoliquors.com/cdn/shop/files/oxcuimdzbczobdeo248f.png?v=1727810919&width=1946",
         ),
+        (
+            "https://www.example.com",
+            "s3://deworkd-local-files/5a9a3ac5-a572-494d-a888-8a065e6e3878/579c900dbfb97dae2fbf329cf6b3411a.pdf;https://example.com/",
+            "s3://deworkd-local-files/5a9a3ac5-a572-494d-a888-8a065e6e3878/579c900dbfb97dae2fbf329cf6b3411a.pdf;https://example.com/",
+        ),
     ],
 )
 def test_normalize_url(base_path, url, expected):

sdk/pyproject.toml

@@ -1,14 +1,14 @@
 [project]
 name = "harambe-sdk"
-version = "0.55.0"
+version = "0.56.0"
 description = "Data extraction SDK for Playwright 🐒🍌"
 authors = [
     { name = "Adam Watkins", email = "[email protected]" }
 ]
 requires-python = ">=3.11,<4.0"
 readme = "README.md"
 dependencies = [
-    "harambe_core==0.55.0",
+    "harambe_core==0.56.0",
     "playwright==1.47.0",
     "beautifulsoup4==4.12.3",
     "requests==2.32.3",