diff --git a/mountaineer/__tests__/client_builder/test_builder.py b/mountaineer/__tests__/client_builder/test_builder.py index 585a8669..c38a2654 100644 --- a/mountaineer/__tests__/client_builder/test_builder.py +++ b/mountaineer/__tests__/client_builder/test_builder.py @@ -8,7 +8,7 @@ from mountaineer.actions import sideeffect from mountaineer.app import AppController -from mountaineer.client_builder.builder import ClientBuilder +from mountaineer.client_builder.builder import APIBuilder from mountaineer.controller import ControllerBase from mountaineer.controller_layout import LayoutControllerBase from mountaineer.render import RenderBase @@ -60,30 +60,30 @@ def simple_app_controller( @pytest.fixture def builder(simple_app_controller: AppController): - return ClientBuilder(simple_app_controller) + return APIBuilder(simple_app_controller) -def test_generate_static_files(builder: ClientBuilder): +def test_generate_static_files(builder: APIBuilder): builder.generate_static_files() -def test_generate_model_definitions(builder: ClientBuilder): +def test_generate_model_definitions(builder: APIBuilder): builder.generate_model_definitions() -def test_generate_action_definitions(builder: ClientBuilder): +def test_generate_action_definitions(builder: APIBuilder): builder.generate_action_definitions() -def test_generate_view_definitions(builder: ClientBuilder): +def test_generate_view_definitions(builder: APIBuilder): builder.generate_link_shortcuts() -def test_generate_link_aggregator(builder: ClientBuilder): +def test_generate_link_aggregator(builder: APIBuilder): builder.generate_link_aggregator() -def test_generate_link_aggregator_ignores_layout(builder: ClientBuilder): +def test_generate_link_aggregator_ignores_layout(builder: APIBuilder): class ExampleLayout(LayoutControllerBase): view_path = "/test.tsx" @@ -100,12 +100,12 @@ async def render(self) -> None: assert "ExampleDetailControllerGetLinks" in global_links -def test_generate_view_servers(builder: ClientBuilder): +def test_generate_view_servers(builder: APIBuilder): builder.generate_view_servers() @pytest.mark.parametrize("empty_links", [True, False]) -def test_generate_index_file_ignores_empty(builder: ClientBuilder, empty_links: bool): +def test_generate_index_file_ignores_empty(builder: APIBuilder, empty_links: bool): # Create some stub files. We simulate a case where the links file # is created but empty file_contents = "import React from 'react';\n" @@ -138,13 +138,13 @@ def test_generate_index_file_ignores_empty(builder: ClientBuilder, empty_links: ] -def test_cache_is_outdated_no_cache(builder: ClientBuilder): +def test_cache_is_outdated_no_cache(builder: APIBuilder): # No cache builder.build_cache = None assert builder.cache_is_outdated() is True -def test_cache_is_outdated_no_existing_data(builder: ClientBuilder, tmp_path: Path): +def test_cache_is_outdated_no_existing_data(builder: APIBuilder, tmp_path: Path): builder.build_cache = tmp_path assert builder.cache_is_outdated() is True @@ -159,7 +159,7 @@ def test_cache_is_outdated_no_existing_data(builder: ClientBuilder, tmp_path: Pa def test_cache_is_outdated_existing_data( - builder: ClientBuilder, + builder: APIBuilder, tmp_path: Path, home_controller: ExampleHomeController, detail_controller: ExampleDetailController, @@ -198,7 +198,7 @@ def test_cache_is_outdated_existing_data( def test_cache_is_outdated_url_change( - builder: ClientBuilder, + builder: APIBuilder, tmp_path: Path, home_controller: ExampleHomeController, detail_controller: ExampleDetailController, @@ -243,7 +243,7 @@ def test_cache_is_outdated_url_change( def test_validate_unique_paths_exact_definition( - builder: ClientBuilder, + builder: APIBuilder, ): """ Two controllers can't manage the same view path. @@ -267,7 +267,7 @@ def render(self) -> None: def test_validate_unique_paths_conflicting_layout( - builder: ClientBuilder, + builder: APIBuilder, ): """ Layouts need to be placed in their own directory. Even if the literal paths @@ -291,7 +291,7 @@ def render(self) -> None: def test_generate_controller_schema_sideeffect_required_attributes( - builder: ClientBuilder, + builder: APIBuilder, ): """ Ensure that we treat @sideeffect and @passthrough return models like diff --git a/mountaineer/__tests__/client_compiler/test_compile.py b/mountaineer/__tests__/client_compiler/test_compile.py new file mode 100644 index 00000000..7cae9ef7 --- /dev/null +++ b/mountaineer/__tests__/client_compiler/test_compile.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from mountaineer.app import AppController +from mountaineer.client_compiler.compile import ClientCompiler + + +def test_build_static_metadata(tmpdir: Path): + app = AppController(view_root=tmpdir) + compiler = ClientCompiler(app=app) + + # Write test files to the view path to determine if we're able + # to parse the whole file tree + static_dir = compiler.view_root.get_managed_static_dir() + + (static_dir / "test_css.css").write_text("CSS_TEXT") + + (static_dir / "nested").mkdir(exist_ok=True) + (static_dir / "nested" / "test_nested.css").write_text("CSS_TEXT") + + # File contents are the same - shas should be the same as well + metadata = compiler._build_static_metadata() + assert "test_css.css" in metadata.static_artifact_shas + assert "nested/test_nested.css" in metadata.static_artifact_shas + assert ( + metadata.static_artifact_shas["test_css.css"] + == metadata.static_artifact_shas["nested/test_nested.css"] + ) diff --git a/mountaineer/__tests__/test_render.py b/mountaineer/__tests__/test_render.py index 079c941a..76ca5570 100644 --- a/mountaineer/__tests__/test_render.py +++ b/mountaineer/__tests__/test_render.py @@ -1,5 +1,6 @@ import pytest +from mountaineer.client_compiler.build_metadata import BuildMetadata from mountaineer.render import ( LinkAttribute, MetaAttribute, @@ -90,7 +91,159 @@ ], ) def test_build_header(metadata: Metadata, expected_tags: list[str]): - assert metadata.build_header() == expected_tags + assert metadata.build_header(build_metadata=None) == expected_tags + + +@pytest.mark.parametrize( + "metadata, build_metadata, expected_tags", + [ + # Test static file with matching SHA + ( + Metadata( + links=[ + LinkAttribute( + rel="stylesheet", + href="/static/css/style.css", + add_static_sha=True, + ) + ] + ), + BuildMetadata(static_artifact_shas={"css/style.css": "abc123"}), + [''], + ), + # Test multiple static files with matching SHAs + ( + Metadata( + links=[ + LinkAttribute( + rel="stylesheet", + href="/static/css/style.css", + add_static_sha=True, + ), + LinkAttribute( + rel="stylesheet", + href="/static/css/other.css", + add_static_sha=True, + ), + ] + ), + BuildMetadata( + static_artifact_shas={ + "css/style.css": "abc123", + "css/other.css": "def456", + } + ), + [ + '', + '', + ], + ), + # Test static file without matching SHA (should not modify URL) + ( + Metadata( + links=[ + LinkAttribute( + rel="stylesheet", + href="/static/css/nonexistent.css", + add_static_sha=True, + ) + ] + ), + BuildMetadata(static_artifact_shas={"css/style.css": "abc123"}), + [''], + ), + # Test mix of static and non-static files + ( + Metadata( + links=[ + LinkAttribute( + rel="stylesheet", + href="/static/css/style.css", + add_static_sha=True, + ), + LinkAttribute( + rel="stylesheet", href="/css/external.css", add_static_sha=True + ), + ] + ), + BuildMetadata(static_artifact_shas={"css/style.css": "abc123"}), + [ + '', + '', + ], + ), + # Test with add_static_sha=False + ( + Metadata( + links=[ + LinkAttribute( + rel="stylesheet", + href="/static/css/style.css", + add_static_sha=False, + ) + ] + ), + BuildMetadata(static_artifact_shas={"css/style.css": "abc123"}), + [''], + ), + # Test with no build_metadata + ( + Metadata( + links=[ + LinkAttribute( + rel="stylesheet", + href="/static/css/style.css", + add_static_sha=True, + ) + ] + ), + None, + [''], + ), + # Test with existing query parameters + ( + Metadata( + links=[ + LinkAttribute( + rel="stylesheet", + href="/static/css/style.css?v=1.0", + add_static_sha=True, + ) + ] + ), + BuildMetadata(static_artifact_shas={"css/style.css": "abc123"}), + [''], + ), + # Test with mixed attributes and SHA + ( + Metadata( + links=[ + LinkAttribute( + rel="stylesheet", + href="/static/css/style.css", + add_static_sha=True, + optional_attributes={ + "media": "screen", + "crossorigin": "anonymous", + }, + ) + ] + ), + BuildMetadata(static_artifact_shas={"css/style.css": "abc123"}), + [ + '' + ], + ), + ], +) +def test_build_header_with_sha( + metadata: Metadata, build_metadata: BuildMetadata | None, expected_tags: list[str] +): + """ + Test the SHA addition logic for static files in the build_header method. + + """ + assert metadata.build_header(build_metadata=build_metadata) == expected_tags COMPLEX_METADATA = Metadata( @@ -177,3 +330,54 @@ def test_merge_metadatas(metadatas: list[Metadata], expected_metadata: Metadata) metadata = metadata.merge(other_metadata) assert metadata == expected_metadata + + +@pytest.mark.parametrize( + "initial_url,new_sha,expected_url", + [ + # No existing query parameters + ("https://example.com/path", "abc123", "https://example.com/path?sha=abc123"), + # Has existing sha parameter + ( + "https://example.com/path?sha=old123", + "new456", + "https://example.com/path?sha=new456", + ), + # Has other query parameters but no sha + ( + "https://example.com/path?param1=value1¶m2=value2", + "def789", + "https://example.com/path?param1=value1¶m2=value2&sha=def789", + ), + # Has multiple query parameters including sha + ( + "https://example.com/path?param1=value1&sha=old123¶m2=value2", + "ghi012", + "https://example.com/path?param1=value1&sha=ghi012¶m2=value2", + ), + # URL with special characters + ( + "https://example.com/path?param=special+value&sha=old", + "jkl345", + "https://example.com/path?param=special+value&sha=jkl345", + ), + # URL with fragment + ( + "https://example.com/path?param=value#fragment", + "mno678", + "https://example.com/path?param=value&sha=mno678#fragment", + ), + # URL with empty query string + ("https://example.com/path?", "pqr901", "https://example.com/path?sha=pqr901"), + # URL with port number + ( + "https://example.com:8080/path?param=value", + "stu234", + "https://example.com:8080/path?param=value&sha=stu234", + ), + ], +) +def test_link_attribute_set_sha(initial_url, new_sha, expected_url): + link = LinkAttribute(rel="test", href=initial_url) + link.set_sha(new_sha) + assert link.href == expected_url diff --git a/mountaineer/app.py b/mountaineer/app.py index 346ed0df..f4056624 100644 --- a/mountaineer/app.py +++ b/mountaineer/app.py @@ -25,7 +25,8 @@ ) from mountaineer.actions.fields import FunctionMetadata from mountaineer.annotation_helpers import MountaineerUnsetValue -from mountaineer.client_compiler.base import ClientBuilderBase +from mountaineer.client_compiler.base import APIBuilderBase +from mountaineer.client_compiler.build_metadata import BuildMetadata from mountaineer.config import ConfigBase from mountaineer.controller import ControllerBase from mountaineer.controller_layout import LayoutControllerBase @@ -93,7 +94,7 @@ class AppController: """ - builders: list[ClientBuilderBase] + builders: list[APIBuilderBase] global_metadata: Metadata | None def __init__( @@ -103,7 +104,7 @@ def __init__( version: str = "0.1.0", view_root: Path | None = None, global_metadata: Metadata | None = None, - custom_builders: list[ClientBuilderBase] | None = None, + custom_builders: list[APIBuilderBase] | None = None, config: ConfigBase | None = None, fastapi_args: dict[str, Any] | None = None, ): @@ -214,7 +215,8 @@ async def generate_controller_html(*args, **kwargs): render_overhead_by_controller = {} render_output = {} for node in direct_hierarchy: - # Must be a layout-only component + # If not set, must be a layout-only component. In this case we don't + # need to do any rendering of internal data if not node.controller: continue @@ -359,7 +361,7 @@ async def generate_controller_html(*args, **kwargs): # This allows each view to avoid having to find these on disk, as well as gives # a proactive error if any view will be unable to render when their script files # are missing - if self.config and self.config.ENVIRONMENT != "development": + if self.development_enabled: controller.resolve_paths(self.view_root, force=True) if not controller.bundled_scripts: raise ValueError( @@ -596,11 +598,15 @@ def compile_html( metadata = page_metadata.metadata if not metadata.ignore_global_metadata and self.global_metadata: metadata = metadata.merge(self.global_metadata) - header_str = "\n".join(metadata.build_header()) + header_str = "\n".join( + metadata.build_header(build_metadata=self.get_build_metadata()) + ) else: if self.global_metadata: metadata = self.global_metadata - header_str = "\n".join(metadata.build_header()) + header_str = "\n".join( + metadata.build_header(build_metadata=self.get_build_metadata()) + ) else: header_str = "" @@ -969,3 +975,27 @@ def definition_for_controller( if controller_definition.controller == controller: return controller_definition raise ValueError(f"Controller {controller} not found") + + def get_build_metadata(self): + """ + Will cache the build metadata in production but not in development, since + we expect production developments will compile their metadata once and then + use it for all endpoints. + + """ + if not self.development_enabled: + # Determine if we've already cached the build + if hasattr(self, "_build_metadata"): + return getattr(self, "_build_metadata") + + metadata_path = self.view_root.get_managed_metadata_dir() / "metadata.json" + if not metadata_path.exists(): + return None + self._build_metadata = BuildMetadata.model_validate_json( + metadata_path.read_text() + ) + return self._build_metadata + + @property + def development_enabled(self): + return self.config and self.config.ENVIRONMENT != "development" diff --git a/mountaineer/app_manager.py b/mountaineer/app_manager.py index c38bdea9..17be360d 100644 --- a/mountaineer/app_manager.py +++ b/mountaineer/app_manager.py @@ -11,7 +11,7 @@ from fastapi import Request from mountaineer.app import AppController -from mountaineer.client_builder.builder import ClientBuilder +from mountaineer.client_builder.builder import APIBuilder from mountaineer.client_compiler.compile import ClientCompiler from mountaineer.controllers.exception_controller import ( ExceptionController, @@ -55,7 +55,7 @@ def __init__( self.mount_exceptions(app_controller) global_build_cache = Path(mkdtemp()) - self.js_compiler = ClientBuilder( + self.js_compiler = APIBuilder( app_controller, live_reload_port=live_reload_port, build_cache=global_build_cache, diff --git a/mountaineer/cli.py b/mountaineer/cli.py index e77cb86a..0b9d5990 100644 --- a/mountaineer/cli.py +++ b/mountaineer/cli.py @@ -16,7 +16,7 @@ find_packages_with_prefix, package_path_to_module, ) -from mountaineer.client_builder.builder import ClientBuilder +from mountaineer.client_builder.builder import APIBuilder from mountaineer.console import CONSOLE from mountaineer.constants import KNOWN_JS_EXTENSIONS from mountaineer.hotreload import HotReloader @@ -58,7 +58,7 @@ def handle_watch( global_build_cache = Path(mkdtemp()) app_manager = DevAppManager.from_webcontroller(webcontroller) - js_compiler = ClientBuilder( + js_compiler = APIBuilder( app_manager.app_controller, live_reload_port=None, build_cache=global_build_cache, diff --git a/mountaineer/client_builder/builder.py b/mountaineer/client_builder/builder.py index 6c0748ba..6094875a 100644 --- a/mountaineer/client_builder/builder.py +++ b/mountaineer/client_builder/builder.py @@ -45,7 +45,7 @@ class RenderSpec: spec: dict[Any, Any] | None -class ClientBuilder: +class APIBuilder: """ Main entrypoint for building the auto-generated typescript code. This includes the server provided API used by useServer. diff --git a/mountaineer/client_compiler/base.py b/mountaineer/client_compiler/base.py index 15b0680d..2d043205 100644 --- a/mountaineer/client_compiler/base.py +++ b/mountaineer/client_compiler/base.py @@ -19,7 +19,7 @@ class ClientBundleMetadata: live_reload_port: int | None = None -class ClientBuilderBase(ABC): +class APIBuilderBase(ABC): """ Base class for client builders. When mounted to an AppController, these build plugins will be called for every file defined in the view/app directory. It's up to the plugin diff --git a/mountaineer/client_compiler/build_metadata.py b/mountaineer/client_compiler/build_metadata.py new file mode 100644 index 00000000..23c68d0b --- /dev/null +++ b/mountaineer/client_compiler/build_metadata.py @@ -0,0 +1,11 @@ +from pydantic import BaseModel + + +class BuildMetadata(BaseModel): + """ + Metadata added during compile_time that should be maintained in the + bundle for production hosting. + + """ + + static_artifact_shas: dict[str, str] diff --git a/mountaineer/client_compiler/compile.py b/mountaineer/client_compiler/compile.py index f423c9dd..5e09cd3f 100644 --- a/mountaineer/client_compiler/compile.py +++ b/mountaineer/client_compiler/compile.py @@ -1,3 +1,4 @@ +from hashlib import md5 from pathlib import Path from shutil import move as shutil_move from tempfile import mkdtemp @@ -5,6 +6,7 @@ from mountaineer.app import AppController from mountaineer.client_compiler.base import ClientBundleMetadata +from mountaineer.client_compiler.build_metadata import BuildMetadata from mountaineer.client_compiler.exceptions import BuildProcessException from mountaineer.console import CONSOLE from mountaineer.io import gather_with_concurrency @@ -88,6 +90,13 @@ async def run_builder_plugins( # directory, we want to merge this with the project directory self._move_build_artifacts_into_project() + # Now that we have prepared the static folder in its final form, we + # should get the md5 hash of the content for our archive + metadata = self._build_static_metadata() + (self.view_root.get_managed_metadata_dir() / "metadata.json").write_text( + metadata.model_dump_json() + ) + def _init_builders(self): metadata = ClientBundleMetadata( live_reload_port=None, @@ -184,3 +193,17 @@ def _move_build_artifacts_into_project(self): for tmp_file in tmp_path.glob("*"): final_file = final_path / tmp_file.name shutil_move(tmp_file, final_file) + + def _build_static_metadata(self): + static_artifact_shas: dict[str, str] = {} + static_dir = self.view_root.get_managed_static_dir() + + for base_path, dirs, files in static_dir.walk(): + for file in files: + static_path = base_path / file + file_contents = static_path.read_bytes() + static_artifact_shas[str(static_path.relative_to(static_dir))] = md5( + file_contents + ).hexdigest() + + return BuildMetadata(static_artifact_shas=static_artifact_shas) diff --git a/mountaineer/client_compiler/postcss.py b/mountaineer/client_compiler/postcss.py index 7e5c73b0..0db64fb2 100644 --- a/mountaineer/client_compiler/postcss.py +++ b/mountaineer/client_compiler/postcss.py @@ -7,14 +7,14 @@ from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn -from mountaineer.client_compiler.base import ClientBuilderBase +from mountaineer.client_compiler.base import APIBuilderBase from mountaineer.client_compiler.exceptions import BuildProcessException from mountaineer.console import CONSOLE from mountaineer.logging import LOGGER from mountaineer.paths import ManagedViewPath -class PostCSSBundler(ClientBuilderBase): +class PostCSSBundler(APIBuilderBase): """ Support PostCSS processing for CSS files. diff --git a/mountaineer/hotreload.py b/mountaineer/hotreload.py index c1d9bb1b..cc6373d5 100644 --- a/mountaineer/hotreload.py +++ b/mountaineer/hotreload.py @@ -10,9 +10,9 @@ from pathlib import Path from types import ModuleType -from mountaineer.logging import setup_logger +from mountaineer.logging import setup_internal_logger -logger = setup_logger(__name__) +logger = setup_internal_logger(__name__) @dataclass diff --git a/mountaineer/logging.py b/mountaineer/logging.py index cbcea9a7..44c73df8 100644 --- a/mountaineer/logging.py +++ b/mountaineer/logging.py @@ -81,8 +81,18 @@ def log_time_duration(message: str): LOGGER.debug(f"{message} : Took {(monotonic_ns() - start)/1e9:.2f}s") -# Our global logger should only surface warnings and above by default -LOGGER = setup_logger( - __name__, - log_level=VERBOSITY_MAPPING[environ.get("MOUNTAINEER_LOG_LEVEL", "WARNING")], -) +def setup_internal_logger(name: str): + """ " + Our global logger should only surface warnings and above by default. + + To adjust Mountaineer logging, set the MOUNTAINEER_LOG_LEVEL environment + variable in your local session. By default it is set to WARNING and above. + + """ + return setup_logger( + name, + log_level=VERBOSITY_MAPPING[environ.get("MOUNTAINEER_LOG_LEVEL", "WARNING")], + ) + + +LOGGER = setup_internal_logger(__name__) diff --git a/mountaineer/paths.py b/mountaineer/paths.py index 33cb4ddf..d08d7c93 100644 --- a/mountaineer/paths.py +++ b/mountaineer/paths.py @@ -114,6 +114,20 @@ def get_managed_ssr_dir(self, tmp_build: bool = False, create_dir: bool = True): path.mkdir(exist_ok=True) return path + def get_managed_metadata_dir( + self, tmp_build: bool = False, create_dir: bool = True + ): + # Only root paths can have SSR directories + if not self.is_root_link: + raise ValueError( + "Cannot get SSR directory from a non-root linked view path" + ) + path = self.get_managed_dir_common("_metadata", create_dir=create_dir) + if tmp_build: + path = path / "tmp" + path.mkdir(exist_ok=True) + return path + def get_managed_dir_common( self, managed_dir: str, diff --git a/mountaineer/render.py b/mountaineer/render.py index c3cd72c8..67e86542 100644 --- a/mountaineer/render.py +++ b/mountaineer/render.py @@ -1,6 +1,7 @@ from hashlib import sha256 from json import dumps as json_dumps from typing import TYPE_CHECKING, Any, Mapping, Type, TypeVar +from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from fastapi import Response from pydantic import BaseModel, model_validator @@ -8,6 +9,8 @@ from pydantic.fields import Field, FieldInfo from typing_extensions import dataclass_transform +from mountaineer.client_compiler.build_metadata import BuildMetadata + T = TypeVar("T") @@ -141,6 +144,35 @@ class LinkAttribute(HashableAttribute, BaseModel): href: str optional_attributes: dict[str, str] = {} + # A sha will only be added automatically for link imports + # that reference files built into the _static directory at runtime. Other + # links will remain as-is without a resolved sha. + add_static_sha: bool = True + + def set_sha(self, sha: str): + """ + Updates the URL by adding or modifying the 'sha' query parameter. If a sha + is already provided as part of href, will override the existing sha. + + """ + # Existing URL components + parsed_url = urlparse(self.href) + query_params = parse_qs(parsed_url.query) + + query_params["sha"] = [sha] + new_query = urlencode(query_params, doseq=True) + + self.href = urlunparse( + ( + parsed_url.scheme, + parsed_url.netloc, + parsed_url.path, + parsed_url.params, + new_query, + parsed_url.fragment, + ) + ) + class ScriptAttribute(HashableAttribute, BaseModel): src: str @@ -195,7 +227,7 @@ def merge_item(a: list[T], b: list[T]): ignore_global_metadata=self.ignore_global_metadata, ) - def build_header(self) -> list[str]: + def build_header(self, build_metadata: BuildMetadata | None) -> list[str]: """ Builds the header for this controller. Returns the list of tags that will be injected into the
tag of the rendered page. @@ -239,11 +271,27 @@ def format_optional_keys(payload: Mapping[str, str | bool | None]) -> str: tags.append(f"") for link_definition in self.links: + if build_metadata and link_definition.add_static_sha: + # By convention, static files should be mounted to the application in a /static endpoint. These + # may be served outside of mountaineer (via a CDN or similar) but we still expect + # these paths to reference the proper paths + link_sha: str | None = None + for ( + static_path, + static_sha, + ) in build_metadata.static_artifact_shas.items(): + # Allow for alternative endings to support existing query parameters + if link_definition.href.startswith(f"/static/{static_path}"): + link_sha = static_sha + if link_sha: + link_definition.set_sha(link_sha) + link_attributes = { "rel": link_definition.rel, "href": link_definition.href, **link_definition.optional_attributes, } + tags.append(f"") return tags