diff --git a/charmcraft/application/commands/__init__.py b/charmcraft/application/commands/__init__.py index 52c624fce..ed8e8f666 100644 --- a/charmcraft/application/commands/__init__.py +++ b/charmcraft/application/commands/__init__.py @@ -30,6 +30,7 @@ from charmcraft.application.commands.remote import RemoteBuild from charmcraft.application.commands.store import ( # auth + FetchLibs, LoginCommand, LogoutCommand, WhoamiCommand, @@ -102,6 +103,7 @@ def fill_command_groups(app: craft_application.Application) -> None: CreateLibCommand, PublishLibCommand, ListLibCommand, + FetchLibs, FetchLibCommand, ], ) diff --git a/charmcraft/application/commands/store.py b/charmcraft/application/commands/store.py index cfd241ecd..7669b5e28 100644 --- a/charmcraft/application/commands/store.py +++ b/charmcraft/application/commands/store.py @@ -28,9 +28,10 @@ import zipfile from collections.abc import Collection from operator import attrgetter -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import yaml +from craft_application import util from craft_cli import ArgumentParsingError, emit from craft_cli.errors import CraftError from craft_parts import Step @@ -41,7 +42,7 @@ from tabulate import tabulate import charmcraft.store.models -from charmcraft import const, env, parts, utils +from charmcraft import const, env, errors, parts, utils from charmcraft.application.commands.base import CharmcraftCommand from charmcraft.models import project from charmcraft.store import ImageHandler, LocalDockerdInterface, OCIRegistry, Store @@ -1510,6 +1511,7 @@ class FetchLibCommand(CharmcraftCommand): ) format_option = True always_load_project = True + hidden = True def fill_parser(self, parser): """Add own parameters to the general parser.""" @@ -1520,7 +1522,7 @@ def fill_parser(self, parser): help="Library to fetch (e.g. charms.mycharm.v2.foo.); optional, default to all", ) - def run(self, parsed_args): + def run(self, parsed_args: argparse.Namespace) -> None: """Run the command.""" if parsed_args.library: local_libs_data = [utils.get_lib_info(full_name=parsed_args.library)] @@ -1534,10 +1536,9 @@ def run(self, parsed_args): to_query = [] for lib in local_libs_data: if lib.lib_id is None: - item = {"charm_name": lib.charm_name, "lib_name": lib.lib_name} + item = {"charm_name": lib.charm_name, "lib_name": lib.lib_name, "api": lib.api} else: - item = {"lib_id": lib.lib_id} - item["api"] = lib.api + item = {"lib_id": lib.lib_id, "api": lib.api} to_query.append(item) libs_tips = store.get_libraries_tips(to_query) @@ -1617,7 +1618,7 @@ def run(self, parsed_args): if parsed_args.format: output_data = [] for lib_data, error_message in full_lib_data: - datum = { + datum: dict[str, Any] = { "charm_name": lib_data.charm_name, "library_name": lib_data.lib_name, "library_id": lib_data.lib_id, @@ -1634,6 +1635,94 @@ def run(self, parsed_args): emit.message(cli.format_content(output_data, parsed_args.format)) +class FetchLibs(CharmcraftCommand): + """Fetch libraries defined in charmcraft.yaml.""" + + name = "fetch-libs" + help_msg = "Fetch one or more charm libraries" + overview = textwrap.dedent( + """ + Fetch charm libraries defined in charmcraft.yaml. + + For each library in the top-level `charm-libs` key, fetch the latest library + version matching those requirements. + + For example: + charm-libs: + # Fetch lib with API version 0. + # If `fetch-libs` is run and a newer minor version is available, + # it will be fetched from the store. + - lib: postgresql.postgres_client + version: "0" + # Always fetch precisely version 0.57. + - lib: mysql.client + version: "0.57" + """ + ) + format_option = True + always_load_project = True + + def run(self, parsed_args: argparse.Namespace) -> None: + """Fetch libraries.""" + store = self._services.store + charm_libs = self._services.project.charm_libs + if not charm_libs: + raise errors.LibraryError( + message="No dependent libraries declared in charmcraft.yaml.", + resolution="Add a 'charm-libs' section to charmcraft.yaml.", + retcode=78, # EX_CONFIG: configuration error + ) + emit.progress("Getting library metadata from charmhub") + libs_metadata = store.get_libraries_metadata_by_name(charm_libs) + declared_libs = {lib.lib: lib for lib in charm_libs} + missing_store_libs = declared_libs.keys() - libs_metadata.keys() + if missing_store_libs: + missing_libs_source = [declared_libs[lib].dict() for lib in sorted(missing_store_libs)] + libs_yaml = util.dump_yaml(missing_libs_source) + raise errors.CraftError( + f"Could not find the following libraries on charmhub:\n{libs_yaml}", + resolution="Use 'charmcraft list-lib' to check library names and versions.", + reportable=False, + logpath_report=False, + ) + + emit.trace(f"Library metadata retrieved: {libs_metadata}") + local_libs = { + f"{lib.charm_name}.{lib.lib_name}": lib for lib in utils.get_libs_from_tree() + } + emit.trace(f"Local libraries: {local_libs}") + + downloaded_libs = 0 + for lib_md in libs_metadata.values(): + lib_name = f"{lib_md.charm_name}.{lib_md.lib_name}" + local_lib = local_libs.get(lib_name) + if local_lib and local_lib.content_hash == lib_md.content_hash: + emit.debug( + f"Skipping {lib_name} because the same file already exists on " + f"disk (hash: {lib_md.content_hash}). " + "Delete the file and re-run 'charmcraft fetch-libs' to force re-download." + ) + continue + lib_name = utils.get_lib_module_name(lib_md.charm_name, lib_md.lib_name, lib_md.api) + emit.progress(f"Downloading {lib_name}") + lib = store.get_library( + charm_name=lib_md.charm_name, + library_id=lib_md.lib_id, + api=lib_md.api, + patch=lib_md.patch, + ) + if lib.content is None: + raise errors.CraftError( + f"Store returned no content for '{lib.charm_name}.{lib.lib_name}'" + ) + downloaded_libs += 1 + lib_path = utils.get_lib_path(lib_md.charm_name, lib_md.lib_name, lib_md.api) + lib_path.parent.mkdir(exist_ok=True, parents=True) + lib_path.write_text(lib.content) + + emit.message(f"Downloaded {downloaded_libs} charm libraries.") + + class ListLibCommand(CharmcraftCommand): """List all libraries belonging to a charm.""" diff --git a/charmcraft/errors.py b/charmcraft/errors.py index ac7cd8439..a12af5c04 100644 --- a/charmcraft/errors.py +++ b/charmcraft/errors.py @@ -50,7 +50,11 @@ def __init__( ) -class BadLibraryPathError(CraftError): +class LibraryError(CraftError): + """Errors related to charm libraries.""" + + +class BadLibraryPathError(LibraryError): """Subclass to provide a specific error for a bad library path.""" def __init__(self, path): @@ -59,7 +63,7 @@ def __init__(self, path): ) -class BadLibraryNameError(CraftError): +class BadLibraryNameError(LibraryError): """Subclass to provide a specific error for a bad library name.""" def __init__(self, name): diff --git a/charmcraft/models/__init__.py b/charmcraft/models/__init__.py index 05515b33f..0131f9348 100644 --- a/charmcraft/models/__init__.py +++ b/charmcraft/models/__init__.py @@ -24,6 +24,7 @@ from .project import ( CharmBuildInfo, CharmcraftBuildPlanner, + CharmLib, CharmcraftProject, BasesCharm, PlatformCharm, @@ -48,6 +49,7 @@ "CharmBuildInfo", "CharmcraftBuildPlanner", "CharmcraftProject", + "CharmLib", "BundleMetadata", "CharmMetadata", "CharmMetadataLegacy", diff --git a/charmcraft/models/project.py b/charmcraft/models/project.py index 51ae653f8..cc32425e8 100644 --- a/charmcraft/models/project.py +++ b/charmcraft/models/project.py @@ -17,6 +17,7 @@ import abc import datetime import pathlib +import re from collections.abc import Iterable, Iterator from typing import ( Any, @@ -120,6 +121,76 @@ def _listify_architectures(cls, value: str | list[str]) -> list[str]: return value +class CharmLib(models.CraftBaseModel): + """A Charm library dependency for this charm.""" + + lib: str = pydantic.Field( + title="Library Path (e.g. my_charm.my_library)", + regex=r"[a-z0-9_]+\.[a-z0-9_]+", + ) + version: str = pydantic.Field( + title="Version filter for the charm. Either an API version or a specific [api].[patch].", + regex=r"[0-9]+(\.[0-9]+)?", + ) + + @pydantic.validator("lib", pre=True) + def _validate_name(cls, value: str) -> str: + """Validate the lib field, providing a useful error message on failure.""" + charm_name, _, lib_name = str(value).partition(".") + if not charm_name or not lib_name: + raise ValueError( + f"Library name invalid. Expected '[charm_name].[lib_name]', got {value!r}" + ) + if not re.fullmatch("[a-z0-9_]+", charm_name): + if "-" in charm_name: + raise ValueError( + f"Invalid charm name in lib {value!r}. Try replacing hyphens ('-') with underscores ('_')." + ) + raise ValueError( + f"Invalid charm name for lib {value!r}. Value {charm_name!r} is invalid." + ) + if not re.fullmatch("[a-z0-9_]+", lib_name): + raise ValueError(f"Library name {lib_name!r} is invalid.") + return str(value) + + @pydantic.validator("version", pre=True) + def _validate_api_version(cls, value: str) -> str: + """Validate the API version field, providing a useful error message on failure.""" + api, *_ = str(value).partition(".") + try: + int(api) + except ValueError: + raise ValueError(f"API version not valid. Expected an integer, got {api!r}") from None + return str(value) + + @pydantic.validator("version", pre=True) + def _validate_patch_version(cls, value: str) -> str: + """Validate the optional patch version, providing a useful error message.""" + api, separator, patch = value.partition(".") + if not separator: + return value + try: + int(patch) + except ValueError: + raise ValueError( + f"Patch version not valid. Expected an integer, got {patch!r}" + ) from None + return value + + @property + def api_version(self) -> int: + """The API version needed for this library.""" + return int(self.version.partition(".")[0]) + + @property + def patch_version(self) -> int | None: + """The patch version needed for this library, or None if no patch version is specified.""" + api, _, patch = self.version.partition(".") + if not patch: + return None + return int(patch) + + @dataclasses.dataclass class CharmBuildInfo(models.BuildInfo): """Information about a single build option, with charmcraft-specific info. @@ -381,6 +452,9 @@ class CharmcraftProject(models.Project, metaclass=abc.ABCMeta): contact: None = None issues: None = None source_code: None = None + charm_libs: list[CharmLib] = pydantic.Field( + default_factory=list, title="List of libraries to use for this charm" + ) # These private attributes are not part of the project model but are attached here # because Charmcraft uses this metadata. diff --git a/charmcraft/services/store.py b/charmcraft/services/store.py index b6c299462..785a9da0b 100644 --- a/charmcraft/services/store.py +++ b/charmcraft/services/store.py @@ -17,14 +17,16 @@ from __future__ import annotations import platform -from collections.abc import Collection, Sequence +from collections.abc import Collection, Mapping, Sequence import craft_application import craft_store from craft_store import models -from charmcraft import const, env, errors +from charmcraft import const, env, errors, store +from charmcraft.models import CharmLib from charmcraft.store import AUTH_DEFAULT_PERMISSIONS, AUTH_DEFAULT_TTL +from charmcraft.store.models import Library, LibraryMetadataRequest class BaseStoreService(craft_application.AppService): @@ -33,6 +35,7 @@ class BaseStoreService(craft_application.AppService): This service should be easily adjustable for craft-application. """ + ClientClass: type[craft_store.StoreClient] = craft_store.StoreClient client: craft_store.StoreClient _endpoints: craft_store.endpoints.Endpoints = craft_store.endpoints.CHARMHUB _environment_auth: str = const.ALTERNATE_AUTH_ENV_VAR @@ -73,7 +76,7 @@ def setup(self) -> None: """Set up the store service.""" super().setup() - self.client = craft_store.StoreClient( + self.client = self.ClientClass( application_name=self._app.name, base_url=self._base_url, storage_base_url=self._storage_url, @@ -159,6 +162,9 @@ def get_credentials( class StoreService(BaseStoreService): """A Store service specifically for Charmcraft.""" + ClientClass = store.Client + client: store.Client # pyright: ignore[reportIncompatibleVariableOverride] + def set_resource_revisions_architectures( self, name: str, resource_name: str, updates: dict[int, list[str]] ) -> Collection[models.resource_revision_model.CharmResourceRevision]: @@ -182,3 +188,42 @@ def set_resource_revisions_architectures( ) new_revisions = self.client.list_resource_revisions(name=name, resource_name=resource_name) return [rev for rev in new_revisions if int(rev.revision) in updates] + + def get_libraries_metadata(self, libraries: Sequence[CharmLib]) -> Sequence[Library]: + """Get the metadata for one or more charm libraries. + + :param libraries: A sequence of libraries to request. + :returns: A sequence of the libraries' metadata in the store. + """ + store_libs = [] + for lib in libraries: + charm_name, _, lib_name = lib.lib.partition(".") + store_lib = LibraryMetadataRequest( + { + "charm-name": charm_name, + "library-name": lib_name, + "api": lib.api_version, + } + ) + if (patch_version := lib.patch_version) is not None: + store_lib["patch"] = patch_version + store_libs.append(store_lib) + + return self.client.fetch_libraries_metadata(store_libs) + + def get_libraries_metadata_by_name( + self, libraries: Sequence[CharmLib] + ) -> Mapping[str, Library]: + """Get a mapping of [charm_name].[library_name] to the requested libraries.""" + return { + f"{lib.charm_name}.{lib.lib_name}": lib + for lib in self.get_libraries_metadata(libraries) + } + + def get_library( + self, charm_name: str, *, library_id: str, api: int | None = None, patch: int | None = None + ) -> Library: + """Get a library by charm name and ID from charmhub.""" + return self.client.get_library( + charm_name=charm_name, library_id=library_id, api=api, patch=patch + ) diff --git a/charmcraft/store/__init__.py b/charmcraft/store/__init__.py index 0e4d3597c..5bf9cc4d4 100644 --- a/charmcraft/store/__init__.py +++ b/charmcraft/store/__init__.py @@ -17,6 +17,7 @@ from charmcraft.store.client import build_user_agent, AnonymousClient, Client from charmcraft.store import models +from charmcraft.store.models import LibraryMetadataRequest from charmcraft.store.registry import ( OCIRegistry, HashingTemporaryFile, @@ -37,4 +38,5 @@ "AUTH_DEFAULT_TTL", "Store", "models", + "LibraryMetadataRequest", ] diff --git a/charmcraft/store/client.py b/charmcraft/store/client.py index 8d0bf44b9..54a0915df 100644 --- a/charmcraft/store/client.py +++ b/charmcraft/store/client.py @@ -18,6 +18,7 @@ import os import platform +from collections.abc import Sequence from json.decoder import JSONDecodeError from typing import Any @@ -31,6 +32,7 @@ ) from charmcraft import __version__, const, utils +from charmcraft.store.models import Library, LibraryMetadataRequest TESTING_ENV_PREFIXES = ["TRAVIS", "AUTOPKGTEST_TMP"] @@ -72,17 +74,36 @@ def request_urlpath_json(self, method: str, urlpath: str, *args, **kwargs) -> di class Client(craft_store.StoreClient): """Lightweight layer above StoreClient.""" - def __init__(self, api_base_url: str, storage_base_url: str, ephemeral: bool = False): + def __init__( + self, + api_base_url: str = "", + storage_base_url: str = "", + ephemeral: bool = False, + application_name: str = "charmcraft", + *, + base_url: str = "", + endpoints: endpoints.Endpoints = endpoints.CHARMHUB, + environment_auth: str = const.ALTERNATE_AUTH_ENV_VAR, + user_agent: str = build_user_agent(), + ): + """Initialise a Charmcraft store client. + + Supports both charmcraft 2.x style init and compatibility with upstream. + """ + if base_url and api_base_url or not base_url and not api_base_url: + raise ValueError("Either base_url or api_base_url must be set, but not both.") + if base_url: + api_base_url = base_url self.api_base_url = api_base_url.rstrip("/") self.storage_base_url = storage_base_url.rstrip("/") super().__init__( base_url=api_base_url, storage_base_url=storage_base_url, - endpoints=endpoints.CHARMHUB, - application_name="charmcraft", - user_agent=build_user_agent(), - environment_auth=const.ALTERNATE_AUTH_ENV_VAR, + endpoints=endpoints, + application_name=application_name, + user_agent=user_agent, + environment_auth=environment_auth, ephemeral=ephemeral, ) @@ -150,3 +171,37 @@ def _storage_push(self, monitor) -> requests.Response: headers={"Content-Type": monitor.content_type, "Accept": "application/json"}, data=monitor, ) + + def get_library( + self, *, charm_name: str, library_id: str, api: int | None = None, patch: int | None = None + ) -> Library: + """Fetch a library attached to a charm. + + http://api.charmhub.io/docs/libraries.html#fetch_library + """ + params = {} + if api is not None: + params["api"] = api + if patch is not None: + params["patch"] = patch + return Library.from_dict( + self.request_urlpath_json( + "GET", + f"/v1/charm/libraries/{charm_name}/{library_id}", + params=params, + ) + ) + + def fetch_libraries_metadata( + self, libs: Sequence[LibraryMetadataRequest] + ) -> Sequence[Library]: + """Fetch the metadata for one or more charm libraries. + + http://api.charmhub.io/docs/libraries.html#fetch_libraries + """ + response = self.request_urlpath_json("POST", "/v1/charm/libraries/bulk", json=libs) + if "libraries" not in response: + raise CraftError( + "Server returned invalid response while querying libraries", details=str(response) + ) + return [Library.from_dict(lib) for lib in response["libraries"]] diff --git a/charmcraft/store/models.py b/charmcraft/store/models.py index dba16702e..a2bab62bb 100644 --- a/charmcraft/store/models.py +++ b/charmcraft/store/models.py @@ -20,9 +20,10 @@ import enum import functools from dataclasses import dataclass -from typing import Literal +from typing import Any, Literal, TypedDict from craft_cli import CraftError +from typing_extensions import NotRequired, Self @dataclasses.dataclass(frozen=True) @@ -170,10 +171,7 @@ class Channel: @dataclasses.dataclass(frozen=True) class Library: - """Charmcraft-specific store library model. - - Deprecated in favour of implementation in craft-store. - """ + """Charmcraft-specific store library model.""" lib_id: str lib_name: str @@ -183,6 +181,24 @@ class Library: content: str | None content_hash: str + @classmethod + def from_dict(cls, value: dict[str, Any]) -> Self: + """Convert a dictionary of this type to the type itself. + + Dictionary should match a single item of the `libraries` attribute in + the response to fetch_libraries or fetch_library + http://api.charmhub.io/docs/libraries.html#fetch_libraries + """ + return cls( + charm_name=value["charm-name"], + lib_name=value["library-name"], + lib_id=value["library-id"], + api=value["api"], + patch=value["patch"], + content_hash=value["hash"], + content=value.get("content"), + ) + @dataclasses.dataclass(frozen=True) class RegistryCredentials: @@ -265,3 +281,14 @@ def name(self) -> str: """Get the channel name as a string.""" risk = self.risk.name.lower() return "/".join(i for i in (self.track, risk, self.branch) if i is not None) + + +LibraryMetadataRequest = TypedDict( + "LibraryMetadataRequest", + { + "charm-name": str, + "library-name": str, + "api": int, + "patch": NotRequired[int], + }, +) diff --git a/charmcraft/utils/__init__.py b/charmcraft/utils/__init__.py index 312f032d9..f9bac33a1 100644 --- a/charmcraft/utils/__init__.py +++ b/charmcraft/utils/__init__.py @@ -23,6 +23,8 @@ create_charm_name_from_importable, create_importable_name, get_lib_internals, + get_lib_path, + get_lib_module_name, get_lib_info, get_libs_from_tree, collect_charmlib_pydeps, @@ -70,6 +72,8 @@ "create_charm_name_from_importable", "create_importable_name", "get_lib_internals", + "get_lib_path", + "get_lib_module_name", "get_lib_info", "get_libs_from_tree", "collect_charmlib_pydeps", diff --git a/charmcraft/utils/charmlibs.py b/charmcraft/utils/charmlibs.py index 8640f8313..5d044e1a7 100644 --- a/charmcraft/utils/charmlibs.py +++ b/charmcraft/utils/charmlibs.py @@ -1,4 +1,4 @@ -# Copyright 2023 Canonical Ltd. +# Copyright 2023-2024 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import os import pathlib from dataclasses import dataclass +from typing import overload import yaml from craft_cli import CraftError @@ -168,7 +169,35 @@ def _api_patch_validator(value): ) -def get_lib_info(*, full_name=None, lib_path=None): +def get_lib_path(charm: str, lib_name: str, api: int) -> pathlib.Path: + """Get a relative path for a library based on its home charm, name and API version. + + :param charm: The name of the charm that owns this library + :param lib_name: The name of the library + :param api: The API version of the library + :returns: A relative path to the library python file. + """ + return ( + pathlib.Path("lib/charms") / create_importable_name(charm) / f"v{api}" / f"{lib_name}.py" + ) + + +def get_lib_module_name(charm: str, lib_name: str, api: int) -> str: + """Get a Python module path for a library based on its home charm, name and API version. + + :param charm: The name of the charm that owns this library + :param lib_name: The name of the library + :param api: The API version of the library + :returns: A string of the full path to the charm. + """ + return f"charms.{create_importable_name(charm)}.v{api}.{lib_name}" + + +@overload +def get_lib_info(*, full_name: str) -> LibData: ... +@overload +def get_lib_info(*, lib_path: pathlib.Path) -> LibData: ... +def get_lib_info(*, full_name: str | None = None, lib_path: pathlib.Path | None = None) -> LibData: """Get the whole lib info from the path/file. This will perform mutation of the charm name to create importable paths. @@ -178,7 +207,7 @@ def get_lib_info(*, full_name=None, lib_path=None): This function needs to be called standing on the root directory of the project. """ - if full_name is None: + if lib_path: # get it from the lib_path try: libsdir, charmsdir, importable_charm_name, v_api = lib_path.parts[:-1] @@ -187,8 +216,7 @@ def get_lib_info(*, full_name=None, lib_path=None): if libsdir != "lib" or charmsdir != "charms" or lib_path.suffix != ".py": raise errors.BadLibraryPathError(lib_path) full_name = ".".join((charmsdir, importable_charm_name, v_api, lib_path.stem)) - - else: + elif full_name: # build the path! convert a lib name with dots to the full path, including lib # dir and Python extension. # e.g.: charms.mycharm.v4.foo -> lib/charms/mycharm/v4/foo.py @@ -204,6 +232,8 @@ def get_lib_info(*, full_name=None, lib_path=None): raise errors.BadLibraryNameError(full_name) path = pathlib.Path("lib") lib_path = path / charmsdir / importable_charm_name / v_api / (libfile + ".py") + else: + raise TypeError("get_lib_info needs either a full name or a lib path") # charm names in the path can contain '_' to be importable # these should be '-', so change them back diff --git a/tests/commands/test_store_client.py b/tests/commands/test_store_client.py index 36f65c152..4dfa78607 100644 --- a/tests/commands/test_store_client.py +++ b/tests/commands/test_store_client.py @@ -140,7 +140,7 @@ def test_client_init(): with patch("craft_store.StoreClient.__init__") as mock_client_init: with patch("charmcraft.store.client.build_user_agent") as mock_ua: mock_ua.return_value = user_agent - Client(api_url, storage_url) + Client(api_url, storage_url, user_agent=user_agent) mock_client_init.assert_called_with( base_url=api_url, storage_base_url=storage_url, diff --git a/tests/conftest.py b/tests/conftest.py index 170b28484..679eaf76c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,6 @@ from unittest.mock import Mock import craft_parts -import craft_store import pytest import responses as responses_module import yaml @@ -35,7 +34,7 @@ from craft_providers import Executor, Provider, bases import charmcraft.parts -from charmcraft import const, deprecations, instrum, parts, services, utils +from charmcraft import const, deprecations, instrum, parts, services, store, utils from charmcraft.application.main import APP_METADATA from charmcraft.bases import get_host_as_base from charmcraft.models import charmcraft as config_module @@ -67,7 +66,7 @@ def service_factory( factory.project = simple_charm - factory.store.client = mock.Mock(spec_set=craft_store.StoreClient) + factory.store.client = mock.Mock(spec_set=store.Client) return factory diff --git a/tests/unit/commands/test_store.py b/tests/unit/commands/test_store.py index 78b34aef4..ee76f93b4 100644 --- a/tests/unit/commands/test_store.py +++ b/tests/unit/commands/test_store.py @@ -14,16 +14,26 @@ # # For further info, check https://github.com/canonical/charmcraft """Unit tests for store commands.""" +import argparse import datetime import textwrap +import craft_cli.pytest_plugin import pytest from craft_store import models +from charmcraft import errors, store from charmcraft.application.commands import SetResourceArchitecturesCommand +from charmcraft.application.commands.store import FetchLibs +from charmcraft.application.main import APP_METADATA +from charmcraft.models.project import CharmLib from charmcraft.utils import cli from tests import get_fake_revision +BASIC_CHARMCRAFT_YAML = """\ +type: charm +""" + @pytest.mark.parametrize( ("updates", "expected"), @@ -72,3 +82,139 @@ def test_set_resource_architectures_output_table(emitter, updates, expected): SetResourceArchitecturesCommand.write_output(cli.OutputFormat.TABLE, updates) emitter.assert_message(expected) + + +def test_fetch_libs_no_charm_libs( + emitter: craft_cli.pytest_plugin.RecordingEmitter, service_factory +): + fetch_libs = FetchLibs({"app": APP_METADATA, "services": service_factory}) + + with pytest.raises(errors.LibraryError) as exc_info: + fetch_libs.run(argparse.Namespace()) + + assert exc_info.value.resolution == "Add a 'charm-libs' section to charmcraft.yaml." + + +@pytest.mark.parametrize( + ("libs", "expected"), + [ + ( + [CharmLib(lib="mysql.mysql", version="1")], + textwrap.dedent( + """\ + Could not find the following libraries on charmhub: + - lib: mysql.mysql + version: '1' + """ + ), + ), + ( + [ + CharmLib(lib="mysql.mysql", version="1"), + CharmLib(lib="some_charm.lib", version="1.2"), + ], + textwrap.dedent( + """\ + Could not find the following libraries on charmhub: + - lib: mysql.mysql + version: '1' + - lib: some_charm.lib + version: '1.2' + """ + ), + ), + ], +) +def test_fetch_libs_missing_from_store(service_factory, libs, expected): + service_factory.project.charm_libs = libs + service_factory.store.client.fetch_libraries_metadata.return_value = [] + fetch_libs = FetchLibs({"app": APP_METADATA, "services": service_factory}) + + with pytest.raises(errors.CraftError) as exc_info: + fetch_libs.run(argparse.Namespace()) + + assert exc_info.value.args[0] == expected + + +@pytest.mark.parametrize( + ("libs", "store_libs", "dl_lib", "expected"), + [ + ( + [CharmLib(lib="mysql.backups", version="1")], + [ + store.models.Library( + charm_name="mysql", + lib_name="backups", + lib_id="ididid", + api=1, + patch=2, + content=None, + content_hash="hashhashhash", + ) + ], + store.models.Library( + charm_name="mysql", + lib_name="backups", + lib_id="ididid", + api=1, + patch=2, + content=None, + content_hash="hashhashhash", + ), + "Store returned no content for 'mysql.backups'", + ), + ], +) +def test_fetch_libs_no_content(new_path, service_factory, libs, store_libs, dl_lib, expected): + service_factory.project.charm_libs = libs + service_factory.store.client.fetch_libraries_metadata.return_value = store_libs + service_factory.store.client.get_library.return_value = dl_lib + fetch_libs = FetchLibs({"app": APP_METADATA, "services": service_factory}) + + with pytest.raises(errors.CraftError, match=expected) as exc_info: + fetch_libs.run(argparse.Namespace()) + + assert exc_info.value.args[0] == expected + + +@pytest.mark.parametrize( + ("libs", "store_libs", "dl_lib", "expected"), + [ + ( + [CharmLib(lib="mysql.backups", version="1")], + [ + store.models.Library( + charm_name="mysql", + lib_name="backups", + lib_id="ididid", + api=1, + patch=2, + content=None, + content_hash="hashhashhash", + ) + ], + store.models.Library( + charm_name="mysql", + lib_name="backups", + lib_id="ididid", + api=1, + patch=2, + content="I am a library.", + content_hash="hashhashhash", + ), + "Store returned no content for 'mysql.backups'", + ), + ], +) +def test_fetch_libs_success( + new_path, emitter, service_factory, libs, store_libs, dl_lib, expected +): + service_factory.project.charm_libs = libs + service_factory.store.client.fetch_libraries_metadata.return_value = store_libs + service_factory.store.client.get_library.return_value = dl_lib + fetch_libs = FetchLibs({"app": APP_METADATA, "services": service_factory}) + + fetch_libs.run(argparse.Namespace()) + + emitter.assert_progress("Getting library metadata from charmhub") + emitter.assert_message("Downloaded 1 charm libraries.") diff --git a/tests/unit/models/test_project.py b/tests/unit/models/test_project.py index 08a5873a8..f4c2cc457 100644 --- a/tests/unit/models/test_project.py +++ b/tests/unit/models/test_project.py @@ -20,6 +20,7 @@ from textwrap import dedent from typing import Any +import hypothesis import pydantic import pyfakefs.fake_filesystem import pytest @@ -29,6 +30,7 @@ from craft_application.util import safe_yaml_load from craft_cli import CraftError from craft_providers import bases +from hypothesis import strategies from charmcraft import const, utils from charmcraft.models import project @@ -190,6 +192,67 @@ def test_build_info_from_build_on_run_on_basic( pytest_check.equal(info.base.version, build_on_base.channel) +@pytest.mark.parametrize( + "lib_name", + ["charm.lib", "charm_with_hyphens.lib", "charm.lib_with_hyphens", "charm0.number_0_lib"], +) +@pytest.mark.parametrize("lib_version", ["0", "1", "2.0", "2.1", "3.14"]) +def test_create_valid_charm_lib(lib_name, lib_version): + project.CharmLib.unmarshal({"lib": lib_name, "version": lib_version}) + + +@pytest.mark.parametrize( + ("name", "error_match"), + [ + ("boop", r"Library name invalid. Expected '\[charm_name\].\[lib_name\]', got 'boop'"), + ( + "raw-charm-name.valid_lib", + r"Invalid charm name in lib 'raw-charm-name.valid_lib'. Try replacing hyphens \('-'\) with underscores \('_'\).", + ), + ( + "Invalid charm name.valid_lib", + "Invalid charm name for lib 'Invalid charm name.valid_lib'. Value 'Invalid charm name' is invalid", + ), + ("my_charm.invalid library name", "Library name 'invalid library name' is invalid."), + ], +) +def test_invalid_charm_lib_name(name: str, error_match: str): + with pytest.raises(pydantic.ValidationError, match=error_match): + project.CharmLib.unmarshal({"lib": name, "version": "0"}) + + +@hypothesis.given( + strategies.one_of( + strategies.floats( + min_value=0.001, + max_value=2**32, + allow_nan=False, + allow_infinity=False, + allow_subnormal=False, + ), + strategies.integers(min_value=0, max_value=2**32), + ) +) +def test_valid_library_version(version: float): + project.CharmLib.unmarshal({"lib": "charm_name.lib_name", "version": str(version)}) + + +@pytest.mark.parametrize("version", [".1", "NaN", ""]) +def test_invalid_api_version(version: str): + with pytest.raises( + pydantic.ValidationError, match="API version not valid. Expected an integer, got '" + ): + project.CharmLib(lib="charm_name.lib_name", version=version) + + +@pytest.mark.parametrize("version", ["1.", "1.number"]) +def test_invalid_patch_version(version: str): + with pytest.raises( + pydantic.ValidationError, match="Patch version not valid. Expected an integer, got '" + ): + project.CharmLib(lib="charm_name.lib_name", version=version) + + @pytest.mark.parametrize( ("run_on", "expected"), [ diff --git a/tests/unit/models/valid_charms_yaml/full-bases.yaml b/tests/unit/models/valid_charms_yaml/full-bases.yaml index b6261af65..170fafd8d 100644 --- a/tests/unit/models/valid_charms_yaml/full-bases.yaml +++ b/tests/unit/models/valid_charms_yaml/full-bases.yaml @@ -24,6 +24,11 @@ parts: charm-strict-dependencies: false another-part: plugin: nil +charm-libs: + - lib: my_charm.my_lib + version: '1' + - lib: other_charm.lib + version: '0.123' bases: - build-on: - name: ubuntu diff --git a/tests/unit/models/valid_charms_yaml/full-platforms.yaml b/tests/unit/models/valid_charms_yaml/full-platforms.yaml index 2a709f95b..855e44561 100644 --- a/tests/unit/models/valid_charms_yaml/full-platforms.yaml +++ b/tests/unit/models/valid_charms_yaml/full-platforms.yaml @@ -24,6 +24,11 @@ parts: charm-strict-dependencies: false another-part: plugin: nil +charm-libs: + - lib: my_charm.my_lib + version: '1' + - lib: other_charm.lib + version: '0.123' base: ubuntu@24.04 build-base: ubuntu@devel platforms: diff --git a/tests/unit/services/test_store.py b/tests/unit/services/test_store.py index df72dbb04..ec4730d07 100644 --- a/tests/unit/services/test_store.py +++ b/tests/unit/services/test_store.py @@ -26,13 +26,15 @@ import charmcraft from charmcraft import application, errors, services +from charmcraft.models.project import CharmLib +from charmcraft.store import client from tests import get_fake_revision @pytest.fixture() -def store(): - store = services.StoreService(app=application.APP_METADATA, services=None) - store.client = mock.Mock(spec_set=craft_store.StoreClient) +def store(service_factory) -> services.StoreService: + store = services.StoreService(app=application.APP_METADATA, services=service_factory) + store.client = mock.Mock(spec_set=client.Client) return store @@ -222,3 +224,24 @@ def test_get_credentials(monkeypatch, store): packages=None, channels=None, ) + + +@pytest.mark.parametrize( + ("libs", "expected_call"), + [ + ([], []), + ( + [CharmLib(lib="my_charm.my_lib", version="1")], + [{"charm-name": "my_charm", "library-name": "my_lib", "api": 1}], + ), + ( + [CharmLib(lib="my_charm.my_lib", version="1.0")], + [{"charm-name": "my_charm", "library-name": "my_lib", "api": 1, "patch": 0}], + ), + ], +) +def test_fetch_libraries_metadata(monkeypatch, store, libs, expected_call): + + store.get_libraries_metadata(libs) + + store.client.fetch_libraries_metadata.assert_called_once_with(expected_call) diff --git a/tests/unit/store/test_client.py b/tests/unit/store/test_client.py new file mode 100644 index 000000000..146a02ed0 --- /dev/null +++ b/tests/unit/store/test_client.py @@ -0,0 +1,109 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""Unit tests for store client.""" + +from unittest import mock + +import pytest + +from charmcraft import store + + +@pytest.fixture() +def client() -> store.Client: + return store.Client(api_base_url="http://charmhub.local") + + +@pytest.mark.parametrize( + ("charm", "lib_id", "api", "patch", "expected_call"), + [ + ( + "my-charm", + "abcdefg", + None, + None, + mock.call("GET", "/v1/charm/libraries/my-charm/abcdefg", params={}), + ), + ( + "my-charm", + "abcdefg", + 0, + 0, + mock.call( + "GET", "/v1/charm/libraries/my-charm/abcdefg", params={"api": 0, "patch": 0} + ), + ), + ], +) +def test_get_library_success(monkeypatch, client, charm, lib_id, api, patch, expected_call): + mock_get_urlpath_json = mock.Mock( + return_value={ + "charm-name": charm, + "library-name": "my_lib", + "library-id": lib_id, + "api": api or 1, + "patch": patch or 2, + "hash": "hashy!", + } + ) + monkeypatch.setattr(client, "request_urlpath_json", mock_get_urlpath_json) + + client.get_library(charm_name=charm, library_id=lib_id, api=api, patch=patch) + + mock_get_urlpath_json.assert_has_calls([expected_call]) + + +@pytest.mark.parametrize( + ("libs", "json_response", "expected"), + [ + ([], {"libraries": []}, []), + ( + [{"charm-name": "my-charm", "library-name": "my_lib"}], + { + "libraries": [ + { + "charm-name": "my-charm", + "library-name": "my_lib", + "library-id": "ididid", + "api": 1, + "patch": 2, + "hash": "hashhashhash", + }, + ], + }, + [ + store.models.Library( + charm_name="my-charm", + lib_name="my_lib", + lib_id="ididid", + api=1, + patch=2, + content=None, + content_hash="hashhashhash", + ), + ], + ), + ], +) +def test_fetch_libraries_metadata(monkeypatch, client, libs, json_response, expected): + mock_get_urlpath_json = mock.Mock(return_value=json_response) + monkeypatch.setattr(client, "request_urlpath_json", mock_get_urlpath_json) + + assert client.fetch_libraries_metadata(libs) == expected + + mock_get_urlpath_json.assert_has_calls( + [mock.call("POST", "/v1/charm/libraries/bulk", json=libs)] + ) diff --git a/tests/unit/store/test_models.py b/tests/unit/store/test_models.py new file mode 100644 index 000000000..04b9de198 --- /dev/null +++ b/tests/unit/store/test_models.py @@ -0,0 +1,68 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""Tests for store models.""" + +import pytest + +from charmcraft.store import models + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + ( + { + "charm-name": "abc", + "library-name": "def", + "library-id": "id", + "api": 1, + "patch": 123, + "hash": "hashyhash", + }, + models.Library( + charm_name="abc", + lib_name="def", + lib_id="id", + api=1, + patch=123, + content_hash="hashyhash", + content=None, + ), + ), + ( + { + "charm-name": "abc", + "library-name": "def", + "library-id": "id", + "api": 1, + "patch": 123, + "hash": "hashyhash", + "content": "I am a library.", + }, + models.Library( + charm_name="abc", + lib_name="def", + lib_id="id", + api=1, + patch=123, + content_hash="hashyhash", + content="I am a library.", + ), + ), + ], +) +def test_library_from_dict(data, expected): + assert models.Library.from_dict(data) == expected diff --git a/tests/unit/utils/test_charmlibs.py b/tests/unit/utils/test_charmlibs.py index 9ff02fb97..34a678df2 100644 --- a/tests/unit/utils/test_charmlibs.py +++ b/tests/unit/utils/test_charmlibs.py @@ -1,4 +1,4 @@ -# Copyright 2023 Canonical Ltd. +# Copyright 2023-2024 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,6 +27,8 @@ collect_charmlib_pydeps, get_lib_info, get_lib_internals, + get_lib_module_name, + get_lib_path, get_libs_from_tree, get_name_from_metadata, ) @@ -79,6 +81,26 @@ def test_get_name_from_metadata_bad_content_no_name(tmp_path, monkeypatch): assert result is None +@pytest.mark.parametrize( + ("charm", "lib", "api", "expected"), + [ + ("my-charm", "some_lib", 0, pathlib.Path("lib/charms/my_charm/v0/some_lib.py")), + ], +) +def test_get_lib_path(charm: str, lib: str, api: int, expected: pathlib.Path): + assert get_lib_path(charm, lib, api) == expected + + +@pytest.mark.parametrize( + ("charm", "lib", "api", "expected"), + [ + ("my-charm", "some_lib", 0, "charms.my_charm.v0.some_lib"), + ], +) +def test_get_lib_module_name(charm: str, lib: str, api: int, expected: str): + assert get_lib_module_name(charm, lib, api) == expected + + # endregion # region getlibinfo tests