diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3d8bc7679f..972222e4ec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,16 +50,16 @@ jobs: build: name: Build charm uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v29.0.5 + with: + cache: false integration-test: strategy: fail-fast: false matrix: juju: - - agent: 2.9.51 # renovate: juju-agent-pin-minor - libjuju: ==2.9.49.1 # renovate: latest libjuju 2 - allure_on_amd64: false - agent: 3.6.2 # renovate: juju-agent-pin-minor + libjuju: ==3.6.1.0 # renovate: latest libjuju 2 allure_on_amd64: true architecture: - amd64 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 905433c52d..aa1a8ce700 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,7 +5,7 @@ name: Release to Charmhub on: push: branches: - - main + - 16/edge paths-ignore: - 'tests/**' - 'docs/**' @@ -46,7 +46,7 @@ jobs: - ci-tests uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v29.0.5 with: - channel: 14/edge + channel: 16/edge artifact-prefix: ${{ needs.ci-tests.outputs.artifact-prefix }} secrets: charmhub-token: ${{ secrets.CHARMHUB_TOKEN }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 30d41cfae8..b2cbbce897 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,7 @@ juju model-config logging-config="=INFO;unit=DEBUG" microk8s enable rbac # Deploy the charm -juju deploy ./postgresql-k8s_ubuntu-22.04-amd64.charm --trust \ +juju deploy ./postgresql-k8s_ubuntu-24.04-amd64.charm --trust \ --resource postgresql-image=$(yq '(.resources.postgresql-image.upstream-source)' metadata.yaml) ``` diff --git a/README.md b/README.md index e5abc71343..3a5583dcf4 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Bootstrap a Kubernetes (e.g. [Multipass-based MicroK8s](https://discourse.charmh ```shell juju add-model postgresql-k8s -juju deploy postgresql-k8s --channel 14 --trust +juju deploy postgresql-k8s --channel 16/edge --trust ``` **Note:** the `--trust` flag is required because the charm and Patroni need to create some K8s resources. @@ -62,7 +62,7 @@ Adding a relation is accomplished with `juju relate` (or `juju integrate` for Ju ```shell # Deploy Charmed PostgreSQL cluster with 3 nodes -juju deploy postgresql-k8s -n 3 --trust --channel 14 +juju deploy postgresql-k8s -n 3 --trust --channel 16/edge # Deploy the relevant application charms juju deploy mycharm @@ -87,7 +87,7 @@ juju status --relations This charm supports legacy interface `pgsql` from the previous [PostgreSQL charm](https://launchpad.net/postgresql-charm): ```shell -juju deploy postgresql-k8s --trust --channel 14 +juju deploy postgresql-k8s --trust --channel 16/edge juju deploy finos-waltz-k8s --channel edge juju relate postgresql-k8s:db finos-waltz-k8s ``` diff --git a/charmcraft.yaml b/charmcraft.yaml index b4cddfe1ff..6f2f412380 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -3,8 +3,8 @@ type: charm platforms: - ubuntu@22.04:amd64: - ubuntu@22.04:arm64: + ubuntu@24.04:amd64: + ubuntu@24.04:arm64: # Files implicitly created by charmcraft without a part: # - dispatch (https://github.com/canonical/charmcraft/pull/1898) # - manifest.yaml diff --git a/lib/charms/postgresql_k8s/v0/postgresql.py b/lib/charms/postgresql_k8s/v0/postgresql.py index bdfef9afbb..2fd8219bca 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql.py +++ b/lib/charms/postgresql_k8s/v0/postgresql.py @@ -35,7 +35,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 42 +LIBPATCH = 43 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" @@ -317,7 +317,7 @@ def delete_user(self, user: str) -> None: logger.error(f"Failed to delete user: {e}") raise PostgreSQLDeleteUserError() from e - def enable_disable_extensions( + def enable_disable_extensions( # noqa: C901 self, extensions: Dict[str, bool], database: Optional[str] = None ) -> None: """Enables or disables a PostgreSQL extension. @@ -347,10 +347,12 @@ def enable_disable_extensions( # Enable/disabled the extension in each database. for database in databases: - with self._connect_to_database( - database=database - ) as connection, connection.cursor() as cursor: + connection = self._connect_to_database(database=database) + connection.autocommit = True + with connection.cursor() as cursor: for extension, enable in ordered_extensions.items(): + if extension == "postgis": + cursor.execute("SET pgaudit.log = 'none';") cursor.execute( f"CREATE EXTENSION IF NOT EXISTS {extension};" if enable @@ -372,6 +374,7 @@ def _generate_database_privileges_statements( ) -> List[Composed]: """Generates a list of databases privileges statements.""" statements = [] + statements.append(SQL("GRANT USAGE, CREATE ON SCHEMA public TO PUBLIC;")) if relations_accessing_this_database == 1: statements.append( SQL( @@ -428,8 +431,10 @@ def _generate_database_privileges_statements( SQL("GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA {} TO {};").format( schema, Identifier(user) ), - SQL("GRANT USAGE ON SCHEMA {} TO {};").format(schema, Identifier(user)), - SQL("GRANT CREATE ON SCHEMA {} TO {};").format(schema, Identifier(user)), + SQL("GRANT USAGE, CREATE ON SCHEMA {} TO {};").format( + schema, Identifier(user) + ), + SQL("GRANT USAGE, CREATE ON SCHEMA {} TO admin;").format(schema), ]) return statements diff --git a/metadata.yaml b/metadata.yaml index 6b4192d904..c4025ba124 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -28,7 +28,7 @@ resources: postgresql-image: type: oci-image description: OCI image for PostgreSQL - upstream-source: ghcr.io/canonical/charmed-postgresql@sha256:7e41d7f60e45ee2f5463aa9aafcd3c35121423423ee08c26a174b99ad0235b7e # renovate: oci-image tag: 14.15-22.04_edge + upstream-source: ghcr.io/canonical/charmed-postgresql:16.4-24.04_edge peers: database-peers: diff --git a/src/charm.py b/src/charm.py index a77f2790df..c4fc83cc1e 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1887,6 +1887,12 @@ def update_config(self, is_creating_backup: bool = False) -> bool: return True if not self._patroni.member_started: + if self.is_tls_enabled: + logger.debug( + "Early exit update_config: patroni not responding but TLS is enabled." + ) + self._handle_postgresql_restart_need() + return True logger.debug("Early exit update_config: Patroni not started yet") return False @@ -1949,8 +1955,14 @@ def _validate_config_options(self) -> None: def _handle_postgresql_restart_need(self): """Handle PostgreSQL restart need based on the TLS configuration and configuration changes.""" - restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled() - self._patroni.reload_patroni_configuration() + if self._can_connect_to_postgresql: + restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled() + else: + restart_postgresql = False + try: + self._patroni.reload_patroni_configuration() + except Exception as e: + logger.error(f"Reload patroni call failed! error: {e!s}") # Wait for some more time than the Patroni's loop_wait default value (10 seconds), # which tells how much time Patroni will wait before checking the configuration # file again to reload it. diff --git a/src/dependency.json b/src/dependency.json index fbe4dc6884..1f87a03a6d 100644 --- a/src/dependency.json +++ b/src/dependency.json @@ -8,7 +8,7 @@ "rock": { "dependencies": {}, "name": "charmed-postgresql", - "upgrade_supported": "^14", - "version": "14.11" + "upgrade_supported": "^16", + "version": "16.6" } } diff --git a/src/patroni.py b/src/patroni.py index 148c77f865..c2c659837a 100644 --- a/src/patroni.py +++ b/src/patroni.py @@ -534,7 +534,7 @@ def render_patroni_yml_file( ) self._render_file(f"{self._storage_path}/patroni.yml", rendered, 0o644) - @retry(stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30)) + @retry(stop=stop_after_attempt(20), wait=wait_exponential(multiplier=1, min=2, max=30)) def reload_patroni_configuration(self) -> None: """Reloads the configuration after it was updated in the file.""" requests.post( diff --git a/templates/patroni.yml.j2 b/templates/patroni.yml.j2 index 0921fcfda5..cf62de7df3 100644 --- a/templates/patroni.yml.j2 +++ b/templates/patroni.yml.j2 @@ -156,11 +156,26 @@ postgresql: authentication: replication: password: {{ replication_password }} + {%- if enable_tls %} + sslrootcert: {{ conf_path }}/ca.pem + sslcert: {{ conf_path }}/cert.pem + sslkey: {{ conf_path }}/key.pem + {%- endif %} rewind: username: {{ rewind_user }} password: {{ rewind_password }} + {%- if enable_tls %} + sslrootcert: {{ conf_path }}/ca.pem + sslcert: {{ conf_path }}/cert.pem + sslkey: {{ conf_path }}/key.pem + {%- endif %} superuser: password: {{ superuser_password }} + {%- if enable_tls %} + sslrootcert: {{ conf_path }}/ca.pem + sslcert: {{ conf_path }}/cert.pem + sslkey: {{ conf_path }}/key.pem + {%- endif %} use_endpoints: true use_unix_socket: true {%- if is_no_sync_member or is_creating_backup %} diff --git a/terraform/variables.tf b/terraform/variables.tf index f69bd70d37..5a841c32b5 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -12,7 +12,7 @@ variable "app_name" { variable "channel" { description = "Charm channel to use when deploying" type = string - default = "14/stable" + default = "16/stable" } variable "revision" { @@ -24,7 +24,7 @@ variable "revision" { variable "base" { description = "Application base" type = string - default = "ubuntu@22.04" + default = "ubuntu@24.04" } variable "units" { diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index abacfd3269..151dd30486 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,9 +4,11 @@ import pytest from pytest_operator.plugin import OpsTest +from .helpers import build_charm + @pytest.fixture(scope="module") async def database_charm(ops_test: OpsTest): """Build the database charm.""" - charm = await ops_test.build_charm(".") + charm = await build_charm(".") return charm diff --git a/tests/integration/ha_tests/test_async_replication.py b/tests/integration/ha_tests/test_async_replication.py index df04ee61fb..46d6019aab 100644 --- a/tests/integration/ha_tests/test_async_replication.py +++ b/tests/integration/ha_tests/test_async_replication.py @@ -17,7 +17,6 @@ from .. import architecture, markers from ..helpers import ( APPLICATION_NAME, - CHARM_BASE, DATABASE_APP_NAME, build_and_deploy, get_leader_unit, @@ -109,12 +108,8 @@ async def test_deploy_async_replication_setup( """Build and deploy two PostgreSQL cluster in two separate models to test async replication.""" await build_and_deploy(ops_test, CLUSTER_SIZE, wait_for_idle=False) await build_and_deploy(ops_test, CLUSTER_SIZE, wait_for_idle=False, model=second_model) - await ops_test.model.deploy( - APPLICATION_NAME, channel="latest/edge", num_units=1, base=CHARM_BASE - ) - await second_model.deploy( - APPLICATION_NAME, channel="latest/edge", num_units=1, base=CHARM_BASE - ) + await ops_test.model.deploy(APPLICATION_NAME, channel="latest/edge", num_units=1) + await second_model.deploy(APPLICATION_NAME, channel="latest/edge", num_units=1) async with ops_test.fast_forward(), fast_forward(second_model): await gather( diff --git a/tests/integration/ha_tests/test_replication.py b/tests/integration/ha_tests/test_replication.py index fcc88cd1d8..06aa34080a 100644 --- a/tests/integration/ha_tests/test_replication.py +++ b/tests/integration/ha_tests/test_replication.py @@ -9,7 +9,6 @@ from ..helpers import ( APPLICATION_NAME, - CHARM_BASE, app_name, build_and_deploy, db_connect, @@ -43,7 +42,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.deploy( APPLICATION_NAME, application_name=APPLICATION_NAME, - base=CHARM_BASE, channel="edge", ) diff --git a/tests/integration/ha_tests/test_rollback_to_master_label.py b/tests/integration/ha_tests/test_rollback_to_master_label.py index 7e6639b0ad..80b33bd130 100644 --- a/tests/integration/ha_tests/test_rollback_to_master_label.py +++ b/tests/integration/ha_tests/test_rollback_to_master_label.py @@ -15,9 +15,9 @@ from ..architecture import architecture from ..helpers import ( APPLICATION_NAME, - CHARM_BASE, DATABASE_APP_NAME, METADATA, + build_charm, get_leader_unit, get_primary, get_unit_by_index, @@ -37,6 +37,7 @@ @pytest.mark.group(1) +@pytest.mark.unstable @markers.juju3 @pytest.mark.unstable @markers.amd64_only # TODO: remove after arm64 stable release @@ -47,16 +48,14 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: ops_test.model.deploy( DATABASE_APP_NAME, num_units=3, - channel="14/stable", + channel="16/stable", revision=LABEL_REVISION, - base=CHARM_BASE, trust=True, ), ops_test.model.deploy( APPLICATION_NAME, num_units=1, channel="latest/edge", - base=CHARM_BASE, ), ) logger.info("Wait for applications to become active") @@ -72,6 +71,7 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: @pytest.mark.group(1) +@pytest.mark.unstable @markers.juju3 @pytest.mark.unstable @markers.amd64_only # TODO: remove after arm64 stable release @@ -98,7 +98,7 @@ async def test_fail_and_rollback(ops_test, continuous_writes) -> None: primary_name = await get_primary(ops_test, DATABASE_APP_NAME) assert primary_name == f"{DATABASE_APP_NAME}/0" - local_charm = await ops_test.build_charm(".") + local_charm = await build_charm(".") filename = local_charm.split("/")[-1] if isinstance(local_charm, str) else local_charm.name fault_charm = Path("/tmp/", filename) shutil.copy(local_charm, fault_charm) diff --git a/tests/integration/ha_tests/test_self_healing.py b/tests/integration/ha_tests/test_self_healing.py index 1afd64239c..7a94d50016 100644 --- a/tests/integration/ha_tests/test_self_healing.py +++ b/tests/integration/ha_tests/test_self_healing.py @@ -12,7 +12,6 @@ from .. import markers from ..helpers import ( APPLICATION_NAME, - CHARM_BASE, METADATA, app_name, build_and_deploy, @@ -69,7 +68,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.deploy( APPLICATION_NAME, application_name=APPLICATION_NAME, - base=CHARM_BASE, channel="edge", ) diff --git a/tests/integration/ha_tests/test_smoke.py b/tests/integration/ha_tests/test_smoke.py index 2509bcc521..6373f11e92 100644 --- a/tests/integration/ha_tests/test_smoke.py +++ b/tests/integration/ha_tests/test_smoke.py @@ -12,7 +12,6 @@ from .. import markers from ..helpers import ( - CHARM_BASE, DATABASE_APP_NAME, scale_application, ) @@ -45,6 +44,7 @@ @pytest.mark.group(1) @markers.amd64_only # TODO: remove after arm64 stable release +@pytest.mark.unstable @pytest.mark.abort_on_fail async def test_app_force_removal(ops_test: OpsTest): """Remove unit with force while storage is alive.""" @@ -55,8 +55,7 @@ async def test_app_force_removal(ops_test: OpsTest): DATABASE_APP_NAME, application_name=DATABASE_APP_NAME, num_units=1, - channel="14/stable", - base=CHARM_BASE, + channel="16/stable", trust=True, config={"profile": "testing"}, ) @@ -106,6 +105,7 @@ async def test_app_force_removal(ops_test: OpsTest): @pytest.mark.group(1) @markers.amd64_only # TODO: remove after arm64 stable release +@pytest.mark.unstable @pytest.mark.abort_on_fail async def test_app_garbage_ignorance(ops_test: OpsTest): """Test charm deploy in dirty environment with garbage storage.""" @@ -159,6 +159,7 @@ async def test_app_garbage_ignorance(ops_test: OpsTest): @pytest.mark.group(1) @markers.amd64_only # TODO: remove after arm64 stable release +@pytest.mark.unstable @pytest.mark.abort_on_fail async def test_app_resources_conflicts(ops_test: OpsTest): """Test application deploy in dirty environment with garbage storage from another application.""" @@ -168,8 +169,7 @@ async def test_app_resources_conflicts(ops_test: OpsTest): DATABASE_APP_NAME, application_name=DUP_DATABASE_APP_NAME, num_units=1, - channel="14/stable", - base=CHARM_BASE, + channel="16/stable", trust=True, config={"profile": "testing"}, ) diff --git a/tests/integration/ha_tests/test_upgrade.py b/tests/integration/ha_tests/test_upgrade.py index 9c19c60f26..8863562bdd 100644 --- a/tests/integration/ha_tests/test_upgrade.py +++ b/tests/integration/ha_tests/test_upgrade.py @@ -14,9 +14,9 @@ from ..helpers import ( APPLICATION_NAME, - CHARM_BASE, DATABASE_APP_NAME, METADATA, + build_charm, count_switchovers, get_leader_unit, get_primary, @@ -35,6 +35,7 @@ @pytest.mark.group(1) +@pytest.mark.unstable @pytest.mark.abort_on_fail async def test_deploy_latest(ops_test: OpsTest) -> None: """Simple test to ensure that the PostgreSQL and application charms get deployed.""" @@ -42,16 +43,14 @@ async def test_deploy_latest(ops_test: OpsTest) -> None: ops_test.model.deploy( DATABASE_APP_NAME, num_units=3, - channel="14/edge", + channel="16/edge", trust=True, config={"profile": "testing"}, - base=CHARM_BASE, ), ops_test.model.deploy( APPLICATION_NAME, num_units=1, channel="latest/edge", - base=CHARM_BASE, ), ) logger.info("Wait for applications to become active") @@ -66,6 +65,7 @@ async def test_deploy_latest(ops_test: OpsTest) -> None: @pytest.mark.group(1) +@pytest.mark.unstable @pytest.mark.abort_on_fail async def test_pre_upgrade_check(ops_test: OpsTest) -> None: """Test that the pre-upgrade-check action runs successfully.""" @@ -93,6 +93,7 @@ async def test_pre_upgrade_check(ops_test: OpsTest) -> None: @pytest.mark.group(1) +@pytest.mark.unstable @pytest.mark.abort_on_fail async def test_upgrade_from_edge(ops_test: OpsTest, continuous_writes) -> None: # Start an application that continuously writes data to the database. @@ -110,7 +111,7 @@ async def test_upgrade_from_edge(ops_test: OpsTest, continuous_writes) -> None: application = ops_test.model.applications[DATABASE_APP_NAME] logger.info("Build charm locally") - charm = await ops_test.build_charm(".") + charm = await build_charm(".") logger.info("Refresh the charm") await application.refresh(path=charm, resources=resources) @@ -159,6 +160,7 @@ async def test_upgrade_from_edge(ops_test: OpsTest, continuous_writes) -> None: @pytest.mark.group(1) +@pytest.mark.unstable @pytest.mark.abort_on_fail async def test_fail_and_rollback(ops_test, continuous_writes) -> None: # Start an application that continuously writes data to the database. @@ -183,7 +185,7 @@ async def test_fail_and_rollback(ops_test, continuous_writes) -> None: primary_name = await get_primary(ops_test, DATABASE_APP_NAME) assert primary_name == f"{DATABASE_APP_NAME}/0" - local_charm = await ops_test.build_charm(".") + local_charm = await build_charm(".") filename = local_charm.split("/")[-1] if isinstance(local_charm, str) else local_charm.name fault_charm = Path("/tmp/", filename) shutil.copy(local_charm, fault_charm) diff --git a/tests/integration/ha_tests/test_upgrade_from_stable.py b/tests/integration/ha_tests/test_upgrade_from_stable.py index ac221930e1..54adb435de 100644 --- a/tests/integration/ha_tests/test_upgrade_from_stable.py +++ b/tests/integration/ha_tests/test_upgrade_from_stable.py @@ -13,9 +13,9 @@ from .. import markers from ..helpers import ( APPLICATION_NAME, - CHARM_BASE, DATABASE_APP_NAME, METADATA, + build_charm, count_switchovers, get_leader_unit, get_primary, @@ -33,6 +33,7 @@ @pytest.mark.group(1) +@pytest.mark.unstable @markers.amd64_only # TODO: remove after arm64 stable release @pytest.mark.abort_on_fail async def test_deploy_stable(ops_test: OpsTest) -> None: @@ -41,15 +42,13 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: ops_test.model.deploy( DATABASE_APP_NAME, num_units=3, - channel="14/stable", + channel="16/stable", trust=True, - base=CHARM_BASE, ), ops_test.model.deploy( APPLICATION_NAME, num_units=1, channel="latest/edge", - base=CHARM_BASE, ), ) logger.info("Wait for applications to become active") @@ -61,13 +60,14 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: @pytest.mark.group(1) +@pytest.mark.unstable @markers.amd64_only # TODO: remove after arm64 stable release @pytest.mark.abort_on_fail async def test_pre_upgrade_check(ops_test: OpsTest) -> None: """Test that the pre-upgrade-check action runs successfully.""" application = ops_test.model.applications[DATABASE_APP_NAME] if "pre-upgrade-check" not in await application.get_actions(): - logger.info("skipping the test because the charm from 14/stable doesn't support upgrade") + logger.info("skipping the test because the charm from 16/stable doesn't support upgrade") return logger.info("Get leader unit") @@ -94,6 +94,7 @@ async def test_pre_upgrade_check(ops_test: OpsTest) -> None: @pytest.mark.group(1) +@pytest.mark.unstable @markers.amd64_only # TODO: remove after arm64 stable release @pytest.mark.abort_on_fail async def test_upgrade_from_stable(ops_test: OpsTest, continuous_writes): @@ -114,7 +115,7 @@ async def test_upgrade_from_stable(ops_test: OpsTest, continuous_writes): actions = await application.get_actions() logger.info("Build charm locally") - charm = await ops_test.build_charm(".") + charm = await build_charm(".") logger.info("Refresh the charm") await application.refresh(path=charm, resources=resources) diff --git a/tests/integration/ha_tests/test_upgrade_to_primary_label.py b/tests/integration/ha_tests/test_upgrade_to_primary_label.py index 0d76c08f2b..952ba52b9f 100644 --- a/tests/integration/ha_tests/test_upgrade_to_primary_label.py +++ b/tests/integration/ha_tests/test_upgrade_to_primary_label.py @@ -17,6 +17,7 @@ CHARM_SERIES, DATABASE_APP_NAME, METADATA, + build_charm, get_leader_unit, get_primary, get_unit_by_index, @@ -35,6 +36,7 @@ @pytest.mark.group(1) +@pytest.mark.unstable @markers.amd64_only # TODO: remove after arm64 stable release @pytest.mark.unstable @pytest.mark.abort_on_fail @@ -50,7 +52,7 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: ops_test.model.deploy( DATABASE_APP_NAME, num_units=3, - channel="14/stable", + channel="16/stable", revision=(280 if architecture == "arm64" else 281), trust=True, **database_additional_params, @@ -59,7 +61,6 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: APPLICATION_NAME, num_units=1, channel="latest/edge", - base=CHARM_BASE, ), ) logger.info("Wait for applications to become active") @@ -75,6 +76,7 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: @pytest.mark.group(1) +@pytest.mark.unstable @markers.amd64_only # TODO: remove after arm64 stable release @pytest.mark.unstable async def test_upgrade(ops_test, continuous_writes) -> None: @@ -100,7 +102,7 @@ async def test_upgrade(ops_test, continuous_writes) -> None: primary_name = await get_primary(ops_test, DATABASE_APP_NAME) assert primary_name == f"{DATABASE_APP_NAME}/0" - local_charm = await ops_test.build_charm(".") + local_charm = await build_charm(".") application = ops_test.model.applications[DATABASE_APP_NAME] resources = {"postgresql-image": METADATA["resources"]["postgresql-image"]["upstream-source"]} diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 9fb6a2160f..d98c77f0a8 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -7,7 +7,7 @@ from datetime import datetime from multiprocessing import ProcessError from pathlib import Path -from subprocess import check_call +from subprocess import check_call, run import botocore import psycopg2 @@ -32,8 +32,8 @@ wait_fixed, ) -CHARM_BASE = "ubuntu@22.04" -CHARM_SERIES = "jammy" +CHARM_BASE = "ubuntu@24.04" +CHARM_SERIES = "noble" METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) DATABASE_APP_NAME = METADATA["name"] APPLICATION_NAME = "postgresql-test-app" @@ -70,6 +70,37 @@ async def app_name( return None +async def build_charm(charm_path) -> Path: + charm_path = Path(charm_path) + architecture = run( + ["dpkg", "--print-architecture"], + capture_output=True, + check=True, + encoding="utf-8", + ).stdout.strip() + assert architecture in ("amd64", "arm64") + # 24.04 pin is temporary solution while multi-base integration testing not supported by data-platform-workflows + packed_charms = list(charm_path.glob(f"*ubuntu@24.04-{architecture}.charm")) + if len(packed_charms) == 1: + # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms + # to begin with `./` or `/` to distinguish them from Charmhub charms. + # Therefore, we need to return an absolute path—a relative `pathlib.Path` does not start + # with `./` when cast to a str. + # (python-libjuju model.deploy() expects a str but will cast any input to a str as a + # workaround for pytest-operator's non-compliant `build_charm` return type of + # `pathlib.Path`.) + return packed_charms[0].resolve(strict=True) + elif len(packed_charms) > 1: + raise ValueError( + f"More than one matching .charm file found at {charm_path=} for {architecture=} and " + f"Ubuntu 24.04: {packed_charms}." + ) + else: + raise ValueError( + f"Unable to find .charm file for {architecture=} and Ubuntu 24.04 at {charm_path=}" + ) + + async def build_and_deploy( ops_test: OpsTest, num_units: int, @@ -89,7 +120,7 @@ async def build_and_deploy( global charm if not charm: - charm = await ops_test.build_charm(".") + charm = await build_charm(".") resources = { "postgresql-image": METADATA["resources"]["postgresql-image"]["upstream-source"], } @@ -100,7 +131,6 @@ async def build_and_deploy( application_name=database_app_name, trust=True, num_units=num_units, - base=CHARM_BASE, config={"profile": "testing"}, ), ) @@ -832,10 +862,8 @@ async def backup_operations( ) -> None: """Basic set of operations for backup testing in different cloud providers.""" # Deploy S3 Integrator and TLS Certificates Operator. - await ops_test.model.deploy(s3_integrator_app_name, base=CHARM_BASE) - await ops_test.model.deploy( - tls_certificates_app_name, config=tls_config, channel=tls_channel, base=CHARM_BASE - ) + await ops_test.model.deploy(s3_integrator_app_name) + await ops_test.model.deploy(tls_certificates_app_name, config=tls_config, channel=tls_channel) # Deploy and relate PostgreSQL to S3 integrator (one database app for each cloud for now # as archivo_mode is disabled after restoring the backup) and to TLS Certificates Operator # (to be able to create backups from replicas). diff --git a/tests/integration/new_relations/test_new_relations.py b/tests/integration/new_relations/test_new_relations.py index 61bcbfdaa4..0c24eb8442 100644 --- a/tests/integration/new_relations/test_new_relations.py +++ b/tests/integration/new_relations/test_new_relations.py @@ -16,7 +16,6 @@ from .. import markers from ..helpers import ( - CHARM_BASE, check_database_users_existence, scale_application, ) @@ -56,7 +55,6 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest, databas APPLICATION_APP_NAME, application_name=APPLICATION_APP_NAME, num_units=2, - base=CHARM_BASE, channel="edge", ), ops_test.model.deploy( @@ -68,7 +66,6 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest, databas }, application_name=DATABASE_APP_NAME, num_units=3, - base=CHARM_BASE, trust=True, config={"profile": "testing"}, ), @@ -81,7 +78,6 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest, databas }, application_name=ANOTHER_DATABASE_APP_NAME, num_units=3, - base=CHARM_BASE, trust=True, config={"profile": "testing"}, ), @@ -196,7 +192,6 @@ async def test_two_applications_doesnt_share_the_same_relation_data(ops_test: Op await ops_test.model.deploy( APPLICATION_APP_NAME, application_name=another_application_app_name, - base=CHARM_BASE, channel="edge", ) await ops_test.model.wait_for_idle(apps=all_app_names, status="active") @@ -453,7 +448,7 @@ async def test_admin_role(ops_test: OpsTest): all_app_names = [DATA_INTEGRATOR_APP_NAME] all_app_names.extend(APP_NAMES) async with ops_test.fast_forward(): - await ops_test.model.deploy(DATA_INTEGRATOR_APP_NAME, base=CHARM_BASE) + await ops_test.model.deploy(DATA_INTEGRATOR_APP_NAME) await ops_test.model.wait_for_idle(apps=[DATA_INTEGRATOR_APP_NAME], status="blocked") await ops_test.model.applications[DATA_INTEGRATOR_APP_NAME].set_config({ "database-name": DATA_INTEGRATOR_APP_NAME.replace("-", "_"), @@ -544,7 +539,6 @@ async def test_invalid_extra_user_roles(ops_test: OpsTest): await ops_test.model.deploy( DATA_INTEGRATOR_APP_NAME, application_name=another_data_integrator_app_name, - base=CHARM_BASE, ) await ops_test.model.wait_for_idle( apps=[another_data_integrator_app_name], status="blocked" @@ -609,7 +603,6 @@ async def test_database_deploy_clientapps(ops_test: OpsTest, database_charm): }, application_name=DATABASE_APP_NAME, num_units=3, - base=CHARM_BASE, trust=True, config={"profile": "testing"}, ), @@ -630,7 +623,7 @@ async def test_discourse(ops_test: OpsTest): await gather( ops_test.model.deploy(DISCOURSE_APP_NAME, application_name=DISCOURSE_APP_NAME), ops_test.model.deploy( - REDIS_APP_NAME, application_name=REDIS_APP_NAME, channel="latest/edge", base=CHARM_BASE + REDIS_APP_NAME, application_name=REDIS_APP_NAME, channel="latest/edge" ), ) diff --git a/tests/integration/new_relations/test_relations_coherence.py b/tests/integration/new_relations/test_relations_coherence.py index aa489473d1..4dc301bdd3 100644 --- a/tests/integration/new_relations/test_relations_coherence.py +++ b/tests/integration/new_relations/test_relations_coherence.py @@ -9,7 +9,7 @@ import pytest from pytest_operator.plugin import OpsTest -from ..helpers import CHARM_BASE, DATABASE_APP_NAME, build_and_deploy +from ..helpers import DATABASE_APP_NAME, build_and_deploy from .helpers import build_connection_string from .test_new_relations import DATA_INTEGRATOR_APP_NAME @@ -30,7 +30,7 @@ async def test_relations(ops_test: OpsTest, database_charm): await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=3000) # Creating first time relation with user role - await ops_test.model.deploy(DATA_INTEGRATOR_APP_NAME, base=CHARM_BASE) + await ops_test.model.deploy(DATA_INTEGRATOR_APP_NAME) await ops_test.model.applications[DATA_INTEGRATOR_APP_NAME].set_config({ "database-name": DATA_INTEGRATOR_APP_NAME.replace("-", "_"), }) diff --git a/tests/integration/relations/test_relations.py b/tests/integration/relations/test_relations.py index 0d33cf38ee..4bbb867def 100644 --- a/tests/integration/relations/test_relations.py +++ b/tests/integration/relations/test_relations.py @@ -7,7 +7,6 @@ import pytest from pytest_operator.plugin import OpsTest -from ..helpers import CHARM_BASE from ..new_relations.test_new_relations import ( APPLICATION_APP_NAME, DATABASE_APP_METADATA, @@ -34,7 +33,6 @@ async def test_deploy_charms(ops_test: OpsTest, database_charm): APPLICATION_APP_NAME, application_name=APPLICATION_APP_NAME, num_units=1, - base=CHARM_BASE, channel="edge", ), ops_test.model.deploy( @@ -46,7 +44,6 @@ async def test_deploy_charms(ops_test: OpsTest, database_charm): }, application_name=APP_NAME, num_units=1, - base=CHARM_BASE, config={ "profile": "testing", "plugin_unaccent_enable": "True", diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 9e29ac16d0..664b0fbf24 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -14,10 +14,10 @@ from tenacity import Retrying, stop_after_delay, wait_fixed from .helpers import ( - CHARM_BASE, METADATA, STORAGE_PATH, build_and_deploy, + build_charm, convert_records_to_dict, db_connect, get_application_units, @@ -379,7 +379,7 @@ async def test_application_removal(ops_test: OpsTest) -> None: @pytest.mark.group(1) async def test_redeploy_charm_same_model(ops_test: OpsTest): """Redeploy the charm in the same model to test that it works.""" - charm = await ops_test.build_charm(".") + charm = await build_charm(".") async with ops_test.fast_forward(): await ops_test.model.deploy( charm, @@ -388,7 +388,6 @@ async def test_redeploy_charm_same_model(ops_test: OpsTest): }, application_name=APP_NAME, num_units=len(UNIT_IDS), - base=CHARM_BASE, trust=True, config={"profile": "testing"}, ) @@ -423,7 +422,7 @@ async def test_redeploy_charm_same_model_after_forcing_removal(ops_test: OpsTest assert set(existing_resources) == set(expected_resources) # Check that the charm can be deployed again. - charm = await ops_test.build_charm(".") + charm = await build_charm(".") async with ops_test.fast_forward(): await ops_test.model.deploy( charm, @@ -432,7 +431,6 @@ async def test_redeploy_charm_same_model_after_forcing_removal(ops_test: OpsTest }, application_name=APP_NAME, num_units=len(UNIT_IDS), - base=CHARM_BASE, trust=True, config={"profile": "testing"}, ) @@ -453,7 +451,7 @@ async def test_storage_with_more_restrictive_permissions(ops_test: OpsTest): app_name = f"test-storage-{APP_NAME}" async with ops_test.fast_forward(): # Deploy and wait for the charm to get into the install hook (maintenance status). - charm = await ops_test.build_charm(".") + charm = await build_charm(".") async with ops_test.fast_forward(): await ops_test.model.deploy( charm, @@ -464,7 +462,6 @@ async def test_storage_with_more_restrictive_permissions(ops_test: OpsTest): }, application_name=app_name, num_units=1, - base=CHARM_BASE, trust=True, config={"profile": "testing"}, ) diff --git a/tests/integration/test_db.py b/tests/integration/test_db.py index 658aa6b713..158c82b120 100644 --- a/tests/integration/test_db.py +++ b/tests/integration/test_db.py @@ -10,7 +10,6 @@ from . import markers from .helpers import ( APPLICATION_NAME, - CHARM_BASE, DATABASE_APP_NAME, build_and_deploy, check_database_creation, @@ -113,13 +112,11 @@ async def test_extensions_blocking(ops_test: OpsTest) -> None: await ops_test.model.deploy( APPLICATION_NAME, application_name=APPLICATION_NAME, - base=CHARM_BASE, channel="edge", ) await ops_test.model.deploy( APPLICATION_NAME, application_name=f"{APPLICATION_NAME}2", - base=CHARM_BASE, channel="edge", ) diff --git a/tests/integration/test_plugins.py b/tests/integration/test_plugins.py index a628f28915..43c62a3dec 100644 --- a/tests/integration/test_plugins.py +++ b/tests/integration/test_plugins.py @@ -62,8 +62,10 @@ HYPOPG_EXTENSION_STATEMENT = "CREATE TABLE hypopg_test (id integer, val text); SELECT hypopg_create_index('CREATE INDEX ON hypopg_test (id)');" IP4R_EXTENSION_STATEMENT = "CREATE TABLE ip4r_test (ip ip4);" JSONB_PLPERL_EXTENSION_STATEMENT = "CREATE OR REPLACE FUNCTION jsonb_plperl_test(val jsonb) RETURNS jsonb TRANSFORM FOR TYPE jsonb LANGUAGE plperl as $$ return $_[0]; $$;" -ORAFCE_EXTENSION_STATEMENT = "SELECT add_months(date '2005-05-31',1);" -PG_SIMILARITY_EXTENSION_STATEMENT = "SHOW pg_similarity.levenshtein_threshold;" +ORAFCE_EXTENSION_STATEMENT = "SELECT oracle.add_months(date '2005-05-31',1);" +PG_SIMILARITY_EXTENSION_STATEMENT = ( + "SET pg_similarity.levenshtein_threshold = 0.7; SELECT 'aaa', 'aab', lev('aaa','aab');" +) PLPERL_EXTENSION_STATEMENT = "CREATE OR REPLACE FUNCTION plperl_test(name text) RETURNS text AS $$ return $_SHARED{$_[0]}; $$ LANGUAGE plperl;" PREFIX_EXTENSION_STATEMENT = "SELECT '123'::prefix_range @> '123456';" RDKIT_EXTENSION_STATEMENT = "SELECT is_valid_smiles('CCC');" diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 71a04eaf06..d67ab7b8e0 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -13,7 +13,6 @@ change_patroni_setting, ) from .helpers import ( - CHARM_BASE, DATABASE_APP_NAME, build_and_deploy, check_database_creation, @@ -78,7 +77,7 @@ async def test_tls(ops_test: OpsTest) -> None: async with ops_test.fast_forward(): # Deploy TLS Certificates operator. await ops_test.model.deploy( - tls_certificates_app_name, config=tls_config, channel=tls_channel, base=CHARM_BASE + tls_certificates_app_name, config=tls_config, channel=tls_channel ) # Relate it to the PostgreSQL to enable TLS. await ops_test.model.relate(DATABASE_APP_NAME, tls_certificates_app_name) @@ -123,7 +122,7 @@ async def test_tls(ops_test: OpsTest) -> None: await run_command_on_unit( ops_test, replica, - 'su postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /var/lib/postgresql/data/pgdata promote"', + 'su postgres -c "/usr/lib/postgresql/16/bin/pg_ctl -D /var/lib/postgresql/data/pgdata promote"', ) # Check that the replica was promoted. diff --git a/tests/integration/test_trust.py b/tests/integration/test_trust.py index 8de2491ed8..ca26e03770 100644 --- a/tests/integration/test_trust.py +++ b/tests/integration/test_trust.py @@ -10,7 +10,6 @@ from pytest_operator.plugin import OpsTest from .helpers import ( - CHARM_BASE, KUBECTL, METADATA, get_leader_unit, @@ -97,7 +96,6 @@ async def test_deploy_without_trust(ops_test: OpsTest, database_charm): application_name=APP_NAME, num_units=3, trust=False, - base=CHARM_BASE, ) await ops_test.model.block_until( diff --git a/tests/integration/test_wrong_arch.py b/tests/integration/test_wrong_arch.py index 4b9980ca3d..0261707bc6 100644 --- a/tests/integration/test_wrong_arch.py +++ b/tests/integration/test_wrong_arch.py @@ -11,7 +11,7 @@ from pytest_operator.plugin import OpsTest from . import markers -from .helpers import CHARM_BASE, DATABASE_APP_NAME, METADATA +from .helpers import DATABASE_APP_NAME, METADATA logger = logging.getLogger(__name__) @@ -40,7 +40,6 @@ async def test_arm_charm_on_amd_host(ops_test: OpsTest) -> None: application_name=DATABASE_APP_NAME, trust=True, num_units=1, - base=CHARM_BASE, config={"profile": "testing"}, ) @@ -63,7 +62,6 @@ async def test_amd_charm_on_arm_host(ops_test: OpsTest) -> None: application_name=DATABASE_APP_NAME, trust=True, num_units=1, - base=CHARM_BASE, config={"profile": "testing"}, ) diff --git a/tests/unit/test_backups.py b/tests/unit/test_backups.py index 9ef8aec9ee..5d37458ebf 100644 --- a/tests/unit/test_backups.py +++ b/tests/unit/test_backups.py @@ -206,7 +206,7 @@ def test_can_use_s3_repository(harness): patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, patch( "charm.Patroni.rock_postgresql_version", - new_callable=PropertyMock(return_value="14.10"), + new_callable=PropertyMock(return_value="16.6"), ) as _rock_postgresql_version, patch("charm.PostgreSQLBackups._execute_command") as _execute_command, patch( diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index c26f9eae36..e1d2318c81 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -198,7 +198,7 @@ def test_on_postgresql_pebble_ready(harness): patch("charm.PostgresqlOperatorCharm._on_leader_elected"), patch("charm.PostgresqlOperatorCharm._create_pgdata") as _create_pgdata, ): - _rock_postgresql_version.return_value = "14.7" + _rock_postgresql_version.return_value = "16.6" # Mock the primary endpoint ready property values. _primary_endpoint_ready.side_effect = [False, True, True] @@ -255,7 +255,7 @@ def test_on_postgresql_pebble_ready_no_connection(harness): ): mock_event = MagicMock() mock_event.workload = harness.model.unit.get_container(POSTGRESQL_CONTAINER) - _rock_postgresql_version.return_value = "14.7" + _rock_postgresql_version.return_value = "16.6" harness.charm._on_postgresql_pebble_ready(mock_event) @@ -1670,6 +1670,7 @@ def test_update_config(harness): harness.update_relation_data( rel_id, harness.charm.unit.name, {"tls": ""} ) # Mock some data in the relation to test that it doesn't change. + _is_tls_enabled.return_value = False harness.charm.update_config() _handle_postgresql_restart_need.assert_not_called() assert "tls" not in harness.get_relation_data(rel_id, harness.charm.unit.name) diff --git a/tests/unit/test_db.py b/tests/unit/test_db.py index ddcbec8390..9b631c4922 100644 --- a/tests/unit/test_db.py +++ b/tests/unit/test_db.py @@ -19,7 +19,7 @@ DATABASE = "test_database" RELATION_NAME = "db" -POSTGRESQL_VERSION = "14" +POSTGRESQL_VERSION = "16" @pytest.fixture(autouse=True) diff --git a/tests/unit/test_patroni.py b/tests/unit/test_patroni.py index e000f2bef5..9be6ec6357 100644 --- a/tests/unit/test_patroni.py +++ b/tests/unit/test_patroni.py @@ -202,7 +202,7 @@ def test_render_patroni_yml_file(harness, patroni): ) as _rock_postgresql_version, patch("charm.Patroni._render_file") as _render_file, ): - _rock_postgresql_version.return_value = "14.7" + _rock_postgresql_version.return_value = "16.6" # Get the expected content from a file. with open("templates/patroni.yml.j2") as file: @@ -217,7 +217,7 @@ def test_render_patroni_yml_file(harness, patroni): rewind_user=REWIND_USER, rewind_password=patroni._rewind_password, minority_count=patroni._members_count // 2, - version="14", + version="16", patroni_password=patroni._patroni_password, ) @@ -252,7 +252,7 @@ def test_render_patroni_yml_file(harness, patroni): rewind_user=REWIND_USER, rewind_password=patroni._rewind_password, minority_count=patroni._members_count // 2, - version="14", + version="16", patroni_password=patroni._patroni_password, ) assert expected_content_with_tls != expected_content diff --git a/tests/unit/test_postgresql.py b/tests/unit/test_postgresql.py index ed6665d9d6..2698b2154d 100644 --- a/tests/unit/test_postgresql.py +++ b/tests/unit/test_postgresql.py @@ -163,6 +163,7 @@ def test_generate_database_privileges_statements(harness): assert harness.charm.postgresql._generate_database_privileges_statements( 1, ["test_schema_1", "test_schema_2"], "test_user" ) == [ + SQL("GRANT USAGE, CREATE ON SCHEMA public TO PUBLIC;"), Composed([ SQL( "DO $$\nDECLARE r RECORD;\nBEGIN\n FOR r IN (SELECT statement FROM (SELECT 1 AS index,'ALTER TABLE '|| schemaname || '.\"' || tablename ||'\" OWNER TO " @@ -220,6 +221,7 @@ def test_generate_database_privileges_statements(harness): assert harness.charm.postgresql._generate_database_privileges_statements( 2, ["test_schema_1", "test_schema_2"], "test_user" ) == [ + SQL("GRANT USAGE, CREATE ON SCHEMA public TO PUBLIC;"), Composed([ SQL("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA "), Identifier("test_schema_1"), @@ -242,18 +244,16 @@ def test_generate_database_privileges_statements(harness): SQL(";"), ]), Composed([ - SQL("GRANT USAGE ON SCHEMA "), + SQL("GRANT USAGE, CREATE ON SCHEMA "), Identifier("test_schema_1"), SQL(" TO "), Identifier("test_user"), SQL(";"), ]), Composed([ - SQL("GRANT CREATE ON SCHEMA "), + SQL("GRANT USAGE, CREATE ON SCHEMA "), Identifier("test_schema_1"), - SQL(" TO "), - Identifier("test_user"), - SQL(";"), + SQL(" TO admin;"), ]), Composed([ SQL("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA "), @@ -277,18 +277,16 @@ def test_generate_database_privileges_statements(harness): SQL(";"), ]), Composed([ - SQL("GRANT USAGE ON SCHEMA "), + SQL("GRANT USAGE, CREATE ON SCHEMA "), Identifier("test_schema_2"), SQL(" TO "), Identifier("test_user"), SQL(";"), ]), Composed([ - SQL("GRANT CREATE ON SCHEMA "), + SQL("GRANT USAGE, CREATE ON SCHEMA "), Identifier("test_schema_2"), - SQL(" TO "), - Identifier("test_user"), - SQL(";"), + SQL(" TO admin;"), ]), ] diff --git a/tests/unit/test_postgresql_provider.py b/tests/unit/test_postgresql_provider.py index e56b392387..95db7bda57 100644 --- a/tests/unit/test_postgresql_provider.py +++ b/tests/unit/test_postgresql_provider.py @@ -20,7 +20,7 @@ DATABASE = "test_database" EXTRA_USER_ROLES = "CREATEDB,CREATEROLE" RELATION_NAME = "database" -POSTGRESQL_VERSION = "14" +POSTGRESQL_VERSION = "16" @pytest.fixture(autouse=True)