From 55ebc46e4b962ad2ec56f6be3a306575dddd40a9 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Thu, 5 Oct 2023 18:25:01 +0000 Subject: [PATCH 01/11] python3.pkgs.python3-saml: fix more expired tests Upstream has now fixed the tests we were previously disabling ourselves, but in the meantime yet another test has started failing. --- .../python-modules/python3-saml/default.nix | 15 ++++++++-- .../python3-saml/skip-broken-tests.patch | 28 ------------------- 2 files changed, 12 insertions(+), 31 deletions(-) delete mode 100644 pkgs/development/python-modules/python3-saml/skip-broken-tests.patch diff --git a/pkgs/development/python-modules/python3-saml/default.nix b/pkgs/development/python-modules/python3-saml/default.nix index 981f963db389d..7a5d8bccaa316 100644 --- a/pkgs/development/python-modules/python3-saml/default.nix +++ b/pkgs/development/python-modules/python3-saml/default.nix @@ -24,14 +24,23 @@ buildPythonPackage rec { }; patches = [ + # skip tests with expired test data + # upstream issue: https://github.com/SAML-Toolkits/python3-saml/issues/373 (fetchpatch { name = "test-expired.patch"; url = "https://github.com/SAML-Toolkits/python3-saml/commit/bd65578e5a21494c89320094c61c1c77250bea33.diff"; hash = "sha256-9Trew6R5JDjtc0NRGoklqMVDEI4IEqFOdK3ezyBU6gI="; }) - # skip tests with expired test data - # upstream issue: https://github.com/SAML-Toolkits/python3-saml/issues/373 - ./skip-broken-tests.patch + (fetchpatch { + name = "test-expired.patch"; + url = "https://github.com/SAML-Toolkits/python3-saml/commit/ea3a6d4ee6ea0c5cfb0f698d8c0ed25638150f47.patch"; + hash = "sha256-Q9+GM+mCEZK0QVp7ulH2hORVig2411OvkC4+o36DeXg="; + }) + (fetchpatch { + name = "test-expired.patch"; + url = "https://github.com/SAML-Toolkits/python3-saml/commit/feb0d1d954ee4d0ad1ad1d7d536bf9e83fa9431b.patch"; + hash = "sha256-NURGI4FUnFlWRZfkioU9IYmZ+Zk9FKfZchjdn7N9abU="; + }) ]; propagatedBuildInputs = [ diff --git a/pkgs/development/python-modules/python3-saml/skip-broken-tests.patch b/pkgs/development/python-modules/python3-saml/skip-broken-tests.patch deleted file mode 100644 index ea845e7da3de8..0000000000000 --- a/pkgs/development/python-modules/python3-saml/skip-broken-tests.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/tests/src/OneLogin/saml2_tests/response_test.py b/tests/src/OneLogin/saml2_tests/response_test.py -index fbe714f..bbed3c2 100644 ---- a/tests/src/OneLogin/saml2_tests/response_test.py -+++ b/tests/src/OneLogin/saml2_tests/response_test.py -@@ -562,6 +562,7 @@ class OneLogin_Saml2_Response_Test(unittest.TestCase): - response_2 = OneLogin_Saml2_Response(settings, xml_2) - self.assertTrue(response_2.check_one_condition()) - -+ @unittest.skip("test data expired") - def testCheckOneAuthnStatement(self): - """ - Tests the check_one_authnstatement method of SamlResponse -@@ -970,6 +971,7 @@ class OneLogin_Saml2_Response_Test(unittest.TestCase): - with self.assertRaisesRegex(Exception, 'Could not validate timestamp: expired. Check system clock.'): - response_2.is_valid(self.get_request_data(), raise_exceptions=True) - -+ @unittest.skip("test data expired") - def testIsInValidNoStatement(self): - """ - Tests the is_valid method of the OneLogin_Saml2_Response -@@ -1080,6 +1082,7 @@ class OneLogin_Saml2_Response_Test(unittest.TestCase): - with self.assertRaisesRegex(Exception, 'Found an Attribute element with duplicated Name'): - response.get_attributes() - -+ @unittest.skip("test data expired") - def testIsInValidDestination(self): - """ - Tests the is_valid method of the OneLogin_Saml2_Response class From d72bc8a58859f3261085ae2c1e489e95100b4746 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Tue, 10 Oct 2023 10:08:05 +0000 Subject: [PATCH 02/11] python3.pkgs.django-allauth: 0.55.2 -> 0.57.0 --- .../python-modules/django-allauth/default.nix | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pkgs/development/python-modules/django-allauth/default.nix b/pkgs/development/python-modules/django-allauth/default.nix index 738cb1c58d32f..49864b0b9bbb5 100644 --- a/pkgs/development/python-modules/django-allauth/default.nix +++ b/pkgs/development/python-modules/django-allauth/default.nix @@ -28,7 +28,7 @@ buildPythonPackage rec { pname = "django-allauth"; - version = "0.55.2"; + version = "0.57.0"; format = "pyproject"; disabled = pythonOlder "3.7"; @@ -37,7 +37,7 @@ buildPythonPackage rec { owner = "pennersr"; repo = pname; rev = version; - hash = "sha256-i0thQymrEDkx2Yt9kM10j4LxL7yChHkG9vsS0508EQA="; + hash = "sha256-zhKqvm43rw28UKNFdfJ2C1dIeZfPqmchb1rJykm1lx4="; }; nativeBuildInputs = [ @@ -73,11 +73,6 @@ buildPythonPackage rec { ] ++ lib.flatten (builtins.attrValues passthru.optional-dependencies); - disabledTestPaths = [ - # tests are out of date - "allauth/socialaccount/providers/cern/tests.py" - ]; - passthru.tests = { inherit dj-rest-auth; }; @@ -85,7 +80,7 @@ buildPythonPackage rec { meta = with lib; { changelog = "https://github.com/pennersr/django-allauth/blob/${version}/ChangeLog.rst"; description = "Integrated set of Django applications addressing authentication, registration, account management as well as 3rd party (social) account authentication"; - downloadPage = " https://github.com/pennersr/django-allauth"; + downloadPage = "https://github.com/pennersr/django-allauth"; homepage = "https://www.intenct.nl/projects/django-allauth"; license = licenses.mit; maintainers = with maintainers; [ derdennisop ]; From a4c356692514ebbf88c8d44f0dfeb9cfe078ea5e Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Wed, 11 Oct 2023 06:57:14 +0000 Subject: [PATCH 03/11] mailman: remove importlib-resources dependency mailman is incompatible with importlib-resources 6.x, and upstream have decided to remove the dependency. Fixes: 3c86835248f5 ("python3Packages.importlib-resources: 5.12.0 -> 6.0.1") Link: https://gitlab.com/mailman/mailman/-/issues/1093 --- pkgs/servers/mail/mailman/package.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/mail/mailman/package.nix b/pkgs/servers/mail/mailman/package.nix index 20c1771d089fc..9bb138ab1ee1a 100644 --- a/pkgs/servers/mail/mailman/package.nix +++ b/pkgs/servers/mail/mailman/package.nix @@ -30,7 +30,6 @@ buildPythonPackage rec { flufl_i18n flufl_lock gunicorn - importlib-resources lazr_config passlib requests @@ -52,6 +51,11 @@ buildPythonPackage rec { url = "https://gitlab.com/mailman/mailman/-/commit/9613154f3c04fa2383fbf017031ef263c291418d.patch"; sha256 = "0vyw87s857vfxbf7kihwb6w094xyxmxbi1bpdqi3ybjamjycp55r"; }) + (fetchpatch { + url = "https://gitlab.com/mailman/mailman/-/commit/5e4431af6bb7d672a7ed7e3329f8fac7812d47f8.patch"; + excludes = [ ".gitlab-ci.yml" ]; + hash = "sha256-y2AE9hU4Z1BpBlJywxMWiuRvltWkk+R9YgMkpemvlIo="; + }) ./log-stderr.patch ]; From ec58ad152a8970c74822890d5dc13f11b2fde627 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Wed, 11 Oct 2023 06:46:46 +0000 Subject: [PATCH 04/11] mailman: use sphinx from python3.pkgs At least since e161990d40c9 ("mailman: remove docutils input"), we rely on sphinx being a Python library package for our Python version, so that it propagates a compatible docutils. Taking sphinx from the top level therefore won't work right if it's using a different Python version. --- pkgs/servers/mail/mailman/package.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/servers/mail/mailman/package.nix b/pkgs/servers/mail/mailman/package.nix index 9bb138ab1ee1a..3350ad73611d8 100644 --- a/pkgs/servers/mail/mailman/package.nix +++ b/pkgs/servers/mail/mailman/package.nix @@ -2,7 +2,6 @@ , fetchpatch , python3 , fetchPypi -, sphinx , postfix , lynx }: From 21e3908ea35e5a98b0951a579f08b979d1909d9b Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Wed, 11 Oct 2023 20:04:37 +0000 Subject: [PATCH 05/11] nixos/mailman: ensure uwsgi uses mailman's python If they differ, uwsgi will fail to start, because it won't be able to find the appropriate libraries. --- nixos/modules/services/mail/mailman.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/mail/mailman.nix b/nixos/modules/services/mail/mailman.nix index ec2a19f58bb11..17129b3aef479 100644 --- a/nixos/modules/services/mail/mailman.nix +++ b/nixos/modules/services/mail/mailman.nix @@ -592,7 +592,7 @@ in { # Since the mailman-web settings.py obstinately creates a logs # dir in the cwd, change to the (writable) runtime directory before # starting uwsgi. - ExecStart = "${pkgs.coreutils}/bin/env -C $RUNTIME_DIRECTORY ${pkgs.uwsgi.override { plugins = ["python3"]; }}/bin/uwsgi --json ${uwsgiConfigFile}"; + ExecStart = "${pkgs.coreutils}/bin/env -C $RUNTIME_DIRECTORY ${pkgs.uwsgi.override { plugins = ["python3"]; python3 = webEnv.python; }}/bin/uwsgi --json ${uwsgiConfigFile}"; User = cfg.webUser; Group = "mailman"; RuntimeDirectory = "mailman-uwsgi"; From 516bafd406daec3fb8a801d0199bbb82cb237fd3 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Thu, 5 Oct 2023 18:09:01 +0000 Subject: [PATCH 06/11] mailmanPackages: pin to Python 3.10 We have Mailman's Python package set pinned to Django 3, which doesn't support Python 3.11. --- pkgs/top-level/all-packages.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index dfe9da5bf9b0e..e81687bf73cf0 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -26600,7 +26600,9 @@ with pkgs; mackerel-agent = callPackage ../servers/monitoring/mackerel-agent { }; - mailmanPackages = callPackage ../servers/mail/mailman { }; + mailmanPackages = callPackage ../servers/mail/mailman { + python3 = python310; + }; inherit (mailmanPackages) mailman mailman-hyperkitty; mailman-web = mailmanPackages.web; From 755ef90835c768bdb18aff9f6c96ccbb8667b290 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Mon, 9 Oct 2023 10:13:39 +0000 Subject: [PATCH 07/11] mailmanPackages: pin redis to 4.5.4 --- pkgs/servers/mail/mailman/python.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkgs/servers/mail/mailman/python.nix b/pkgs/servers/mail/mailman/python.nix index 7c6edbd5e9ea2..60e97313e4f6c 100644 --- a/pkgs/servers/mail/mailman/python.nix +++ b/pkgs/servers/mail/mailman/python.nix @@ -29,6 +29,15 @@ python3.override { hash = "sha256-AkYRPLJJLbh1xr5Wl0p8iTMzvybNkokchfYxUc7gnTQ="; }; }); + + # django-q tests fail with redis 5.0.0. + redis = super.redis.overridePythonAttrs ({ pname, ... }: rec { + version = "4.5.4"; + src = fetchPypi { + inherit pname version; + hash = "sha256-c+w12k2iZ9aEfkf2hzD91fYuLKaePvWIXGp4qTdMOJM="; + }; + }); }) overlay; From 0d910a60ef242e5827c64914a39b89f9ff6c3b70 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Tue, 10 Oct 2023 12:06:30 +0000 Subject: [PATCH 08/11] mailmanPackages: pin elasticsearch to 7.17.9 Hyperkitty's tests fail with 8.x, because it tries and fails to import elastic_transport. There are 7.17.10 and 7.17.11 releases, but for unknown reasons they're not available on PyPI. Link: https://github.com/elastic/elasticsearch-py/issues/2336 --- pkgs/servers/mail/mailman/python.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkgs/servers/mail/mailman/python.nix b/pkgs/servers/mail/mailman/python.nix index 60e97313e4f6c..e4f96db0224d1 100644 --- a/pkgs/servers/mail/mailman/python.nix +++ b/pkgs/servers/mail/mailman/python.nix @@ -20,6 +20,14 @@ python3.override { */ django = super.django_3; + elasticsearch = super.elasticsearch.overridePythonAttrs ({ pname, ... }: rec { + version = "7.17.9"; + src = fetchPypi { + inherit pname version; + hash = "sha256-ZsTs4q3+fMEg4rameYof1cd3rs+C7sObuVzvfPx+orM="; + }; + }); + # https://gitlab.com/mailman/hyperkitty/-/merge_requests/541 mistune = super.mistune.overridePythonAttrs (old: rec { version = "2.0.5"; From 92f2d1e99cdda163ec2838917ca5e1d5a6271c72 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Tue, 10 Oct 2023 10:08:33 +0000 Subject: [PATCH 09/11] python3.pkgs.django-mailman3: apply allauth fixes Fixes: 37ead196e0ba ("python311Packages.django-allauth: 0.54.0 -> 0.55.2") --- .../python-modules/django-mailman3/default.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkgs/development/python-modules/django-mailman3/default.nix b/pkgs/development/python-modules/django-mailman3/default.nix index 6639257740bc9..c68c554b9b9e6 100644 --- a/pkgs/development/python-modules/django-mailman3/default.nix +++ b/pkgs/development/python-modules/django-mailman3/default.nix @@ -1,6 +1,7 @@ { lib , buildPythonPackage , fetchPypi +, fetchpatch # propagates , django-gravatar2 @@ -24,6 +25,17 @@ buildPythonPackage rec { hash = "sha256-GpI1W0O9aJpLF/mcS23ktJDZsP69S2zQy7drOiWBnTM="; }; + patches = [ + (fetchpatch { + url = "https://gitlab.com/mailman/django-mailman3/-/commit/840d0d531a0813de9a30e72427e202aea21b40fe.patch"; + hash = "sha256-vltvsIP/SWpQZeXDUB+GWlTu+ghFMUqIT8i6CrYcmGo="; + }) + (fetchpatch { + url = "https://gitlab.com/mailman/django-mailman3/-/commit/25c55e31d28f2fa8eb23f0e83c12f9b0a05bfbf0.patch"; + hash = "sha256-ug5tBmnVfJTn5ufDDVg/cEtsZM59jQYJpQZV51T3qIc="; + }) + ]; + postPatch = '' substituteInPlace setup.py \ --replace 'django>=3.2,<4.2' 'django>=3.2,<4.3' From 91bbbbabe03f6ec4c1d953e0c13c9eb09925bc37 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Tue, 10 Oct 2023 12:07:10 +0000 Subject: [PATCH 10/11] mailmanPackages.hyperkitty: apply allauth fix --- pkgs/servers/mail/mailman/hyperkitty.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/servers/mail/mailman/hyperkitty.nix b/pkgs/servers/mail/mailman/hyperkitty.nix index 334a24b1ed003..ba2a894e12b89 100644 --- a/pkgs/servers/mail/mailman/hyperkitty.nix +++ b/pkgs/servers/mail/mailman/hyperkitty.nix @@ -1,6 +1,7 @@ { lib , python3 , fetchPypi +, fetchpatch }: with python3.pkgs; @@ -17,6 +18,10 @@ buildPythonPackage rec { patches = [ ./0001-Disable-broken-test_help_output-testcase.patch + (fetchpatch { + url = "https://gitlab.com/mailman/hyperkitty/-/commit/5bb394662882bfc73c3e877458da44343aa06922.patch"; + hash = "sha256-9vcY6nu3txDftH6aYpdh9qSrLzZceGjVFxuD1Ux18gw="; + }) ]; postPatch = '' From 09e0a34f92cb8145a8e1dd24b367af9ca9d4dbf9 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Tue, 10 Oct 2023 12:09:39 +0000 Subject: [PATCH 11/11] mailmanPackages.web: apply allauth fix --- pkgs/servers/mail/mailman/web.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/mail/mailman/web.nix b/pkgs/servers/mail/mailman/web.nix index ed5c267ad0c2d..21fa64085d57a 100644 --- a/pkgs/servers/mail/mailman/web.nix +++ b/pkgs/servers/mail/mailman/web.nix @@ -1,4 +1,4 @@ -{ lib, python3, fetchPypi +{ lib, python3, fetchPypi, fetchpatch , sassc, hyperkitty, postorius }: @@ -14,6 +14,13 @@ buildPythonPackage rec { sha256 = "sha256-UWdqrcx529r6kwgf0YEHiDrpZlGoUBR6OdYtHMTPMGY="; }; + patches = [ + (fetchpatch { + url = "https://gitlab.com/mailman/mailman-web/-/commit/448bba249d39c09c0cef5e059415cc07a3ce569c.patch"; + hash = "sha256-rs1vaV4YyLyJ0+EGY70CirvjArpGQr29DOTvgj68wgs="; + }) + ]; + postPatch = '' # Django is depended on transitively by hyperkitty and postorius, # and mailman_web has overly restrictive version bounds on it, so