From d5205d4001f0e83a661d9faba61c92babf419323 Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Mon, 25 Nov 2024 14:32:51 +1100 Subject: [PATCH 01/10] Add Jetbrains IDE files to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dd3ad7275..626350fbd 100644 --- a/.gitignore +++ b/.gitignore @@ -91,6 +91,7 @@ ENV/ env.bak/ venv.bak/ .vscode +.idea # Rope project settings .ropeproject From daeb1474fa87ed447d307d5d4a2302cfd2b19789 Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Mon, 25 Nov 2024 14:33:07 +1100 Subject: [PATCH 02/10] Pin dependencies during hatch builds --- hatch_build.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 23 +++++++++----- 2 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 hatch_build.py diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 000000000..3fc4f15cd --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,83 @@ +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.metadata.plugin.interface import MetadataHookInterface + +import tomlkit +import pathlib + + +class PinVersionMetadataHook(MetadataHookInterface): + """Update the dependency versions so they are pinned metadata of the wheel artefact produced by a wheel.""" + def update(self, metadata): + change_count = 0 + # Overlay the pinned dependencies, replacing dependencies if they already exist. + for pinned_dep in self.config["config"]["pinned_dependencies"]: + pinned_dep_name = pinned_dep.split(" ")[0] + for dep in metadata["dependencies"]: + dep_name = dep.split(" ")[0] + # If the dependency is already in the list, replace it with the pinned version. + if dep_name == pinned_dep_name: + index = metadata["dependencies"].index(dep) + metadata["dependencies"][index] = pinned_dep + change_count += 1 + break + print(f"Updated {change_count} dependencies to pinned versions in hatch's metadata " + f"for building wheel artefact metadata.") + + +class PinMetadataBuildHook(BuildHookInterface): + """Update the dependency versions so they are pinned in the pyproject.toml of the sdist artefact produced by a build.""" + PLUGIN_NAME = "pin-during-build" + + original_dependencies: list = None + made_changes: bool = False + + def initialize(self, version, build_data): + # Get the pinned dependencies, this is a list of package specifier strings that are the dependencies to pin. + pinned_dependencies = self.metadata.hatch.metadata.hook_config["custom"]["config"]["pinned_dependencies"] + + # Load the toml file + pyproject_file = pathlib.Path("pyproject.toml") + toml_data = tomlkit.loads(pyproject_file.read_text()) + + # Get the dependencies from the toml file, this is a list of package specifier strings. + dependencies = toml_data["project"]["dependencies"] + + # Save the original dependencies for later. + self.original_dependencies = dependencies.copy() + + change_count = 0 + + # Overlay the pinned dependencies, replacing dependencies if they already exist. + for pinned_dep in pinned_dependencies: + pinned_dep_name = pinned_dep.split(" ")[0] + for dep in dependencies: + dep_name = dep.split(" ")[0] + # If the dependency is already in the list, replace it with the pinned version. + if dep_name == pinned_dep_name: + index = dependencies.index(dep) + dependencies[index] = pinned_dep + change_count += 1 + break + + # If we made any changes, write the changes back to the pyroject.toml file for the build. + if change_count > 0: + # Write the changes back to the file. + pyproject_file.write_text(tomlkit.dumps(toml_data)) + print(f"Updated {change_count} dependencies to pinned versions in the pyproject.toml " + f"for building sdist artefact.") + # Set a flag to restore the original dependencies after the build. + self.made_changes = True + + # Update the build data with the pinned dependencies so it populates the METADATA file in the wheel artefact. + build_data["dependencies"] = dependencies + + return super().initialize(version, build_data) + + def finalize(self, version, build_data, artefact_path): + # If we have made changes restore the original dependencies after the build to keep the git repo clean. + if self.made_changes: + pyproject_file = pathlib.Path("pyproject.toml") + toml_data = tomlkit.loads(pyproject_file.read_text()) + toml_data["project"]["dependencies"] = self.original_dependencies + pyproject_file.write_text(tomlkit.dumps(toml_data)) + print("Restored original dependencies in pyproject.toml file after building.") diff --git a/pyproject.toml b/pyproject.toml index 89859465e..1dcbb4366 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,7 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ] -dependencies = [ - "xarray", - "pandas", - "scipy", - "bottleneck", - "scikit-learn", -] +dependencies = ["xarray", "pandas", "scipy", "bottleneck", "scikit-learn"] [project.optional-dependencies] dev = [ @@ -77,6 +71,21 @@ exclude = [ "/docs/", "/tests/" ] +dependencies = [ + "tomlkit", +] + +[tool.hatch.build.hooks.custom] +override = true # This is required to activate the build hook. + +[tool.hatch.metadata.hooks.custom.config] +pinned_dependencies = [ + "xarray ~= 2024.1", + "pandas ~= 2.0", + "scipy ~= 1.1", + "bottleneck ~= 1.3", + "scikit-learn ~= 1.4", +] [tool.hatch.version] path = "src/scores/__init__.py" From a73b6267cf45c8ff2b4e32f77caf657b59854d1f Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Mon, 25 Nov 2024 19:38:15 +1100 Subject: [PATCH 03/10] Add more extensive docstrings. --- hatch_build.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index 3fc4f15cd..b3b35712d 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -6,7 +6,12 @@ class PinVersionMetadataHook(MetadataHookInterface): - """Update the dependency versions so they are pinned metadata of the wheel artefact produced by a wheel.""" + """ + Update the dependency versions so they are pinned metadata of the wheel artefact produced by a wheel. + + This is invoked by `hatch` when `hatch build` is run as part of constructing the package metadata + before it moves on to executing the next appropriate build step, be it sdist or wheels. + """ def update(self, metadata): change_count = 0 # Overlay the pinned dependencies, replacing dependencies if they already exist. @@ -25,7 +30,14 @@ def update(self, metadata): class PinMetadataBuildHook(BuildHookInterface): - """Update the dependency versions so they are pinned in the pyproject.toml of the sdist artefact produced by a build.""" + """ + Update the dependency versions so they are pinned in the pyproject.toml of the sdist artefact produced by a build. + + When `hatch` performs a build in response to running the `hatch build` build command, its plugin system + will call the initialize method of this hook before the build process starts. This allows the hook to + update the dependencies in the pyproject.toml file before the build starts. The finalize method is called + after the build has completed to restore the original dependencies. + """ PLUGIN_NAME = "pin-during-build" original_dependencies: list = None From 1d5fcc30ab4ff4bd636ba960563c2e133a9e4515 Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Tue, 26 Nov 2024 03:25:15 +1100 Subject: [PATCH 04/10] Improve logic, info text, naming, and comments. --- hatch_build.py | 36 +++++++++++++++++++++++++++++------- pyproject.toml | 2 ++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index b3b35712d..27efa5f50 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -5,7 +5,7 @@ import pathlib -class PinVersionMetadataHook(MetadataHookInterface): +class PinVersionsMetadataHook(MetadataHookInterface): """ Update the dependency versions so they are pinned metadata of the wheel artefact produced by a wheel. @@ -25,11 +25,11 @@ def update(self, metadata): metadata["dependencies"][index] = pinned_dep change_count += 1 break - print(f"Updated {change_count} dependencies to pinned versions in hatch's metadata " - f"for building wheel artefact metadata.") + print(f"Updated {change_count} dependencies in hatch's internal dependency metadata" + f" to pinned versions from config.") -class PinMetadataBuildHook(BuildHookInterface): +class PinVersionsBuildHook(BuildHookInterface): """ Update the dependency versions so they are pinned in the pyproject.toml of the sdist artefact produced by a build. @@ -44,6 +44,21 @@ class PinMetadataBuildHook(BuildHookInterface): made_changes: bool = False def initialize(self, version, build_data): + # To avoid affecting the wheel METADATA file, we only run this hook's logic when + # the build system is building a sdist artefact. + + print() + if self.build_config.builder.PLUGIN_NAME != "sdist": + print("Building wheel artefact. ") + print("This uses hatch's internal dependency metadata.") + print() + return + else: + print("Building sdist artefact.") + print("This does not use hatch's internal dependency metadata.") + print() + print("Updating pyproject.toml to contain the correct versions of pinned dependencies.") + # Get the pinned dependencies, this is a list of package specifier strings that are the dependencies to pin. pinned_dependencies = self.metadata.hatch.metadata.hook_config["custom"]["config"]["pinned_dependencies"] @@ -59,7 +74,8 @@ def initialize(self, version, build_data): change_count = 0 - # Overlay the pinned dependencies, replacing dependencies if they already exist. + # Update dependencies with pinned versions if they are in the configuration and + # only if the un-pinned version is present. for pinned_dep in pinned_dependencies: pinned_dep_name = pinned_dep.split(" ")[0] for dep in dependencies: @@ -76,9 +92,11 @@ def initialize(self, version, build_data): # Write the changes back to the file. pyproject_file.write_text(tomlkit.dumps(toml_data)) print(f"Updated {change_count} dependencies to pinned versions in the pyproject.toml " - f"for building sdist artefact.") + f"which will be incorporated into the final sdist artefact.") # Set a flag to restore the original dependencies after the build. self.made_changes = True + else: + print("No dependencies were changed to pinned version.") # Update the build data with the pinned dependencies so it populates the METADATA file in the wheel artefact. build_data["dependencies"] = dependencies @@ -92,4 +110,8 @@ def finalize(self, version, build_data, artefact_path): toml_data = tomlkit.loads(pyproject_file.read_text()) toml_data["project"]["dependencies"] = self.original_dependencies pyproject_file.write_text(tomlkit.dumps(toml_data)) - print("Restored original dependencies in pyproject.toml file after building.") + print() + print("Build of sdist artefact completed.") + print("Restored original dependencies in pyproject.toml file.") + print("This puts the git repo back to its original state.") + print() diff --git a/pyproject.toml b/pyproject.toml index 1dcbb4366..ce610ad18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,8 @@ dependencies = [ override = true # This is required to activate the build hook. [tool.hatch.metadata.hooks.custom.config] +# The pinned versions of these dependencies will be subsituted during package builds. +# Pinned versions will only be substitued for packages that are already in this package's dependencies list. pinned_dependencies = [ "xarray ~= 2024.1", "pandas ~= 2.0", From 225a714fd1a5341daff258154815696caeabffca Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Tue, 26 Nov 2024 09:00:36 +1100 Subject: [PATCH 05/10] Docstring improvements --- hatch_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index 27efa5f50..2ce37c38a 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -31,7 +31,8 @@ def update(self, metadata): class PinVersionsBuildHook(BuildHookInterface): """ - Update the dependency versions so they are pinned in the pyproject.toml of the sdist artefact produced by a build. + Temporarily edit the dependencies in the pyproject.toml file to use the desired pinned versions + when using `hatch` to build a package for release. When `hatch` performs a build in response to running the `hatch build` build command, its plugin system will call the initialize method of this hook before the build process starts. This allows the hook to From d0a0433106a7886d4661a021ce7a78cda9178118 Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Tue, 26 Nov 2024 11:36:27 +1100 Subject: [PATCH 06/10] Add zenodo entry --- .zenodo.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.zenodo.json b/.zenodo.json index 33aba14c6..40759c31a 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -56,7 +56,12 @@ "orcid": "https://orcid.org/0000-0002-5545-1736", "affiliation": "CSIRO, Australia", "name": "Shrestha, Durga" - } + }, + { + "orcid": "https://orcid.org/0009-0002-8569-1439", + "affiliation": "Independent Contributor, Australia", + "name": "Bishop, Sam" + } ], "license": "Apache-2.0", From f08215ed35a1726faba825727204a1753c4c120c Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Tue, 26 Nov 2024 11:36:46 +1100 Subject: [PATCH 07/10] Reword docstring --- hatch_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index 2ce37c38a..c1a97a30a 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -7,7 +7,7 @@ class PinVersionsMetadataHook(MetadataHookInterface): """ - Update the dependency versions so they are pinned metadata of the wheel artefact produced by a wheel. + Update the dependency metadata that `hatch` uses to set the dependencies of wheel artefacts. This is invoked by `hatch` when `hatch build` is run as part of constructing the package metadata before it moves on to executing the next appropriate build step, be it sdist or wheels. From 2e9965ed11ea0b66e151835e7b3d48242b87ea11 Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Tue, 26 Nov 2024 11:36:58 +1100 Subject: [PATCH 08/10] Formatted with black --- hatch_build.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index c1a97a30a..e9ee33bb1 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -12,6 +12,7 @@ class PinVersionsMetadataHook(MetadataHookInterface): This is invoked by `hatch` when `hatch build` is run as part of constructing the package metadata before it moves on to executing the next appropriate build step, be it sdist or wheels. """ + def update(self, metadata): change_count = 0 # Overlay the pinned dependencies, replacing dependencies if they already exist. @@ -25,8 +26,10 @@ def update(self, metadata): metadata["dependencies"][index] = pinned_dep change_count += 1 break - print(f"Updated {change_count} dependencies in hatch's internal dependency metadata" - f" to pinned versions from config.") + print( + f"Updated {change_count} dependencies in hatch's internal dependency metadata" + f" to pinned versions from config." + ) class PinVersionsBuildHook(BuildHookInterface): @@ -39,6 +42,7 @@ class PinVersionsBuildHook(BuildHookInterface): update the dependencies in the pyproject.toml file before the build starts. The finalize method is called after the build has completed to restore the original dependencies. """ + PLUGIN_NAME = "pin-during-build" original_dependencies: list = None @@ -92,8 +96,10 @@ def initialize(self, version, build_data): if change_count > 0: # Write the changes back to the file. pyproject_file.write_text(tomlkit.dumps(toml_data)) - print(f"Updated {change_count} dependencies to pinned versions in the pyproject.toml " - f"which will be incorporated into the final sdist artefact.") + print( + f"Updated {change_count} dependencies to pinned versions in the pyproject.toml " + f"which will be incorporated into the final sdist artefact." + ) # Set a flag to restore the original dependencies after the build. self.made_changes = True else: From 9c4d52d269e40cf16cf9e1a560c80d9bbf265154 Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Tue, 26 Nov 2024 11:49:50 +1100 Subject: [PATCH 09/10] Sort imports with isort --- hatch_build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index e9ee33bb1..ac3aa83ac 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -1,8 +1,8 @@ -from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.metadata.plugin.interface import MetadataHookInterface +import pathlib import tomlkit -import pathlib +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.metadata.plugin.interface import MetadataHookInterface class PinVersionsMetadataHook(MetadataHookInterface): From df7496d6d36598e5d69bd70a99348b3be2a4e910 Mon Sep 17 00:00:00 2001 From: Sam Bishop Date: Tue, 26 Nov 2024 11:51:08 +1100 Subject: [PATCH 10/10] Fix type hint. --- hatch_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hatch_build.py b/hatch_build.py index ac3aa83ac..0fe9410da 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -1,4 +1,5 @@ import pathlib +from typing import Optional import tomlkit from hatchling.builders.hooks.plugin.interface import BuildHookInterface @@ -45,7 +46,7 @@ class PinVersionsBuildHook(BuildHookInterface): PLUGIN_NAME = "pin-during-build" - original_dependencies: list = None + original_dependencies: Optional[list] = None made_changes: bool = False def initialize(self, version, build_data):