Improve the upgrade-python script

ddev/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+***Added***:
+
+* Improve the upgrade-python script ([#16021](https://github.com/DataDog/integrations-core/pull/16021))
+
 ## 5.2.1 / 2023-10-12
 
 ***Fixed***:

ddev/src/ddev/cli/meta/scripts/upgrade_python.py

@@ -9,6 +9,7 @@
 
 if TYPE_CHECKING:
     from ddev.cli.application import Application
+    from ddev.src.ddev.validation.tracker import ValidationTracker
 
 
 @click.command('upgrade-python', short_help='Upgrade the Python version throughout the repository')
@@ -20,57 +21,200 @@ def upgrade_python(app: Application, version: str):
     \b
     `$ ddev meta scripts upgrade-python 3.11`
     """
-    import tomlkit
 
     from ddev.repo.constants import PYTHON_VERSION as old_version
 
     tracker = app.create_validation_tracker('Python upgrades')
 
     for target in app.repo.integrations.iter_testable(['all']):
-        config_file = target.path / 'hatch.toml'
-        test_config = tomlkit.parse(config_file.read_text())
+        update_hatch_file(app, target.path, version, old_version, tracker)
+        update_pyproject_file(target, version, old_version, tracker)
+        update_setup_file(target, version, old_version, tracker)
+
+    update_ci_files(app, version, old_version, tracker)
+
+    if app.repo.name == 'core':
+        update_ddev_pyproject_file(app, version, old_version, tracker)
+        update_constants_file(app, version, old_version, tracker)
+        update_ddev_template_files(app, version, old_version, tracker)
+        app.display_warning("Documentation files have not been updated. Please modify them manually.")
+
+    tracker.display()
+
+    if tracker.errors:  # no cov
+        app.abort()
+
+
+def update_ci_files(app: Application, new_version: str, old_version: str, tracker: ValidationTracker):
+    for file in (app.repo.path / ".github" / "workflows").glob("*.yml"):
         changed = False
+        content = file.read_text()
 
-        for env in test_config.get('envs', {}).values():
-            default_python = env.get('python', '')
-            if default_python == old_version:
-                env['python'] = version
-                tracker.success()
+        for pattern in ("python-version: '{}'", 'PYTHON_VERSION: "{}"', "'{}'"):
+            if pattern.format(old_version) in content:
+                content = content.replace(pattern.format(old_version), pattern.format(new_version))
                 changed = True
 
-            for variables in env.get('matrix', []):
-                pythons = variables.get('python', [])
-                for i, python in enumerate(pythons):
-                    if python == old_version:
-                        pythons[i] = version
-                        tracker.success()
-                        changed = True
-
-            for overrides in env.get('overrides', {}).get('matrix', {}).get('python', {}).values():
-                for override in overrides:
-                    pythons = override.get('if', [])
-                    for i, python in enumerate(pythons):
-                        if python == old_version:
-                            pythons[i] = version
-                            tracker.success()
-                            changed = True
-
         if changed:
-            config_file.write_text(tomlkit.dumps(test_config))
+            file.write_text(content)
+            tracker.success()
+
+
+def update_ddev_template_files(app: Application, new_version: str, old_version: str, tracker: ValidationTracker):
+    for check_type in ("check", "jmx", "logs"):
+        folder_path = (
+            app.repo.path
+            / 'datadog_checks_dev'
+            / 'datadog_checks'
+            / 'dev'
+            / 'tooling'
+            / 'templates'
+            / 'integration'
+            / check_type
+            / '{check_name}'
+        )
+        pyproject_file = folder_path / 'pyproject.toml'
+        changed = False
 
-    if app.repo.name == 'core':
-        constant_file = app.repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py'
+        if pyproject_file.is_file():
+            content = pyproject_file.read_text()
+
+            for pattern in ('requires-python = ">={}"', "Programming Language :: Python :: {}"):
+                if pattern.format(old_version) in content:
+                    content = content.replace(pattern.format(old_version), pattern.format(new_version))
+                    changed = True
+
+            if changed:
+                pyproject_file.write_text(content)
+                tracker.success()
+
+        if (folder_path / 'hatch.toml').is_file():
+            update_hatch_file(app, folder_path, new_version, old_version, tracker)
+
+
+def update_ddev_pyproject_file(app: Application, new_version: str, old_version: str, tracker: ValidationTracker):
+    import tomlkit
+
+    config_file = app.repo.path / 'ddev' / 'pyproject.toml'
+    config = tomlkit.parse(config_file.read_text())
+    changed = False
+    new_version = f"py{new_version.replace('.', '')}"
+    old_version = f"py{old_version.replace('.', '')}"
 
-        lines = constant_file.read_text().splitlines(keepends=True)
-        for i, line in enumerate(lines):
-            if line.startswith('PYTHON_VERSION = '):
-                lines[i] = line.replace(old_version, version)
+    if black_config := config.get('tool', {}).get('black', {}):
+        target_version = black_config.get('target-version', [])
+
+        for index, version in enumerate(target_version):
+            if version == old_version:
+                target_version[index] = new_version
+                tracker.success()
+                changed = True
                 break
 
-        constant_file.write_text(''.join(lines))
+    if ruff_config := config.get('tool', {}).get('ruff', {}):
+        if ruff_config.get('target-version') == old_version:
+            ruff_config['target-version'] = new_version
+            tracker.success()
+            changed = True
+
+    if changed:
+        config_file.write_text(tomlkit.dumps(config))
+
+
+def update_setup_file(target, new_version: str, old_version: str, tracker: ValidationTracker):
+    setup_file = target.path / 'setup.py'
+
+    if setup_file.is_file():
+        content = setup_file.read_text()
+
+        if f"Programming Language :: Python :: {old_version}" in content:
+            content = content.replace(
+                f"Programming Language :: Python :: {old_version}", f"Programming Language :: Python :: {new_version}"
+            )
+
+            setup_file.write_text(content)
+            tracker.success()
+
+
+def update_constants_file(app: Application, new_version: str, old_version: str, tracker: ValidationTracker):
+    constant_file = app.repo.path / 'ddev' / 'src' / 'ddev' / 'repo' / 'constants.py'
+
+    lines = constant_file.read_text().splitlines(keepends=True)
+    for i, line in enumerate(lines):
+        if line.startswith('PYTHON_VERSION = '):
+            lines[i] = line.replace(old_version, new_version)
+            break
+
+    constant_file.write_text(''.join(lines))
+    tracker.success()
+
+
+def update_pyproject_file(target, new_version: str, old_version: str, tracker: ValidationTracker):
+    import tomlkit
+
+    config_file = target.path / 'pyproject.toml'
+    config = tomlkit.parse(config_file.read_text())
+    changed = False
+
+    classifiers = config.get('project', {}).get('classifiers', [])
+    for index, classifier in enumerate(classifiers):
+        if classifier == f"Programming Language :: Python :: {old_version}":
+            classifiers[index] = f"Programming Language :: Python :: {new_version}"
+            changed = True
+            tracker.success()
+            break
+
+    if changed:
+        config_file.write_text(tomlkit.dumps(config))
+
+
+def update_hatch_file(app: Application, target_path, new_version: str, old_version: str, tracker: ValidationTracker):
+    import tomlkit
+
+    config_file = target_path / 'hatch.toml'
+    test_config = tomlkit.parse(config_file.read_text())
+    changed = False
+
+    for env in test_config.get('envs', {}).values():
+        if update_hatch_env(app, env, new_version, old_version, config_file, tracker):
+            changed = True
+
+    if changed:
+        config_file.write_text(tomlkit.dumps(test_config))
+
+
+def update_hatch_env(
+    app: Application, env, new_version: str, old_version: str, config_file, tracker: ValidationTracker
+) -> bool:
+    changed = False
+
+    default_python = env.get('python', '')
+    if default_python == old_version:
+        env['python'] = new_version
         tracker.success()
+        changed = True
 
-    tracker.display()
+    for variables in env.get('matrix', []):
+        pythons = variables.get('python', [])
+        for i, python in enumerate(pythons):
+            if python == old_version:
+                pythons[i] = new_version
+                tracker.success()
+                changed = True
 
-    if tracker.errors:  # no cov
-        app.abort()
+    for overrides in env.get('overrides', {}).get('matrix', {}).get('python', {}).values():
+        for override in overrides:
+            pythons = override.get('if', [])
+            for i, python in enumerate(pythons):
+                if python == old_version:
+                    pythons[i] = new_version
+                    tracker.success()
+                    changed = True
+
+    if isinstance(env.get('overrides', {}), dict):
+        for name in list(env.get('overrides', {}).get('name', {}).keys()):
+            if f"py{old_version}" in name:
+                # TODO I don't find a way to keep the exact same format when I modify this.
+                app.display_warning(f'An override has been found in {config_file}. Please manually update it.')
+
+    return changed

ddev/tests/cli/meta/scripts/conftest.py

@@ -42,6 +42,72 @@ def fake_repo(tmp_path_factory, config_file, ddev):
 [[envs.default.matrix]]
 python = ["2.7", "3.9"]
 
+""",
+    )
+
+    write_file(
+        repo_path / 'dummy',
+        'pyproject.toml',
+        """[project]
+name = "dummy"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: BSD License",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3.9",
+]
+""",
+    )
+
+    write_file(
+        repo_path / '.github' / 'workflows',
+        'build-ddev.yml',
+        """name: build ddev
+env:
+  APP_NAME: ddev
+  PYTHON_VERSION: "3.9"
+  PYOXIDIZER_VERSION: "0.24.0"
+""",
+    )
+
+    write_file(
+        repo_path / 'ddev',
+        'pyproject.toml',
+        """[tool.black]
+target-version = ["py39"]
+
+[tool.ruff]
+target-version = "py39"
+""",
+    )
+
+    write_file(
+        repo_path
+        / 'datadog_checks_dev'
+        / 'datadog_checks'
+        / 'dev'
+        / 'tooling'
+        / 'templates'
+        / 'integration'
+        / 'check'
+        / '{check_name}',
+        'pyproject.toml',
+        """[project]
+name = "dummy"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: BSD License",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3.9",
+]
 """,
     )
 

ddev/tests/cli/meta/scripts/test_upgrade_python.py

@@ -16,13 +16,39 @@ def test_upgrade_python(fake_repo, ddev):
     result = ddev('meta', 'scripts', 'upgrade-python', new_version)
 
     assert result.exit_code == 0, result.output
-    assert result.output == 'Python upgrades\n\nPassed: 2\n'
+    assert result.output.endswith('Python upgrades\n\nPassed: 7\n')
 
     contents = constant_file.read_text()
     assert f'PYTHON_VERSION = {old_version!r}' not in contents
     assert f'PYTHON_VERSION = {new_version!r}' in contents
 
+    ci_file = fake_repo.path / '.github' / 'workflows' / 'build-ddev.yml'
+    contents = ci_file.read_text()
+    assert f'PYTHON_VERSION: "{old_version}"' not in contents
+    assert f'PYTHON_VERSION: "{new_version}"' in contents
+
     hatch_file = fake_repo.path / 'dummy' / 'hatch.toml'
     contents = hatch_file.read_text()
     assert f'python = ["2.7", "{old_version}"]' not in contents
     assert f'python = ["2.7", "{new_version}"]' in contents
+
+    pyproject_file = fake_repo.path / 'dummy' / 'pyproject.toml'
+    contents = pyproject_file.read_text()
+    assert f'Programming Language :: Python :: {old_version}' not in contents
+    assert f'Programming Language :: Python :: {new_version}' in contents
+
+    template_file = (
+        fake_repo.path
+        / 'datadog_checks_dev'
+        / 'datadog_checks'
+        / 'dev'
+        / 'tooling'
+        / 'templates'
+        / 'integration'
+        / 'check'
+        / '{check_name}'
+        / 'pyproject.toml'
+    )
+    contents = template_file.read_text()
+    assert f'Programming Language :: Python :: {old_version}' not in contents
+    assert f'Programming Language :: Python :: {new_version}' in contents