fix: setup.py update using script (#77)

* fix: setup.py update using script * fix: fix issues preventing setup.py updates * no longer need backports.zoneinfo now that the project supports python version >= 3.9 * fix quality problems flagged by pylint --------- Co-authored-by: Justin Hynes <[email protected]>
openedx · Jan 30, 2025 · 07e5051 · 07e5051
1 parent a715bb5
commit 07e5051
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 38 deletions.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -2,3 +2,4 @@ include LICENSE
 include README.md
 include requirements/base.in
 include requirements/test.in
+include requirements/constraints.txt
diff --git a/requirements/base.in b/requirements/base.in
@@ -6,4 +6,3 @@ django-model-utils                         # Provides TimeStampedModel abstract
 edx-opaque-keys>=0.2.1                     # Create and introspect course and xblock identities
 six                                        # Utilities for supporting Python 2 & 3 in the same codebase
 setuptools
-backports.zoneinfo; python_version<'3.9'
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -6,7 +6,7 @@
 #
 asgiref==3.8.1
     # via django
-django==4.2.17
+django==4.2.18
     # via
     #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
     #   -r requirements/base.in
@@ -19,7 +19,7 @@ edx-opaque-keys==2.11.0
     # via -r requirements/base.in
 pbr==6.1.0
     # via stevedore
-pymongo==4.10.1
+pymongo==4.11
     # via edx-opaque-keys
 six==1.17.0
     # via -r requirements/base.in

diff --git a/requirements/ci.txt b/requirements/ci.txt
@@ -4,15 +4,15 @@
 #
 #    make upgrade
 #
-cachetools==5.5.0
+cachetools==5.5.1
     # via tox
 chardet==5.2.0
     # via tox
 colorama==0.4.6
     # via tox
 distlib==0.3.9
     # via virtualenv
-filelock==3.16.1
+filelock==3.17.0
     # via
     #   tox
     #   virtualenv
@@ -26,9 +26,9 @@ platformdirs==4.3.6
     #   virtualenv
 pluggy==1.5.0
     # via tox
-pyproject-api==1.8.0
+pyproject-api==1.9.0
     # via tox
-tox==4.23.2
+tox==4.24.1
     # via -r requirements/ci.in
-virtualenv==20.28.1
+virtualenv==20.29.1
     # via tox
diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -17,7 +17,7 @@ build==1.2.2.post1
     # via
     #   -r requirements/pip-tools.txt
     #   pip-tools
-cachetools==5.5.0
+cachetools==5.5.1
     # via
     #   -r requirements/ci.txt
     #   tox
@@ -37,7 +37,7 @@ click-log==0.4.0
     # via
     #   -r requirements/quality.txt
     #   edx-lint
-code-annotations==2.1.0
+code-annotations==2.2.0
     # via
     #   -r requirements/quality.txt
     #   edx-lint
@@ -57,7 +57,7 @@ distlib==0.3.9
     # via
     #   -r requirements/ci.txt
     #   virtualenv
-django==4.2.17
+django==4.2.18
     # via
     #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
     #   -r requirements/quality.txt
@@ -68,11 +68,11 @@ dnspython==2.7.0
     # via
     #   -r requirements/quality.txt
     #   pymongo
-edx-lint==5.4.1
+edx-lint==5.6.0
     # via -r requirements/quality.txt
 edx-opaque-keys==2.11.0
     # via -r requirements/quality.txt
-filelock==3.16.1
+filelock==3.17.0
     # via
     #   -r requirements/ci.txt
     #   tox
@@ -81,7 +81,7 @@ iniconfig==2.0.0
     # via
     #   -r requirements/quality.txt
     #   pytest
-isort==5.13.2
+isort==6.0.0
     # via
     #   -r requirements/quality.txt
     #   pylint
@@ -129,7 +129,7 @@ pycodestyle==2.12.1
     # via -r requirements/quality.txt
 pydocstyle==6.3.0
     # via -r requirements/quality.txt
-pylint==3.3.3
+pylint==3.3.4
     # via
     #   -r requirements/quality.txt
     #   edx-lint
@@ -149,11 +149,11 @@ pylint-plugin-utils==0.8.2
     #   -r requirements/quality.txt
     #   pylint-celery
     #   pylint-django
-pymongo==4.10.1
+pymongo==4.11
     # via
     #   -r requirements/quality.txt
     #   edx-opaque-keys
-pyproject-api==1.8.0
+pyproject-api==1.9.0
     # via
     #   -r requirements/ci.txt
     #   tox
@@ -204,13 +204,13 @@ tomlkit==0.13.2
     # via
     #   -r requirements/quality.txt
     #   pylint
-tox==4.23.2
+tox==4.24.1
     # via -r requirements/ci.txt
 typing-extensions==4.12.2
     # via
     #   -r requirements/quality.txt
     #   edx-opaque-keys
-virtualenv==20.28.1
+virtualenv==20.29.1
     # via
     #   -r requirements/ci.txt
     #   tox

diff --git a/requirements/quality.txt b/requirements/quality.txt
@@ -20,7 +20,7 @@ click==8.1.8
     #   edx-lint
 click-log==0.4.0
     # via edx-lint
-code-annotations==2.1.0
+code-annotations==2.2.0
     # via
     #   -r requirements/test.txt
     #   edx-lint
@@ -30,7 +30,7 @@ coverage[toml]==7.6.10
     #   pytest-cov
 dill==0.3.9
     # via pylint
-django==4.2.17
+django==4.2.18
     # via
     #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
     #   -r requirements/test.txt
@@ -41,15 +41,15 @@ dnspython==2.7.0
     # via
     #   -r requirements/test.txt
     #   pymongo
-edx-lint==5.4.1
+edx-lint==5.6.0
     # via -r requirements/quality.in
 edx-opaque-keys==2.11.0
     # via -r requirements/test.txt
 iniconfig==2.0.0
     # via
     #   -r requirements/test.txt
     #   pytest
-isort==5.13.2
+isort==6.0.0
     # via
     #   -r requirements/quality.in
     #   pylint
@@ -81,7 +81,7 @@ pycodestyle==2.12.1
     # via -r requirements/quality.in
 pydocstyle==6.3.0
     # via -r requirements/quality.in
-pylint==3.3.3
+pylint==3.3.4
     # via
     #   edx-lint
     #   pylint-celery
@@ -95,7 +95,7 @@ pylint-plugin-utils==0.8.2
     # via
     #   pylint-celery
     #   pylint-django
-pymongo==4.10.1
+pymongo==4.11
     # via
     #   -r requirements/test.txt
     #   edx-opaque-keys

diff --git a/requirements/test.txt b/requirements/test.txt
@@ -10,7 +10,7 @@ asgiref==3.8.1
     #   django
 click==8.1.8
     # via code-annotations
-code-annotations==2.1.0
+code-annotations==2.2.0
     # via -r requirements/test.in
 coverage[toml]==7.6.10
     # via
@@ -42,7 +42,7 @@ pbr==6.1.0
     #   stevedore
 pluggy==1.5.0
     # via pytest
-pymongo==4.10.1
+pymongo==4.11
     # via
     #   -r requirements/base.txt
     #   edx-opaque-keys

diff --git a/setup.py b/setup.py
@@ -29,27 +29,95 @@ def load_requirements(*requirements_paths):
     """
     Load all requirements from the specified requirements files.
 
-    Returns:
-        list: Requirements file relative path strings
+    Requirements will include any constraints from files specified
+    with -c in the requirements files.
+    Returns a list of requirement strings.
     """
-    requirements = set()
+    # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why.
+
+    # e.g. {"django": "Django", "confluent-kafka": "confluent_kafka[avro]"}
+    by_canonical_name = {}
+
+    def check_name_consistent(package):
+        """
+        Raise exception if package is named different ways.
+
+        This ensures that packages are named consistently so we can match
+        constraints to packages. It also ensures that if we require a package
+        with extras we don't constrain it without mentioning the extras (since
+        that too would interfere with matching constraints.)
+        """
+        canonical = package.lower().replace('_', '-').split('[')[0]
+        seen_spelling = by_canonical_name.get(canonical)
+        if seen_spelling is None:
+            by_canonical_name[canonical] = package
+        elif seen_spelling != package:
+            raise Exception(
+                f'Encountered both "{seen_spelling}" and "{package}" in requirements '
+                'and constraints files; please use just one or the other.'
+            )
+
+    requirements = {}
+    constraint_files = set()
+
+    # groups "pkg<=x.y.z,..." into ("pkg", "<=x.y.z,...")
+    re_package_name_base_chars = r"a-zA-Z0-9\-_."  # chars allowed in base package name
+    # Two groups: name[maybe,extras], and optionally a constraint
+    requirement_line_regex = re.compile(
+        r"([%s]+(?:\[[%s,\s]+\])?)([<>=][^#\s]+)?"  # pylint: disable=consider-using-f-string
+        % (re_package_name_base_chars, re_package_name_base_chars)
+    )
+
+    def add_version_constraint_or_raise(current_line, current_requirements, add_if_not_present):
+        regex_match = requirement_line_regex.match(current_line)
+        if regex_match:
+            package = regex_match.group(1)
+            version_constraints = regex_match.group(2)
+            check_name_consistent(package)
+            existing_version_constraints = current_requirements.get(package, None)
+            # It's fine to add constraints to an unconstrained package,
+            # but raise an error if there are already constraints in place.
+            if existing_version_constraints and existing_version_constraints != version_constraints:
+                raise BaseException(f'Multiple constraint definitions found for {package}:'
+                                    f' "{existing_version_constraints}" and "{version_constraints}".'
+                                    f'Combine constraints into one location with {package}'
+                                    f'{existing_version_constraints},{version_constraints}.')
+            if add_if_not_present or package in current_requirements:
+                current_requirements[package] = version_constraints
+
+    # Read requirements from .in files and store the path to any
+    # constraint files that are pulled in.
     for path in requirements_paths:
-        requirements.update(
-            line.split('#')[0].strip()
-            for line in open(path, encoding='utf-8').readlines()  # pylint: disable=consider-using-with
-            if is_requirement(line.strip())
-        )
-    return list(requirements)
+        with open(path, encoding='utf-8') as reqs:
+            for line in reqs:
+                if is_requirement(line):
+                    add_version_constraint_or_raise(line, requirements, True)
+                if line and line.startswith('-c') and not line.startswith('-c http'):
+                    constraint_files.add(os.path.dirname(path) + '/' + line.split('#')[0].replace('-c', '').strip())
+
+    # process constraint files: add constraints to existing requirements
+    for constraint_file in constraint_files:
+        with open(constraint_file, encoding='utf-8') as reader:
+            for line in reader:
+                if is_requirement(line):
+                    add_version_constraint_or_raise(line, requirements, False)
+
+    # process back into list of pkg><=constraints strings
+    constrained_requirements = [f'{pkg}{version or ""}' for (pkg, version) in sorted(requirements.items())]
+    return constrained_requirements
 
 
 def is_requirement(line):
     """
     Return True if the requirement line is a package requirement.
 
     Returns:
-        bool: True if the line is not blank, a comment, a URL, or an included file
+        bool: True if the line is not blank, a comment,
+        a URL, or an included file
     """
-    return line and not line.startswith(('-r', '#', '-e', 'git+', '-c'))
+    # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why
+
+    return line and line.strip() and not line.startswith(('-r', '#', '-e', 'git+', '-c'))
 
 
 if sys.argv[-1] == 'tag':