[WIP] Refactoring on Source Parsing Related Functions from #316

conda_lock/conda_lock.py

@@ -63,7 +63,6 @@
 except ImportError:
     PIP_SUPPORT = False
 from conda_lock.lockfile import (
-    Dependency,
     GitMeta,
     InputMeta,
     LockedDependency,
@@ -76,12 +75,8 @@
     write_conda_lock_file,
 )
 from conda_lock.lookup import set_lookup_location
-from conda_lock.src_parser import LockSpecification, aggregate_lock_specs
-from conda_lock.src_parser.environment_yaml import parse_environment_file
-from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file
-from conda_lock.src_parser.pyproject_toml import parse_pyproject_toml
+from conda_lock.src_parser import LockSpecification, make_lock_spec
 from conda_lock.virtual_package import (
-    FakeRepoData,
     default_virtual_package_repodata,
     virtual_package_repo_from_specification,
 )
@@ -114,8 +109,6 @@
     sys.exit(1)
 
 
-DEFAULT_PLATFORMS = ["osx-64", "linux-64", "win-64"]
-
 KIND_EXPLICIT: Literal["explicit"] = "explicit"
 KIND_LOCK: Literal["lock"] = "lock"
 KIND_ENV: Literal["env"] = "env"
@@ -242,44 +235,6 @@ def fn_to_dist_name(fn: str) -> str:
     return fn
 
 
-def make_lock_spec(
-    *,
-    src_files: List[pathlib.Path],
-    virtual_package_repo: FakeRepoData,
-    channel_overrides: Optional[Sequence[str]] = None,
-    platform_overrides: Optional[Sequence[str]] = None,
-    required_categories: Optional[AbstractSet[str]] = None,
-) -> LockSpecification:
-    """Generate the lockfile specs from a set of input src_files.  If required_categories is set filter out specs that do not match those"""
-    lock_specs = parse_source_files(
-        src_files=src_files, platform_overrides=platform_overrides
-    )
-
-    lock_spec = aggregate_lock_specs(lock_specs)
-    lock_spec.virtual_package_repo = virtual_package_repo
-    lock_spec.channels = (
-        [Channel.from_string(co) for co in channel_overrides]
-        if channel_overrides
-        else lock_spec.channels
-    )
-    lock_spec.platforms = (
-        list(platform_overrides) if platform_overrides else lock_spec.platforms
-    ) or list(DEFAULT_PLATFORMS)
-
-    if required_categories is not None:
-
-        def dep_has_category(d: Dependency, categories: AbstractSet[str]) -> bool:
-            return d.category in categories
-
-        lock_spec.dependencies = [
-            d
-            for d in lock_spec.dependencies
-            if dep_has_category(d, categories=required_categories)
-        ]
-
-    return lock_spec
-
-
 def make_lock_files(
     *,
     conda: PathLike,
@@ -357,6 +312,7 @@ def make_lock_files(
             platform_overrides=platform_overrides,
             virtual_package_repo=virtual_package_repo,
             required_categories=required_categories if filter_categories else None,
+            pip_support=PIP_SUPPORT,
         )
         lock_content: Optional[Lockfile] = None
 
@@ -866,42 +822,6 @@ def create_lockfile_from_spec(
     )
 
 
-def parse_source_files(
-    src_files: List[pathlib.Path],
-    platform_overrides: Optional[Sequence[str]],
-) -> List[LockSpecification]:
-    """
-    Parse a sequence of dependency specifications from source files
-
-    Parameters
-    ----------
-    src_files :
-        Files to parse for dependencies
-    platform_overrides :
-        Target platforms to render environment.yaml and meta.yaml files for
-    """
-    desired_envs: List[LockSpecification] = []
-    for src_file in src_files:
-        if src_file.name == "meta.yaml":
-            desired_envs.append(
-                parse_meta_yaml_file(
-                    src_file, list(platform_overrides or DEFAULT_PLATFORMS)
-                )
-            )
-        elif src_file.name == "pyproject.toml":
-            desired_envs.append(parse_pyproject_toml(src_file))
-        else:
-            desired_envs.append(
-                parse_environment_file(
-                    src_file,
-                    platform_overrides,
-                    default_platforms=DEFAULT_PLATFORMS,
-                    pip_support=PIP_SUPPORT,
-                )
-            )
-    return desired_envs
-
-
 def _add_auth_to_line(line: str, auth: Dict[str, str]) -> str:
     matching_auths = [a for a in auth if a in line]
     if not matching_auths:

conda_lock/src_parser/__init__.py

@@ -5,7 +5,7 @@
 import typing
 
 from itertools import chain
-from typing import Dict, List, Optional, Tuple, Union
+from typing import AbstractSet, Dict, List, Optional, Sequence, Tuple, Union
 
 from pydantic import BaseModel, validator
 from typing_extensions import Literal
@@ -17,6 +17,8 @@
 from conda_lock.virtual_package import FakeRepoData
 
 
+DEFAULT_PLATFORMS = ["osx-64", "linux-64", "win-64"]
+
 logger = logging.getLogger(__name__)
 
 
@@ -136,3 +138,85 @@ def aggregate_lock_specs(
         platforms=ordered_union(lock_spec.platforms or [] for lock_spec in lock_specs),
         sources=ordered_union(lock_spec.sources or [] for lock_spec in lock_specs),
     )
+
+
+def parse_source_files(
+    src_files: List[pathlib.Path],
+    platform_overrides: Optional[Sequence[str]],
+    pip_support: bool = True,
+) -> List[LockSpecification]:
+    """
+    Parse a sequence of dependency specifications from source files
+
+    Parameters
+    ----------
+    src_files :
+        Files to parse for dependencies
+    platform_overrides :
+        Target platforms to render environment.yaml and meta.yaml files for
+    """
+    from conda_lock.src_parser.environment_yaml import parse_environment_file
+    from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file
+    from conda_lock.src_parser.pyproject_toml import parse_pyproject_toml
+
+    desired_envs: List[LockSpecification] = []
+    for src_file in src_files:
+        if src_file.name == "meta.yaml":
+            desired_envs.append(
+                parse_meta_yaml_file(
+                    src_file, list(platform_overrides or DEFAULT_PLATFORMS)
+                )
+            )
+        elif src_file.name == "pyproject.toml":
+            desired_envs.append(parse_pyproject_toml(src_file))
+        else:
+            desired_envs.append(
+                parse_environment_file(
+                    src_file,
+                    platform_overrides,
+                    default_platforms=DEFAULT_PLATFORMS,
+                    pip_support=pip_support,
+                )
+            )
+    return desired_envs
+
+
+def make_lock_spec(
+    *,
+    src_files: List[pathlib.Path],
+    virtual_package_repo: FakeRepoData,
+    channel_overrides: Optional[Sequence[str]] = None,
+    platform_overrides: Optional[Sequence[str]] = None,
+    required_categories: Optional[AbstractSet[str]] = None,
+    pip_support: bool = True,
+) -> LockSpecification:
+    """Generate the lockfile specs from a set of input src_files.  If required_categories is set filter out specs that do not match those"""
+    lock_specs = parse_source_files(
+        src_files=src_files,
+        platform_overrides=platform_overrides,
+        pip_support=pip_support,
+    )
+
+    lock_spec = aggregate_lock_specs(lock_specs)
+    lock_spec.virtual_package_repo = virtual_package_repo
+    lock_spec.channels = (
+        [Channel.from_string(co) for co in channel_overrides]
+        if channel_overrides
+        else lock_spec.channels
+    )
+    lock_spec.platforms = (
+        list(platform_overrides) if platform_overrides else lock_spec.platforms
+    ) or list(DEFAULT_PLATFORMS)
+
+    if required_categories is not None:
+
+        def dep_has_category(d: Dependency, categories: AbstractSet[str]) -> bool:
+            return d.category in categories
+
+        lock_spec.dependencies = [
+            d
+            for d in lock_spec.dependencies
+            if dep_has_category(d, categories=required_categories)
+        ]
+
+    return lock_spec

conda_lock/src_parser/pyproject_toml.py

@@ -235,44 +235,6 @@ def to_match_spec(conda_dep_name: str, conda_version: Optional[str]) -> str:
     return spec
 
 
-def parse_pyproject_toml(
-    pyproject_toml: pathlib.Path,
-) -> LockSpecification:
-    with pyproject_toml.open("rb") as fp:
-        contents = toml_load(fp)
-    build_system = get_in(["build-system", "build-backend"], contents)
-    pep_621_probe = get_in(["project", "dependencies"], contents)
-    pdm_probe = get_in(["tool", "pdm"], contents)
-    parse = parse_poetry_pyproject_toml
-    if pep_621_probe is not None:
-        if pdm_probe is None:
-            parse = partial(
-                parse_requirements_pyproject_toml,
-                prefix=("project",),
-                main_tag="dependencies",
-                optional_tag="optional-dependencies",
-            )
-        else:
-            parse = parse_pdm_pyproject_toml
-    elif build_system.startswith("poetry"):
-        parse = parse_poetry_pyproject_toml
-    elif build_system.startswith("flit"):
-        parse = partial(
-            parse_requirements_pyproject_toml,
-            prefix=("tool", "flit", "metadata"),
-            main_tag="requires",
-            optional_tag="requires-extra",
-        )
-    else:
-        import warnings
-
-        warnings.warn(
-            "Could not detect build-system in pyproject.toml.  Assuming poetry"
-        )
-
-    return parse(pyproject_toml, contents)
-
-
 def parse_python_requirement(
     requirement: str,
     manager: Literal["conda", "pip"] = "conda",
@@ -383,3 +345,41 @@ def parse_pdm_pyproject_toml(
     res.dependencies.extend(dev_reqs)
 
     return res
+
+
+def parse_pyproject_toml(
+    pyproject_toml: pathlib.Path,
+) -> LockSpecification:
+    with pyproject_toml.open("rb") as fp:
+        contents = toml_load(fp)
+    build_system = get_in(["build-system", "build-backend"], contents)
+    pep_621_probe = get_in(["project", "dependencies"], contents)
+    pdm_probe = get_in(["tool", "pdm"], contents)
+    parse = parse_poetry_pyproject_toml
+    if pep_621_probe is not None:
+        if pdm_probe is None:
+            parse = partial(
+                parse_requirements_pyproject_toml,
+                prefix=("project",),
+                main_tag="dependencies",
+                optional_tag="optional-dependencies",
+            )
+        else:
+            parse = parse_pdm_pyproject_toml
+    elif build_system.startswith("poetry"):
+        parse = parse_poetry_pyproject_toml
+    elif build_system.startswith("flit"):
+        parse = partial(
+            parse_requirements_pyproject_toml,
+            prefix=("tool", "flit", "metadata"),
+            main_tag="requires",
+            optional_tag="requires-extra",
+        )
+    else:
+        import warnings
+
+        warnings.warn(
+            "Could not detect build-system in pyproject.toml.  Assuming poetry"
+        )
+
+    return parse(pyproject_toml, contents)

tests/test_conda_lock.py

@@ -30,20 +30,17 @@
 from conda_lock.conda_lock import (
     DEFAULT_FILES,
     DEFAULT_LOCKFILE_NAME,
-    DEFAULT_PLATFORMS,
     _add_auth_to_line,
     _add_auth_to_lockfile,
     _extract_domain,
     _strip_auth_from_line,
     _strip_auth_from_lockfile,
-    aggregate_lock_specs,
     create_lockfile_from_spec,
     default_virtual_package_repodata,
     determine_conda_executable,
     extract_input_hash,
     main,
     make_lock_spec,
-    parse_meta_yaml_file,
     run_lock,
 )
 from conda_lock.conda_solver import extract_json_object, fake_conda_environment
@@ -66,8 +63,15 @@
 )
 from conda_lock.models.channel import Channel
 from conda_lock.pypi_solver import parse_pip_requirement, solve_pypi
-from conda_lock.src_parser import LockSpecification, Selectors, VersionedDependency
+from conda_lock.src_parser import (
+    DEFAULT_PLATFORMS,
+    LockSpecification,
+    Selectors,
+    VersionedDependency,
+    aggregate_lock_specs,
+)
 from conda_lock.src_parser.environment_yaml import parse_environment_file
+from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file
 from conda_lock.src_parser.pyproject_toml import (
     parse_pyproject_toml,
     poetry_version_to_conda_version,