Merge pull request #1244 from isuruf/migration_cfp

If conda-forge-pinning package has migrations installed, use those

conda_smithy/configure_feedstock.py

@@ -9,6 +9,7 @@
 from collections import OrderedDict
 import copy
 import hashlib
+import time
 
 import conda_build.api
 import conda_build.utils
@@ -486,7 +487,7 @@ def _get_fast_finish_script(
     return fast_finish_text
 
 
-def migrate_combined_spec(combined_spec, forge_dir, config):
+def migrate_combined_spec(combined_spec, forge_dir, config, forge_config):
     """CFEP-9 variant migrations
 
     Apply the list of migrations configurations to the build (in the correct sequence)
@@ -499,17 +500,17 @@ def migrate_combined_spec(combined_spec, forge_dir, config):
 
     """
     combined_spec = combined_spec.copy()
-    migrations_root = os.path.join(
-        forge_dir, ".ci_support", "migrations", "*.yaml"
-    )
-    migrations = glob.glob(migrations_root)
+    if "migration_fns" not in forge_config:
+        migrations = set_migration_fns(forge_dir, forge_config)
+    migrations = forge_config["migration_fns"]
 
     from .variant_algebra import parse_variant, variant_add
 
     migration_variants = [
         (fn, parse_variant(open(fn, "r").read(), config=config))
         for fn in migrations
     ]
+
     migration_variants.sort(
         key=lambda fn_v: (fn_v[1]["migration_ts"], fn_v[0])
     )
@@ -566,7 +567,7 @@ def _render_ci_provider(
         )
 
         migrated_combined_variant_spec = migrate_combined_spec(
-            combined_variant_spec, forge_dir, config
+            combined_variant_spec, forge_dir, config, forge_config,
         )
 
         metas = conda_build.api.render(
@@ -1594,6 +1595,84 @@ def make_jinja_env(feedstock_directory):
     return env
 
 
+def get_migrations_in_dir(migrations_root):
+    """
+    Given a directory, return the migrations as a mapping
+    from the timestamp to a tuple of (filename, migration_number)
+    """
+    res = {}
+    for fn in glob.glob(os.path.join(migrations_root, "*.yaml")):
+        with open(fn, "r") as f:
+            contents = f.read()
+            migration_yaml = (
+                yaml.load(contents, Loader=yaml.loader.BaseLoader) or {}
+            )
+            # Use a object as timestamp to not delete it
+            ts = migration_yaml.get("migrator_ts", object())
+            migration_number = migration_yaml.get("__migrator", {}).get(
+                "migration_number", 1
+            )
+            res[ts] = (fn, migration_number)
+    return res
+
+
+def set_migration_fns(forge_dir, forge_config):
+    """
+    This will calculate the migration files and set migration_fns
+    in the forge_config as a list.
+    
+    First, this will look in the conda-forge-pinning (CFP) package
+    to see if it has migrations installed. If not, the filenames of
+    the migrations the feedstock are used.
+    
+    Then, this will look at migrations in the feedstock and if they
+    have a timestamp and doesn't exist in the CFP package, the
+    migration is considered old and deleted.
+    
+    Then, if there is a migration in the feedstock with the same
+    migration number and timestamp in the CFP package, the filename of
+    the migration in the CFP package is used.
+    
+    Finally, if none of the conditions are met for a migration in the
+    feedstock, the filename of the migration in the feedstock is used.
+    """
+    exclusive_config_file = forge_config["exclusive_config_file"]
+    cfp_migrations_dir = os.path.join(
+        os.path.dirname(exclusive_config_file),
+        "share",
+        "conda-forge",
+        "migrations",
+    )
+
+    migrations_root = os.path.join(forge_dir, ".ci_support", "migrations")
+    migrations_in_feedstock = get_migrations_in_dir(migrations_root)
+
+    if not os.path.exists(cfp_migrations_dir):
+        migration_fns = [fn for fn, _ in migrations_in_feedstock.values()]
+        forge_config["migration_fns"] = migration_fns
+        return
+
+    migrations_in_cfp = get_migrations_in_dir(cfp_migrations_dir)
+
+    result = []
+    for ts, (fn, num) in migrations_in_feedstock.items():
+        if ts == object:
+            # This file doesn't have a timestamp. Use it as it is.
+            result.append(fn)
+        elif ts in migrations_in_cfp:
+            # Use the one from cfp if migration_numbers match
+            new_fn, new_num = migrations_in_cfp[ts]
+            if num == new_num:
+                result.append(new_fn)
+            else:
+                result.append(fn)
+        else:
+            # Delete this as this migration is over.
+            remove_file(fn)
+    forge_config["migration_fns"] = result
+    return
+
+
 def main(
     forge_file_directory,
     no_check_uptodate=False,
@@ -1655,6 +1734,7 @@ def main(
         set_exe_file(os.path.join(forge_dir, "build-locally.py"))
     clear_variants(forge_dir)
     clear_scripts(forge_dir)
+    set_migration_fns(forge_dir, config)
 
     # the order of these calls appears to matter
     render_info = []

conda_smithy/variant_algebra.py

@@ -41,7 +41,7 @@ def parse_variant(
     Parameters
     ----------
     variant_file_content : str
-        The loaded vaiant contents.  This can include selectors etc.
+        The loaded variant contents.  This can include selectors etc.
     """
     if not config:
         from conda_build.config import Config

news/migration_cfp.rst

@@ -0,0 +1,28 @@
+**Added:**
+
+* <news item>
+
+**Changed:**
+
+* If conda-forge-pinning package has migrations installed, use those
+  migration yaml files instead of the ones from the feedstock if the
+  timestamp field match and remove if the migration yaml has a
+  timestamp and there's no corresponding one in conda-forge-pinning
+  which indicates that the migration is over.
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>
+

tests/conftest.py

@@ -245,6 +245,7 @@ def recipe_migration_cfep9(config_yaml, request):
     ) as fh:
         fh.write(
             """
+migrator_ts: 1
 zlib:
     - 1000
 """

tests/test_configure_feedstock.py

@@ -4,6 +4,7 @@
 import pytest
 import copy
 import yaml
+import textwrap
 
 
 def test_noarch_skips_appveyor(noarch_recipe, jinja_env):
@@ -446,6 +447,68 @@ def test_migrator_recipe(recipe_migration_cfep9, jinja_env):
         assert variant["zlib"] == ["1000"]
 
 
+def test_migrator_cfp_override(recipe_migration_cfep9, jinja_env):
+    cfp_file = recipe_migration_cfep9.config["exclusive_config_file"]
+    cfp_migration_dir = os.path.join(
+        os.path.dirname(cfp_file), "share", "conda-forge", "migrations"
+    )
+    os.makedirs(cfp_migration_dir, exist_ok=True)
+    with open(os.path.join(cfp_migration_dir, "zlib2.yaml"), "w") as f:
+        f.write(
+            textwrap.dedent(
+                """
+                migrator_ts: 1
+                zlib:
+                   - 1001
+                """
+            )
+        )
+    cnfgr_fdstk.render_azure(
+        jinja_env=jinja_env,
+        forge_config=recipe_migration_cfep9.config,
+        forge_dir=recipe_migration_cfep9.recipe,
+    )
+
+    with open(
+        os.path.join(
+            recipe_migration_cfep9.recipe,
+            ".ci_support",
+            "linux_python2.7.yaml",
+        )
+    ) as fo:
+        variant = yaml.safe_load(fo)
+        assert variant["zlib"] == ["1001"]
+
+
+def test_migrator_delete_old(recipe_migration_cfep9, jinja_env):
+    cfp_file = recipe_migration_cfep9.config["exclusive_config_file"]
+    cfp_migration_dir = os.path.join(
+        os.path.dirname(cfp_file), "share", "conda-forge", "migrations"
+    )
+    assert os.path.exists(
+        os.path.join(
+            recipe_migration_cfep9.recipe,
+            ".ci_support",
+            "migrations",
+            "zlib.yaml",
+        )
+    )
+    os.makedirs(cfp_migration_dir, exist_ok=True)
+    cnfgr_fdstk.render_azure(
+        jinja_env=jinja_env,
+        forge_config=recipe_migration_cfep9.config,
+        forge_dir=recipe_migration_cfep9.recipe,
+    )
+    assert not os.path.exists(
+        os.path.join(
+            recipe_migration_cfep9.recipe,
+            ".ci_support",
+            "migrations",
+            "zlib.yaml",
+        )
+    )
+
+
 def test_migrator_downgrade_recipe(
     recipe_migration_cfep9_downgrade, jinja_env
 ):