generate the eden_scm github actions (#103)

Summary:
Pull Request resolved: facebook/sapling#103

Automate maintenance of the edenscm_* github actions yamls

Add job file and name options and support for the Rust install section

Reviewed By: fanzeyi

Differential Revision: D34044422

fbshipit-source-id: 7d5f07d37bab1eff5de30a88e710dbf7479ca192

.github/workflows/getdeps_linux.yml

@@ -17,8 +17,12 @@ jobs:
     - uses: actions/checkout@v2
     - name: Fetch boost
       run: python3 build/fbcode_builder/getdeps.py fetch --no-tests boost
-    - name: Fetch rust
-      run: python3 build/fbcode_builder/getdeps.py fetch --no-tests rust
+    - name: Install Rust Stable
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        default: true
+        profile: minimal
     - name: Fetch ninja
       run: python3 build/fbcode_builder/getdeps.py fetch --no-tests ninja
     - name: Fetch cmake
@@ -77,8 +81,6 @@ jobs:
       run: python3 build/fbcode_builder/getdeps.py fetch --no-tests fb303
     - name: Build boost
       run: python3 build/fbcode_builder/getdeps.py build --no-tests boost
-    - name: Build rust
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests rust
     - name: Build ninja
       run: python3 build/fbcode_builder/getdeps.py build --no-tests ninja
     - name: Build cmake

.github/workflows/getdeps_mac.yml

@@ -17,8 +17,12 @@ jobs:
     - uses: actions/checkout@v2
     - name: Fetch boost
       run: python3 build/fbcode_builder/getdeps.py fetch --no-tests boost
-    - name: Fetch rust
-      run: python3 build/fbcode_builder/getdeps.py fetch --no-tests rust
+    - name: Install Rust Stable
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        default: true
+        profile: minimal
     - name: Fetch ninja
       run: python3 build/fbcode_builder/getdeps.py fetch --no-tests ninja
     - name: Fetch cmake
@@ -75,8 +79,6 @@ jobs:
       run: python3 build/fbcode_builder/getdeps.py fetch --no-tests fb303
     - name: Build boost
       run: python3 build/fbcode_builder/getdeps.py build --no-tests boost
-    - name: Build rust
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests rust
     - name: Build ninja
       run: python3 build/fbcode_builder/getdeps.py build --no-tests ninja
     - name: Build cmake

.github/workflows/getdeps_windows.yml

@@ -22,8 +22,12 @@ jobs:
       run: git config --system core.longpaths true
     - name: Fetch boost
       run: python build/fbcode_builder/getdeps.py fetch --no-tests boost
-    - name: Fetch rust
-      run: python build/fbcode_builder/getdeps.py fetch --no-tests rust
+    - name: Install Rust Stable
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        default: true
+        profile: minimal
     - name: Fetch ninja
       run: python build/fbcode_builder/getdeps.py fetch --no-tests ninja
     - name: Fetch cmake
@@ -76,8 +80,6 @@ jobs:
       run: python build/fbcode_builder/getdeps.py fetch --no-tests fb303
     - name: Build boost
       run: python build/fbcode_builder/getdeps.py build --no-tests boost
-    - name: Build rust
-      run: python build/fbcode_builder/getdeps.py build --no-tests rust
     - name: Build ninja
       run: python build/fbcode_builder/getdeps.py build --no-tests ninja
     - name: Build cmake

build/fbcode_builder/getdeps.py

@@ -900,24 +900,35 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
         py3 = "python3"
 
         if build_opts.is_linux():
-            job_name = "linux"
+            artifacts = "linux"
             runs_on = f"ubuntu-{args.ubuntu_version}"
         elif build_opts.is_windows():
             # We're targeting the windows-2016 image because it has
             # Visual Studio 2017 installed, and at the time of writing,
             # the version of boost in the manifests (1.69) is not
             # buildable with Visual Studio 2019
-            job_name = "windows"
+            artifacts = "windows"
             runs_on = "windows-2016"
             # The windows runners are python 3 by default; python2.exe
             # is available if needed.
             py3 = "python"
         else:
-            job_name = "mac"
+            artifacts = "mac"
             runs_on = "macOS-latest"
 
         os.makedirs(args.output_dir, exist_ok=True)
-        output_file = os.path.join(args.output_dir, f"getdeps_{job_name}.yml")
+
+        job_file_prefix = "getdeps_"
+        if args.job_file_prefix:
+            job_file_prefix = args.job_file_prefix
+
+        output_file = os.path.join(args.output_dir, f"{job_file_prefix}{artifacts}.yml")
+
+        if args.job_name_prefix:
+            job_name = args.job_name_prefix + artifacts.capitalize()
+        else:
+            job_name = artifacts
+
         with open(output_file, "w") as out:
             # Deliberate line break here because the @ and the generated
             # symbols are meaningful to our internal tooling when they
@@ -963,15 +974,42 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
 
             projects = loader.manifests_in_dependency_order()
 
+            allow_sys_arg = ""
+            if (
+                build_opts.allow_system_packages
+                and build_opts.is_linux()
+                and build_opts.host_type.get_package_manager()
+            ):
+                allow_sys_arg = " --allow-system-packages"
+                out.write("    - name: Install system deps\n")
+                out.write(
+                    f"      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
+                )
+
             for m in projects:
                 if m != manifest:
-                    out.write("    - name: Fetch %s\n" % m.name)
-                    out.write(f"      run: {getdepscmd} fetch --no-tests {m.name}\n")
+                    if m.name == "rust":
+                        out.write("    - name: Install Rust Stable\n")
+                        out.write("      uses: actions-rs/toolchain@v1\n")
+                        out.write("      with:\n")
+                        out.write("        toolchain: stable\n")
+                        out.write("        default: true\n")
+                        out.write("        profile: minimal\n")
+                    else:
+                        out.write("    - name: Fetch %s\n" % m.name)
+                        out.write(
+                            f"      run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
+                        )
 
             for m in projects:
                 if m != manifest:
-                    out.write("    - name: Build %s\n" % m.name)
-                    out.write(f"      run: {getdepscmd} build --no-tests {m.name}\n")
+                    if m.name == "rust":
+                        continue
+                    else:
+                        out.write("    - name: Build %s\n" % m.name)
+                        out.write(
+                            f"      run: {getdepscmd}{allow_sys_arg} build --no-tests {m.name}\n"
+                        )
 
             out.write("    - name: Build %s\n" % manifest.name)
 
@@ -982,7 +1020,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 )
 
             out.write(
-                f"      run: {getdepscmd} build --src-dir=. {manifest.name} {project_prefix}\n"
+                f"      run: {getdepscmd}{allow_sys_arg} build --src-dir=. {manifest.name} {project_prefix}\n"
             )
 
             out.write("    - name: Copy artifacts\n")
@@ -996,8 +1034,8 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 strip = ""
 
             out.write(
-                f"      run: {getdepscmd} fixup-dyn-deps{strip} "
-                f"--src-dir=. {manifest.name} _artifacts/{job_name} {project_prefix} "
+                f"      run: {getdepscmd}{allow_sys_arg} fixup-dyn-deps{strip} "
+                f"--src-dir=. {manifest.name} _artifacts/{artifacts} {project_prefix} "
                 f"--final-install-prefix /usr/local\n"
             )
 
@@ -1008,7 +1046,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
 
             out.write("    - name: Test %s\n" % manifest.name)
             out.write(
-                f"      run: {getdepscmd} test --src-dir=. {manifest.name} {project_prefix}\n"
+                f"      run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
             )
 
     def setup_project_cmd_parser(self, parser):
@@ -1042,6 +1080,18 @@ def setup_project_cmd_parser(self, parser):
             dest="os_types",
             default=[],
         )
+        parser.add_argument(
+            "--job-file-prefix",
+            type=str,
+            help="add a prefix to all job file names",
+            default=None,
+        )
+        parser.add_argument(
+            "--job-name-prefix",
+            type=str,
+            help="add a prefix to all job names",
+            default=None,
+        )
 
 
 def get_arg_var_name(args):

build/fbcode_builder/manifests/cmake

@@ -4,6 +4,10 @@ name = cmake
 [homebrew]
 cmake
 
+# 18.04 cmake is too old
+[debs.not(all(distro=ubuntu,distro_vers="18.04"))]
+cmake
+
 [rpms]
 cmake
 

build/fbcode_builder/manifests/eden_scm

@@ -47,6 +47,8 @@ fbcode/fboss/common = common
 \.pyc$
 
 [dependencies]
+fb303
+fbthrift
 fb303-source
 fbthrift-source
 python