Create exclusion list of integration tests and cover easiest cases

.github/workflows/mononoke-integration_linux.yml

@@ -75,8 +75,9 @@ jobs:
       run: df -h
     - name: Run Monononke integration tests
       run: |
-        for dir in /tmp/build/installed/python-click-*/lib/fb-py-libs/python-click/click; do
-            export PYTHONPATH="${dir}${PYTHONPATH:+:${PYTHONPATH}}"
-        done
         python3 eden/mononoke/tests/integration/run_tests_getdeps.py /tmp/build/installed /tmp/build/build/mononoke_integration_test
       continue-on-error: true
+    - name: Rerun failed Monononke integration tests (reduce flakiness)
+      run: |
+        cat eden/mononoke/tests/integration/.test* || true
+        python3 eden/mononoke/tests/integration/run_tests_getdeps.py /tmp/build/installed /tmp/build/build/mononoke_integration_test --rerun-failed

.github/workflows/mononoke-integration_mac.yml

@@ -69,12 +69,18 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install click
+    - name: Install Brew dependencies
+      run: |
+        brew install coreutils curl-openssl jq
     - name: Check space
       run: df -h
     - name: Run Monononke integration tests
       run: |
-        for dir in /tmp/build/installed/python-click-*/lib/fb-py-libs/python-click/click; do
-            export PYTHONPATH="${dir}${PYTHONPATH:+:${PYTHONPATH}}"
-        done
+        export PATH="/usr/local/opt/curl-openssl/bin:$PATH"
         python3 eden/mononoke/tests/integration/run_tests_getdeps.py /tmp/build/installed /tmp/build/build/mononoke_integration_test
       continue-on-error: true
+    - name: Rerun failed Monononke integration tests (reduce flakiness)
+      run: |
+        cat eden/mononoke/tests/integration/.test* || true
+        export PATH="/usr/local/opt/curl-openssl/bin:$PATH"
+        python3 eden/mononoke/tests/integration/run_tests_getdeps.py /tmp/build/installed /tmp/build/build/mononoke_integration_test --rerun-failed

eden/mononoke/permission_checker/src/oss.rs

@@ -22,8 +22,23 @@ impl MononokeIdentity {
         bail!("Decoding from JSON is not yet implemented for MononokeIdentity")
     }
 
-    pub fn try_from_x509(_: &X509) -> Result<MononokeIdentitySet> {
-        bail!("Decoding from x509 is not yet implemented for MononokeIdentity")
+    pub fn try_from_x509(cert: &X509) -> Result<MononokeIdentitySet> {
+        let subject_vec: Result<Vec<_>> = cert
+            .subject_name()
+            .entries()
+            .map(|entry| {
+                Ok(format!(
+                    "{}={}",
+                    entry.object().nid().short_name()?,
+                    entry.data().as_utf8()?
+                ))
+            })
+            .collect();
+        let subject_name = subject_vec?.as_slice().join(",");
+
+        let mut idents = MononokeIdentitySet::new();
+        idents.insert(MononokeIdentity::new("X509_SUBJECT_NAME", subject_name)?);
+        Ok(idents)
     }
 }
 

eden/mononoke/tests/integration/library-push-redirector.sh

@@ -134,7 +134,7 @@ function start_large_small_repo {
   wait_for_mononoke
   # Setting XREPOSYNC allows the user of this fn to set x-repo sync mutable counter instead of the backsync one
   # This is useful, if the intention is to use x-repo sync instead of backsync after the setup
-  if [[ -v XREPOSYNC ]]; then
+  if [[ -n "${XREPOSYNC:-}" ]]; then
     sqlite3 "$TESTTMP/monsql/sqlite_dbs" "INSERT INTO mutable_counters (repo_id, name, value) VALUES ($REPOIDLARGE, 'xreposync_from_$REPOIDSMALL', 2)";
   else
     sqlite3 "$TESTTMP/monsql/sqlite_dbs" "INSERT INTO mutable_counters (repo_id, name, value) VALUES ($REPOIDSMALL, 'backsync_from_$REPOIDLARGE', 2)";

eden/mononoke/tests/integration/library.sh

@@ -6,6 +6,14 @@
 
 # Library routines and initial setup for Mononoke-related tests.
 
+if [ -f "$TEST_FIXTURES/facebook/fb_library.sh" ]; then
+  # shellcheck source=fbcode/eden/mononoke/tests/integration/facebook/fb_library.sh
+  . "$TEST_FIXTURES/facebook/fb_library.sh"
+fi
+
+ALLOWED_IDENTITY_TYPE="${FB_ALLOWED_IDENTITY_TYPE:-X509_SUBJECT_NAME}"
+ALLOWED_IDENTITY_DATA="${FB_ALLOWED_IDENTITY_DATA:-CN=localhost,O=Mononoke,C=US,ST=CA}"
+
 if [[ -n "$DB_SHARD_NAME" ]]; then
   MONONOKE_DEFAULT_START_TIMEOUT=60
 else
@@ -17,6 +25,39 @@ REPONAME=repo
 COMMON_ARGS=(--skip-caching --mysql-master-only --tunables-config "file:$TESTTMP/mononoke_tunables.json")
 TEST_CERTDIR="${HGTEST_CERTDIR:-"$TEST_CERTS"}"
 
+case "$(uname -s)" in
+  # Workarounds for running tests on MacOS
+  Darwin*)
+    # MacOS builtin "du" command has very different flags than the GNU version.
+    # Adding this as env variable rather than a function, because this is called
+    # mostly in "find -exec" context and it is hard to use bash functions there,
+    # because of https://unix.stackexchange.com/questions/50692/executing-user-defined-function-in-a-find-exec-call
+    GNU_DU=gdu
+
+    # As opposed to the GNU version wc -l outputs number of lines prefixed by
+    # whitespace. This function will strip the whitespaces on MacOS.
+    function count_stdin_lines {
+      wc -l "$@" | sed -E "s/^[[:space:]]+//"
+    }
+  ;;
+  *)
+    GNU_DU=du
+
+    function count_stdin_lines {
+      wc -l "$@"
+    }
+esac
+
+function silentkill {
+  # sends KILL to the given process and waits for it so that nothing is printed
+  # to the terminal on MacOS
+  { kill -9 $1 && wait $1; } > /dev/null 2>&1
+  # wait exit code is 137, if the process was already killed when wait was
+  # called, or the process' exit code. Because we don't need it and we want
+  # determinism in tests lets return 0 here
+  true
+}
+
 function get_free_socket {
 
 # From https://unix.stackexchange.com/questions/55913/whats-the-easiest-way-to-find-an-unused-local-port
@@ -308,7 +349,7 @@ function write_stub_log_entry {
 # Remove the glog prefix
 function strip_glog {
   # based on https://our.internmc.facebook.com/intern/wiki/LogKnock/Log_formats/#regex-for-glog
-  sed -E -e 's%^[VDIWECF][[:digit:]]{4} [[:digit:]]{2}:?[[:digit:]]{2}:?[[:digit:]]{2}(\.[[:digit:]]+)?\s+(([0-9a-f]+)\s+)?(\[([^]]+)\]\s+)?(\(([^\)]+)\)\s+)?(([a-zA-Z0-9_./-]+):([[:digit:]]+))\]\s+%%'
+  sed -E -e 's%^[VDIWECF][[:digit:]]{4} [[:digit:]]{2}:?[[:digit:]]{2}:?[[:digit:]]{2}(\.[[:digit:]]+)?[[:space:]]+(([0-9a-f]+)[[:space:]]+)?(\[([^]]+)\][[:space:]]+)?(\(([^\)]+)\)[[:space:]]+)?(([a-zA-Z0-9_./-]+):([[:digit:]]+))\][[:space:]]+%%'
 }
 
 function wait_for_json_record_count {
@@ -481,15 +522,13 @@ EOF
 
   echo "{}" > "$TESTTMP/mononoke_tunables.json"
 
-  ALLOWED_USERNAME="${ALLOWED_USERNAME:-myusername0}"
-
   cd mononoke-config || exit 1
   mkdir -p common
   touch common/commitsyncmap.toml
   cat > common/common.toml <<CONFIG
 [[whitelist_entry]]
-identity_type = "USER"
-identity_data = "$ALLOWED_USERNAME"
+identity_type = "$ALLOWED_IDENTITY_TYPE"
+identity_data = "$ALLOWED_IDENTITY_DATA"
 CONFIG
 
   echo "# Start new config" > common/storage.toml

eden/mononoke/tests/integration/run_tests_getdeps.py

@@ -9,7 +9,9 @@
 import os
 import subprocess
 import sys
-from os.path import abspath, dirname, join
+from glob import iglob
+from os.path import abspath, basename, dirname, join
+from pathlib import Path
 
 
 parser = argparse.ArgumentParser(
@@ -22,6 +24,15 @@
 parser.add_argument(
     "build_dir", help="Location where to put generated manifest.json file"
 )
+parser.add_argument(
+    "tests", nargs="*", help="Optional list of tests to run. Run all if none provided"
+)
+parser.add_argument(
+    "-r",
+    "--rerun-failed",
+    action="store_true",
+    help="Rerun failed tests based on '.testfailed' file",
+)
 args = parser.parse_args()
 
 install_dir = args.install_dir
@@ -45,20 +56,118 @@
 with open(join(build_dir, "manifest.json"), "w") as f:
     f.write(json.dumps(MANIFEST_DEPS, sort_keys=True, indent=4))
 
-tests = ["test-init.t"]
+if args.tests or args.rerun_failed:
+    tests = {basename(p) for p in args.tests or []}
+
+    if args.rerun_failed:
+        # Based on eden/scm/tests/run-tests.py
+        for title in ("failed", "errored"):
+            failed = Path(repo_root) / "eden/mononoke/tests/integration/.test{}".format(
+                title
+            )
+            if failed.is_file():
+                tests.update(t for t in failed.read_text().splitlines() if t)
+
+    tests = list(tests)
+else:
+    excluded_tests = {
+        "test-backsync-forever.t",  # Unknown issue
+        "test-backsyncer-merges.t",  # Missing BACKSYNCER
+        "test-blobimport-lfs.t",  # Timed out
+        "test-blobimport.t",  # Case insensitivity of paths in MacOS
+        "test-blobstore_healer.t",  # PANIC not implemented in sql_ext
+        "test-bookmarks-filler.t",  # Probably missing binary
+        "test-cmd-manual-scrub.t",  # Just wrong outout
+        "test-cross-repo-commit-sync-live.t",  # Unknown issue
+        "test-cross-repo-commit-sync-merge.t",  # requires FB-specific command
+        "test-cross-repo-commit-sync.t",  # requires FB-specific command
+        "test-cross-repo-commit-validator.t",  # requires FB-specific command
+        "test-edenapi-server-complete-trees.t",  # Missing eden/scm's commands
+        "test-edenapi-server-files.t",  # Missing eden/scm's commands
+        "test-edenapi-server-history.t",  # Missing eden/scm's commands
+        "test-edenapi-server-trees.t",  # Missing eden/scm's commands
+        "test-fastreplay-inline-args.t",  # Returns different data in OSS
+        "test-gitimport-octopus.t",  # Missing MONONOKE_REGENERATE_HG_FILENODES
+        "test-gitimport.t",  # Issue with hggit extension
+        "test-hook-tailer.t",  # Issue with hggit extension
+        "test-hooks.t",  # Hooks are not in OSS yet
+        "test-infinitepush-lfs.t",  # Timed out
+        "test-large-path-and-content.t",  # Complex bash issues
+        "test-lfs-copytracing.t",  # Timed out
+        "test-lfs-server-acl-check.t",  # Timed out
+        "test-lfs-server-consistent-hashing.t",  # Timed out
+        "test-lfs-server-disabled-hostname-resolution.t",  # Timed out
+        "test-lfs-server-identity-parsing-from-header.t",  # Timed out
+        "test-lfs-server-identity-parsing-untrusted.t",  # Timed out
+        "test-lfs-server-identity-parsing.t",  # Timed out
+        "test-lfs-server-max-upload-size.t",  # Timed out
+        "test-lfs-server-proxy-skip-upstream.t",  # Complex bash issues
+        "test-lfs-server-proxy-sync.t",  # Timed out
+        "test-lfs-server-proxy.t",  # Timed out
+        "test-lfs-server-rate-limiting.t",  # Timed out
+        "test-lfs-server-scuba-logging.t",  # Timed out
+        "test-lfs-server.t",  # Timed out
+        "test-lfs-to-mononoke.t",  # Timed out
+        "test-lfs-wantslfspointers.t",  # Timed out
+        "test-lfs.t",  # Timed out
+        "test-megarepo-tool.t",  # Missing MONONOKE_HG_SYNC
+        "test-mononoke-admin.t",  # Missing MEGAREPO_TOOL
+        "test-mononoke-hg-sync-job-generate-bundles-force.t",  # Missing MONONOKE_HG_SYNC
+        "test-mononoke-hg-sync-job-generate-bundles-lfs-verification.t",  # Timed out
+        "test-mononoke-hg-sync-job-generate-bundles-lfs.t",  # Timed out
+        "test-mononoke-hg-sync-job-generate-bundles-loop.t",  # Missing MONONOKE_HG_SYNC
+        "test-mononoke-hg-sync-job-generate-bundles-other-books.t",  # Missing MONONOKE_HG_SYNC
+        "test-mononoke-hg-sync-job-generate-bundles.t",  # Missing MONONOKE_HG_SYNC
+        "test-mononoke-hg-sync-job-sync-globalrevs.t",  # Missing MONONOKE_HG_SYNC
+        "test-mononoke-hg-sync-job-with-copies.t",  # Missing MONONOKE_HG_SYNC
+        "test-mononoke-hg-sync-job.t",  # Missing MONONOKE_HG_SYNC
+        "test-push-protocol-lfs.t",  # Timed out
+        "test-push-redirector-pushrebase-hooks.t",  # Hooks are not in OSS yet
+        "test-push-redirector-pushrebase-onesided.t",  # Missing MONONOKE_X_REPO_SYNC
+        "test-push-redirector-sync-job.t",  # Missing BACKSYNCER
+        "test-pushrebase-block-casefolding.t",  # Most likely MacOS path case insensitivity
+        "test-pushrebase-discovery.t",  # Hooks are not in OSS yet
+        "test-remotefilelog-lfs.t",  # Timed out
+        "test-scs-blame.t",  # Missing SCS_SERVER
+        "test-scs-common-base.t",  # Missing SCS_SERVER
+        "test-scs-diff.t",  # Missing SCS_SERVER
+        "test-scs-list-bookmarks.t",  # Missing SCS_SERVER
+        "test-scs-log.t",  # Missing SCS_SERVER
+        "test-scs-lookup.t",  # Missing SCS_SERVER
+        "test-scs-x-repo.t",  # Missing SCS_SERVER
+        "test-scs.t",  # Missing SCS_SERVER
+        "test-server.t",  # Returns different data in OSS
+        "test-traffic-replay.t",  # Missing TRAFFIC_REPLAY
+        "test-unbundle-replay-hg-recording.t",  # Returns different data in OSS
+    }
+
+    tests = [
+        t
+        for t in (
+            basename(p)
+            for p in iglob(join(repo_root, "eden/mononoke/tests/integration/*.t"))
+        )
+        if t not in excluded_tests
+    ]
 
 env = dict(os.environ.items())
 env["NO_LOCAL_PATHS"] = "1"
 eden_scm_packages = join(install_dir, "eden_scm/lib/python2.7/site-packages")
 pythonpath = env.get("PYTHONPATH")
 env["PYTHONPATH"] = eden_scm_packages + (":{}".format(pythonpath) if pythonpath else "")
 
-subprocess.run(
-    [
-        sys.executable,
-        join(repo_root, "eden/mononoke/tests/integration/integration_runner_real.py"),
-        join(build_dir, "manifest.json"),
-    ]
-    + tests,
-    env=env,
-)
+if tests:
+    sys.exit(
+        subprocess.run(
+            [
+                sys.executable,
+                join(
+                    repo_root,
+                    "eden/mononoke/tests/integration/integration_runner_real.py",
+                ),
+                join(build_dir, "manifest.json"),
+            ]
+            + tests,
+            env=env,
+        ).returncode
+    )

eden/mononoke/tests/integration/test-alias-verify.t

@@ -48,7 +48,7 @@
 
 
   $ rm -rf $TESTTMP/blobstore/blobs/blob-repo0000.alias.*
-  $ ls $TESTTMP/blobstore/blobs | grep "alias" | wc -l
+  $ ls $TESTTMP/blobstore/blobs | grep "alias" | count_stdin_lines
   0
 
   $ aliasverify verify 2>&1 | grep "Alias Verification"
@@ -60,7 +60,7 @@
   * Missing alias blob: alias Sha256(b9a294f298d0ed2b65ca4488a42b473ff5f75d0b9843cbea84e1b472f9a514d1), content_id ContentId(Blake2(1af04efffa454f843420a538617f0c4166550da421b65a59ed95a85b43a25ada)) (glob)
   * Missing alias blob: alias Sha256(2ba85baaa7922ff4c0dfdbc00fd07bd69dcb1dce745c6a8c676fe8b5642a0d66), content_id ContentId(Blake2(1a3f1094cdae123ec6999b7baf4211ffd94f47970bedd71e13ec07f24a9aba6a)) (glob)
 
-  $ ls $TESTTMP/blobstore/blobs | grep "alias" | wc -l
+  $ ls $TESTTMP/blobstore/blobs | grep "alias" | count_stdin_lines
   0
 
   $ aliasverify generate --debug 2>&1 | grep "Missing alias blob"

eden/mononoke/tests/integration/test-backfill-derived-data.t

@@ -47,5 +47,5 @@ backfill derived data
   * using repo "repo" repoid RepositoryId(0) (glob)
   * changeset resolved as: * (glob)
   * derived fsnodes in * (glob)
-  $ backfill_derived_data single c3384961b16276f2db77df9d7c874bbe981cf0525bd6f84a502f919044f2dabd --all-types 2>&1 | grep derived | wc -l
+  $ backfill_derived_data single c3384961b16276f2db77df9d7c874bbe981cf0525bd6f84a502f919044f2dabd --all-types 2>&1 | grep derived | count_stdin_lines
   8

eden/mononoke/tests/integration/test-backfill-git-mapping.t

@@ -39,9 +39,9 @@ import testing repo to mononoke
   * using repo "repo" repoid RepositoryId(0) (glob)
 
 check that mapping is populated
-  $ echo ${HG_HASH_1^^}
+  $ tr "[:lower:]" "[:upper:]" <<< "${HG_HASH_1}"
   D000F571737066778CC230F7DC9A763180FDE257
-  $ echo ${HG_HASH_2^^}
+  $ tr "[:lower:]" "[:upper:]" <<< "${HG_HASH_2}"
   87B89069092550479FDF0EB22E632E031AF9C3D9
 
   $ get_bonsai_git_mapping

eden/mononoke/tests/integration/test-blobimport-derived-data.t

@@ -31,7 +31,7 @@
   $ mononoke_admin derived-data exists blame master 2> /dev/null
   Not Derived: c5b2b396b5dd503d2b9ca8c19db1cb0e733c48f43c0ae79f2f174866e11ea38a
 
-  $ blobimport --log repo-hg/.hg repo --derived-data-type fsnodes --derived-data-type blame |& grep Deriving
+  $ blobimport --log repo-hg/.hg repo --derived-data-type fsnodes --derived-data-type blame 2>&1 | grep Deriving
   * Deriving data for: ["fsnodes", "blame", "filenodes"] (glob)
 
   $ mononoke_admin derived-data exists fsnodes master 2> /dev/null

eden/mononoke/tests/integration/test-blobimport-inline.t

@@ -29,7 +29,7 @@
   $ for (( i=0; i < $lines_cnt; i++ ))
   > do
   >  LINE_LENGTH=$(random_int $max_line_length)
-  >  echo $(head -c 10000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1) >> file
+  >  echo $(head -c 10000 /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1) >> file
   > done
 
   $ hg ci -Aqm "commit"$c
@@ -41,8 +41,8 @@
   >   do
   >     LINE_LENGTH=$(random_int $max_line_length)
   >     LINE_NUMBER=$(random_int $lines_cnt)
-  >     CONTENT=$(head -c 10000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1)
-  >     sed -i "$LINE_NUMBER""s/.*/$CONTENT/" file
+  >     CONTENT=$(head -c 10000 /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w $LINE_LENGTH 2>/dev/null | head -n 1)
+  >     sed -ie "$LINE_NUMBER""s/.*/$CONTENT/" file
   >   done
   >   hg ci -Aqm "commit"$c
   > done