[mono] Integrate xharness tool (iOS) (#35316)

* Integrate xharness

.config/dotnet-tools.json

@@ -15,7 +15,7 @@
       ]
     },
     "microsoft.dotnet.xharness.cli": {
-      "version": "1.0.0-prerelease.20214.1",
+      "version": "1.0.0-prerelease.20227.9",
       "commands": [
         "xharness"
       ]

eng/testing/MobileRunnerTemplate.sh

@@ -2,9 +2,30 @@
 
 EXECUTION_DIR=$(dirname $0)
 TEST_NAME=$1
-SIMULATOR_NAME="iPhone 11"
-
-# TODO: to be replaced with xharness cli tool
+TARGET_ARCH=$2
+TARGET=
+SCHEME_SDK=
+
+if [ "$TARGET_ARCH" == "arm" ]; then
+    TARGET=ios-device
+    SCHEME_SDK=Release-iphoneos
+elif [ "$TARGET_ARCH" == "arm64" ]; then
+    TARGET=ios-device
+    SCHEME_SDK=Release-iphoneos
+elif [ "$TARGET_ARCH" == "x64" ]; then
+    TARGET=ios-simulator-64
+    SCHEME_SDK=Release-iphonesimulator
+elif [ "$TARGET_ARCH" == "x86" ]; then
+    TARGET=ios-simulator-32
+    SCHEME_SDK=Release-iphonesimulator
+else
+    echo "Unknown architecture: $TARGET_ARCH"
+    exit 1
+fi
+
+# "Release" in SCHEME_SDK is what xcode produces (see "bool Optimized" property in AppleAppBuilderTask)
+
+APP_BUNDLE=$EXECUTION_DIR/Bundle/$TEST_NAME/$SCHEME_SDK/$TEST_NAME.app
 
 # it doesn't support parallel execution yet, so, here is a hand-made semaphore:
 LOCKDIR=/tmp/runonsim.lock
@@ -18,20 +39,10 @@ while true; do
     fi
 done
 
-# Release here is what xcode produces (see "bool Optimized" property in AppleAppBuilderTask)
-AppBundlePath=$EXECUTION_DIR/Bundle/$TEST_NAME/Release-iphonesimulator/$TEST_NAME.app
-
-# kill a simulator if it exists
-xcrun simctl shutdown "$SIMULATOR_NAME" || true
-
-# boot it again
-xcrun simctl boot "$SIMULATOR_NAME"
-
-# open UI (this step is not neccessary)
-open -a Simulator
+XHARNESS_OUT="$EXECUTION_DIR/Bundle/xharness-output"
 
-# install the *.app bundle
-xcrun simctl install "$SIMULATOR_NAME" "$SIMULATOR_NAME"
+dotnet xharness ios test --app="$APP_BUNDLE" \
+    --targets=$TARGET \
+    --output-directory=$XHARNESS_OUT
 
-# launch the app, redirect logs to console and quite once tests are completed
-xcrun simctl launch --console booted net.dot.$TEST_NAME testlib:$TEST_NAME.dll --auto-exit
+echo "Xharness artifacts: $XHARNESS_OUT"

eng/testing/tests.targets

@@ -98,7 +98,7 @@
     <PropertyGroup>
       <RunTestsCommand>"$(RunScriptOutputPath)" --runtime-path "$(TestHostRootPath.TrimEnd('\/'))"</RunTestsCommand>
       <RunTestsCommand Condition="'$(TestRspFile)' != '' and '$(RuntimeFlavor)' != 'Mono'">$(RunTestsCommand) --rsp-file "$(TestRspFile)"</RunTestsCommand>
-      <RunTestsCommand Condition="'$(TargetOS)'=='iOS'">"$(RunScriptOutputPath)" $(AssemblyName)</RunTestsCommand>
+      <RunTestsCommand Condition="'$(TargetOS)'=='iOS'">"$(RunScriptOutputPath)" $(AssemblyName) $(TargetArchitecture)</RunTestsCommand>
     </PropertyGroup>
 
     <!-- Invoke the run script with the test host as the runtime path. -->
@@ -141,27 +141,15 @@
       <!-- remove PDBs to save some space until we integrate ILLink -->
       <BclBinaries Remove="$(RuntimePackDir)\lib\$(NetCoreAppCurrent)\*.pdb" />
     </ItemGroup>
+    <Error Condition="$(TargetArchitecture.StartsWith('arm')) and '$(DevTeamProvisioning)' == ''" 
+           Text="'DevTeamProvisioning' shouldn't be empty for arm64" />
     <Error Condition="!Exists('$(AppleTestRunner)')" Text="AppleTestRunner=$(AppleTestRunner) doesn't exist" />
     <Error Condition="!Exists('$(RuntimePackDir)')" Text="RuntimePackDir=$(RuntimePackDir) doesn't exist" />
     <RemoveDir Directories="$(BundleDir)" />
     <Copy SourceFiles="@(TestBinaries)" DestinationFolder="$(BundleDir)" SkipUnchangedFiles="true"/>
     <Copy SourceFiles="@(AppleTestRunnerBinaries)" DestinationFolder="$(BundleDir)\%(RecursiveDir)" SkipUnchangedFiles="true"/>
     <Copy SourceFiles="@(BclBinaries)" DestinationFolder="$(BundleDir)\%(RecursiveDir)" SkipUnchangedFiles="true"/>
 
-    <!-- xharness expects 'xunit-excludes.txt' with xunit categories to ignore in the bundle -->
-    <ItemGroup>
-      <XunitIgnoreCategories Include="category=OuterLoop"/>
-      <XunitIgnoreCategories Include="category=failing"/>
-      <XunitIgnoreCategories Include="category=nonnetcoreapptests"/>
-      <XunitIgnoreCategories Include="category=nonosxtests"/>
-      <XunitIgnoreCategories Include="category=nonmonotests"/>
-    </ItemGroup>
-    <WriteLinesToFile
-        File="$(BundleDir)\xunit-excludes.txt"
-        Lines="@(XunitIgnoreCategories)"
-        Overwrite="true"
-        Encoding="Unicode"/>
-
     <!-- Run App bundler, it should AOT libs (if needed), link all native bits, compile simple UI (written in ObjC)
          and produce an app bundle (with xcode project) -->
     <AppleAppBuilderTask 

src/mono/msbuild/AppleAppBuilder/Templates/CMakeLists.txt.template

@@ -19,7 +19,7 @@ include_directories("%MonoInclude%")
 
 set_target_properties(%ProjectName% PROPERTIES
     MACOSX_BUNDLE TRUE
-    MACOSX_BUNDLE_INFO_PLIST ${CMAKE_CURRENT_SOURCE_DIR}/Info.plist.in
+    MACOSX_BUNDLE_INFO_PLIST ${CMAKE_CURRENT_SOURCE_DIR}/Info.plist
     XCODE_ATTRIBUTE_ENABLE_BITCODE "NO"
     XCODE_ATTRIBUTE_DEAD_CODE_STRIPPING "NO"
     RESOURCE "${APP_RESOURCES}"

src/mono/msbuild/AppleAppBuilder/Templates/runtime.m

@@ -212,19 +212,12 @@
     // setenv ("MONO_LOG_LEVEL", "debug", TRUE);
     // setenv ("MONO_LOG_MASK", "all", TRUE);
 
-    bool auto_exit = FALSE;
     id args_array = [[NSProcessInfo processInfo] arguments];
     assert ([args_array count] <= 128);
     const char *managed_argv [128];
     int argi;
     for (argi = 0; argi < [args_array count]; argi++) {
         NSString* arg = [args_array objectAtIndex: argi];
-        assert ([arg length] >= 3);
-        if ([arg hasPrefix:@"--setenv="]) {
-            // TODO: setenv
-        } else if ([arg isEqualToString:@"--auto-exit"]) {
-            auto_exit = TRUE;
-        }
         managed_argv[argi] = [arg UTF8String];
     }
 
@@ -273,6 +266,5 @@
     // Print this so apps parsing logs can detect when we exited
     os_log_info (OS_LOG_DEFAULT, "Exit code: %d.", res);
 
-    if (auto_exit)
-        exit (res);
+    exit (res);
 }

src/mono/msbuild/AppleAppBuilder/Xcode.cs

@@ -76,7 +76,7 @@ public static string GenerateXCode(
         string plist = Utils.GetEmbeddedResource("Info.plist.template")
             .Replace("%BundleIdentifier%", projectName);
 
-        File.WriteAllText(Path.Combine(binDir, "Info.plist.in"), plist);
+        File.WriteAllText(Path.Combine(binDir, "Info.plist"), plist);
         File.WriteAllText(Path.Combine(binDir, "CMakeLists.txt"), cmakeLists);
 
         var cmakeArgs = new StringBuilder();

src/mono/msbuild/AppleTestRunner/AppleTestRunner.cs

@@ -126,9 +126,9 @@ protected override void TerminateWithSuccess()
 
     protected override TestRunnerType TestRunner => TestRunnerType.Xunit;
 
-    protected override string? IgnoreFilesDirectory => Environment.CurrentDirectory;
+    protected override string? IgnoreFilesDirectory => null;
 
-    public string BundleIdentifier => "net.dot.test-runner";
+    public string BundleIdentifier => "net.dot." + s_MainTestName;
 
     public string? UniqueIdentifier { get; }