Add a Mac Catalyst device test run (#16260)

* Add a Mac Catalyst device test run

* Clean tests for maccat

* skip entitlements for now

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "dotnet.defaultSolution": "Microsoft.Maui-dev.sln"
+    "dotnet.defaultSolution": "Microsoft.Maui-vscode.sln"
 }

eng/devices/catalyst.cake

@@ -3,32 +3,31 @@
 
 string TARGET = Argument("target", "Test");
 
-const string defaultVersion = "14.4";
 const string dotnetVersion = "net7.0";
 
 // required
 FilePath PROJECT = Argument("project", EnvironmentVariable("MAC_TEST_PROJECT") ?? "");
-string TEST_DEVICE = Argument("device", EnvironmentVariable("MAC_TEST_DEVICE") ?? $"ios-simulator-64_{defaultVersion}"); // comma separated in the form <platform>-<device|simulator>[-<32|64>][_<version>] (eg: ios-simulator-64_13.4,[...])
+string TEST_DEVICE = Argument("device", EnvironmentVariable("MAC_TEST_DEVICE") ?? "maccatalyst");
 
 // optional
 var DOTNET_PATH = Argument("dotnet-path", EnvironmentVariable("DOTNET_PATH"));
 var TARGET_FRAMEWORK = Argument("tfm", EnvironmentVariable("TARGET_FRAMEWORK") ?? $"{dotnetVersion}-maccatalyst");
-var BINLOG_ARG = Argument("binlog", EnvironmentVariable("IOS_TEST_BINLOG") ?? "");
+var BINLOG_ARG = Argument("binlog", EnvironmentVariable("MAC_TEST_BINLOG") ?? "");
 DirectoryPath BINLOG_DIR = string.IsNullOrEmpty(BINLOG_ARG) && !string.IsNullOrEmpty(PROJECT.FullPath) ? PROJECT.GetDirectory() : BINLOG_ARG;
 var TEST_APP = Argument("app", EnvironmentVariable("MAC_TEST_APP") ?? "");
 FilePath TEST_APP_PROJECT = Argument("appproject", EnvironmentVariable("MAC_TEST_APP_PROJECT") ?? "");
 var TEST_RESULTS = Argument("results", EnvironmentVariable("MAC_TEST_RESULTS") ?? "");
 
 // other
-string PLATFORM = "mac";
-string DOTNET_PLATFORM = "maccatalyst-x64";
+string RUNTIME_IDENTIFIER = Argument("rid", EnvironmentVariable("MAC_RUNTIME_IDENTIFIER") ?? "maccatalyst-x64");
 string CONFIGURATION = Argument("configuration", "Release");
 bool DEVICE_CLEANUP = Argument("cleanup", true);
 
 Information("Project File: {0}", PROJECT);
 Information("Build Binary Log (binlog): {0}", BINLOG_DIR);
-Information("Build Platform: {0}", PLATFORM);
 Information("Build Configuration: {0}", CONFIGURATION);
+Information("Build Runtime Identifier: {0}", RUNTIME_IDENTIFIER);
+Information("Build Target Framework: {0}", TARGET_FRAMEWORK);
 
 Setup(context =>
 {
@@ -48,13 +47,104 @@ void Cleanup()
 
 Task("Cleanup");
 
+Task("Build")
+	.WithCriteria(!string.IsNullOrEmpty(PROJECT.FullPath))
+	.Does(() =>
+{
+	var name = System.IO.Path.GetFileNameWithoutExtension(PROJECT.FullPath);
+	var binlog = $"{BINLOG_DIR}/{name}-{CONFIGURATION}-catalyst.binlog";
+
+	SetDotNetEnvironmentVariables(DOTNET_PATH);
+
+	DotNetBuild(PROJECT.FullPath, new DotNetBuildSettings {
+		Configuration = CONFIGURATION,
+		Framework = TARGET_FRAMEWORK,
+		MSBuildSettings = new DotNetMSBuildSettings {
+			MaxCpuCount = 0
+		},
+		ToolPath = DOTNET_PATH,
+		ArgumentCustomization = args => args
+			.Append("/p:BuildIpa=true")
+			.Append("/p:RuntimeIdentifier=" + RUNTIME_IDENTIFIER)
+			.Append("/bl:" + binlog)
+	});
+});
+
+Task("Test")
+	.IsDependentOn("Build")
+	.Does(() =>
+{
+	if (string.IsNullOrEmpty(TEST_APP)) {
+		if (string.IsNullOrEmpty(PROJECT.FullPath))
+			throw new Exception("If no app was specified, an app must be provided.");
+		var binDir = PROJECT.GetDirectory().Combine("bin").Combine(CONFIGURATION + "/" + TARGET_FRAMEWORK).Combine(RUNTIME_IDENTIFIER).FullPath;
+		var apps = GetDirectories(binDir + "/*.app");
+		TEST_APP = apps.First().FullPath;
+	}
+	if (string.IsNullOrEmpty(TEST_RESULTS)) {
+		TEST_RESULTS = TEST_APP + "-results";
+	}
+
+	Information("Test Device: {0}", TEST_DEVICE);
+	Information("Test App: {0}", TEST_APP);
+	Information("Test Results Directory: {0}", TEST_RESULTS);
+
+	if (!IsCIBuild())
+		CleanDirectories(TEST_RESULTS);
+	else
+	{
+		// Because we retry on CI we don't want to delete the previous failures
+		// We want to publish those files for reference
+		DeleteFiles(Directory(TEST_RESULTS).Path.Combine("*.*").FullPath);
+	}
+
+	var settings = new DotNetToolSettings {
+		DiagnosticOutput = true,
+		ArgumentCustomization = args => args.Append("run xharness apple test " +
+		$"--app=\"{TEST_APP}\" " +
+		$"--targets=\"{TEST_DEVICE}\" " +
+		$"--output-directory=\"{TEST_RESULTS}\" " +
+		$"--verbosity=\"Debug\" ")
+	};
+
+	bool testsFailed = true;
+	try {
+		DotNetTool("tool", settings);
+		testsFailed = false;
+	} finally {
+		// catalyst test result files are weirdly named, so fix it up
+		var resultsFile = GetFiles($"{TEST_RESULTS}/xunit-test-*.xml").FirstOrDefault();
+		if (FileExists(resultsFile)) {
+			CopyFile(resultsFile, resultsFile.GetDirectory().CombineWithFilePath("TestResults.xml"));
+		}
+
+		if (testsFailed && IsCIBuild())
+		{
+			var failurePath = $"{TEST_RESULTS}/TestResultsFailures/{Guid.NewGuid()}";
+			EnsureDirectoryExists(failurePath);
+			// The tasks will retry the tests and overwrite the failed results each retry
+			// we want to retain the failed results for diagnostic purposes
+			CopyFiles($"{TEST_RESULTS}/*.*", failurePath);
+
+			// We don't want these to upload
+			MoveFile($"{failurePath}/TestResults.xml", $"{failurePath}/Results.xml");
+		}
+	}
+
+	// this _may_ not be needed, but just in case
+	var failed = XmlPeek($"{TEST_RESULTS}/TestResults.xml", "/assemblies/assembly[@failed > 0 or @errors > 0]/@failed");
+	if (!string.IsNullOrEmpty(failed)) {
+		throw new Exception($"At least {failed} test(s) failed.");
+	}
+});
+
 Task("uitest")
 	.Does(() =>
 {
 	if (string.IsNullOrEmpty(TEST_APP) ) {
 		if (string.IsNullOrEmpty(TEST_APP_PROJECT.FullPath))
 			throw new Exception("If no app was specified, an app must be provided.");
-		var binDir = TEST_APP_PROJECT.GetDirectory().Combine("bin").Combine(CONFIGURATION + "/" + TARGET_FRAMEWORK).Combine(DOTNET_PLATFORM).FullPath;
+		var binDir = TEST_APP_PROJECT.GetDirectory().Combine("bin").Combine(CONFIGURATION + "/" + TARGET_FRAMEWORK).Combine(RUNTIME_IDENTIFIER).FullPath;
 		Information("BinDir: {0}", binDir);
 		var apps = GetDirectories(binDir + "/*.app");
 		TEST_APP = apps.First().FullPath;

eng/pipelines/common/device-tests-steps.yml

@@ -1,5 +1,5 @@
 parameters:
-  platform: '' # [ android, ios, windows ]
+  platform: '' # [ android, ios, catalyst, windows ]
   path: '' # path to csproj
   device: '' # the xharness device to use
   cakeArgs: '' # additional cake args
@@ -15,7 +15,7 @@ steps:
     parameters:
       ${{ if eq(parameters.platform, 'windows')}}:
         platform: windows
-      ${{ if or(eq(parameters.platform, 'ios'), eq(parameters.platform, 'android'))}}:
+      ${{ if or(eq(parameters.platform, 'ios'), eq(parameters.platform, 'catalyst'), eq(parameters.platform, 'android'))}}:
         platform: macos
       skipXcode: ${{ or(eq(parameters.platform, 'android'), eq(parameters.platform, 'windows')) }}
       skipProvisioning: ${{ eq(parameters.platform, 'windows') }}

eng/pipelines/common/device-tests.yml

@@ -1,8 +1,10 @@
 parameters:
   androidPool: { }
   iosPool: { }
+  catalystPool: { }
   androidApiLevels: [ 30 ]
   iosVersions: [ 'latest' ]
+  catalystVersions: [ 'latest' ]
   provisionatorChannel: 'latest'
   agentPoolAccessToken: ''
   artifactName: 'nuget'
@@ -14,6 +16,7 @@ parameters:
       desc: Human Description
       android: /optional/path/to/android.csproj
       ios: /optional/path/to/ios.csproj
+      catalyst: /optional/path/to/catalyst.csproj
 
 stages:
   - stage: android_device_tests
@@ -91,6 +94,43 @@ stages:
             checkoutDirectory: ${{ parameters.checkoutDirectory }}
             useArtifacts: ${{ parameters.useArtifacts }}
 
+  - stage: catalyst_device_tests
+    displayName: macOS Device Tests
+    dependsOn: []
+    jobs:
+    - job: catalyst_device_tests
+      workspace:
+        clean: all
+      displayName: "macOS tests"
+      pool: ${{ parameters.catalystPool }}
+      strategy:
+        matrix:
+          # create all the variables used for the matrix
+          ${{ each project in parameters.projects }}:
+            ${{ if ne(project.catalyst, '') }}:
+              ${{ each version in parameters.catalystVersions }}:
+                ${{ if not(containsValue(project.catalystVersionsExclude, version)) }}:
+                  ${{ replace(coalesce(project.desc, project.name), ' ', '_') }}_V_${{ version }}:
+                    REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE)
+                    PROJECT_PATH: ${{ project.catalyst }}
+                    ${{ if eq(version, 'latest') }}:
+                      DEVICE: maccatalyst
+                    ${{ else }}:
+                      DEVICE: maccatalyst_${{ version }}
+      steps:
+        - template: device-tests-steps.yml
+          parameters:
+            platform: catalyst
+            path: $(PROJECT_PATH)
+            device: $(DEVICE)
+            windowsPackageId: catalyst # Only needed for Windows, will be ignored
+            provisionatorChannel: ${{ parameters.provisionatorChannel }}
+            agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }}
+            artifactName: ${{ parameters.artifactName }}
+            artifactItemPattern: ${{ parameters.artifactItemPattern }}
+            checkoutDirectory: ${{ parameters.checkoutDirectory }}
+            useArtifacts: ${{ parameters.useArtifacts }}
+
   - stage: windows_device_tests
     displayName: Windows Device Tests
     dependsOn: []

eng/pipelines/device-tests.yml

@@ -69,6 +69,11 @@ parameters:
     default:
       name: $(iosTestsVmPool)
       vmImage: $(iosTestsVmImage)
+  - name: catalystPool
+    type: object
+    default:
+      name: $(iosTestsVmPool)
+      vmImage: $(iosTestsVmImage)
 
 resources:
   repositories:
@@ -84,16 +89,19 @@ stages:
     parameters:
       androidPool: ${{ parameters.androidPool }}
       iosPool: ${{ parameters.iosPool }}
+      catalystPool: ${{ parameters.catalystPool }}
       agentPoolAccessToken: $(AgentPoolAccessToken)
       ${{ if or(parameters.BuildEverything, and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'devdiv'))) }}:
         androidApiLevels: [ 30, 29, 28, 27, 26, 25, 24, 23 ]
         # androidApiLevels: [ 30, 29, 28, 27, 26, 25, 24, 23, 22, 21 ] # fix the issue of getting the test results off
         iosVersions: [ 'latest', '15.5', '14.5', '13.7']
+        catalystVersions: [ 'latest' ]
         provisionatorChannel: ${{ parameters.provisionatorChannel }}
       ${{ if not(or(parameters.BuildEverything, and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'devdiv')))) }}:
         androidApiLevels: [ 30, 23 ]
         # androidApiLevels: [ 30, 21 ] # fix the issue of getting the test results off
         iosVersions: [ 'latest' ]
+        catalystVersions: [ 'latest' ]
         provisionatorChannel: ${{ parameters.provisionatorChannel }}
       projects:
         - name: essentials
@@ -102,28 +110,33 @@ stages:
           windowsPackageId: 'com.microsoft.maui.essentials.devicetests'
           android: $(System.DefaultWorkingDirectory)/src/Essentials/test/DeviceTests/Essentials.DeviceTests.csproj
           ios: $(System.DefaultWorkingDirectory)/src/Essentials/test/DeviceTests/Essentials.DeviceTests.csproj
+          catalyst: $(System.DefaultWorkingDirectory)/src/Essentials/test/DeviceTests/Essentials.DeviceTests.csproj
           windows: $(System.DefaultWorkingDirectory)/src/Essentials/test/DeviceTests/Essentials.DeviceTests.csproj
         - name: graphics
           desc: Graphics
           androidApiLevelsExclude: [25] # Ignore for now API25 since the runs's are not stable
           windowsPackageId: 'com.microsoft.maui.graphics.devicetests'
           android: $(System.DefaultWorkingDirectory)/src/Graphics/tests/DeviceTests/Graphics.DeviceTests.csproj
           ios: $(System.DefaultWorkingDirectory)/src/Graphics/tests/DeviceTests/Graphics.DeviceTests.csproj
+          catalyst: $(System.DefaultWorkingDirectory)/src/Graphics/tests/DeviceTests/Graphics.DeviceTests.csproj
           windows: $(System.DefaultWorkingDirectory)/src/Graphics/tests/DeviceTests/Graphics.DeviceTests.csproj
         - name: core
           desc: Core
           androidApiLevelsExclude: [25] # Ignore for now API25 since the runs's are not stable
           windowsPackageId: 'com.microsoft.maui.core.devicetests'
           android: $(System.DefaultWorkingDirectory)/src/Core/tests/DeviceTests/Core.DeviceTests.csproj
           ios: $(System.DefaultWorkingDirectory)/src/Core/tests/DeviceTests/Core.DeviceTests.csproj
-          # Skip this one for Windows for now, it's crashing sometimes
+          # Skip this one for Mac Catalyst for now, it's crashing
+          catalyst: #$(System.DefaultWorkingDirectory)/src/Core/tests/DeviceTests/Core.DeviceTests.csproj
           windows: $(System.DefaultWorkingDirectory)/src/Core/tests/DeviceTests/Core.DeviceTests.csproj
         - name: controls
           desc: Controls
           androidApiLevelsExclude: [25] # Ignore for now API25 since the runs's are not stable
           windowsPackageId: 'com.microsoft.maui.controls.devicetests'
           android: $(System.DefaultWorkingDirectory)/src/Controls/tests/DeviceTests/Controls.DeviceTests.csproj
           ios: $(System.DefaultWorkingDirectory)/src/Controls/tests/DeviceTests/Controls.DeviceTests.csproj
+          # Skip this one for Mac Catalyst for now, it's crashing
+          catalyst: #$(System.DefaultWorkingDirectory)/src/Controls/tests/DeviceTests/Controls.DeviceTests.csproj
           # Skip this one for Windows for now, it's crashing
           windows: #$(System.DefaultWorkingDirectory)/src/Controls/tests/DeviceTests/Controls.DeviceTests.csproj
         - name: blazorwebview
@@ -132,5 +145,6 @@ stages:
           windowsPackageId: 'Microsoft.Maui.MauiBlazorWebView.DeviceTests'
           android: $(System.DefaultWorkingDirectory)/src/BlazorWebView/tests/MauiDeviceTests/MauiBlazorWebView.DeviceTests.csproj
           ios: $(System.DefaultWorkingDirectory)/src/BlazorWebView/tests/MauiDeviceTests/MauiBlazorWebView.DeviceTests.csproj
+          catalyst: $(System.DefaultWorkingDirectory)/src/BlazorWebView/tests/MauiDeviceTests/MauiBlazorWebView.DeviceTests.csproj
           windows: $(System.DefaultWorkingDirectory)/src/BlazorWebView/tests/MauiDeviceTests/MauiBlazorWebView.DeviceTests.csproj
 

src/Essentials/test/DeviceTests/Platforms/MacCatalyst/Info.plist

@@ -26,5 +26,9 @@
 	</array>
 	<key>XSAppIconAssets</key>
 	<string>Assets.xcassets/appicon.appiconset</string>
+	<key>NSLocationWhenInUseUsageDescription</key>
+	<string>Access to your location is required for cool things to happen!</string>
+	<key>NSContactsUsageDescription</key>
+	<string>Contacts</string>
 </dict>
 </plist>

src/Essentials/test/DeviceTests/Startup.cs

@@ -32,9 +32,6 @@ public static MauiApp CreateMauiApp()
 					},
 					SkipCategories = Traits
 						.GetSkipTraits()
-#if __ANDROID__
-						.Append($"{Traits.FileProvider}={Traits.FeatureSupport.ToExclude(OperatingSystem.IsAndroidVersionAtLeast(24))}")
-#endif
 						.ToList(),
 				})
 				.UseHeadlessRunner(new HeadlessRunnerOptions

src/Essentials/test/DeviceTests/Tests/DeviceInfo_Tests.cs

@@ -53,6 +53,8 @@ public void Platform_Is_Correct()
 		{
 #if WINDOWS_UWP || WINDOWS
 			Assert.Equal(DevicePlatform.WinUI, DeviceInfo.Platform);
+#elif MACCATALYST
+			Assert.Equal(DevicePlatform.MacCatalyst, DeviceInfo.Platform);
 #elif __IOS__
 			Assert.Equal(DevicePlatform.iOS, DeviceInfo.Platform);
 #elif __ANDROID__

src/Essentials/test/DeviceTests/Tests/HardwareSupport.cs

@@ -9,6 +9,8 @@ static class HardwareSupport
 #if __ANDROID__
 			// android emulates the accelerometer
 			true;
+#elif MACCATALYST
+			false;
 #elif __IOS__
 			// all iOS devices (and only devices) have an accelerometer
 			DeviceInfo.DeviceType == DeviceType.Physical;
@@ -21,6 +23,8 @@ static class HardwareSupport
 #if __ANDROID__
 			// android emulates the magnetometer
 			true;
+#elif MACCATALYST
+			false;
 #elif __IOS__
 			// all iOS devices (and only devices) have a magnetometer
 			DeviceInfo.DeviceType == DeviceType.Physical;
@@ -34,6 +38,8 @@ static class HardwareSupport
 			// Android emulators and devices have gyros
 			Android.App.Application.Context.GetSystemService(Android.Content.Context.SensorService) is Android.Hardware.SensorManager sensorManager &&
 			sensorManager.GetDefaultSensor(Android.Hardware.SensorType.Gyroscope) is not null;
+#elif MACCATALYST
+			false;
 #elif __IOS__
 			// all iOS devices (and only devices) have a gyroscope
 			DeviceInfo.DeviceType == DeviceType.Physical;
@@ -46,6 +52,8 @@ static class HardwareSupport
 #if __ANDROID__
 			// android emulates the compass
 			true;
+#elif MACCATALYST
+			false;
 #elif __IOS__
 			// all iOS devices (and only devices) have a compass
 			DeviceInfo.DeviceType == DeviceType.Physical;
@@ -71,6 +79,8 @@ static class HardwareSupport
 #if __ANDROID__
 			// TODO: android emulates the lamp, I think...
 			PlatformUtils.HasSystemFeature(Android.Content.PM.PackageManager.FeatureCameraFlash);
+#elif MACCATALYST
+			false;
 #elif __IOS__
 			// all iOS devices (and only devices) have a camera
 			DeviceInfo.DeviceType == DeviceType.Physical;
@@ -82,6 +92,8 @@ static class HardwareSupport
 		public static bool HasBarometer =>
 #if __ANDROID__
 			true;
+#elif MACCATALYST
+			false;
 #elif __IOS__
 			// iphone 6 and never have a barometer. looking in how to test this.
 			DeviceInfo.DeviceType == DeviceType.Physical;