[CI] Collect diagnostic info on LaunchTimedOut simulator failures. Ref #25299

README.md

@@ -68,3 +68,4 @@ For legacy Xamarin.iOS and Xamarin.Mac downloads (discontinued), see [Downloads]
 
 Copyright (c) .NET Foundation Contributors. All rights reserved.
 Licensed under the [MIT](https://github.com/dotnet/macios/blob/main/LICENSE) License.
+

tests/xharness/AppRunner.cs

@@ -432,9 +432,75 @@ public async Task<int> RunAsync ()
 				FailureMessage = "Test app failed to launch.";
 			}
 
+			// Collect diagnostic info when the launch times out to help diagnose the issue
+			if (Result == TestExecutingResult.LaunchTimedOut)
+				await CollectLaunchTimedOutDiagnostics ();
+
 			return testReporter?.Success == true ? 0 : 1;
 		}
 
+		async Task CollectLaunchTimedOutDiagnostics ()
+		{
+			MainLog.WriteLine ("Launch timed out. Collecting diagnostic info...");
+
+			try {
+				var diagnosticLog = Logs.Create ($"launch-timeout-diagnostics-{Harness.Helpers.Timestamp}.log", "Launch timeout diagnostics");
+
+				diagnosticLog.WriteLine ($"Launch timed out for {AppInformation.AppName} ({AppInformation.BundleIdentifier})");
+				diagnosticLog.WriteLine ($"Target: {target}");
+				diagnosticLog.WriteLine ($"Simulator UDID: {simulator?.UDID ?? "N/A"}");
+				diagnosticLog.WriteLine ($"Simulator Name: {simulator?.Name ?? "N/A"}");
+				diagnosticLog.WriteLine ($"Launch timeout: {harness.LaunchTimeout} minutes");
+				diagnosticLog.WriteLine ("");
+
+				// Collect load average (high load is correlated with this issue)
+				diagnosticLog.WriteLine ("=== System Load ===");
+				var uptimeResult = await processManager.ExecuteCommandAsync ("uptime", Array.Empty<string> (), diagnosticLog, TimeSpan.FromSeconds (5));
+				diagnosticLog.WriteLine ($"uptime exit code: {uptimeResult.ExitCode}");
+				diagnosticLog.WriteLine ("");
+
+				// Check for recent crash reports (sqlite3 crashes are correlated with LaunchTimedOut)
+				diagnosticLog.WriteLine ("=== Recent Crash Reports (last 10 minutes) ===");
+				var crashReportsDir = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), "Library/Logs/DiagnosticReports");
+				if (Directory.Exists (crashReportsDir)) {
+					var tenMinutesAgo = DateTime.UtcNow.AddMinutes (-10);
+					var recentCrashes = Directory.GetFiles (crashReportsDir, "*.ips")
+						.Where (f => File.GetCreationTimeUtc (f) > tenMinutesAgo)
+						.OrderByDescending (f => File.GetCreationTimeUtc (f))
+						.ToList ();
+					diagnosticLog.WriteLine ($"Found {recentCrashes.Count} recent crash reports:");
+					foreach (var crash in recentCrashes)
+						diagnosticLog.WriteLine ($"  {File.GetCreationTimeUtc (crash):O} {Path.GetFileName (crash)}");
+				} else {
+					diagnosticLog.WriteLine ($"Crash reports directory not found: {crashReportsDir}");
+				}
+				diagnosticLog.WriteLine ("");
+
+				// List booted simulators and their state
+				diagnosticLog.WriteLine ("=== Simulator List ===");
+				var simListResult = await processManager.ExecuteXcodeCommandAsync ("simctl", new [] { "list" }, diagnosticLog, TimeSpan.FromMinutes (1));
+				diagnosticLog.WriteLine ($"simctl list exit code: {simListResult.ExitCode}");
+				diagnosticLog.WriteLine ("");
+
+				// Check if the specific simulator is booted
+				if (simulator?.UDID is not null) {
+					diagnosticLog.WriteLine ($"=== Simulator Status for {simulator.UDID} ===");
+					var statusResult = await processManager.ExecuteXcodeCommandAsync ("simctl", new [] { "bootstatus", simulator.UDID }, diagnosticLog, TimeSpan.FromSeconds (10));
+					diagnosticLog.WriteLine ($"bootstatus exit code: {statusResult.ExitCode}");
+					diagnosticLog.WriteLine ("");
+
+					// List installed apps on the simulator
+					diagnosticLog.WriteLine ($"=== Installed Apps on {simulator.UDID} ===");
+					var listAppsResult = await processManager.ExecuteXcodeCommandAsync ("simctl", new [] { "listapps", simulator.UDID }, diagnosticLog, TimeSpan.FromSeconds (30));
+					diagnosticLog.WriteLine ($"listapps exit code: {listAppsResult.ExitCode}");
+				}
+
+				MainLog.WriteLine ($"Launch timeout diagnostics written to {diagnosticLog.FullPath}");
+			} catch (Exception e) {
+				MainLog.WriteLine ($"Failed to collect launch timeout diagnostics: {e.Message}");
+			}
+		}
+
 		static bool IsLaunchFailure (IFileBackedLog log)
 		{
 			try {

tests/xharness/Jenkins/TestTasks/RunSimulator.cs

@@ -123,6 +123,23 @@ public async Task RunTestAsync ()
 				await testTask.Runner!.RunAsync ();
 			}
 			testTask.ExecutionResult = testTask.Runner.Result;
+			testTask.FailureMessage = testTask.Runner.FailureMessage;
+
+			// Retry once on LaunchTimedOut - this is a transient failure typically caused by
+			// high system load or simulator instability (e.g. sqlite3 crashes). Ref #25299.
+			if (testTask.ExecutionResult == TestExecutingResult.LaunchTimedOut && testTask.Harness.InCI) {
+				mainLog.WriteLine ($"Test launch timed out for {testTask.ProjectFile} on {testTask.Device?.Name} ({testTask.Device?.UDID}). Retrying once...");
+				testTask.Runner = null;
+				using (var resource = await testTask.NotifyBlockingWaitAsync (testTask.AcquireResourceAsync ())) {
+					await SelectSimulatorAsync ();
+					await testTask.Runner!.RunAsync ();
+				}
+				testTask.ExecutionResult = testTask.Runner.Result;
+				testTask.FailureMessage = testTask.Runner.FailureMessage;
+			}
+
+			if (testTask.ExecutionResult == TestExecutingResult.LaunchTimedOut)
+				mainLog.WriteLine ($"Test launch timed out for {testTask.ProjectFile} on {testTask.Device?.Name} ({testTask.Device?.UDID}). See the 'Launch timeout diagnostics' log for more info. Ref: https://github.com/dotnet/macios/issues/25299");
 
 			testTask.KnownFailure = null;
 			if (errorKnowledgeBase.IsKnownTestIssue (testTask.Runner.MainLog, out var failure)) {

tools/devops/automation/templates/tests/run-tests.yml

@@ -149,7 +149,7 @@ steps:
     fi
 
   displayName: 'Collect diagnostic info from simulators'
-  condition: eq(variables['system.debug'], true)
+  condition: or(eq(variables['system.debug'], true), failed())
   continueOnError: true
   name: collectSimulatorInfo
   timeoutInMinutes: 30
@@ -160,7 +160,7 @@ steps:
   inputs:
     targetPath: $(System.DefaultWorkingDirectory)/diagnostic-sim-output
     artifactName: '${{ parameters.uploadPrefix }}diagnostic-simulator-info-$(Build.BuildId)-$(System.StageAttempt)-$(System.JobAttempt)-${{ parameters.labelWithPlatform }}'
-  condition: and(eq(variables['system.debug'], true), succeededOrFailed())
+  condition: or(eq(variables['system.debug'], true), failed())
   continueOnError: true
 
 # Upload TestSummary as an artifact.