diff --git a/README.md b/README.md index 9832dd23e805..6ee9d03482fd 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,4 @@ For legacy Xamarin.iOS and Xamarin.Mac downloads (discontinued), see [Downloads] Copyright (c) .NET Foundation Contributors. All rights reserved. Licensed under the [MIT](https://github.com/dotnet/macios/blob/main/LICENSE) License. + diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 4cc1a11b2690..70583e1ae6c9 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -432,9 +432,75 @@ public async Task RunAsync () FailureMessage = "Test app failed to launch."; } + // Collect diagnostic info when the launch times out to help diagnose the issue + if (Result == TestExecutingResult.LaunchTimedOut) + await CollectLaunchTimedOutDiagnostics (); + return testReporter?.Success == true ? 0 : 1; } + async Task CollectLaunchTimedOutDiagnostics () + { + MainLog.WriteLine ("Launch timed out. Collecting diagnostic info..."); + + try { + var diagnosticLog = Logs.Create ($"launch-timeout-diagnostics-{Harness.Helpers.Timestamp}.log", "Launch timeout diagnostics"); + + diagnosticLog.WriteLine ($"Launch timed out for {AppInformation.AppName} ({AppInformation.BundleIdentifier})"); + diagnosticLog.WriteLine ($"Target: {target}"); + diagnosticLog.WriteLine ($"Simulator UDID: {simulator?.UDID ?? "N/A"}"); + diagnosticLog.WriteLine ($"Simulator Name: {simulator?.Name ?? "N/A"}"); + diagnosticLog.WriteLine ($"Launch timeout: {harness.LaunchTimeout} minutes"); + diagnosticLog.WriteLine (""); + + // Collect load average (high load is correlated with this issue) + diagnosticLog.WriteLine ("=== System Load ==="); + var uptimeResult = await processManager.ExecuteCommandAsync ("uptime", Array.Empty (), diagnosticLog, TimeSpan.FromSeconds (5)); + diagnosticLog.WriteLine ($"uptime exit code: {uptimeResult.ExitCode}"); + diagnosticLog.WriteLine (""); + + // Check for recent crash reports (sqlite3 crashes are correlated with LaunchTimedOut) + diagnosticLog.WriteLine ("=== Recent Crash Reports (last 10 minutes) ==="); + var crashReportsDir = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), "Library/Logs/DiagnosticReports"); + if (Directory.Exists (crashReportsDir)) { + var tenMinutesAgo = DateTime.UtcNow.AddMinutes (-10); + var recentCrashes = Directory.GetFiles (crashReportsDir, "*.ips") + .Where (f => File.GetCreationTimeUtc (f) > tenMinutesAgo) + .OrderByDescending (f => File.GetCreationTimeUtc (f)) + .ToList (); + diagnosticLog.WriteLine ($"Found {recentCrashes.Count} recent crash reports:"); + foreach (var crash in recentCrashes) + diagnosticLog.WriteLine ($" {File.GetCreationTimeUtc (crash):O} {Path.GetFileName (crash)}"); + } else { + diagnosticLog.WriteLine ($"Crash reports directory not found: {crashReportsDir}"); + } + diagnosticLog.WriteLine (""); + + // List booted simulators and their state + diagnosticLog.WriteLine ("=== Simulator List ==="); + var simListResult = await processManager.ExecuteXcodeCommandAsync ("simctl", new [] { "list" }, diagnosticLog, TimeSpan.FromMinutes (1)); + diagnosticLog.WriteLine ($"simctl list exit code: {simListResult.ExitCode}"); + diagnosticLog.WriteLine (""); + + // Check if the specific simulator is booted + if (simulator?.UDID is not null) { + diagnosticLog.WriteLine ($"=== Simulator Status for {simulator.UDID} ==="); + var statusResult = await processManager.ExecuteXcodeCommandAsync ("simctl", new [] { "bootstatus", simulator.UDID }, diagnosticLog, TimeSpan.FromSeconds (10)); + diagnosticLog.WriteLine ($"bootstatus exit code: {statusResult.ExitCode}"); + diagnosticLog.WriteLine (""); + + // List installed apps on the simulator + diagnosticLog.WriteLine ($"=== Installed Apps on {simulator.UDID} ==="); + var listAppsResult = await processManager.ExecuteXcodeCommandAsync ("simctl", new [] { "listapps", simulator.UDID }, diagnosticLog, TimeSpan.FromSeconds (30)); + diagnosticLog.WriteLine ($"listapps exit code: {listAppsResult.ExitCode}"); + } + + MainLog.WriteLine ($"Launch timeout diagnostics written to {diagnosticLog.FullPath}"); + } catch (Exception e) { + MainLog.WriteLine ($"Failed to collect launch timeout diagnostics: {e.Message}"); + } + } + static bool IsLaunchFailure (IFileBackedLog log) { try { diff --git a/tests/xharness/Jenkins/TestTasks/RunSimulator.cs b/tests/xharness/Jenkins/TestTasks/RunSimulator.cs index e94c372e0534..23301361514b 100644 --- a/tests/xharness/Jenkins/TestTasks/RunSimulator.cs +++ b/tests/xharness/Jenkins/TestTasks/RunSimulator.cs @@ -123,6 +123,25 @@ public async Task RunTestAsync () await testTask.Runner!.RunAsync (); } testTask.ExecutionResult = testTask.Runner.Result; + testTask.FailureMessage = testTask.Runner.FailureMessage; + + // Retry once on LaunchTimedOut - this is a transient failure typically caused by + // high system load or simulator instability (e.g. sqlite3 crashes). Ref #25299. + if (testTask.ExecutionResult == TestExecutingResult.LaunchTimedOut && testTask.Harness.InCI) { + mainLog.WriteLine ($"Test launch timed out for {testTask.ProjectFile} on {testTask.Device?.Name} ({testTask.Device?.UDID}). Retrying once..."); + testTask.Runner = null; + // Reset execution result so SelectSimulatorAsync doesn't bail due to Finished flag + testTask.ExecutionResult = testTask.ExecutionResult & ~TestExecutingResult.StateMask | TestExecutingResult.Running; + using (var resource = await testTask.NotifyBlockingWaitAsync (testTask.AcquireResourceAsync ())) { + await SelectSimulatorAsync (); + await testTask.Runner!.RunAsync (); + } + testTask.ExecutionResult = testTask.Runner.Result; + testTask.FailureMessage = testTask.Runner.FailureMessage; + } + + if (testTask.ExecutionResult == TestExecutingResult.LaunchTimedOut) + mainLog.WriteLine ($"Test launch timed out for {testTask.ProjectFile} on {testTask.Device?.Name} ({testTask.Device?.UDID}). See the 'Launch timeout diagnostics' log for more info. Ref: https://github.com/dotnet/macios/issues/25299"); testTask.KnownFailure = null; if (errorKnowledgeBase.IsKnownTestIssue (testTask.Runner.MainLog, out var failure)) { diff --git a/tools/devops/automation/templates/tests/run-tests.yml b/tools/devops/automation/templates/tests/run-tests.yml index 0ff0d43d39cb..651f9ea97762 100644 --- a/tools/devops/automation/templates/tests/run-tests.yml +++ b/tools/devops/automation/templates/tests/run-tests.yml @@ -149,7 +149,7 @@ steps: fi displayName: 'Collect diagnostic info from simulators' - condition: eq(variables['system.debug'], true) + condition: or(eq(variables['system.debug'], true), failed()) continueOnError: true name: collectSimulatorInfo timeoutInMinutes: 30 @@ -160,7 +160,7 @@ steps: inputs: targetPath: $(System.DefaultWorkingDirectory)/diagnostic-sim-output artifactName: '${{ parameters.uploadPrefix }}diagnostic-simulator-info-$(Build.BuildId)-$(System.StageAttempt)-$(System.JobAttempt)-${{ parameters.labelWithPlatform }}' - condition: and(eq(variables['system.debug'], true), succeededOrFailed()) + condition: or(eq(variables['system.debug'], true), failed()) continueOnError: true # Upload TestSummary as an artifact.