From 5f36cd427d181cea2de5644979c852bab1be31b5 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Wed, 14 May 2025 21:33:37 -0700 Subject: [PATCH 01/36] WIP of PoC of tracing. --- Directory.Packages.props | 16 +- src/Client/Grpc/GrpcDurableTaskClient.cs | 20 +- src/Grpc/orchestrator_service.proto | 7 + .../Grpc/DiagnosticActivityExtensions.cs | 95 ++++++++++ src/Shared/Grpc/FieldInfoExtensionMethods.cs | 55 ++++++ src/Shared/Grpc/ProtoUtils.cs | 14 +- src/Shared/Grpc/Schema.cs | 26 +++ src/Shared/Grpc/TraceActivityConstants.cs | 16 ++ src/Shared/Grpc/TraceHelper.cs | 171 ++++++++++++++++++ src/Shared/Shared.csproj | 3 +- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 19 +- src/Worker/Grpc/GrpcOrchestrationRunner.cs | 4 +- .../GrpcSidecar/Grpc/TaskHubGrpcServer.cs | 5 + .../IntegrationTestBase.cs | 11 +- .../TracingIntegrationTests.cs | 90 +++++++++ 15 files changed, 515 insertions(+), 37 deletions(-) create mode 100644 src/Shared/Grpc/DiagnosticActivityExtensions.cs create mode 100644 src/Shared/Grpc/FieldInfoExtensionMethods.cs create mode 100644 src/Shared/Grpc/Schema.cs create mode 100644 src/Shared/Grpc/TraceActivityConstants.cs create mode 100644 src/Shared/Grpc/TraceHelper.cs create mode 100644 test/Grpc.IntegrationTests/TracingIntegrationTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index cfa91868..06d6c9f7 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,32 +6,28 @@ --> true - - + - - - @@ -40,7 +36,6 @@ - @@ -53,22 +48,20 @@ - - + - + - @@ -78,5 +71,4 @@ - - + \ No newline at end of file diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index a4960ae9..2a02b35b 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -105,24 +105,6 @@ public override async Task ScheduleNewOrchestrationInstanceAsync( } } - if (Activity.Current?.Id != null || Activity.Current?.TraceStateString != null) - { - if (request.ParentTraceContext == null) - { - request.ParentTraceContext = new P.TraceContext(); - } - - if (Activity.Current?.Id != null) - { - request.ParentTraceContext.TraceParent = Activity.Current?.Id; - } - - if (Activity.Current?.TraceStateString != null) - { - request.ParentTraceContext.TraceState = Activity.Current?.TraceStateString; - } - } - DateTimeOffset? startAt = options?.StartAt; this.logger.SchedulingOrchestration( request.InstanceId, @@ -136,6 +118,8 @@ public override async Task ScheduleNewOrchestrationInstanceAsync( request.ScheduledStartTimestamp = Timestamp.FromDateTimeOffset(startAt.Value.ToUniversalTime()); } + using Activity? newActivity = TraceHelper.StartActivityForNewOrchestration(request); + P.CreateInstanceResponse? result = await this.sidecarClient.StartInstanceAsync( request, cancellationToken: cancellation); return result.InstanceId; diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 88928c3b..6ffb90b7 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -77,6 +77,8 @@ message ExecutionStartedEvent { TraceContext parentTraceContext = 7; google.protobuf.StringValue orchestrationSpanID = 8; map tags = 9; + google.protobuf.StringValue orchestrationID = 10; + google.protobuf.Timestamp activityStartTIme = 11; } message ExecutionCompletedEvent { @@ -256,6 +258,7 @@ message ScheduleTaskAction { string name = 1; google.protobuf.StringValue version = 2; google.protobuf.StringValue input = 3; + TraceContext parentTraceContext = 4; } message CreateSubOrchestrationAction { @@ -331,6 +334,10 @@ message OrchestratorResponse { // The number of work item events that were processed by the orchestrator. // This field is optional. If not set, the service should assume that the orchestrator processed all events. google.protobuf.Int32Value numEventsProcessed = 5; + + google.protobuf.StringValue orchestrationSpanID = 6; + google.protobuf.StringValue orchestrationID = 7; + google.protobuf.Timestamp activityStartTime = 8; } message CreateInstanceRequest { diff --git a/src/Shared/Grpc/DiagnosticActivityExtensions.cs b/src/Shared/Grpc/DiagnosticActivityExtensions.cs new file mode 100644 index 00000000..1e9fa466 --- /dev/null +++ b/src/Shared/Grpc/DiagnosticActivityExtensions.cs @@ -0,0 +1,95 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace Microsoft.DurableTask +{ + using System; + using System.Diagnostics; + using System.Linq; + using System.Linq.Expressions; + using System.Reflection; + + /// + /// Replica from System.Diagnostics.DiagnosticSource >= 6.0.0 + /// + internal enum ActivityStatusCode + { + Unset = 0, + OK = 1, + Error = 2, + } + + /// + /// Extensions for . + /// + internal static class DiagnosticActivityExtensions + { + private static readonly Action s_setSpanId; + private static readonly Action s_setId; + private static readonly Action s_setStatus; + + static DiagnosticActivityExtensions() + { + BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance; + s_setSpanId = typeof(Activity).GetField("_spanId", flags).CreateSetter(); + s_setId = typeof(Activity).GetField("_id", flags).CreateSetter(); + s_setStatus = CreateSetStatus(); + } + + public static void SetId(this Activity activity, string id) + => s_setId(activity, id); + + public static void SetSpanId(this Activity activity, string spanId) + => s_setSpanId(activity, spanId); + + public static void SetStatus(this Activity activity, ActivityStatusCode status, string description) + => s_setStatus(activity, status, description); + + private static Action CreateSetStatus() + { + MethodInfo method = typeof(Activity).GetMethod("SetStatus"); + if (method is null) + { + return (activity, status, description) => { + if (activity is null) + { + throw new ArgumentNullException(nameof(activity)); + } + string str = status switch + { + ActivityStatusCode.Unset => "UNSET", + ActivityStatusCode.OK => "OK", + ActivityStatusCode.Error => "ERROR", + _ => null, + }; + activity.SetTag("otel.status_code", str); + activity.SetTag("otel.status_description", description); + }; + } + + /* + building expression tree to effectively perform: + (activity, status, description) => activity.SetStatus((ActivityStatusCode)(int)status, description); + */ + + ParameterExpression targetExp = Expression.Parameter(typeof(Activity), "target"); + ParameterExpression status = Expression.Parameter(typeof(ActivityStatusCode), "status"); + ParameterExpression description = Expression.Parameter(typeof(string), "description"); + UnaryExpression convert = Expression.Convert(status, typeof(int)); + convert = Expression.Convert(convert, method.GetParameters().First().ParameterType); + MethodCallExpression callExp = Expression.Call(targetExp, method, convert, description); + return Expression.Lambda>(callExp, targetExp, status, description) + .Compile(); + } + } +} diff --git a/src/Shared/Grpc/FieldInfoExtensionMethods.cs b/src/Shared/Grpc/FieldInfoExtensionMethods.cs new file mode 100644 index 00000000..a0bcde77 --- /dev/null +++ b/src/Shared/Grpc/FieldInfoExtensionMethods.cs @@ -0,0 +1,55 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Linq.Expressions; +using System.Reflection; + +namespace Microsoft.DurableTask +{ + /// + /// Extensions for . + /// + internal static class FieldInfoExtensionMethods + { + /// + /// Create a re-usable setter for a . + /// When cached and reused, This is quicker than using . + /// + /// The target type of the object. + /// The value type of the field. + /// The field info. + /// A re-usable action to set the field. + internal static Action CreateSetter(this FieldInfo fieldInfo) + { + if (fieldInfo == null) + { + throw new ArgumentNullException(nameof(fieldInfo)); + } + + ParameterExpression targetExp = Expression.Parameter(typeof(TTarget), "target"); + Expression source = targetExp; + + if (typeof(TTarget) != fieldInfo.DeclaringType) + { + source = Expression.Convert(targetExp, fieldInfo.DeclaringType); + } + + // Creating the setter to set the value to the field + ParameterExpression valueExp = Expression.Parameter(typeof(TValue), "value"); + MemberExpression fieldExp = Expression.Field(source, fieldInfo); + BinaryExpression assignExp = Expression.Assign(fieldExp, valueExp); + return Expression.Lambda>(assignExp, targetExp, valueExp).Compile(); + } + } +} diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index 56907bc9..8608c59e 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Buffers.Text; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Text; @@ -279,7 +280,8 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( string? customStatus, IEnumerable actions, string completionToken, - EntityConversionState? entityConversionState) + EntityConversionState? entityConversionState, + Activity? parentActivity) { Check.NotNull(actions); var response = new P.OrchestratorResponse @@ -287,6 +289,9 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( InstanceId = instanceId, CustomStatus = customStatus, CompletionToken = completionToken, + OrchestrationID = parentActivity?.Id, + OrchestrationSpanID = parentActivity?.SpanId.ToString(), + ActivityStartTime = parentActivity?.StartTimeUtc.ToTimestamp(), }; foreach (OrchestratorAction action in actions) @@ -302,6 +307,13 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( Name = scheduleTaskAction.Name, Version = scheduleTaskAction.Version, Input = scheduleTaskAction.Input, + ParentTraceContext = parentActivity is not null + ? new P.TraceContext + { + TraceParent = parentActivity.Id, + TraceState = parentActivity.TraceStateString, + } + : null, }; break; case OrchestratorActionType.CreateSubOrchestration: diff --git a/src/Shared/Grpc/Schema.cs b/src/Shared/Grpc/Schema.cs new file mode 100644 index 00000000..c82df136 --- /dev/null +++ b/src/Shared/Grpc/Schema.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask; + +internal static class Schema +{ + internal static class Task + { + internal const string Type = "durabletask.type"; + internal const string Name = "durabletask.task.name"; + internal const string Version = "durabletask.task.version"; + internal const string InstanceId = "durabletask.task.instance_id"; + internal const string ExecutionId = "durabletask.task.execution_id"; + internal const string Status = "durabletask.task.status"; + internal const string TaskId = "durabletask.task.task_id"; + internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; + internal const string FireAt = "durabletask.fire_at"; + } + + internal static class Status + { + internal const string Code = "otel.status_code"; + internal const string Description = "otel.status_description"; + } +} diff --git a/src/Shared/Grpc/TraceActivityConstants.cs b/src/Shared/Grpc/TraceActivityConstants.cs new file mode 100644 index 00000000..ef0154e1 --- /dev/null +++ b/src/Shared/Grpc/TraceActivityConstants.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask; + +internal class TraceActivityConstants +{ + public const string Client = "client"; + public const string Orchestration = "orchestration"; + public const string Activity = "activity"; + public const string Event = "event"; + public const string Timer = "timer"; + + public const string CreateOrchestration = "create_orchestration"; + public const string OrchestrationEvent = "orchestration_event"; +} diff --git a/src/Shared/Grpc/TraceHelper.cs b/src/Shared/Grpc/TraceHelper.cs new file mode 100644 index 00000000..0368c2bf --- /dev/null +++ b/src/Shared/Grpc/TraceHelper.cs @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using Google.Protobuf.WellKnownTypes; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask; + +public class TraceHelper +{ + const string Source = "Microsoft.DurableTask"; + + static readonly ActivitySource ActivityTraceSource = new ActivitySource(Source); + + /// + /// Starts a new trace activity for scheduling an orchestration from the client. + /// + /// The orchestration's execution started event. + /// + /// Returns a newly started with orchestration-specific metadata. + /// + internal static Activity? StartActivityForNewOrchestration(P.CreateInstanceRequest createInstanceRequest) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + name: CreateSpanName(TraceActivityConstants.CreateOrchestration, createInstanceRequest.Name, createInstanceRequest.Version), + kind: ActivityKind.Producer); + + if (newActivity != null) + { + newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); + newActivity.SetTag(Schema.Task.Name, createInstanceRequest.Name); + newActivity.SetTag(Schema.Task.InstanceId, createInstanceRequest.InstanceId); + newActivity.SetTag(Schema.Task.ExecutionId, createInstanceRequest.ExecutionId); + + if (!string.IsNullOrEmpty(createInstanceRequest.Version)) + { + newActivity.SetTag(Schema.Task.Version, createInstanceRequest.Version); + } + } + + if (Activity.Current?.Id != null || Activity.Current?.TraceStateString != null) + { + createInstanceRequest.ParentTraceContext ??= new P.TraceContext(); + + if (Activity.Current?.Id != null) + { + createInstanceRequest.ParentTraceContext.TraceParent = Activity.Current?.Id; + } + + if (Activity.Current?.TraceStateString != null) + { + createInstanceRequest.ParentTraceContext.TraceState = Activity.Current?.TraceStateString; + } + } + + return newActivity; + } + + /// + /// Starts a new trace activity for orchestration execution. + /// + /// The orchestration's execution started event. + /// + /// Returns a newly started with orchestration-specific metadata. + /// + internal static Activity? StartTraceActivityForOrchestrationExecution(P.ExecutionStartedEvent? startEvent) + { + if (startEvent == null) + { + return null; + } + + if (startEvent.ParentTraceContext is null || !ActivityContext.TryParse(startEvent.ParentTraceContext.TraceParent, startEvent.ParentTraceContext.TraceState, out ActivityContext activityContext)) + { + return null; + } + + string activityName = CreateSpanName(TraceActivityConstants.Orchestration, startEvent.Name, startEvent.Version); + ActivityKind activityKind = ActivityKind.Server; + DateTimeOffset startTime = startEvent.ActivityStartTIme?.ToDateTimeOffset() ?? default; + + Activity? activity = ActivityTraceSource.StartActivity( + activityName, + kind: activityKind, + parentContext: activityContext, + startTime: startTime); + + if (activity == null) + { + return null; + } + + activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); + activity.SetTag(Schema.Task.Name, startEvent.Name); + activity.SetTag(Schema.Task.InstanceId, startEvent.OrchestrationInstance.InstanceId); + + if (!string.IsNullOrEmpty(startEvent.Version)) + { + activity.SetTag(Schema.Task.Version, startEvent.Version); + } + + if (startEvent.OrchestrationID != null && startEvent.OrchestrationSpanID != null) + { + activity.SetId(startEvent.OrchestrationID!); + activity.SetSpanId(startEvent.OrchestrationSpanID!); + } + else + { + startEvent.OrchestrationID = activity.Id; + startEvent.OrchestrationSpanID = activity.SpanId.ToString(); + startEvent.ActivityStartTIme = Timestamp.FromDateTime(activity.StartTimeUtc); + } + + // DistributedTraceActivity.Current = activity; + // return DistributedTraceActivity.Current; + + return activity; + } + + /// + /// Starts a new trace activity for (task) activity execution. + /// + /// The associated . + /// The associated orchestration instance metadata. + /// + /// Returns a newly started with (task) activity and orchestration-specific metadata. + /// + internal static Activity? StartTraceActivityForTaskExecution( + P.ActivityRequest request) + { + if (request.ParentTraceContext is null || !ActivityContext.TryParse(request.ParentTraceContext.TraceParent, request.ParentTraceContext.TraceState, out ActivityContext activityContext)) + { + return null; + } + + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.Activity, request.Name, request.Version), + kind: ActivityKind.Server, + parentContext: activityContext); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Activity); + newActivity.SetTag(Schema.Task.Name, request.Name); + newActivity.SetTag(Schema.Task.InstanceId, request.OrchestrationInstance.InstanceId); + newActivity.SetTag(Schema.Task.TaskId, request.TaskId); + + if (!string.IsNullOrEmpty(request.Version)) + { + newActivity.SetTag(Schema.Task.Version, request.Version); + } + + return newActivity; + } + + static string CreateSpanName(string spanDescription, string? taskName, string? taskVersion) + { + if (!string.IsNullOrEmpty(taskVersion)) + { + return $"{spanDescription}:{taskName}@({taskVersion})"; + } + else + { + return $"{spanDescription}:{taskName}"; + } + } +} diff --git a/src/Shared/Shared.csproj b/src/Shared/Shared.csproj index 76b3bed6..6424c15b 100644 --- a/src/Shared/Shared.csproj +++ b/src/Shared/Shared.csproj @@ -14,7 +14,8 @@ - + + diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 327bb0cc..b8583288 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Diagnostics; using System.Text; using DurableTask.Core; using DurableTask.Core.Entities; using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; +using DurableTask.Core.Tracing; using Microsoft.DurableTask.Abstractions; using Microsoft.DurableTask.Entities; using Microsoft.DurableTask.Worker.Shims; @@ -356,6 +358,16 @@ async Task OnRunOrchestratorAsync( string completionToken, CancellationToken cancellationToken) { + var executionStartedEvent = + request + .NewEvents + .Concat(request.PastEvents) + .Where(e => e.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.ExecutionStarted) + .Select(e => e.ExecutionStarted) + .FirstOrDefault(); + + using Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution(executionStartedEvent); + OrchestratorExecutionResult? result = null; P.TaskFailureDetails? failureDetails = null; TaskName name = new("(unknown)"); @@ -435,7 +447,8 @@ async Task OnRunOrchestratorAsync( result.CustomStatus, result.Actions, completionToken, - entityConversionState); + entityConversionState, + traceActivity); } else if (versioning != null && failureDetails != null && versionFailure) { @@ -502,8 +515,12 @@ await this.client.AbandonTaskOrchestratorWorkItemAsync( await this.client.CompleteOrchestratorTaskAsync(response, cancellationToken: cancellationToken); } + private static readonly ActivitySource ActivitySource = new ActivitySource("Microsoft.DurableTask.Worker"); + async Task OnRunActivityAsync(P.ActivityRequest request, string completionToken, CancellationToken cancellation) { + using Activity? traceActivity = TraceHelper.StartTraceActivityForTaskExecution(request); + OrchestrationInstance instance = request.OrchestrationInstance.ToCore(); string rawInput = request.Input; diff --git a/src/Worker/Grpc/GrpcOrchestrationRunner.cs b/src/Worker/Grpc/GrpcOrchestrationRunner.cs index 9a6e3c5f..c35256bb 100644 --- a/src/Worker/Grpc/GrpcOrchestrationRunner.cs +++ b/src/Worker/Grpc/GrpcOrchestrationRunner.cs @@ -120,7 +120,9 @@ public static string LoadAndRun( result.CustomStatus, result.Actions, completionToken: string.Empty, /* doesn't apply */ - entityConversionState: null); + entityConversionState: null, + // TODO: Should this activity be created? + parentActivity: null); byte[] responseBytes = response.ToByteArray(); return Convert.ToBase64String(responseBytes); } diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs index 167a437f..6e0563ff 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs @@ -143,6 +143,11 @@ await this.client.CreateTaskOrchestrationAsync( Version = request.Version, OrchestrationInstance = instance, Tags = request.Tags.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), + ParentTraceContext = request.ParentTraceContext is not null + ? new(request.ParentTraceContext.TraceParent, request.ParentTraceContext.TraceState) + { + } + : null }, OrchestrationInstance = instance, }); diff --git a/test/Grpc.IntegrationTests/IntegrationTestBase.cs b/test/Grpc.IntegrationTests/IntegrationTestBase.cs index fdebdb24..642107ef 100644 --- a/test/Grpc.IntegrationTests/IntegrationTestBase.cs +++ b/test/Grpc.IntegrationTests/IntegrationTestBase.cs @@ -37,6 +37,8 @@ public IntegrationTestBase(ITestOutputHelper output, GrpcSidecarFixture sidecarF /// public CancellationToken TimeoutToken => this.testTimeoutSource.Token; + public ICollection ExportedItems = new List(); + void IDisposable.Dispose() { this.testTimeoutSource.Dispose(); @@ -57,14 +59,15 @@ protected async Task StartWorkerAsync(ActionConfigures the durable task client builder. protected IHostBuilder CreateHostBuilder(Action workerConfigure, Action? clientConfigure) { - return Host.CreateDefaultBuilder() + var host = Host.CreateDefaultBuilder() .ConfigureLogging(b => { b.ClearProviders(); b.AddProvider(this.logProvider); b.SetMinimumLevel(LogLevel.Debug); - }) - .ConfigureServices(services => + + }) + .ConfigureServices((context, services) => { services.AddDurableTaskWorker(b => { @@ -79,6 +82,8 @@ protected IHostBuilder CreateHostBuilder(Action worke clientConfigure?.Invoke(b); }); }); + + return host; } protected IReadOnlyCollection GetLogs() diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs new file mode 100644 index 00000000..648782e8 --- /dev/null +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Worker; +using Xunit.Abstractions; + +namespace Microsoft.DurableTask.Grpc.Tests; + +public class TracingIntegrationTests : IntegrationTestBase +{ + public TracingIntegrationTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) + : base(output, sidecarFixture) + { + } + + static readonly ActivitySource TestActivitySource = new ActivitySource(nameof(TracingIntegrationTests)); + + [Fact] + public async Task Orchestration_Traces() + { + var activities = new List(); + + using var listener = new ActivityListener(); + + listener.ShouldListenTo = s => s.Name == nameof(TracingIntegrationTests) || s.Name == "Microsoft.DurableTask"; + listener.Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData; + listener.ActivityStopped = a => activities.Add(a); + + ActivitySource.AddActivityListener(listener); + + TaskName orchestratorName = nameof(Orchestration_Traces); + + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, (ctx, input) => PageableOrchestrationAsync(ctx, input)) + .AddActivityFunc?>( + nameof(PageableActivityAsync), (_, input) => PageableActivityAsync(input))); + }); + + using (var testActivity = TestActivitySource.StartActivity("Test", ActivityKind.Client)) + { + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync( + orchestratorName, input: string.Empty); + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + } + + var testActivity = activities.Single(a => a.OperationName == "Test"); + + activities.Should() + .ContainSingle(a => a.OperationName == "orchestration:Orchestration_Traces") + .And.Match(a => a.par.Should().Be(testActivity.Id); + + activities.Should().HaveCountGreaterThan(0); + } + + static Task?> PageableActivityAsync(PageRequest? input) + { + int pageSize = input?.PageSize ?? 3; + Page CreatePage(string? next) + => new (Enumerable.Range(0, pageSize).Select(x => $"item_{x}").ToList(), next); + Page? page = input?.Continuation switch + { + null => CreatePage("1"), + "1" => CreatePage("2"), + "2" => CreatePage(null), + _ => null, + }; + + return Task.FromResult(page); + } + + static async Task PageableOrchestrationAsync(TaskOrchestrationContext context, string? input) + { + AsyncPageable pageable = Pageable.Create((continuation, _, _) => + { + return context.CallActivityAsync?>( + nameof(PageableActivityAsync), new PageRequest(continuation))!; + }); + + return await pageable.CountAsync(); + } + + record PageRequest(string? Continuation, int? PageSize = null); +} From 76fdee22d3f77dd7d484dbecf7730588111d535a Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Wed, 14 May 2025 23:39:14 -0700 Subject: [PATCH 02/36] Wiring tracing metadata through test server. --- .../GrpcOrchestratorExecutionResult.cs | 13 +++++++ .../GrpcSidecar/Dispatcher/ITaskExecutor.cs | 2 +- .../Dispatcher/TaskOrchestrationDispatcher.cs | 11 +++++- .../GrpcSidecar/Grpc/ProtobufUtils.cs | 3 ++ .../GrpcSidecar/Grpc/TaskHubGrpcServer.cs | 19 +++++----- .../TracingIntegrationTests.cs | 35 ++++++++++++------- 6 files changed, 60 insertions(+), 23 deletions(-) create mode 100644 test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs new file mode 100644 index 00000000..44b6f073 --- /dev/null +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using DurableTask.Core; + +namespace Microsoft.DurableTask.Sidecar.Dispatcher; + +public class GrpcOrchestratorExecutionResult : OrchestratorExecutionResult +{ + public string? OrchestrationActivityId { get; set; } + public string? OrchestrationActivitySpanId { get; set; } + public DateTimeOffset? OrchestrationActivityStartTime { get; set; } +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITaskExecutor.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITaskExecutor.cs index 535b9502..2a8f144e 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITaskExecutor.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITaskExecutor.cs @@ -18,7 +18,7 @@ interface ITaskExecutor /// Returns a task containing the result of the orchestrator execution. These are effectively the side-effects of the /// orchestrator code, such as calling activities, scheduling timers, etc. /// - Task ExecuteOrchestrator( + Task ExecuteOrchestrator( OrchestrationInstance instance, IEnumerable pastEvents, IEnumerable newEvents); diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs index 20cae03e..e4e983af 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs @@ -62,7 +62,7 @@ protected override async Task ExecuteWorkItemAsync(TaskOrchestrationWorkItem wor // Execute the orchestrator code and get back a set of new actions to take. // IMPORTANT: This IEnumerable may be lazily evaluated and should only be enumerated once! - OrchestratorExecutionResult result = await this.taskExecutor.ExecuteOrchestrator( + GrpcOrchestratorExecutionResult result = await this.taskExecutor.ExecuteOrchestrator( instance, workItem.OrchestrationRuntimeState.PastEvents, workItem.OrchestrationRuntimeState.NewEvents); @@ -89,6 +89,15 @@ protected override async Task ExecuteWorkItemAsync(TaskOrchestrationWorkItem wor continue; } + ExecutionStartedEvent? executionStartedEvent = workItem.OrchestrationRuntimeState.NewEvents.OfType().FirstOrDefault(); + + if (executionStartedEvent?.ParentTraceContext is not null) + { + executionStartedEvent.ParentTraceContext.ActivityStartTime = result.OrchestrationActivityStartTime; + executionStartedEvent.ParentTraceContext.Id = result.OrchestrationActivityId; + executionStartedEvent.ParentTraceContext.SpanId = result.OrchestrationActivitySpanId; + } + // Commit the changes to the durable store await this.service.CompleteTaskOrchestrationWorkItemAsync( workItem, diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs index 6c9ab0b9..167bd941 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs @@ -96,6 +96,9 @@ public static Proto.HistoryEvent ToHistoryEventProto(HistoryEvent e) TraceParent = startedEvent.ParentTraceContext.TraceParent, TraceState = startedEvent.ParentTraceContext.TraceState, }, + ActivityStartTIme = startedEvent.ParentTraceContext?.ActivityStartTime is not null ? Timestamp.FromDateTimeOffset(startedEvent.ParentTraceContext.ActivityStartTime.Value) : null, + OrchestrationID = startedEvent.ParentTraceContext?.Id, + OrchestrationSpanID = startedEvent.ParentTraceContext?.SpanId, }; break; case EventType.ExecutionTerminated: diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs index 6e0563ff..aaeb9e31 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs @@ -20,7 +20,7 @@ public class TaskHubGrpcServer : P.TaskHubSidecarService.TaskHubSidecarServiceBa { static readonly Task EmptyCompleteTaskResponse = Task.FromResult(new P.CompleteTaskResponse()); - readonly ConcurrentDictionary> pendingOrchestratorTasks = new(StringComparer.OrdinalIgnoreCase); + readonly ConcurrentDictionary> pendingOrchestratorTasks = new(StringComparer.OrdinalIgnoreCase); readonly ConcurrentDictionary> pendingActivityTasks = new(StringComparer.OrdinalIgnoreCase); readonly ILogger log; @@ -145,8 +145,6 @@ await this.client.CreateTaskOrchestrationAsync( Tags = request.Tags.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), ParentTraceContext = request.ParentTraceContext is not null ? new(request.ParentTraceContext.TraceParent, request.ParentTraceContext.TraceState) - { - } : null }, OrchestrationInstance = instance, @@ -364,16 +362,19 @@ static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationState state, { if (!this.pendingOrchestratorTasks.TryRemove( request.InstanceId, - out TaskCompletionSource? tcs)) + out TaskCompletionSource? tcs)) { // TODO: Log? throw new RpcException(new Status(StatusCode.NotFound, $"Orchestration not found")); } - OrchestratorExecutionResult result = new() + GrpcOrchestratorExecutionResult result = new() { Actions = request.Actions.Select(ProtobufUtils.ToOrchestratorAction), CustomStatus = request.CustomStatus, + OrchestrationActivityId = request.OrchestrationID, + OrchestrationActivitySpanId = request.OrchestrationSpanID, + OrchestrationActivityStartTime = request.ActivityStartTime?.ToDateTimeOffset(), }; tcs.TrySetResult(result); @@ -463,14 +464,14 @@ public override async Task GetWorkItems(P.GetWorkItemsRequest request, IServerSt /// Invoked by the when a work item is available, proxies the call to execute an orchestrator over a gRPC channel. /// /// - async Task ITaskExecutor.ExecuteOrchestrator( + async Task ITaskExecutor.ExecuteOrchestrator( OrchestrationInstance instance, IEnumerable pastEvents, IEnumerable newEvents) { // Create a task completion source that represents the async completion of the orchestrator execution. // This must be done before we start the orchestrator execution. - TaskCompletionSource tcs = + TaskCompletionSource tcs = this.CreateTaskCompletionSourceForOrchestrator(instance.InstanceId); try @@ -563,9 +564,9 @@ async Task SendWorkItemToClientAsync(P.WorkItem workItem) } } - TaskCompletionSource CreateTaskCompletionSourceForOrchestrator(string instanceId) + TaskCompletionSource CreateTaskCompletionSourceForOrchestrator(string instanceId) { - TaskCompletionSource tcs = new(); + TaskCompletionSource tcs = new(); this.pendingOrchestratorTasks.TryAdd(instanceId, tcs); return tcs; } diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index 648782e8..3fbb6e61 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -18,19 +18,28 @@ public TracingIntegrationTests(ITestOutputHelper output, GrpcSidecarFixture side static readonly ActivitySource TestActivitySource = new ActivitySource(nameof(TracingIntegrationTests)); - [Fact] - public async Task Orchestration_Traces() + static ActivityListener CreateListener(string source, ICollection activities) { - var activities = new List(); - - using var listener = new ActivityListener(); + ActivityListener listener = new(); - listener.ShouldListenTo = s => s.Name == nameof(TracingIntegrationTests) || s.Name == "Microsoft.DurableTask"; + listener.ShouldListenTo = s => s.Name == source; listener.Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData; listener.ActivityStopped = a => activities.Add(a); ActivitySource.AddActivityListener(listener); + return listener; + } + + [Fact] + public async Task Orchestration_Traces() + { + var testActivities = new List(); + var coreActivities = new List(); + + using var testListener = CreateListener(nameof(TracingIntegrationTests), testActivities); + using var coreListener = CreateListener("Microsoft.DurableTask", coreActivities); + TaskName orchestratorName = nameof(Orchestration_Traces); await using HostTestLifetime server = await this.StartWorkerAsync(b => @@ -42,7 +51,7 @@ public async Task Orchestration_Traces() nameof(PageableActivityAsync), (_, input) => PageableActivityAsync(input))); }); - using (var testActivity = TestActivitySource.StartActivity("Test", ActivityKind.Client)) + using (var activity = TestActivitySource.StartActivity("Test", ActivityKind.Client)) { string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync( orchestratorName, input: string.Empty); @@ -50,13 +59,15 @@ public async Task Orchestration_Traces() instanceId, getInputsAndOutputs: true, this.TimeoutToken); } - var testActivity = activities.Single(a => a.OperationName == "Test"); + testActivities.Count.Should().Be(1); - activities.Should() - .ContainSingle(a => a.OperationName == "orchestration:Orchestration_Traces") - .And.Match(a => a.par.Should().Be(testActivity.Id); + var testActivity = testActivities.Single(); + + coreActivities.Count.Should().Be(1); - activities.Should().HaveCountGreaterThan(0); + var coreActivity = coreActivities.Single(); + + coreActivity.ParentId.Should().Be(testActivity.Id); } static Task?> PageableActivityAsync(PageRequest? input) From 4681dc047429658674bbc79619da9530551125d0 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 15 May 2025 09:27:36 -0700 Subject: [PATCH 03/36] Ensure orchestration traces are parented correctly. --- .../TracingIntegrationTests.cs | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index 3fbb6e61..bdc194eb 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -16,13 +16,11 @@ public TracingIntegrationTests(ITestOutputHelper output, GrpcSidecarFixture side { } - static readonly ActivitySource TestActivitySource = new ActivitySource(nameof(TracingIntegrationTests)); - - static ActivityListener CreateListener(string source, ICollection activities) + static ActivityListener CreateListener(string[] sources, ICollection activities) { ActivityListener listener = new(); - listener.ShouldListenTo = s => s.Name == source; + listener.ShouldListenTo = s => sources.Contains(s.Name); listener.Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData; listener.ActivityStopped = a => activities.Add(a); @@ -30,15 +28,20 @@ static ActivityListener CreateListener(string source, ICollection acti return listener; } - + + const string TestActivitySourceName = nameof(TracingIntegrationTests); + const string CoreActivitySourceName = "Microsoft.DurableTask"; + + static readonly string[] ActivitySourceNames = [TestActivitySourceName, CoreActivitySourceName]; + [Fact] public async Task Orchestration_Traces() { - var testActivities = new List(); - var coreActivities = new List(); + ActivitySource testActivitySource = new ActivitySource(nameof(TracingIntegrationTests)); + + var activities = new List(); - using var testListener = CreateListener(nameof(TracingIntegrationTests), testActivities); - using var coreListener = CreateListener("Microsoft.DurableTask", coreActivities); + using var listener = CreateListener(ActivitySourceNames, activities); TaskName orchestratorName = nameof(Orchestration_Traces); @@ -51,7 +54,7 @@ public async Task Orchestration_Traces() nameof(PageableActivityAsync), (_, input) => PageableActivityAsync(input))); }); - using (var activity = TestActivitySource.StartActivity("Test", ActivityKind.Client)) + using (var activity = testActivitySource.StartActivity("Test", ActivityKind.Client)) { string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync( orchestratorName, input: string.Empty); @@ -59,15 +62,19 @@ public async Task Orchestration_Traces() instanceId, getInputsAndOutputs: true, this.TimeoutToken); } - testActivities.Count.Should().Be(1); - - var testActivity = testActivities.Single(); + var testActivity = activities.Single(a => a.Source == testActivitySource && a.OperationName == "Test"); + var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "create_orchestration:Orchestration_Traces"); - coreActivities.Count.Should().Be(1); - - var coreActivity = coreActivities.Single(); + createActivity.ParentId.Should().Be(testActivity.Id); + createActivity.ParentSpanId.Should().Be(testActivity.SpanId); - coreActivity.ParentId.Should().Be(testActivity.Id); + activities + .Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "orchestration:Orchestration_Traces") + .Should().AllSatisfy(a => + { + a.ParentId.Should().Be(createActivity.Id); + a.ParentSpanId.Should().Be(createActivity.SpanId); + }); } static Task?> PageableActivityAsync(PageRequest? input) From 3421304a1bf1e3972e913daefaa5f8a322918aad Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 15 May 2025 09:44:29 -0700 Subject: [PATCH 04/36] Check orchestration activities are same logical activity. --- .../TracingIntegrationTests.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index bdc194eb..0b32d3c8 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -65,12 +65,21 @@ public async Task Orchestration_Traces() var testActivity = activities.Single(a => a.Source == testActivitySource && a.OperationName == "Test"); var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "create_orchestration:Orchestration_Traces"); + // The creation activity should be parented to the test activity. createActivity.ParentId.Should().Be(testActivity.Id); createActivity.ParentSpanId.Should().Be(testActivity.SpanId); - activities - .Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "orchestration:Orchestration_Traces") - .Should().AllSatisfy(a => + var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "orchestration:Orchestration_Traces"); + + // The orchestration activities should be the same "logical" activity. + orchestrationActivities.Select(a => a.StartTimeUtc).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.Id).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.SpanId).Distinct().Should().HaveCount(1); + + // The orchestration activities should be parented to the create activity. + orchestrationActivities + .Should().HaveCountGreaterThan(0) + .And.AllSatisfy(a => { a.ParentId.Should().Be(createActivity.Id); a.ParentSpanId.Should().Be(createActivity.SpanId); From b951b5dd164f9e88207a6d0901890210901f43af Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 15 May 2025 11:36:24 -0700 Subject: [PATCH 05/36] Test activity traces. --- .../GrpcScheduleTaskOrchestratorAction.cs | 12 ++++++++++++ .../Dispatcher/TaskOrchestrationDispatcher.cs | 5 +++++ .../GrpcSidecar/Grpc/ProtobufUtils.cs | 7 ++++++- .../GrpcSidecar/Grpc/TaskHubGrpcServer.cs | 7 +++++++ .../TracingIntegrationTests.cs | 14 +++++++++++++- 5 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs new file mode 100644 index 00000000..bd8a5af4 --- /dev/null +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using DurableTask.Core.Command; +using DurableTask.Core.Tracing; + +namespace Microsoft.DurableTask.Sidecar.Dispatcher; + +public class GrpcScheduleTaskOrchestratorAction : ScheduleTaskOrchestratorAction +{ + public DistributedTraceContext? ParentTraceContext { get; set; } +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs index e4e983af..0cec5a89 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs @@ -206,6 +206,11 @@ void ApplyOrchestratorActions( scheduleTaskAction.Version, scheduleTaskAction.Input); + if (action is GrpcScheduleTaskOrchestratorAction { ParentTraceContext: not null } grpcAction) + { + scheduledEvent.ParentTraceContext ??= new(grpcAction.ParentTraceContext.TraceParent, grpcAction.ParentTraceContext.TraceState); + } + newActivityMessages ??= new List(); newActivityMessages.Add(new TaskMessage { diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs index 167bd941..a0e33d1c 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs @@ -6,8 +6,10 @@ using DurableTask.Core.Command; using DurableTask.Core.History; using DurableTask.Core.Query; +using DurableTask.Core.Tracing; using Google.Protobuf; using Google.Protobuf.WellKnownTypes; +using Microsoft.DurableTask.Sidecar.Dispatcher; using Proto = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.Sidecar.Grpc; @@ -240,12 +242,15 @@ public static OrchestratorAction ToOrchestratorAction(Proto.OrchestratorAction a switch (a.OrchestratorActionTypeCase) { case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.ScheduleTask: - return new ScheduleTaskOrchestratorAction + return new GrpcScheduleTaskOrchestratorAction { Id = a.Id, Input = a.ScheduleTask.Input, Name = a.ScheduleTask.Name, Version = a.ScheduleTask.Version, + ParentTraceContext = a.ScheduleTask.ParentTraceContext is not null + ? new DistributedTraceContext(a.ScheduleTask.ParentTraceContext.TraceParent, a.ScheduleTask.ParentTraceContext.TraceState) + : null, }; case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.CreateSubOrchestration: return new CreateSubOrchestrationAction diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs index aaeb9e31..f1fd2ec3 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs @@ -523,6 +523,13 @@ await this.SendWorkItemToClientAsync(new P.WorkItem InstanceId = instance.InstanceId, ExecutionId = instance.ExecutionId, }, + ParentTraceContext = activityEvent.ParentTraceContext is not null + ? new() + { + TraceParent = activityEvent.ParentTraceContext.TraceParent, + TraceState = activityEvent.ParentTraceContext.TraceState, + } + : null, } }); } diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index 0b32d3c8..2f9a2b8e 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -69,7 +69,7 @@ public async Task Orchestration_Traces() createActivity.ParentId.Should().Be(testActivity.Id); createActivity.ParentSpanId.Should().Be(testActivity.SpanId); - var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "orchestration:Orchestration_Traces"); + var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "orchestration:Orchestration_Traces").ToList(); // The orchestration activities should be the same "logical" activity. orchestrationActivities.Select(a => a.StartTimeUtc).Distinct().Should().HaveCount(1); @@ -84,6 +84,18 @@ public async Task Orchestration_Traces() a.ParentId.Should().Be(createActivity.Id); a.ParentSpanId.Should().Be(createActivity.SpanId); }); + + var orchestrationActivity = orchestrationActivities.First(); + + var activityActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "activity:PageableActivityAsync").ToList(); + + activityActivities + .Should().HaveCountGreaterThan(0) + .And.AllSatisfy(a => + { + a.ParentId.Should().Be(orchestrationActivity.Id); + a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); + }); } static Task?> PageableActivityAsync(PageRequest? input) From 06697bbb6b9197a4246a40e25c2d8d7c8aab3c92 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 15 May 2025 15:45:01 -0700 Subject: [PATCH 06/36] Assert tags of orchestration activities. --- src/Grpc/orchestrator_service.proto | 30 ++++++------- src/Shared/Grpc/ProtoUtils.cs | 6 +-- src/Shared/Grpc/TraceHelper.cs | 10 ++--- .../GrpcSidecar/Grpc/ProtobufUtils.cs | 4 +- .../GrpcSidecar/Grpc/TaskHubGrpcServer.cs | 6 +-- .../TracingIntegrationTests.cs | 44 ++++++++++++++++--- 6 files changed, 66 insertions(+), 34 deletions(-) diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 6ffb90b7..11a34f3c 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -77,8 +77,8 @@ message ExecutionStartedEvent { TraceContext parentTraceContext = 7; google.protobuf.StringValue orchestrationSpanID = 8; map tags = 9; - google.protobuf.StringValue orchestrationID = 10; - google.protobuf.Timestamp activityStartTIme = 11; + google.protobuf.StringValue orchestrationActivityID = 10; + google.protobuf.Timestamp orchestrationActivityStartTime = 11; } message ExecutionCompletedEvent { @@ -194,7 +194,7 @@ message EntityOperationCalledEvent { } message EntityLockRequestedEvent { - string criticalSectionId = 1; + string criticalSectionId = 1; repeated string lockSet = 2; int32 position = 3; google.protobuf.StringValue parentInstanceId = 4; // used only within messages, null in histories @@ -219,7 +219,7 @@ message EntityUnlockSentEvent { message EntityLockGrantedEvent { string criticalSectionId = 1; } - + message HistoryEvent { int32 eventId = 1; google.protobuf.Timestamp timestamp = 2; @@ -246,8 +246,8 @@ message HistoryEvent { ExecutionResumedEvent executionResumed = 22; EntityOperationSignaledEvent entityOperationSignaled = 23; EntityOperationCalledEvent entityOperationCalled = 24; - EntityOperationCompletedEvent entityOperationCompleted = 25; - EntityOperationFailedEvent entityOperationFailed = 26; + EntityOperationCompletedEvent entityOperationCompleted = 25; + EntityOperationFailedEvent entityOperationFailed = 26; EntityLockRequestedEvent entityLockRequested = 27; EntityLockGrantedEvent entityLockGranted = 28; EntityUnlockSentEvent entityUnlockSent = 29; @@ -335,9 +335,9 @@ message OrchestratorResponse { // This field is optional. If not set, the service should assume that the orchestrator processed all events. google.protobuf.Int32Value numEventsProcessed = 5; - google.protobuf.StringValue orchestrationSpanID = 6; - google.protobuf.StringValue orchestrationID = 7; - google.protobuf.Timestamp activityStartTime = 8; + google.protobuf.StringValue orchestrationActivitySpanID = 6; + google.protobuf.StringValue orchestrationActivityID = 7; + google.protobuf.Timestamp orchestrationActivityStartTime = 8; } message CreateInstanceRequest { @@ -500,7 +500,7 @@ message SignalEntityRequest { } message SignalEntityResponse { - // no payload + // no payload } message GetEntityRequest { @@ -666,16 +666,16 @@ service TaskHubSidecarService { // Waits for an orchestration instance to reach a running or completion state. rpc WaitForInstanceStart(GetInstanceRequest) returns (GetInstanceResponse); - + // Waits for an orchestration instance to reach a completion state (completed, failed, terminated, etc.). rpc WaitForInstanceCompletion(GetInstanceRequest) returns (GetInstanceResponse); // Raises an event to a running orchestration instance. rpc RaiseEvent(RaiseEventRequest) returns (RaiseEventResponse); - + // Terminates a running orchestration instance. rpc TerminateInstance(TerminateRequest) returns (TerminateResponse); - + // Suspends a running orchestration instance. rpc SuspendInstance(SuspendRequest) returns (SuspendResponse); @@ -757,7 +757,7 @@ message CompleteTaskResponse { } message HealthPing { - // No payload + // No payload } message StreamInstanceHistoryRequest { @@ -770,4 +770,4 @@ message StreamInstanceHistoryRequest { message HistoryChunk { repeated HistoryEvent events = 1; -} +} \ No newline at end of file diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index 8608c59e..f849e92c 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -289,9 +289,9 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( InstanceId = instanceId, CustomStatus = customStatus, CompletionToken = completionToken, - OrchestrationID = parentActivity?.Id, - OrchestrationSpanID = parentActivity?.SpanId.ToString(), - ActivityStartTime = parentActivity?.StartTimeUtc.ToTimestamp(), + OrchestrationActivityID = parentActivity?.Id, + OrchestrationActivitySpanID = parentActivity?.SpanId.ToString(), + OrchestrationActivityStartTime = parentActivity?.StartTimeUtc.ToTimestamp(), }; foreach (OrchestratorAction action in actions) diff --git a/src/Shared/Grpc/TraceHelper.cs b/src/Shared/Grpc/TraceHelper.cs index 0368c2bf..4630292f 100644 --- a/src/Shared/Grpc/TraceHelper.cs +++ b/src/Shared/Grpc/TraceHelper.cs @@ -78,7 +78,7 @@ public class TraceHelper string activityName = CreateSpanName(TraceActivityConstants.Orchestration, startEvent.Name, startEvent.Version); ActivityKind activityKind = ActivityKind.Server; - DateTimeOffset startTime = startEvent.ActivityStartTIme?.ToDateTimeOffset() ?? default; + DateTimeOffset startTime = startEvent.OrchestrationActivityStartTime?.ToDateTimeOffset() ?? default; Activity? activity = ActivityTraceSource.StartActivity( activityName, @@ -100,16 +100,16 @@ public class TraceHelper activity.SetTag(Schema.Task.Version, startEvent.Version); } - if (startEvent.OrchestrationID != null && startEvent.OrchestrationSpanID != null) + if (startEvent.OrchestrationActivityID != null && startEvent.OrchestrationSpanID != null) { - activity.SetId(startEvent.OrchestrationID!); + activity.SetId(startEvent.OrchestrationActivityID!); activity.SetSpanId(startEvent.OrchestrationSpanID!); } else { - startEvent.OrchestrationID = activity.Id; + startEvent.OrchestrationActivityID = activity.Id; startEvent.OrchestrationSpanID = activity.SpanId.ToString(); - startEvent.ActivityStartTIme = Timestamp.FromDateTime(activity.StartTimeUtc); + startEvent.OrchestrationActivityStartTime = Timestamp.FromDateTime(activity.StartTimeUtc); } // DistributedTraceActivity.Current = activity; diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs index a0e33d1c..e3c27bc0 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs @@ -98,8 +98,8 @@ public static Proto.HistoryEvent ToHistoryEventProto(HistoryEvent e) TraceParent = startedEvent.ParentTraceContext.TraceParent, TraceState = startedEvent.ParentTraceContext.TraceState, }, - ActivityStartTIme = startedEvent.ParentTraceContext?.ActivityStartTime is not null ? Timestamp.FromDateTimeOffset(startedEvent.ParentTraceContext.ActivityStartTime.Value) : null, - OrchestrationID = startedEvent.ParentTraceContext?.Id, + OrchestrationActivityStartTime = startedEvent.ParentTraceContext?.ActivityStartTime is not null ? Timestamp.FromDateTimeOffset(startedEvent.ParentTraceContext.ActivityStartTime.Value) : null, + OrchestrationActivityID = startedEvent.ParentTraceContext?.Id, OrchestrationSpanID = startedEvent.ParentTraceContext?.SpanId, }; break; diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs index f1fd2ec3..76ea9e29 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs @@ -372,9 +372,9 @@ static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationState state, { Actions = request.Actions.Select(ProtobufUtils.ToOrchestratorAction), CustomStatus = request.CustomStatus, - OrchestrationActivityId = request.OrchestrationID, - OrchestrationActivitySpanId = request.OrchestrationSpanID, - OrchestrationActivityStartTime = request.ActivityStartTime?.ToDateTimeOffset(), + OrchestrationActivityId = request.OrchestrationActivityID, + OrchestrationActivitySpanId = request.OrchestrationActivitySpanID, + OrchestrationActivityStartTime = request.OrchestrationActivityStartTime?.ToDateTimeOffset(), }; tcs.TrySetResult(result); diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index 2f9a2b8e..ef64dc4c 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -34,16 +34,16 @@ static ActivityListener CreateListener(string[] sources, ICollection a static readonly string[] ActivitySourceNames = [TestActivitySourceName, CoreActivitySourceName]; + static readonly ActivitySource TestActivitySource = new(TestActivitySourceName); + [Fact] public async Task Orchestration_Traces() { - ActivitySource testActivitySource = new ActivitySource(nameof(TracingIntegrationTests)); - var activities = new List(); using var listener = CreateListener(ActivitySourceNames, activities); - TaskName orchestratorName = nameof(Orchestration_Traces); + string orchestratorName = nameof(Orchestration_Traces); await using HostTestLifetime server = await this.StartWorkerAsync(b => { @@ -54,20 +54,25 @@ public async Task Orchestration_Traces() nameof(PageableActivityAsync), (_, input) => PageableActivityAsync(input))); }); - using (var activity = testActivitySource.StartActivity("Test", ActivityKind.Client)) + string instanceId; + + using (var activity = TestActivitySource.StartActivity("Test", ActivityKind.Client)) { - string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync( + instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync( orchestratorName, input: string.Empty); OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync( instanceId, getInputsAndOutputs: true, this.TimeoutToken); } - var testActivity = activities.Single(a => a.Source == testActivitySource && a.OperationName == "Test"); + var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "create_orchestration:Orchestration_Traces"); // The creation activity should be parented to the test activity. createActivity.ParentId.Should().Be(testActivity.Id); createActivity.ParentSpanId.Should().Be(testActivity.SpanId); + createActivity.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + createActivity.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); + createActivity.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "orchestration:Orchestration_Traces").ToList(); @@ -83,6 +88,10 @@ public async Task Orchestration_Traces() { a.ParentId.Should().Be(createActivity.Id); a.ParentSpanId.Should().Be(createActivity.SpanId); + + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); }); var orchestrationActivity = orchestrationActivities.First(); @@ -95,11 +104,34 @@ public async Task Orchestration_Traces() { a.ParentId.Should().Be(orchestrationActivity.Id); a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); + + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(nameof(PageableActivityAsync)); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("activity"); + }); + + activityActivities + .Zip(Enumerable.Range(0, Int32.MaxValue)) + .Should().AllSatisfy(indexed => + { + indexed.First.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().Be(indexed.Second); + }); + + var activityExecutionActivities = activities.Where(a => a.Source.Name == TestActivitySourceName && a.OperationName == nameof(PageableActivityAsync)).ToList(); + + activityExecutionActivities + .Should().HaveCount(activityActivities.Count) + .And.AllSatisfy(a => + { + a.ParentId.Should().BeOneOf(activityActivities.Select(aa => aa.Id)); + a.ParentSpanId.ToString().Should().BeOneOf(activityActivities.Select(aa => aa.SpanId.ToString())); }); } static Task?> PageableActivityAsync(PageRequest? input) { + using var activity = TestActivitySource.StartActivity(); + int pageSize = input?.PageSize ?? 3; Page CreatePage(string? next) => new (Enumerable.Range(0, pageSize).Select(x => $"item_{x}").ToList(), next); From b8bd6ca59e42b63c26ab4488973739cbe57ff4c5 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 16 May 2025 14:09:47 -0700 Subject: [PATCH 07/36] Sketch emitting client activity trace. --- src/Shared/Grpc/ProtoUtils.cs | 15 ++- src/Shared/Grpc/TraceHelper.cs | 96 +++++++++++++++++++ .../Grpc/GrpcDurableTaskWorker.Processor.cs | 9 ++ 3 files changed, 117 insertions(+), 3 deletions(-) diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index f849e92c..80f32691 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -302,16 +302,25 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( { case OrchestratorActionType.ScheduleOrchestrator: var scheduleTaskAction = (ScheduleTaskOrchestratorAction)action; + + ActivityContext? clientActivityContext = null; + + if (parentActivity != null) + { + ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + clientActivityContext = new(parentActivity.TraceId, clientSpanId, parentActivity.ActivityTraceFlags, parentActivity.TraceStateString); + } + protoAction.ScheduleTask = new P.ScheduleTaskAction { Name = scheduleTaskAction.Name, Version = scheduleTaskAction.Version, Input = scheduleTaskAction.Input, - ParentTraceContext = parentActivity is not null + ParentTraceContext = clientActivityContext is not null ? new P.TraceContext { - TraceParent = parentActivity.Id, - TraceState = parentActivity.TraceStateString, + TraceParent = $"00-{clientActivityContext.Value.TraceId}-{clientActivityContext.Value.SpanId}-0{clientActivityContext.Value.TraceFlags:d}", + TraceState = clientActivityContext.Value.TraceState, } : null, }; diff --git a/src/Shared/Grpc/TraceHelper.cs b/src/Shared/Grpc/TraceHelper.cs index 4630292f..d7b8f7dc 100644 --- a/src/Shared/Grpc/TraceHelper.cs +++ b/src/Shared/Grpc/TraceHelper.cs @@ -157,6 +157,102 @@ public class TraceHelper return newActivity; } + /// + /// Starts a new trace activity for (task) activity that represents the time between when the task message + /// is enqueued and when the response message is received. + /// + /// The associated . + /// The associated . + /// + /// Returns a newly started with (task) activity and orchestration-specific metadata. + /// + internal static Activity? StartTraceActivityForSchedulingTask( + string? instanceId, + P.HistoryEvent historyEvent, + P.TaskScheduledEvent taskScheduledEvent) + { + if (taskScheduledEvent == null) + { + return null; + } + + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.Activity, taskScheduledEvent.Name, taskScheduledEvent.Version), + kind: ActivityKind.Client, + startTime: historyEvent.Timestamp?.ToDateTimeOffset() ?? default, + parentContext: Activity.Current?.Context ?? default); + + if (newActivity == null) + { + return null; + } + + if (taskScheduledEvent.ParentTraceContext != null) + { + if (ActivityContext.TryParse(taskScheduledEvent.ParentTraceContext.TraceParent, taskScheduledEvent.ParentTraceContext?.TraceState, out ActivityContext parentContext)) + { + newActivity.SetSpanId(parentContext.SpanId.ToString()); + } + } + + newActivity.AddTag(Schema.Task.Type, TraceActivityConstants.Activity); + newActivity.AddTag(Schema.Task.Name, taskScheduledEvent.Name); + newActivity.AddTag(Schema.Task.InstanceId, instanceId); + newActivity.AddTag(Schema.Task.TaskId, historyEvent.EventId); + + if (!string.IsNullOrEmpty(taskScheduledEvent.Version)) + { + newActivity.AddTag(Schema.Task.Version, taskScheduledEvent.Version); + } + + return newActivity; + } + + /// + /// Emits a new trace activity for a (task) activity that successfully completes. + /// + /// The associated . + /// The associated . + internal static void EmitTraceActivityForTaskCompleted( + string? instanceId, + P.HistoryEvent historyEvent, + P.TaskScheduledEvent taskScheduledEvent) + { + // The parent of this is the parent orchestration span ID. It should be the client span which started this + Activity? activity = StartTraceActivityForSchedulingTask(instanceId, historyEvent, taskScheduledEvent); + + activity?.Dispose(); + } + + /// + /// Emits a new trace activity for a (task) activity that fails. + /// + /// The associated . + /// The associated . + /// The associated . + /// Specifies the method to propagate unhandled exceptions to parent orchestrations. + internal static void EmitTraceActivityForTaskFailed( + string? instanceId, + P.HistoryEvent historyEvent, + P.TaskScheduledEvent taskScheduledEvent, + P.TaskFailedEvent? failedEvent) + { + Activity? activity = StartTraceActivityForSchedulingTask(instanceId, historyEvent, taskScheduledEvent); + + if (activity is null) + { + return; + } + + if (failedEvent != null) + { + string statusDescription = failedEvent.FailureDetails?.ErrorMessage ?? "Unspecified task activity failure"; + activity?.SetStatus(ActivityStatusCode.Error, statusDescription); + } + + activity?.Dispose(); + } + static string CreateSpanName(string spanDescription, string? taskName, string? taskVersion) { if (!string.IsNullOrEmpty(taskVersion)) diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index b8583288..702023bc 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -368,6 +368,15 @@ async Task OnRunOrchestratorAsync( using Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution(executionStartedEvent); + foreach (var newEvent in request.NewEvents) + { + if (newEvent.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskCompleted) + { + var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); + TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled); + } + } + OrchestratorExecutionResult? result = null; P.TaskFailureDetails? failureDetails = null; TaskName name = new("(unknown)"); From 47e20180027b9986ba1a6ff44ef3274de3a832b9 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Wed, 28 May 2025 10:03:08 -0700 Subject: [PATCH 08/36] Exclude Rider files. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index dfcfd56f..4a122434 100644 --- a/.gitignore +++ b/.gitignore @@ -348,3 +348,6 @@ MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ + +# Rider (cross platform .NET/C# tools) working folder +.idea/ From 588d518c3adcc0d68454cc700bf96845792760e7 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Wed, 28 May 2025 15:10:21 -0700 Subject: [PATCH 09/36] Attempt to capture activity/orchestration failures. --- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 702023bc..1ff84cdb 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -375,6 +375,11 @@ async Task OnRunOrchestratorAsync( var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled); } + else if (newEvent.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskFailed) + { + var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskFailed.TaskScheduledId); + TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled, newEvent.TaskFailed); + } } OrchestratorExecutionResult? result = null; @@ -515,6 +520,16 @@ await this.client.AbandonTaskOrchestratorWorkItemAsync( }; } + var failedAction = response.Actions.FirstOrDefault(a => a.CompleteOrchestration?.OrchestrationStatus == P.OrchestrationStatus.Failed); + + if (failedAction is not null) + { + traceActivity?.SetStatus(ActivityStatusCode.Error, failedAction.CompleteOrchestration.FailureDetails?.ErrorMessage ?? "Orchestrator failed with no error message"); + } + + // Stop the trace activity here to avoid including the completion time in the latency calculation + traceActivity?.Stop(); + this.Logger.SendingOrchestratorResponse( name, response.InstanceId, @@ -568,6 +583,8 @@ async Task OnRunActivityAsync(P.ActivityRequest request, string completionToken, int outputSizeInBytes = 0; if (failureDetails != null) { + traceActivity?.SetStatus(ActivityStatusCode.Error, failureDetails.ErrorMessage); + outputSizeInBytes = failureDetails.GetApproximateByteCount(); } else if (output != null) @@ -588,6 +605,9 @@ async Task OnRunActivityAsync(P.ActivityRequest request, string completionToken, CompletionToken = completionToken, }; + // Stop the trace activity here to avoid including the completion time in the latency calculation + traceActivity?.Stop(); + await this.client.CompleteActivityTaskAsync(response, cancellationToken: cancellation); } From 59d99b2a06ca7c4aa400e8d2b95bfab430982b53 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Tue, 3 Jun 2025 15:01:59 -0700 Subject: [PATCH 10/36] Stop orchestration trace when execution completes. --- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 1ff84cdb..5e91f9f7 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -366,7 +366,7 @@ async Task OnRunOrchestratorAsync( .Select(e => e.ExecutionStarted) .FirstOrDefault(); - using Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution(executionStartedEvent); + Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution(executionStartedEvent); foreach (var newEvent in request.NewEvents) { @@ -520,15 +520,17 @@ await this.client.AbandonTaskOrchestratorWorkItemAsync( }; } - var failedAction = response.Actions.FirstOrDefault(a => a.CompleteOrchestration?.OrchestrationStatus == P.OrchestrationStatus.Failed); + var completeOrchestrationAction = response.Actions.FirstOrDefault(a => a.CompleteOrchestration is not null); - if (failedAction is not null) + if (completeOrchestrationAction is not null) { - traceActivity?.SetStatus(ActivityStatusCode.Error, failedAction.CompleteOrchestration.FailureDetails?.ErrorMessage ?? "Orchestrator failed with no error message"); - } + if (completeOrchestrationAction.CompleteOrchestration.OrchestrationStatus == P.OrchestrationStatus.Failed) + { + traceActivity?.SetStatus(ActivityStatusCode.Error, completeOrchestrationAction.CompleteOrchestration.Result); + } - // Stop the trace activity here to avoid including the completion time in the latency calculation - traceActivity?.Stop(); + traceActivity?.Stop(); + } this.Logger.SendingOrchestratorResponse( name, From e4312387a8c59e1ec55749bf672c5ad4c49dcae0 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Wed, 4 Jun 2025 14:58:16 -0700 Subject: [PATCH 11/36] Update protos to match backend. --- src/Grpc/orchestrator_service.proto | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 11a34f3c..ac2628ad 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -97,6 +97,7 @@ message TaskScheduledEvent { google.protobuf.StringValue version = 2; google.protobuf.StringValue input = 3; TraceContext parentTraceContext = 4; + map tags = 5; } message TaskCompletedEvent { @@ -258,7 +259,8 @@ message ScheduleTaskAction { string name = 1; google.protobuf.StringValue version = 2; google.protobuf.StringValue input = 3; - TraceContext parentTraceContext = 4; + map tags = 4; + TraceContext parentTraceContext = 5; } message CreateSubOrchestrationAction { From 4d2c5816afd8754eb441e18f1d8b738eca7e78fd Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 5 Jun 2025 13:24:36 -0700 Subject: [PATCH 12/36] Add/set partent trace context to suborchestration action. --- src/Grpc/orchestrator_service.proto | 1 + src/Shared/Grpc/ProtoUtils.cs | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index ac2628ad..209fe053 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -268,6 +268,7 @@ message CreateSubOrchestrationAction { string name = 2; google.protobuf.StringValue version = 3; google.protobuf.StringValue input = 4; + TraceContext parentTraceContext = 5; } message CreateTimerAction { diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index 80f32691..6b2d8959 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -333,6 +333,13 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( InstanceId = subOrchestrationAction.InstanceId, Name = subOrchestrationAction.Name, Version = subOrchestrationAction.Version, + ParentTraceContext = parentActivity is not null + ? new P.TraceContext + { + TraceParent = $"00-{parentActivity.Context.TraceId}-{parentActivity.Context.SpanId}-0{parentActivity.Context.TraceFlags:d}", + TraceState = parentActivity.Context.TraceState, + } + : null, }; break; case OrchestratorActionType.CreateTimer: From 5b3501623f43c7f54d181c3a8129677292114a05 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 5 Jun 2025 14:39:37 -0700 Subject: [PATCH 13/36] Port suborchestration tracing. --- src/Shared/Grpc/ProtoUtils.cs | 22 ++--- src/Shared/Grpc/TraceHelper.cs | 96 +++++++++++++++++++ .../Grpc/GrpcDurableTaskWorker.Processor.cs | 49 ++++++++-- 3 files changed, 148 insertions(+), 19 deletions(-) diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index 6b2d8959..b7eb6751 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -298,19 +298,19 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( { var protoAction = new P.OrchestratorAction { Id = action.Id }; + ActivityContext? clientActivityContext = null; + + if (parentActivity != null) + { + ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + clientActivityContext = new(parentActivity.TraceId, clientSpanId, parentActivity.ActivityTraceFlags, parentActivity.TraceStateString); + } + switch (action.OrchestratorActionType) { case OrchestratorActionType.ScheduleOrchestrator: var scheduleTaskAction = (ScheduleTaskOrchestratorAction)action; - ActivityContext? clientActivityContext = null; - - if (parentActivity != null) - { - ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); - clientActivityContext = new(parentActivity.TraceId, clientSpanId, parentActivity.ActivityTraceFlags, parentActivity.TraceStateString); - } - protoAction.ScheduleTask = new P.ScheduleTaskAction { Name = scheduleTaskAction.Name, @@ -333,11 +333,11 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( InstanceId = subOrchestrationAction.InstanceId, Name = subOrchestrationAction.Name, Version = subOrchestrationAction.Version, - ParentTraceContext = parentActivity is not null + ParentTraceContext = clientActivityContext is not null ? new P.TraceContext { - TraceParent = $"00-{parentActivity.Context.TraceId}-{parentActivity.Context.SpanId}-0{parentActivity.Context.TraceFlags:d}", - TraceState = parentActivity.Context.TraceState, + TraceParent = $"00-{clientActivityContext.Value.TraceId}-{clientActivityContext.Value.SpanId}-0{clientActivityContext.Value.TraceFlags:d}", + TraceState = clientActivityContext.Value.TraceState, } : null, }; diff --git a/src/Shared/Grpc/TraceHelper.cs b/src/Shared/Grpc/TraceHelper.cs index d7b8f7dc..69511f20 100644 --- a/src/Shared/Grpc/TraceHelper.cs +++ b/src/Shared/Grpc/TraceHelper.cs @@ -253,6 +253,102 @@ internal static void EmitTraceActivityForTaskFailed( activity?.Dispose(); } + /// + /// Starts a new trace activity for sub-orchestrations. Represents the time between enqueuing + /// the sub-orchestration message and it completing. + /// + /// The associated . + /// The associated . + /// + /// Returns a newly started with (task) activity and orchestration-specific metadata. + /// + internal static Activity? CreateTraceActivityForSchedulingSubOrchestration( + string? instanceId, + P.HistoryEvent historyEvent, + P.SubOrchestrationInstanceCreatedEvent createdEvent) + { + if (instanceId == null || createdEvent == null) + { + return null; + } + + Activity? activity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.Orchestration, createdEvent.Name, createdEvent.Version), + kind: ActivityKind.Client, + startTime: historyEvent.Timestamp?.ToDateTimeOffset() ?? default, + parentContext: Activity.Current?.Context ?? default); + + if (activity == null) + { + return null; + } + + if (createdEvent.ParentTraceContext != null) + { + if (ActivityContext.TryParse(createdEvent.ParentTraceContext.TraceParent, createdEvent.ParentTraceContext?.TraceState, out ActivityContext parentContext)) + { + activity.SetSpanId(parentContext.SpanId.ToString()); + } + } + + activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); + activity.SetTag(Schema.Task.Name, createdEvent.Name); + activity.SetTag(Schema.Task.InstanceId, instanceId); + + if (!string.IsNullOrEmpty(createdEvent.Version)) + { + activity.SetTag(Schema.Task.Version, createdEvent.Version); + } + + return activity; + } + + /// + /// Emits a new trace activity for sub-orchestration execution when the sub-orchestration + /// completes successfully. + /// + /// The associated . + /// The associated . + internal static void EmitTraceActivityForSubOrchestrationCompleted( + string? instanceId, + P.HistoryEvent historyEvent, + P.SubOrchestrationInstanceCreatedEvent createdEvent) + { + // The parent of this is the parent orchestration span ID. It should be the client span which started this + Activity? activity = CreateTraceActivityForSchedulingSubOrchestration(instanceId, historyEvent, createdEvent); + + activity?.Dispose(); + } + + /// + /// Emits a new trace activity for sub-orchestration execution when the sub-orchestration fails. + /// + /// The associated . + /// The associated . + /// The associated . + /// Specifies the method to propagate unhandled exceptions to parent orchestrations. + internal static void EmitTraceActivityForSubOrchestrationFailed( + string? orchestrationInstance, + P.HistoryEvent historyEvent, + P.SubOrchestrationInstanceCreatedEvent createdEvent, + P.SubOrchestrationInstanceFailedEvent? failedEvent) + { + Activity? activity = CreateTraceActivityForSchedulingSubOrchestration(orchestrationInstance, historyEvent, createdEvent); + + if (activity is null) + { + return; + } + + if (failedEvent != null) + { + string statusDescription = failedEvent.FailureDetails.ErrorMessage ?? "Unspecified sub-orchestration failure"; + activity?.SetStatus(ActivityStatusCode.Error, statusDescription); + } + + activity?.Dispose(); + } + static string CreateSpanName(string spanDescription, string? taskName, string? taskVersion) { if (!string.IsNullOrEmpty(taskVersion)) diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 5e91f9f7..4e46f506 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -370,15 +370,48 @@ async Task OnRunOrchestratorAsync( foreach (var newEvent in request.NewEvents) { - if (newEvent.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskCompleted) + switch (newEvent.EventTypeCase) { - var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); - TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled); - } - else if (newEvent.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskFailed) - { - var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskFailed.TaskScheduledId); - TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled, newEvent.TaskFailed); + case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCompleted: + { + var subOrchestrationEvent = request.PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) + .LastOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); + + TraceHelper.EmitTraceActivityForSubOrchestrationCompleted( + request.InstanceId, + subOrchestrationEvent, + subOrchestrationEvent.SubOrchestrationInstanceCreated); + break; + } + + case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceFailed: + { + var subOrchestrationEvent = request.PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) + .LastOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); + + TraceHelper.EmitTraceActivityForSubOrchestrationFailed( + request.InstanceId, + subOrchestrationEvent, + subOrchestrationEvent.SubOrchestrationInstanceCreated, + newEvent.SubOrchestrationInstanceFailed); + break; + } + + case P.HistoryEvent.EventTypeOneofCase.TaskCompleted: + { + var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); + TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled); + break; + } + + case P.HistoryEvent.EventTypeOneofCase.TaskFailed: + { + var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskFailed.TaskScheduledId); + TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled, newEvent.TaskFailed); + break; + } } } From 2a809110fa6e36876230973d8e06e510705f2ddf Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 6 Jun 2025 11:29:01 -0700 Subject: [PATCH 14/36] Add tracing for events sent by worker. --- src/Shared/Grpc/ProtoUtils.cs | 7 +++++ src/Shared/Grpc/TraceHelper.cs | 29 +++++++++++++++++++ .../Grpc/GrpcDurableTaskWorker.Processor.cs | 1 + src/Worker/Grpc/GrpcOrchestrationRunner.cs | 1 + 4 files changed, 38 insertions(+) diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index b7eb6751..c3c45f2d 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -277,6 +277,7 @@ internal static Timestamp ToTimestamp(this DateTime dateTime) /// When an orchestrator action is unknown. internal static P.OrchestratorResponse ConstructOrchestratorResponse( string instanceId, + string executionId, string? customStatus, IEnumerable actions, string completionToken, @@ -394,6 +395,12 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( Name = sendEventAction.EventName, Data = sendEventAction.EventData, }; + + // Distributed Tracing: start a new trace activity derived from the orchestration + // for an EventRaisedEvent (external event) + using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(sendEventAction,instanceId, executionId); + + traceActivity?.Stop(); } break; diff --git a/src/Shared/Grpc/TraceHelper.cs b/src/Shared/Grpc/TraceHelper.cs index 69511f20..245995ed 100644 --- a/src/Shared/Grpc/TraceHelper.cs +++ b/src/Shared/Grpc/TraceHelper.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System.Diagnostics; +using DurableTask.Core.Command; using Google.Protobuf.WellKnownTypes; using P = Microsoft.DurableTask.Protobuf; @@ -349,6 +350,34 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( activity?.Dispose(); } + internal static Activity? StartTraceActivityForEventRaisedFromWorker( + SendEventOrchestratorAction eventRaisedEvent, + string? instanceId, + string? executionId) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaisedEvent.EventName, null), + kind: ActivityKind.Producer, + parentContext: Activity.Current?.Context ?? default); + + if (newActivity == null) + { + return null; + } + + newActivity.AddTag(Schema.Task.Type, TraceActivityConstants.Event); + newActivity.AddTag(Schema.Task.Name, eventRaisedEvent.EventName); + newActivity.AddTag(Schema.Task.InstanceId, instanceId); + newActivity.AddTag(Schema.Task.ExecutionId, executionId); + + if (!string.IsNullOrEmpty(eventRaisedEvent.Instance?.InstanceId)) + { + newActivity.AddTag(Schema.Task.EventTargetInstanceId, eventRaisedEvent.Instance!.InstanceId); + } + + return newActivity; + } + static string CreateSpanName(string spanDescription, string? taskName, string? taskVersion) { if (!string.IsNullOrEmpty(taskVersion)) diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 4e46f506..7378e935 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -491,6 +491,7 @@ async Task OnRunOrchestratorAsync( { response = ProtoUtils.ConstructOrchestratorResponse( request.InstanceId, + request.ExecutionId, result.CustomStatus, result.Actions, completionToken, diff --git a/src/Worker/Grpc/GrpcOrchestrationRunner.cs b/src/Worker/Grpc/GrpcOrchestrationRunner.cs index c35256bb..0cea8056 100644 --- a/src/Worker/Grpc/GrpcOrchestrationRunner.cs +++ b/src/Worker/Grpc/GrpcOrchestrationRunner.cs @@ -117,6 +117,7 @@ public static string LoadAndRun( P.OrchestratorResponse response = ProtoUtils.ConstructOrchestratorResponse( request.InstanceId, + request.ExecutionId, result.CustomStatus, result.Actions, completionToken: string.Empty, /* doesn't apply */ From 889640ebcc7a5a5eb80a8943384285c0152defcd Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 6 Jun 2025 13:28:08 -0700 Subject: [PATCH 15/36] Add tracing for client-sent events. --- src/Client/Grpc/GrpcDurableTaskClient.cs | 2 ++ src/Shared/Grpc/TraceHelper.cs | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 2a02b35b..064ba0bc 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -141,6 +141,8 @@ public override async Task RaiseEventAsync( Input = this.DataConverter.Serialize(eventPayload), }; + using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(request, instanceId); + await this.sidecarClient.RaiseEventAsync(request, cancellationToken: cancellation); } diff --git a/src/Shared/Grpc/TraceHelper.cs b/src/Shared/Grpc/TraceHelper.cs index 245995ed..873c02bb 100644 --- a/src/Shared/Grpc/TraceHelper.cs +++ b/src/Shared/Grpc/TraceHelper.cs @@ -378,6 +378,30 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( return newActivity; } + /// + /// Creates a new trace activity for events created from the client. + /// + /// The associated . + /// The associated . + /// + /// Returns a newly started with (task) activity and orchestration-specific metadata. + /// + internal static Activity? StartActivityForNewEventRaisedFromClient(P.RaiseEventRequest eventRaised, string instanceId) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null), + kind: ActivityKind.Producer, + parentContext: Activity.Current?.Context ?? default, + tags: new KeyValuePair[] + { + new(Schema.Task.Type, TraceActivityConstants.Event), + new(Schema.Task.Name, eventRaised.Name), + new(Schema.Task.EventTargetInstanceId, instanceId), + }); + + return newActivity; + } + static string CreateSpanName(string spanDescription, string? taskName, string? taskVersion) { if (!string.IsNullOrEmpty(taskVersion)) From bc42ee6e9c2f5ce750cff4e9c75a47e8d8345bbb Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Tue, 10 Jun 2025 15:10:54 -0700 Subject: [PATCH 16/36] Add tracing for orchestration timers. --- src/Shared/Grpc/TraceHelper.cs | 36 +++++++++++++++++++ .../Grpc/GrpcDurableTaskWorker.Processor.cs | 7 ++++ 2 files changed, 43 insertions(+) diff --git a/src/Shared/Grpc/TraceHelper.cs b/src/Shared/Grpc/TraceHelper.cs index 873c02bb..00cf36dc 100644 --- a/src/Shared/Grpc/TraceHelper.cs +++ b/src/Shared/Grpc/TraceHelper.cs @@ -402,6 +402,37 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( return newActivity; } + /// + /// Emits a new trace activity for timers. + /// + /// The associated . + /// The name of the orchestration invoking the timer. + /// The timer's start time. + /// The associated . + internal static void EmitTraceActivityForTimer( + string? instanceId, + string orchestrationName, + DateTime startTime, + P.TimerFiredEvent timerFiredEvent) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateTimerSpanName(orchestrationName), + kind: ActivityKind.Internal, + startTime: startTime, + parentContext: Activity.Current?.Context ?? default); + + if (newActivity is not null) + { + newActivity.AddTag(Schema.Task.Type, TraceActivityConstants.Timer); + newActivity.AddTag(Schema.Task.Name, orchestrationName); + newActivity.AddTag(Schema.Task.InstanceId, instanceId); + newActivity.AddTag(Schema.Task.FireAt, timerFiredEvent.FireAt.ToDateTime().ToString("o")); + newActivity.AddTag(Schema.Task.TaskId, timerFiredEvent.TimerId); + + newActivity.Dispose(); + } + } + static string CreateSpanName(string spanDescription, string? taskName, string? taskVersion) { if (!string.IsNullOrEmpty(taskVersion)) @@ -413,4 +444,9 @@ static string CreateSpanName(string spanDescription, string? taskName, string? t return $"{spanDescription}:{taskName}"; } } + + static string CreateTimerSpanName(string orchestrationName) + { + return $"{TraceActivityConstants.Orchestration}:{orchestrationName}:{TraceActivityConstants.Timer}"; + } } diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 7378e935..90e11fbe 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -412,6 +412,13 @@ async Task OnRunOrchestratorAsync( TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled, newEvent.TaskFailed); break; } + + case P.HistoryEvent.EventTypeOneofCase.TimerFired: + { + // We immediately publish the activity span for this timer by creating the activity and immediately calling Dispose() on it. + TraceHelper.EmitTraceActivityForTimer(request.InstanceId, executionStartedEvent?.Name, newEvent.Timestamp.ToDateTime(), newEvent.TimerFired); + break; + } } } From 5b44cef11d2fd7e82692d91e988630ac48cf63cc Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Wed, 11 Jun 2025 16:44:46 -0700 Subject: [PATCH 17/36] Sketch basic tracing test. --- .../TracingIntegrationTests.cs | 97 ++++++++++--------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index ef64dc4c..aff554fc 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -37,31 +37,36 @@ static ActivityListener CreateListener(string[] sources, ICollection a static readonly ActivitySource TestActivitySource = new(TestActivitySourceName); [Fact] - public async Task Orchestration_Traces() + public async Task MultiTaskOrchestration() { var activities = new List(); using var listener = CreateListener(ActivitySourceNames, activities); - string orchestratorName = nameof(Orchestration_Traces); + string orchestratorName = nameof(MultiTaskOrchestration); await using HostTestLifetime server = await this.StartWorkerAsync(b => { b.AddTasks(tasks => tasks - .AddOrchestratorFunc( - orchestratorName, (ctx, input) => PageableOrchestrationAsync(ctx, input)) - .AddActivityFunc?>( - nameof(PageableActivityAsync), (_, input) => PageableActivityAsync(input))); + .AddOrchestratorFunc( + orchestratorName, + async (ctx, input) => + { + await ctx.CallActivityAsync(nameof(TestActivityAsync), true); + await ctx.CallActivityAsync(nameof(TestActivityAsync), true); + + return true; + }) + .AddActivityFunc(nameof(TestActivityAsync), (_, input) => TestActivityAsync(input))); }); string instanceId; - using (var activity = TestActivitySource.StartActivity("Test", ActivityKind.Client)) + using (var activity = TestActivitySource.StartActivity("Test")) { - instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync( - orchestratorName, input: string.Empty); - OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync( - instanceId, getInputsAndOutputs: true, this.TimeoutToken); + instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: true, cancellation: this.TimeoutToken); + + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true, this.TimeoutToken); } var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); @@ -76,15 +81,16 @@ public async Task Orchestration_Traces() var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "orchestration:Orchestration_Traces").ToList(); - // The orchestration activities should be the same "logical" activity. + orchestrationActivities.Should().HaveCountGreaterThan(0); + + // The orchestration activities should be the same "logical" orchestration activity. orchestrationActivities.Select(a => a.StartTimeUtc).Distinct().Should().HaveCount(1); orchestrationActivities.Select(a => a.Id).Distinct().Should().HaveCount(1); orchestrationActivities.Select(a => a.SpanId).Distinct().Should().HaveCount(1); // The orchestration activities should be parented to the create activity. orchestrationActivities - .Should().HaveCountGreaterThan(0) - .And.AllSatisfy(a => + .Should().AllSatisfy(a => { a.ParentId.Should().Be(createActivity.Id); a.ParentSpanId.Should().Be(createActivity.SpanId); @@ -96,66 +102,67 @@ public async Task Orchestration_Traces() var orchestrationActivity = orchestrationActivities.First(); - var activityActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "activity:PageableActivityAsync").ToList(); + var clientActivityActivities = activities.Where(a => a.Kind == ActivityKind.Client && a.Source.Name == CoreActivitySourceName && a.OperationName == "activity:TestActivityAsync").ToList(); - activityActivities - .Should().HaveCountGreaterThan(0) + // The "client" (i.e. scheduled) task activities should be parented to the orchestration activity. + clientActivityActivities + .Should().HaveCount(2) .And.AllSatisfy(a => { a.ParentId.Should().Be(orchestrationActivity.Id); a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); - a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(nameof(PageableActivityAsync)); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(nameof(TestActivityAsync)); a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("activity"); + a.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().NotBeNull(); }); - activityActivities + clientActivityActivities .Zip(Enumerable.Range(0, Int32.MaxValue)) .Should().AllSatisfy(indexed => { indexed.First.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().Be(indexed.Second); }); - var activityExecutionActivities = activities.Where(a => a.Source.Name == TestActivitySourceName && a.OperationName == nameof(PageableActivityAsync)).ToList(); + var serverActivityActivities = activities.Where(a => a.Kind == ActivityKind.Server && a.Source.Name == CoreActivitySourceName && a.OperationName == "activity:TestActivityAsync").ToList(); + + // The "server" (i.e. executed) task activities should be parented to the client activity activities. + serverActivityActivities + .Should().HaveCount(clientActivityActivities.Count) + .And.AllSatisfy(a => + { + a.ParentId.Should().BeOneOf(clientActivityActivities.Select(aa => aa.Id)); + a.ParentSpanId.ToString().Should().BeOneOf(clientActivityActivities.Select(aa => aa.SpanId.ToString())); + }); + + var activityExecutionActivities = activities.Where(a => a.Source.Name == TestActivitySourceName && a.OperationName == nameof(TestActivityAsync)).ToList(); + // The activity execution activities should be parented to the server activity activities. activityExecutionActivities - .Should().HaveCount(activityActivities.Count) + .Should().HaveCount(serverActivityActivities.Count) .And.AllSatisfy(a => { - a.ParentId.Should().BeOneOf(activityActivities.Select(aa => aa.Id)); - a.ParentSpanId.ToString().Should().BeOneOf(activityActivities.Select(aa => aa.SpanId.ToString())); + a.ParentId.Should().BeOneOf(serverActivityActivities.Select(aa => aa.Id)); + a.ParentSpanId.ToString().Should().BeOneOf(serverActivityActivities.Select(aa => aa.SpanId.ToString())); }); } - static Task?> PageableActivityAsync(PageRequest? input) + static Task TestActivityAsync(bool shouldSucceed) { using var activity = TestActivitySource.StartActivity(); - int pageSize = input?.PageSize ?? 3; - Page CreatePage(string? next) - => new (Enumerable.Range(0, pageSize).Select(x => $"item_{x}").ToList(), next); - Page? page = input?.Continuation switch + if (shouldSucceed) { - null => CreatePage("1"), - "1" => CreatePage("2"), - "2" => CreatePage(null), - _ => null, - }; + activity?.SetStatus(ActivityStatusCode.Ok); - return Task.FromResult(page); - } - - static async Task PageableOrchestrationAsync(TaskOrchestrationContext context, string? input) - { - AsyncPageable pageable = Pageable.Create((continuation, _, _) => + return Task.FromResult(true); + } + else { - return context.CallActivityAsync?>( - nameof(PageableActivityAsync), new PageRequest(continuation))!; - }); + activity?.SetStatus(ActivityStatusCode.Error); - return await pageable.CountAsync(); + return Task.FromResult(false); + } } - - record PageRequest(string? Continuation, int? PageSize = null); } From fa62a5fcd72179f9e527e65d6284be1813dd9fcd Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 09:13:46 -0700 Subject: [PATCH 18/36] Test activity types. --- .../TracingIntegrationTests.cs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index aff554fc..bb2a7a68 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -44,6 +44,7 @@ public async Task MultiTaskOrchestration() using var listener = CreateListener(ActivitySourceNames, activities); string orchestratorName = nameof(MultiTaskOrchestration); + string activityName = nameof(TestActivityAsync); await using HostTestLifetime server = await this.StartWorkerAsync(b => { @@ -57,7 +58,7 @@ public async Task MultiTaskOrchestration() return true; }) - .AddActivityFunc(nameof(TestActivityAsync), (_, input) => TestActivityAsync(input))); + .AddActivityFunc(activityName, (_, input) => TestActivityAsync(input))); }); string instanceId; @@ -70,7 +71,7 @@ public async Task MultiTaskOrchestration() } var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); - var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "create_orchestration:Orchestration_Traces"); + var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"create_orchestration:{orchestratorName}"); // The creation activity should be parented to the test activity. createActivity.ParentId.Should().Be(testActivity.Id); @@ -79,7 +80,7 @@ public async Task MultiTaskOrchestration() createActivity.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); createActivity.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); - var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == "orchestration:Orchestration_Traces").ToList(); + var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{orchestratorName}").ToList(); orchestrationActivities.Should().HaveCountGreaterThan(0); @@ -92,6 +93,7 @@ public async Task MultiTaskOrchestration() orchestrationActivities .Should().AllSatisfy(a => { + a.Kind.Should().Be(ActivityKind.Server); a.ParentId.Should().Be(createActivity.Id); a.ParentSpanId.Should().Be(createActivity.SpanId); @@ -102,18 +104,19 @@ public async Task MultiTaskOrchestration() var orchestrationActivity = orchestrationActivities.First(); - var clientActivityActivities = activities.Where(a => a.Kind == ActivityKind.Client && a.Source.Name == CoreActivitySourceName && a.OperationName == "activity:TestActivityAsync").ToList(); + var clientActivityActivities = activities.Where(a => a.Kind == ActivityKind.Client && a.Source.Name == CoreActivitySourceName && a.OperationName == $"activity:{activityName}").ToList(); // The "client" (i.e. scheduled) task activities should be parented to the orchestration activity. clientActivityActivities .Should().HaveCount(2) .And.AllSatisfy(a => { + a.Kind.Should().Be(ActivityKind.Client); a.ParentId.Should().Be(orchestrationActivity.Id); a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); - a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(nameof(TestActivityAsync)); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(activityName); a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("activity"); a.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().NotBeNull(); }); @@ -125,15 +128,21 @@ public async Task MultiTaskOrchestration() indexed.First.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().Be(indexed.Second); }); - var serverActivityActivities = activities.Where(a => a.Kind == ActivityKind.Server && a.Source.Name == CoreActivitySourceName && a.OperationName == "activity:TestActivityAsync").ToList(); + var serverActivityActivities = activities.Where(a => a.Kind == ActivityKind.Server && a.Source.Name == CoreActivitySourceName && a.OperationName == $"activity:{activityName}").ToList(); // The "server" (i.e. executed) task activities should be parented to the client activity activities. serverActivityActivities .Should().HaveCount(clientActivityActivities.Count) .And.AllSatisfy(a => { + a.Kind.Should().Be(ActivityKind.Server); a.ParentId.Should().BeOneOf(clientActivityActivities.Select(aa => aa.Id)); a.ParentSpanId.ToString().Should().BeOneOf(clientActivityActivities.Select(aa => aa.SpanId.ToString())); + + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(activityName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("activity"); + a.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().NotBeNull(); }); var activityExecutionActivities = activities.Where(a => a.Source.Name == TestActivitySourceName && a.OperationName == nameof(TestActivityAsync)).ToList(); From 4ca345df950265e7eb60aa941cb2853dccf67799 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 09:34:43 -0700 Subject: [PATCH 19/36] Add test for sent events. --- .../TracingIntegrationTests.cs | 187 ++++++++++++++++-- 1 file changed, 176 insertions(+), 11 deletions(-) diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index bb2a7a68..1cbed8f7 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -111,21 +111,110 @@ public async Task MultiTaskOrchestration() .Should().HaveCount(2) .And.AllSatisfy(a => { - a.Kind.Should().Be(ActivityKind.Client); a.ParentId.Should().Be(orchestrationActivity.Id); a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); + }); + + var serverActivityActivities = activities.Where(a => a.Kind == ActivityKind.Server && a.Source.Name == CoreActivitySourceName && a.OperationName == $"activity:{activityName}").ToList(); + + // The "server" (i.e. executed) task activities should be parented to the client activity activities. + serverActivityActivities + .Should().HaveCount(clientActivityActivities.Count) + .And.AllSatisfy(a => + { + a.ParentId.Should().BeOneOf(clientActivityActivities.Select(aa => aa.Id)); + a.ParentSpanId.ToString().Should().BeOneOf(clientActivityActivities.Select(aa => aa.SpanId.ToString())); + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(activityName); a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("activity"); a.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().NotBeNull(); }); + var activityExecutionActivities = activities.Where(a => a.Source.Name == TestActivitySourceName && a.OperationName == nameof(TestActivityAsync)).ToList(); + + // The activity execution activities should be parented to the server activity activities. + activityExecutionActivities + .Should().HaveCount(serverActivityActivities.Count) + .And.AllSatisfy(a => + { + a.ParentId.Should().BeOneOf(serverActivityActivities.Select(aa => aa.Id)); + a.ParentSpanId.ToString().Should().BeOneOf(serverActivityActivities.Select(aa => aa.SpanId.ToString())); + }); + } + + [Fact] + public async Task TaskOrchestrationWithActivityFailure() + { + var activities = new List(); + + using var listener = CreateListener(ActivitySourceNames, activities); + + string orchestratorName = nameof(TaskOrchestrationWithActivityFailure); + string activityName = nameof(TestActivityAsync); + + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, + async (ctx, input) => + { + await ctx.CallActivityAsync(nameof(TestActivityAsync), false); + + return true; + }) + .AddActivityFunc(activityName, (_, input) => TestActivityAsync(input))); + }); + + string instanceId; + + using (var activity = TestActivitySource.StartActivity("Test")) + { + instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: true, cancellation: this.TimeoutToken); + + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true, this.TimeoutToken); + } + + var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); + var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"create_orchestration:{orchestratorName}"); + + // The creation activity should be parented to the test activity. + createActivity.ParentId.Should().Be(testActivity.Id); + createActivity.ParentSpanId.Should().Be(testActivity.SpanId); + + var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{orchestratorName}").ToList(); + + orchestrationActivities.Should().HaveCountGreaterThan(0); + + // The orchestration activities should be the same "logical" orchestration activity. + orchestrationActivities.Select(a => a.StartTimeUtc).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.Id).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.SpanId).Distinct().Should().HaveCount(1); + + // The orchestration activities should be parented to the create activity. + orchestrationActivities + .Should().AllSatisfy(a => + { + a.Kind.Should().Be(ActivityKind.Server); + a.ParentId.Should().Be(createActivity.Id); + a.ParentSpanId.Should().Be(createActivity.SpanId); + a.Status.Should().Be(ActivityStatusCode.Error); + }); + + var orchestrationActivity = orchestrationActivities.First(); + + var clientActivityActivities = activities.Where(a => a.Kind == ActivityKind.Client && a.Source.Name == CoreActivitySourceName && a.OperationName == $"activity:{activityName}").ToList(); + + // The "client" (i.e. scheduled) task activities should be parented to the orchestration activity. clientActivityActivities - .Zip(Enumerable.Range(0, Int32.MaxValue)) - .Should().AllSatisfy(indexed => + .Should().HaveCount(1) + .And.AllSatisfy(a => { - indexed.First.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().Be(indexed.Second); + a.ParentId.Should().Be(orchestrationActivity.Id); + a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); + a.Status.Should().Be(ActivityStatusCode.Error); }); var serverActivityActivities = activities.Where(a => a.Kind == ActivityKind.Server && a.Source.Name == CoreActivitySourceName && a.OperationName == $"activity:{activityName}").ToList(); @@ -135,14 +224,9 @@ public async Task MultiTaskOrchestration() .Should().HaveCount(clientActivityActivities.Count) .And.AllSatisfy(a => { - a.Kind.Should().Be(ActivityKind.Server); a.ParentId.Should().BeOneOf(clientActivityActivities.Select(aa => aa.Id)); a.ParentSpanId.ToString().Should().BeOneOf(clientActivityActivities.Select(aa => aa.SpanId.ToString())); - - a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); - a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(activityName); - a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("activity"); - a.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().NotBeNull(); + a.Status.Should().Be(ActivityStatusCode.Error); }); var activityExecutionActivities = activities.Where(a => a.Source.Name == TestActivitySourceName && a.OperationName == nameof(TestActivityAsync)).ToList(); @@ -154,6 +238,87 @@ public async Task MultiTaskOrchestration() { a.ParentId.Should().BeOneOf(serverActivityActivities.Select(aa => aa.Id)); a.ParentSpanId.ToString().Should().BeOneOf(serverActivityActivities.Select(aa => aa.SpanId.ToString())); + a.Status.Should().Be(ActivityStatusCode.Error); + }); + } + + [Fact] + public async Task TaskOrchestrationWithSentEvent() + { + var activities = new List(); + + using var listener = CreateListener(ActivitySourceNames, activities); + + string orchestratorName = nameof(TaskOrchestrationWithActivityFailure); + string activityName = nameof(TestActivityAsync); + string targetInstanceId = "TestInstanceId"; + string eventName = "TestEvent"; + + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, + (ctx, input) => + { + ctx.SendEvent(targetInstanceId, eventName, "TestData"); + + return true; + }) + .AddActivityFunc(activityName, (_, input) => TestActivityAsync(input))); + }); + + string instanceId; + + using (var activity = TestActivitySource.StartActivity("Test")) + { + instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: true, cancellation: this.TimeoutToken); + + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true, this.TimeoutToken); + } + + var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); + var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"create_orchestration:{orchestratorName}"); + + // The creation activity should be parented to the test activity. + createActivity.ParentId.Should().Be(testActivity.Id); + createActivity.ParentSpanId.Should().Be(testActivity.SpanId); + + var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{orchestratorName}").ToList(); + + orchestrationActivities.Should().HaveCountGreaterThan(0); + + // The orchestration activities should be the same "logical" orchestration activity. + orchestrationActivities.Select(a => a.StartTimeUtc).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.Id).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.SpanId).Distinct().Should().HaveCount(1); + + // The orchestration activities should be parented to the create activity. + orchestrationActivities + .Should().AllSatisfy(a => + { + a.Kind.Should().Be(ActivityKind.Server); + a.ParentId.Should().Be(createActivity.Id); + a.ParentSpanId.Should().Be(createActivity.SpanId); + }); + + var orchestrationActivity = orchestrationActivities.First(); + + var sentEventActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration_event:{eventName}").ToList(); + + // The "client" (i.e. scheduled) task activities should be parented to the orchestration activity. + sentEventActivities + .Should().HaveCount(1) + .And.AllSatisfy(a => + { + a.Kind.Should().Be(ActivityKind.Producer); + a.ParentId.Should().Be(orchestrationActivity.Id); + a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); + + a.TagObjects.Should().ContainKey("durabletask.event.target_instance_id").WhoseValue.Should().Be(targetInstanceId); + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(eventName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("event"); }); } @@ -171,7 +336,7 @@ static Task TestActivityAsync(bool shouldSucceed) { activity?.SetStatus(ActivityStatusCode.Error); - return Task.FromResult(false); + throw new InvalidOperationException("Test activity failed."); } } } From 11803e72d9ecce6d69eab62caf03c76698486c11 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 09:52:46 -0700 Subject: [PATCH 20/36] Add test for timer event. --- .../TracingIntegrationTests.cs | 87 ++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index 1cbed8f7..d476df63 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -249,7 +249,7 @@ public async Task TaskOrchestrationWithSentEvent() using var listener = CreateListener(ActivitySourceNames, activities); - string orchestratorName = nameof(TaskOrchestrationWithActivityFailure); + string orchestratorName = nameof(TaskOrchestrationWithSentEvent); string activityName = nameof(TestActivityAsync); string targetInstanceId = "TestInstanceId"; string eventName = "TestEvent"; @@ -322,6 +322,91 @@ public async Task TaskOrchestrationWithSentEvent() }); } + [Fact] + public async Task TaskOrchestrationWithTimer() + { + var activities = new List(); + + using var listener = CreateListener(ActivitySourceNames, activities); + + string orchestratorName = nameof(TaskOrchestrationWithTimer); + string activityName = nameof(TestActivityAsync); + + DateTime fireAt = default; + + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, + async (ctx, input) => + { + fireAt = ctx.CurrentUtcDateTime.AddSeconds(1); + + await ctx.CreateTimer(fireAt, CancellationToken.None); + + return true; + }) + .AddActivityFunc(activityName, (_, input) => TestActivityAsync(input))); + }); + + string instanceId; + + using (var activity = TestActivitySource.StartActivity("Test")) + { + instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: true, cancellation: this.TimeoutToken); + + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true, this.TimeoutToken); + } + + fireAt.Should().NotBe(default); + + var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); + var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"create_orchestration:{orchestratorName}"); + + // The creation activity should be parented to the test activity. + createActivity.ParentId.Should().Be(testActivity.Id); + createActivity.ParentSpanId.Should().Be(testActivity.SpanId); + + var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{orchestratorName}").ToList(); + + orchestrationActivities.Should().HaveCountGreaterThan(0); + + // The orchestration activities should be the same "logical" orchestration activity. + orchestrationActivities.Select(a => a.StartTimeUtc).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.Id).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.SpanId).Distinct().Should().HaveCount(1); + + // The orchestration activities should be parented to the create activity. + orchestrationActivities + .Should().AllSatisfy(a => + { + a.Kind.Should().Be(ActivityKind.Server); + a.ParentId.Should().Be(createActivity.Id); + a.ParentSpanId.Should().Be(createActivity.SpanId); + }); + + var orchestrationActivity = orchestrationActivities.First(); + + var timerActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{orchestratorName}:timer").ToList(); + + // The "client" (i.e. scheduled) task activities should be parented to the orchestration activity. + timerActivities + .Should().HaveCount(1) + .And.AllSatisfy(a => + { + a.Kind.Should().Be(ActivityKind.Internal); + a.ParentId.Should().Be(orchestrationActivity.Id); + a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); + + a.TagObjects.Should().ContainKey("durabletask.fire_at").WhoseValue.Should().Be(fireAt.ToString("O")); + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("timer"); + a.TagObjects.Should().ContainKey("durabletask.task.task_id").WhoseValue.Should().NotBeNull(); + }); + } + static Task TestActivityAsync(bool shouldSucceed) { using var activity = TestActivitySource.StartActivity(); From 19cf60fac7581c721cc85e5da18c8ae0b19452c5 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 10:01:00 -0700 Subject: [PATCH 21/36] Add test for client raised events. --- .../TracingIntegrationTests.cs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index d476df63..d4bcdf59 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -407,6 +407,56 @@ public async Task TaskOrchestrationWithTimer() }); } + [Fact] + public async Task ClientRaiseEvent() + { + var activities = new List(); + + using var listener = CreateListener(ActivitySourceNames, activities); + + string orchestratorName = nameof(this.ClientRaiseEvent); + string eventName = "TestEvent"; + + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, + (ctx, input) => + { + return true; + })); + }); + + string instanceId; + + using (var activity = TestActivitySource.StartActivity("Test")) + { + instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: true, + cancellation: this.TimeoutToken); + + await server.Client.RaiseEventAsync(instanceId, eventName, "TestData", this.TimeoutToken); + + OrchestrationMetadata metadata = + await server.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true, + this.TimeoutToken); + } + + var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); + + var raiseEventActivity = activities.Single(a => + a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration_event:{eventName}"); + + // The raise event activity should be parented to the test activity. + raiseEventActivity.Kind.Should().Be(ActivityKind.Producer); + raiseEventActivity.ParentId.Should().Be(testActivity.Id); + raiseEventActivity.ParentSpanId.Should().Be(testActivity.SpanId); + + raiseEventActivity.TagObjects.Should().ContainKey("durabletask.event.target_instance_id").WhoseValue.Should().Be(instanceId); + raiseEventActivity.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(eventName); + raiseEventActivity.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("event"); + } + static Task TestActivityAsync(bool shouldSucceed) { using var activity = TestActivitySource.StartActivity(); From a37b36450e8731cec01a541a034b705ca51ca72e Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 11:17:43 -0700 Subject: [PATCH 22/36] Move files to tracing namespace. --- src/Client/Grpc/GrpcDurableTaskClient.cs | 1 + .../Grpc/DiagnosticActivityExtensions.cs | 95 ------------------- src/Shared/Grpc/FieldInfoExtensionMethods.cs | 55 ----------- src/Shared/Grpc/ProtoUtils.cs | 1 + .../Tracing/DiagnosticActivityExtensions.cs | 94 ++++++++++++++++++ .../Grpc/Tracing/FieldInfoExtensionMethods.cs | 54 +++++++++++ src/Shared/Grpc/{ => Tracing}/Schema.cs | 11 +-- .../{ => Tracing}/TraceActivityConstants.cs | 5 +- src/Shared/Grpc/{ => Tracing}/TraceHelper.cs | 2 +- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 3 +- 10 files changed, 157 insertions(+), 164 deletions(-) delete mode 100644 src/Shared/Grpc/DiagnosticActivityExtensions.cs delete mode 100644 src/Shared/Grpc/FieldInfoExtensionMethods.cs create mode 100644 src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs create mode 100644 src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs rename src/Shared/Grpc/{ => Tracing}/Schema.cs (69%) rename src/Shared/Grpc/{ => Tracing}/TraceActivityConstants.cs (78%) rename src/Shared/Grpc/{ => Tracing}/TraceHelper.cs (99%) diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 850025eb..c38682a3 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -5,6 +5,7 @@ using System.Text; using Google.Protobuf.WellKnownTypes; using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Tracing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; diff --git a/src/Shared/Grpc/DiagnosticActivityExtensions.cs b/src/Shared/Grpc/DiagnosticActivityExtensions.cs deleted file mode 100644 index 1e9fa466..00000000 --- a/src/Shared/Grpc/DiagnosticActivityExtensions.cs +++ /dev/null @@ -1,95 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -namespace Microsoft.DurableTask -{ - using System; - using System.Diagnostics; - using System.Linq; - using System.Linq.Expressions; - using System.Reflection; - - /// - /// Replica from System.Diagnostics.DiagnosticSource >= 6.0.0 - /// - internal enum ActivityStatusCode - { - Unset = 0, - OK = 1, - Error = 2, - } - - /// - /// Extensions for . - /// - internal static class DiagnosticActivityExtensions - { - private static readonly Action s_setSpanId; - private static readonly Action s_setId; - private static readonly Action s_setStatus; - - static DiagnosticActivityExtensions() - { - BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance; - s_setSpanId = typeof(Activity).GetField("_spanId", flags).CreateSetter(); - s_setId = typeof(Activity).GetField("_id", flags).CreateSetter(); - s_setStatus = CreateSetStatus(); - } - - public static void SetId(this Activity activity, string id) - => s_setId(activity, id); - - public static void SetSpanId(this Activity activity, string spanId) - => s_setSpanId(activity, spanId); - - public static void SetStatus(this Activity activity, ActivityStatusCode status, string description) - => s_setStatus(activity, status, description); - - private static Action CreateSetStatus() - { - MethodInfo method = typeof(Activity).GetMethod("SetStatus"); - if (method is null) - { - return (activity, status, description) => { - if (activity is null) - { - throw new ArgumentNullException(nameof(activity)); - } - string str = status switch - { - ActivityStatusCode.Unset => "UNSET", - ActivityStatusCode.OK => "OK", - ActivityStatusCode.Error => "ERROR", - _ => null, - }; - activity.SetTag("otel.status_code", str); - activity.SetTag("otel.status_description", description); - }; - } - - /* - building expression tree to effectively perform: - (activity, status, description) => activity.SetStatus((ActivityStatusCode)(int)status, description); - */ - - ParameterExpression targetExp = Expression.Parameter(typeof(Activity), "target"); - ParameterExpression status = Expression.Parameter(typeof(ActivityStatusCode), "status"); - ParameterExpression description = Expression.Parameter(typeof(string), "description"); - UnaryExpression convert = Expression.Convert(status, typeof(int)); - convert = Expression.Convert(convert, method.GetParameters().First().ParameterType); - MethodCallExpression callExp = Expression.Call(targetExp, method, convert, description); - return Expression.Lambda>(callExp, targetExp, status, description) - .Compile(); - } - } -} diff --git a/src/Shared/Grpc/FieldInfoExtensionMethods.cs b/src/Shared/Grpc/FieldInfoExtensionMethods.cs deleted file mode 100644 index a0bcde77..00000000 --- a/src/Shared/Grpc/FieldInfoExtensionMethods.cs +++ /dev/null @@ -1,55 +0,0 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- - -using System; -using System.Linq.Expressions; -using System.Reflection; - -namespace Microsoft.DurableTask -{ - /// - /// Extensions for . - /// - internal static class FieldInfoExtensionMethods - { - /// - /// Create a re-usable setter for a . - /// When cached and reused, This is quicker than using . - /// - /// The target type of the object. - /// The value type of the field. - /// The field info. - /// A re-usable action to set the field. - internal static Action CreateSetter(this FieldInfo fieldInfo) - { - if (fieldInfo == null) - { - throw new ArgumentNullException(nameof(fieldInfo)); - } - - ParameterExpression targetExp = Expression.Parameter(typeof(TTarget), "target"); - Expression source = targetExp; - - if (typeof(TTarget) != fieldInfo.DeclaringType) - { - source = Expression.Convert(targetExp, fieldInfo.DeclaringType); - } - - // Creating the setter to set the value to the field - ParameterExpression valueExp = Expression.Parameter(typeof(TValue), "value"); - MemberExpression fieldExp = Expression.Field(source, fieldInfo); - BinaryExpression assignExp = Expression.Assign(fieldExp, valueExp); - return Expression.Lambda>(assignExp, targetExp, valueExp).Compile(); - } - } -} diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index 00bca583..09cdaf5d 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -17,6 +17,7 @@ using Google.Protobuf.WellKnownTypes; using DTCore = DurableTask.Core; using P = Microsoft.DurableTask.Protobuf; +using TraceHelper = Microsoft.DurableTask.Tracing.TraceHelper; namespace Microsoft.DurableTask; diff --git a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs new file mode 100644 index 00000000..ef33fa91 --- /dev/null +++ b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs @@ -0,0 +1,94 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; + +namespace Microsoft.DurableTask.Tracing; + +/// +/// Replica from System.Diagnostics.DiagnosticSource >= 6.0.0 +/// +internal enum ActivityStatusCode +{ + Unset = 0, + OK = 1, + Error = 2, +} + +/// +/// Extensions for . +/// +static class DiagnosticActivityExtensions +{ + static readonly Action s_setSpanId; + static readonly Action s_setId; + static readonly Action s_setStatus; + + static DiagnosticActivityExtensions() + { + BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance; + s_setSpanId = typeof(Activity).GetField("_spanId", flags).CreateSetter(); + s_setId = typeof(Activity).GetField("_id", flags).CreateSetter(); + s_setStatus = CreateSetStatus(); + } + + public static void SetId(this Activity activity, string id) + => s_setId(activity, id); + + public static void SetSpanId(this Activity activity, string spanId) + => s_setSpanId(activity, spanId); + + public static void SetStatus(this Activity activity, ActivityStatusCode status, string description) + => s_setStatus(activity, status, description); + + static Action CreateSetStatus() + { + MethodInfo method = typeof(Activity).GetMethod("SetStatus"); + if (method is null) + { + return (activity, status, description) => { + if (activity is null) + { + throw new ArgumentNullException(nameof(activity)); + } + string str = status switch + { + ActivityStatusCode.Unset => "UNSET", + ActivityStatusCode.OK => "OK", + ActivityStatusCode.Error => "ERROR", + _ => null, + }; + activity.SetTag("otel.status_code", str); + activity.SetTag("otel.status_description", description); + }; + } + + /* + building expression tree to effectively perform: + (activity, status, description) => activity.SetStatus((ActivityStatusCode)(int)status, description); + */ + + ParameterExpression targetExp = Expression.Parameter(typeof(Activity), "target"); + ParameterExpression status = Expression.Parameter(typeof(ActivityStatusCode), "status"); + ParameterExpression description = Expression.Parameter(typeof(string), "description"); + UnaryExpression convert = Expression.Convert(status, typeof(int)); + convert = Expression.Convert(convert, method.GetParameters().First().ParameterType); + MethodCallExpression callExp = Expression.Call(targetExp, method, convert, description); + return Expression.Lambda>(callExp, targetExp, status, description) + .Compile(); + } +} diff --git a/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs b/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs new file mode 100644 index 00000000..230af777 --- /dev/null +++ b/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs @@ -0,0 +1,54 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using System; +using System.Linq.Expressions; +using System.Reflection; + +namespace Microsoft.DurableTask.Tracing; + +/// +/// Extensions for . +/// +static class FieldInfoExtensionMethods +{ + /// + /// Create a re-usable setter for a . + /// When cached and reused, This is quicker than using . + /// + /// The target type of the object. + /// The value type of the field. + /// The field info. + /// A re-usable action to set the field. + internal static Action CreateSetter(this FieldInfo fieldInfo) + { + if (fieldInfo == null) + { + throw new ArgumentNullException(nameof(fieldInfo)); + } + + ParameterExpression targetExp = Expression.Parameter(typeof(TTarget), "target"); + Expression source = targetExp; + + if (typeof(TTarget) != fieldInfo.DeclaringType) + { + source = Expression.Convert(targetExp, fieldInfo.DeclaringType); + } + + // Creating the setter to set the value to the field + ParameterExpression valueExp = Expression.Parameter(typeof(TValue), "value"); + MemberExpression fieldExp = Expression.Field(source, fieldInfo); + BinaryExpression assignExp = Expression.Assign(fieldExp, valueExp); + return Expression.Lambda>(assignExp, targetExp, valueExp).Compile(); + } +} diff --git a/src/Shared/Grpc/Schema.cs b/src/Shared/Grpc/Tracing/Schema.cs similarity index 69% rename from src/Shared/Grpc/Schema.cs rename to src/Shared/Grpc/Tracing/Schema.cs index c82df136..13ca84c5 100644 --- a/src/Shared/Grpc/Schema.cs +++ b/src/Shared/Grpc/Tracing/Schema.cs @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -namespace Microsoft.DurableTask; +namespace Microsoft.DurableTask.Tracing; -internal static class Schema +static class Schema { internal static class Task { @@ -12,15 +12,8 @@ internal static class Task internal const string Version = "durabletask.task.version"; internal const string InstanceId = "durabletask.task.instance_id"; internal const string ExecutionId = "durabletask.task.execution_id"; - internal const string Status = "durabletask.task.status"; internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; } - - internal static class Status - { - internal const string Code = "otel.status_code"; - internal const string Description = "otel.status_description"; - } } diff --git a/src/Shared/Grpc/TraceActivityConstants.cs b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs similarity index 78% rename from src/Shared/Grpc/TraceActivityConstants.cs rename to src/Shared/Grpc/Tracing/TraceActivityConstants.cs index ef0154e1..0ee799f0 100644 --- a/src/Shared/Grpc/TraceActivityConstants.cs +++ b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -namespace Microsoft.DurableTask; +namespace Microsoft.DurableTask.Tracing; -internal class TraceActivityConstants +class TraceActivityConstants { - public const string Client = "client"; public const string Orchestration = "orchestration"; public const string Activity = "activity"; public const string Event = "event"; diff --git a/src/Shared/Grpc/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs similarity index 99% rename from src/Shared/Grpc/TraceHelper.cs rename to src/Shared/Grpc/Tracing/TraceHelper.cs index 00cf36dc..4979a12d 100644 --- a/src/Shared/Grpc/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -6,7 +6,7 @@ using Google.Protobuf.WellKnownTypes; using P = Microsoft.DurableTask.Protobuf; -namespace Microsoft.DurableTask; +namespace Microsoft.DurableTask.Tracing; public class TraceHelper { diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 90e11fbe..389c6076 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -7,13 +7,14 @@ using DurableTask.Core.Entities; using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; -using DurableTask.Core.Tracing; using Microsoft.DurableTask.Abstractions; using Microsoft.DurableTask.Entities; +using Microsoft.DurableTask.Tracing; using Microsoft.DurableTask.Worker.Shims; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; +using ActivityStatusCode = System.Diagnostics.ActivityStatusCode; using DTCore = DurableTask.Core; using P = Microsoft.DurableTask.Protobuf; From 7e8262c730b7093c44d3b0e23f601d5de92002be Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 11:49:39 -0700 Subject: [PATCH 23/36] Clean up files. --- .../Tracing/DiagnosticActivityExtensions.cs | 20 +++++-------------- .../Grpc/Tracing/FieldInfoExtensionMethods.cs | 17 ++++------------ src/Shared/Grpc/Tracing/Schema.cs | 2 ++ .../Grpc/Tracing/TraceActivityConstants.cs | 2 ++ src/Shared/Grpc/Tracing/TraceHelper.cs | 2 ++ 5 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs index ef33fa91..fd22d412 100644 --- a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs +++ b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs @@ -1,19 +1,9 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/DiagnosticActivityExtensions.cs -using System; using System.Diagnostics; -using System.Linq; using System.Linq.Expressions; using System.Reflection; @@ -22,7 +12,7 @@ namespace Microsoft.DurableTask.Tracing; /// /// Replica from System.Diagnostics.DiagnosticSource >= 6.0.0 /// -internal enum ActivityStatusCode +enum ActivityStatusCode { Unset = 0, OK = 1, diff --git a/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs b/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs index 230af777..2d416eda 100644 --- a/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs +++ b/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs @@ -1,17 +1,8 @@ -// ---------------------------------------------------------------------------------- -// Copyright Microsoft Corporation -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ---------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/FieldInfoExtensionMethods.cs -using System; using System.Linq.Expressions; using System.Reflection; diff --git a/src/Shared/Grpc/Tracing/Schema.cs b/src/Shared/Grpc/Tracing/Schema.cs index 13ca84c5..468e72e4 100644 --- a/src/Shared/Grpc/Tracing/Schema.cs +++ b/src/Shared/Grpc/Tracing/Schema.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/Schema.cs + namespace Microsoft.DurableTask.Tracing; static class Schema diff --git a/src/Shared/Grpc/Tracing/TraceActivityConstants.cs b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs index 0ee799f0..55d26715 100644 --- a/src/Shared/Grpc/Tracing/TraceActivityConstants.cs +++ b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/TraceActivityConstants.cs + namespace Microsoft.DurableTask.Tracing; class TraceActivityConstants diff --git a/src/Shared/Grpc/Tracing/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs index 4979a12d..1e679b46 100644 --- a/src/Shared/Grpc/Tracing/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/TraceHelper.cs + using System.Diagnostics; using DurableTask.Core.Command; using Google.Protobuf.WellKnownTypes; From 6b67b9d70d4265ae71fc455f10224a6f221246cd Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 15:22:37 -0700 Subject: [PATCH 24/36] Fixup test for suborchestration tracing. --- .../GrpcCreateSubOrchestrationAction.cs | 12 ++ ...rpcSubOrchestrationInstanceCreatedEvent.cs | 19 +++ .../Dispatcher/TaskOrchestrationDispatcher.cs | 6 +- .../GrpcSidecar/Grpc/ProtobufUtils.cs | 15 ++- .../TracingIntegrationTests.cs | 108 ++++++++++++++++++ 5 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs create mode 100644 test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs new file mode 100644 index 00000000..ce916d7a --- /dev/null +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using DurableTask.Core.Command; +using DurableTask.Core.Tracing; + +namespace Microsoft.DurableTask.Sidecar.Dispatcher; + +public class GrpcCreateSubOrchestrationAction : CreateSubOrchestrationAction +{ + public DistributedTraceContext? ParentTraceContext { get; set; } +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs new file mode 100644 index 00000000..f170b492 --- /dev/null +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.Serialization; +using DurableTask.Core.History; +using DurableTask.Core.Tracing; + +namespace Microsoft.DurableTask.Sidecar.Dispatcher; + +public class GrpcSubOrchestrationInstanceCreatedEvent : SubOrchestrationInstanceCreatedEvent +{ + public GrpcSubOrchestrationInstanceCreatedEvent(int eventId) + : base(eventId) + { + } + + [DataMember] + public DistributedTraceContext? ParentTraceContext { get; set; } +} \ No newline at end of file diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs index 0cec5a89..c88989bc 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs @@ -238,12 +238,15 @@ void ApplyOrchestratorActions( } else if (action is CreateSubOrchestrationAction subOrchestrationAction) { - runtimeState.AddEvent(new SubOrchestrationInstanceCreatedEvent(subOrchestrationAction.Id) + var grpcAction = action as GrpcCreateSubOrchestrationAction; + + runtimeState.AddEvent(new GrpcSubOrchestrationInstanceCreatedEvent(subOrchestrationAction.Id) { Name = subOrchestrationAction.Name, Version = subOrchestrationAction.Version, InstanceId = subOrchestrationAction.InstanceId, Input = subOrchestrationAction.Input, + ParentTraceContext = grpcAction?.ParentTraceContext }); ExecutionStartedEvent startedEvent = new(-1, subOrchestrationAction.Input) @@ -262,6 +265,7 @@ void ApplyOrchestratorActions( Version = runtimeState.Version, TaskScheduleId = subOrchestrationAction.Id, }, + ParentTraceContext = grpcAction?.ParentTraceContext, Tags = subOrchestrationAction.Tags, }; diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs index e3c27bc0..c7fd9100 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs @@ -149,6 +149,16 @@ public static Proto.HistoryEvent ToHistoryEventProto(HistoryEvent e) Name = subOrchestrationCreated.Name, Version = subOrchestrationCreated.Version, }; + + if (subOrchestrationCreated is GrpcSubOrchestrationInstanceCreatedEvent { ParentTraceContext: not null } grpcEvent) + { + payload.SubOrchestrationInstanceCreated.ParentTraceContext = new Proto.TraceContext + { + TraceParent = grpcEvent.ParentTraceContext.TraceParent, + TraceState = grpcEvent.ParentTraceContext.TraceState, + }; + } + break; case EventType.SubOrchestrationInstanceCompleted: var subOrchestrationCompleted = (SubOrchestrationInstanceCompletedEvent)e; @@ -253,12 +263,15 @@ public static OrchestratorAction ToOrchestratorAction(Proto.OrchestratorAction a : null, }; case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.CreateSubOrchestration: - return new CreateSubOrchestrationAction + return new GrpcCreateSubOrchestrationAction { Id = a.Id, Input = a.CreateSubOrchestration.Input, Name = a.CreateSubOrchestration.Name, InstanceId = a.CreateSubOrchestration.InstanceId, + ParentTraceContext = a.CreateSubOrchestration.ParentTraceContext is not null + ? new DistributedTraceContext(a.CreateSubOrchestration.ParentTraceContext.TraceParent, a.CreateSubOrchestration.ParentTraceContext.TraceState) + : null, Tags = null, // TODO Version = a.CreateSubOrchestration.Version, }; diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index d4bcdf59..f4af1229 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -242,6 +242,114 @@ public async Task TaskOrchestrationWithActivityFailure() }); } + [Fact] + public async Task TaskWithSuborchestration() + { + var activities = new List(); + + using var listener = CreateListener(ActivitySourceNames, activities); + + string orchestratorName = nameof(TaskWithSuborchestration); + string subOrchestratorName = "SubOrchestration"; + string activityName = nameof(TestActivityAsync); + + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, + async (ctx, input) => + { + await ctx.CallSubOrchestratorAsync(subOrchestratorName, input: true); + + return true; + }) + .AddOrchestratorFunc( + subOrchestratorName, + async (ctx, input) => + { + await ctx.CallActivityAsync(nameof(TestActivityAsync), input); + + return true; + }) + .AddActivityFunc(activityName, (_, input) => TestActivityAsync(input))); + }); + + string instanceId; + + using (var activity = TestActivitySource.StartActivity("Test")) + { + instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: true, cancellation: this.TimeoutToken); + + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true, this.TimeoutToken); + } + + var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); + var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"create_orchestration:{orchestratorName}"); + + // The creation activity should be parented to the test activity. + createActivity.ParentId.Should().Be(testActivity.Id); + createActivity.ParentSpanId.Should().Be(testActivity.SpanId); + createActivity.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + createActivity.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); + createActivity.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + + var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{orchestratorName}").ToList(); + + orchestrationActivities.Should().HaveCountGreaterThan(0); + + // The orchestration activities should be the same "logical" orchestration activity. + orchestrationActivities.Select(a => a.StartTimeUtc).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.Id).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.SpanId).Distinct().Should().HaveCount(1); + + // The orchestration activities should be parented to the create activity. + orchestrationActivities + .Should().AllSatisfy(a => + { + a.Kind.Should().Be(ActivityKind.Server); + a.ParentId.Should().Be(createActivity.Id); + a.ParentSpanId.Should().Be(createActivity.SpanId); + + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + }); + + var orchestrationActivity = orchestrationActivities.First(); + + var clientSuborchestrationActivities = activities.Where(a => a.Kind == ActivityKind.Client && a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{subOrchestratorName}").ToList(); + + // The client suborchestration activities should be parented to the orchestration activity. + clientSuborchestrationActivities + .Should().HaveCount(1) + .And.AllSatisfy(a => + { + a.ParentId.Should().Be(orchestrationActivity.Id); + a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); + + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(subOrchestratorName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + }); + + var clientSuborchestrationActivity = clientSuborchestrationActivities.First(); + + var serverSuborchestrationActivities = activities.Where(a => a.Kind == ActivityKind.Server && a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{subOrchestratorName}").ToList(); + + // The server suborchestration activities should be parented to the client orchestration activity. + serverSuborchestrationActivities + .Should().HaveCountGreaterThan(0) + .And.AllSatisfy(a => + { + a.ParentId.Should().Be(clientSuborchestrationActivity.Id); + a.ParentSpanId.Should().Be(clientSuborchestrationActivity.SpanId); + + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(subOrchestratorName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + }); + } + [Fact] public async Task TaskOrchestrationWithSentEvent() { From 73c8ee045890f09c5f4bf2aa7008e39dcac93634 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 15:25:59 -0700 Subject: [PATCH 25/36] Add test for suborchestration failure. --- .../TracingIntegrationTests.cs | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index f4af1229..6bbff3df 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -350,6 +350,117 @@ public async Task TaskWithSuborchestration() }); } + [Fact] + public async Task TaskWithSuborchestrationFailure() + { + var activities = new List(); + + using var listener = CreateListener(ActivitySourceNames, activities); + + string orchestratorName = nameof(TaskWithSuborchestrationFailure); + string subOrchestratorName = "SubOrchestration"; + string activityName = nameof(TestActivityAsync); + + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, + async (ctx, input) => + { + await ctx.CallSubOrchestratorAsync(subOrchestratorName, input: false); + + return true; + }) + .AddOrchestratorFunc( + subOrchestratorName, + async (ctx, input) => + { + await ctx.CallActivityAsync(nameof(TestActivityAsync), input); + + return true; + }) + .AddActivityFunc(activityName, (_, input) => TestActivityAsync(input))); + }); + + string instanceId; + + using (var activity = TestActivitySource.StartActivity("Test")) + { + instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: true, cancellation: this.TimeoutToken); + + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true, this.TimeoutToken); + } + + var testActivity = activities.Single(a => a.Source == TestActivitySource && a.OperationName == "Test"); + var createActivity = activities.Single(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"create_orchestration:{orchestratorName}"); + + // The creation activity should be parented to the test activity. + createActivity.ParentId.Should().Be(testActivity.Id); + createActivity.ParentSpanId.Should().Be(testActivity.SpanId); + createActivity.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + createActivity.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); + createActivity.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + + var orchestrationActivities = activities.Where(a => a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{orchestratorName}").ToList(); + + orchestrationActivities.Should().HaveCountGreaterThan(0); + + // The orchestration activities should be the same "logical" orchestration activity. + orchestrationActivities.Select(a => a.StartTimeUtc).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.Id).Distinct().Should().HaveCount(1); + orchestrationActivities.Select(a => a.SpanId).Distinct().Should().HaveCount(1); + + // The orchestration activities should be parented to the create activity. + orchestrationActivities + .Should().AllSatisfy(a => + { + a.Kind.Should().Be(ActivityKind.Server); + a.ParentId.Should().Be(createActivity.Id); + a.ParentSpanId.Should().Be(createActivity.SpanId); + a.Status.Should().Be(ActivityStatusCode.Error); + + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + }); + + var orchestrationActivity = orchestrationActivities.First(); + + var clientSuborchestrationActivities = activities.Where(a => a.Kind == ActivityKind.Client && a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{subOrchestratorName}").ToList(); + + // The client suborchestration activities should be parented to the orchestration activity. + clientSuborchestrationActivities + .Should().HaveCount(1) + .And.AllSatisfy(a => + { + a.ParentId.Should().Be(orchestrationActivity.Id); + a.ParentSpanId.Should().Be(orchestrationActivity.SpanId); + a.Status.Should().Be(ActivityStatusCode.Error); + + a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(subOrchestratorName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + }); + + var clientSuborchestrationActivity = clientSuborchestrationActivities.First(); + + var serverSuborchestrationActivities = activities.Where(a => a.Kind == ActivityKind.Server && a.Source.Name == CoreActivitySourceName && a.OperationName == $"orchestration:{subOrchestratorName}").ToList(); + + // The server suborchestration activities should be parented to the client orchestration activity. + serverSuborchestrationActivities + .Should().HaveCountGreaterThan(0) + .And.AllSatisfy(a => + { + a.ParentId.Should().Be(clientSuborchestrationActivity.Id); + a.ParentSpanId.Should().Be(clientSuborchestrationActivity.SpanId); + a.Status.Should().Be(ActivityStatusCode.Error); + + a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(subOrchestratorName); + a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + }); + } + [Fact] public async Task TaskOrchestrationWithSentEvent() { From 9497ad56df28d7d9425cabdde6255e051b500c64 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 12 Jun 2025 15:50:40 -0700 Subject: [PATCH 26/36] Revert change. --- src/Shared/Shared.csproj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Shared/Shared.csproj b/src/Shared/Shared.csproj index 6424c15b..76b3bed6 100644 --- a/src/Shared/Shared.csproj +++ b/src/Shared/Shared.csproj @@ -14,8 +14,7 @@ - - + From a51a235c302275a55e8b4dac9b352c0fad3aaf4c Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 13 Jun 2025 11:06:43 -0700 Subject: [PATCH 27/36] Eliminate the unnecessary activity ID (as it can be derived). --- src/Grpc/orchestrator_service.proto | 8 +++----- src/Shared/Grpc/ProtoUtils.cs | 5 ++--- src/Shared/Grpc/Tracing/TraceHelper.cs | 11 +++-------- .../Dispatcher/GrpcOrchestratorExecutionResult.cs | 1 - .../Dispatcher/TaskOrchestrationDispatcher.cs | 1 - .../GrpcSidecar/Grpc/ProtobufUtils.cs | 3 +-- .../GrpcSidecar/Grpc/TaskHubGrpcServer.cs | 5 ++--- 7 files changed, 11 insertions(+), 23 deletions(-) diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 0b448eea..c0c4d18b 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -77,8 +77,7 @@ message ExecutionStartedEvent { TraceContext parentTraceContext = 7; google.protobuf.StringValue orchestrationSpanID = 8; map tags = 9; - google.protobuf.StringValue orchestrationActivityID = 10; - google.protobuf.Timestamp orchestrationActivityStartTime = 11; + google.protobuf.Timestamp orchestrationSpanStartTime = 10; } message ExecutionCompletedEvent { @@ -338,9 +337,8 @@ message OrchestratorResponse { // This field is optional. If not set, the service should assume that the orchestrator processed all events. google.protobuf.Int32Value numEventsProcessed = 5; - google.protobuf.StringValue orchestrationActivitySpanID = 6; - google.protobuf.StringValue orchestrationActivityID = 7; - google.protobuf.Timestamp orchestrationActivityStartTime = 8; + google.protobuf.StringValue orchestrationSpanID = 6; + google.protobuf.Timestamp orchestrationSpanStartTime = 7; } message CreateInstanceRequest { diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index 51a91e07..bb72700b 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -294,9 +294,8 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( InstanceId = instanceId, CustomStatus = customStatus, CompletionToken = completionToken, - OrchestrationActivityID = parentActivity?.Id, - OrchestrationActivitySpanID = parentActivity?.SpanId.ToString(), - OrchestrationActivityStartTime = parentActivity?.StartTimeUtc.ToTimestamp(), + OrchestrationSpanID = parentActivity?.SpanId.ToString(), + OrchestrationSpanStartTime = parentActivity?.StartTimeUtc.ToTimestamp(), }; foreach (OrchestratorAction action in actions) diff --git a/src/Shared/Grpc/Tracing/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs index 1e679b46..098b8c24 100644 --- a/src/Shared/Grpc/Tracing/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -81,7 +81,7 @@ public class TraceHelper string activityName = CreateSpanName(TraceActivityConstants.Orchestration, startEvent.Name, startEvent.Version); ActivityKind activityKind = ActivityKind.Server; - DateTimeOffset startTime = startEvent.OrchestrationActivityStartTime?.ToDateTimeOffset() ?? default; + DateTimeOffset startTime = startEvent.OrchestrationSpanStartTime?.ToDateTimeOffset() ?? default; Activity? activity = ActivityTraceSource.StartActivity( activityName, @@ -103,21 +103,16 @@ public class TraceHelper activity.SetTag(Schema.Task.Version, startEvent.Version); } - if (startEvent.OrchestrationActivityID != null && startEvent.OrchestrationSpanID != null) + if (startEvent.OrchestrationSpanID != null) { - activity.SetId(startEvent.OrchestrationActivityID!); activity.SetSpanId(startEvent.OrchestrationSpanID!); } else { - startEvent.OrchestrationActivityID = activity.Id; startEvent.OrchestrationSpanID = activity.SpanId.ToString(); - startEvent.OrchestrationActivityStartTime = Timestamp.FromDateTime(activity.StartTimeUtc); + startEvent.OrchestrationSpanStartTime = Timestamp.FromDateTime(activity.StartTimeUtc); } - // DistributedTraceActivity.Current = activity; - // return DistributedTraceActivity.Current; - return activity; } diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs index 44b6f073..302e13d4 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs @@ -7,7 +7,6 @@ namespace Microsoft.DurableTask.Sidecar.Dispatcher; public class GrpcOrchestratorExecutionResult : OrchestratorExecutionResult { - public string? OrchestrationActivityId { get; set; } public string? OrchestrationActivitySpanId { get; set; } public DateTimeOffset? OrchestrationActivityStartTime { get; set; } } diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs index c88989bc..64074fb2 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs @@ -94,7 +94,6 @@ protected override async Task ExecuteWorkItemAsync(TaskOrchestrationWorkItem wor if (executionStartedEvent?.ParentTraceContext is not null) { executionStartedEvent.ParentTraceContext.ActivityStartTime = result.OrchestrationActivityStartTime; - executionStartedEvent.ParentTraceContext.Id = result.OrchestrationActivityId; executionStartedEvent.ParentTraceContext.SpanId = result.OrchestrationActivitySpanId; } diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs index c7fd9100..6d6f0277 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs @@ -98,9 +98,8 @@ public static Proto.HistoryEvent ToHistoryEventProto(HistoryEvent e) TraceParent = startedEvent.ParentTraceContext.TraceParent, TraceState = startedEvent.ParentTraceContext.TraceState, }, - OrchestrationActivityStartTime = startedEvent.ParentTraceContext?.ActivityStartTime is not null ? Timestamp.FromDateTimeOffset(startedEvent.ParentTraceContext.ActivityStartTime.Value) : null, - OrchestrationActivityID = startedEvent.ParentTraceContext?.Id, OrchestrationSpanID = startedEvent.ParentTraceContext?.SpanId, + OrchestrationSpanStartTime = startedEvent.ParentTraceContext?.ActivityStartTime is not null ? Timestamp.FromDateTimeOffset(startedEvent.ParentTraceContext.ActivityStartTime.Value) : null }; break; case EventType.ExecutionTerminated: diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs index 76ea9e29..38998fa3 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs @@ -372,9 +372,8 @@ static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationState state, { Actions = request.Actions.Select(ProtobufUtils.ToOrchestratorAction), CustomStatus = request.CustomStatus, - OrchestrationActivityId = request.OrchestrationActivityID, - OrchestrationActivitySpanId = request.OrchestrationActivitySpanID, - OrchestrationActivityStartTime = request.OrchestrationActivityStartTime?.ToDateTimeOffset(), + OrchestrationActivitySpanId = request.OrchestrationSpanID, + OrchestrationActivityStartTime = request.OrchestrationSpanStartTime?.ToDateTimeOffset(), }; tcs.TrySetResult(result); From 14afe63805dfe77fac474d2e04dd41a530c91d0d Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 18 Jul 2025 13:31:18 -0700 Subject: [PATCH 28/36] Fix up a number of warnings. --- src/Shared/Grpc/ProtoUtils.cs | 14 ++- .../Tracing/DiagnosticActivityExtensions.cs | 7 +- src/Shared/Grpc/Tracing/Schema.cs | 44 ++++++- src/Shared/Grpc/Tracing/TraceHelper.cs | 115 ++++++++++-------- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 99 +++++++-------- src/Worker/Grpc/GrpcOrchestrationRunner.cs | 3 +- 6 files changed, 162 insertions(+), 120 deletions(-) diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index bb72700b..f5c28dd2 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -270,6 +270,7 @@ internal static Timestamp ToTimestamp(this DateTime dateTime) /// Constructs a . /// /// The orchestrator instance ID. + /// The orchestrator execution ID. /// The orchestrator customer status or null if no custom status. /// The orchestrator actions. /// @@ -277,6 +278,7 @@ internal static Timestamp ToTimestamp(this DateTime dateTime) /// value that was provided by the corresponding that triggered the orchestrator execution. /// /// The entity conversion state, or null if no conversion is required. + /// The that represents orchestration execution. /// The orchestrator response. /// When an orchestrator action is unknown. internal static P.OrchestratorResponse ConstructOrchestratorResponse( @@ -286,7 +288,7 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( IEnumerable actions, string completionToken, EntityConversionState? entityConversionState, - Activity? parentActivity) + Activity? orchestrationActivity) { Check.NotNull(actions); var response = new P.OrchestratorResponse @@ -294,8 +296,8 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( InstanceId = instanceId, CustomStatus = customStatus, CompletionToken = completionToken, - OrchestrationSpanID = parentActivity?.SpanId.ToString(), - OrchestrationSpanStartTime = parentActivity?.StartTimeUtc.ToTimestamp(), + OrchestrationSpanID = orchestrationActivity?.SpanId.ToString(), + OrchestrationSpanStartTime = orchestrationActivity?.StartTimeUtc.ToTimestamp(), }; foreach (OrchestratorAction action in actions) @@ -304,10 +306,10 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( ActivityContext? clientActivityContext = null; - if (parentActivity != null) + if (orchestrationActivity != null) { ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); - clientActivityContext = new(parentActivity.TraceId, clientSpanId, parentActivity.ActivityTraceFlags, parentActivity.TraceStateString); + clientActivityContext = new(orchestrationActivity.TraceId, clientSpanId, orchestrationActivity.ActivityTraceFlags, orchestrationActivity.TraceStateString); } switch (action.OrchestratorActionType) @@ -410,7 +412,7 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( // Distributed Tracing: start a new trace activity derived from the orchestration // for an EventRaisedEvent (external event) - using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(sendEventAction,instanceId, executionId); + using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(sendEventAction, instanceId, executionId); traceActivity?.Stop(); } diff --git a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs index fd22d412..2eb877ca 100644 --- a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs +++ b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs @@ -25,20 +25,15 @@ enum ActivityStatusCode static class DiagnosticActivityExtensions { static readonly Action s_setSpanId; - static readonly Action s_setId; static readonly Action s_setStatus; static DiagnosticActivityExtensions() { BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance; - s_setSpanId = typeof(Activity).GetField("_spanId", flags).CreateSetter(); - s_setId = typeof(Activity).GetField("_id", flags).CreateSetter(); + s_setSpanId = (typeof(Activity).GetField("_spanId", flags) ?? throw new InvalidOperationException("The field Activity._spanId was not found.")).CreateSetter(); s_setStatus = CreateSetStatus(); } - public static void SetId(this Activity activity, string id) - => s_setId(activity, id); - public static void SetSpanId(this Activity activity, string spanId) => s_setSpanId(activity, spanId); diff --git a/src/Shared/Grpc/Tracing/Schema.cs b/src/Shared/Grpc/Tracing/Schema.cs index 468e72e4..ef684aea 100644 --- a/src/Shared/Grpc/Tracing/Schema.cs +++ b/src/Shared/Grpc/Tracing/Schema.cs @@ -1,21 +1,59 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/Schema.cs - namespace Microsoft.DurableTask.Tracing; +/// +/// Schema for tracing events related to Durable Task operations. +/// +/// +/// Adapted from "https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/Schema.cs". +/// static class Schema { - internal static class Task + /// + /// Tags for tracing events related to orchestrations. + /// + public static class Task { + /// + /// The type of activity being executed, such as "orchestration", "activity", or "event". + /// internal const string Type = "durabletask.type"; + + /// + /// The name of the orchestration, activity, or event associated with the tracing event. + /// internal const string Name = "durabletask.task.name"; + + /// + /// The version of the orchestration or activity being executed. + /// internal const string Version = "durabletask.task.version"; + + /// + /// The ID of the orchestration instance associated with the tracing event. + /// internal const string InstanceId = "durabletask.task.instance_id"; + + /// + /// The execution ID of the orchestration instance associated with the tracing event. + /// internal const string ExecutionId = "durabletask.task.execution_id"; + + /// + /// The event ID of the task being executed. + /// internal const string TaskId = "durabletask.task.task_id"; + + /// + /// The ID of the orchestration instance for which the event will be raised. + /// internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; + + /// + /// The time at which the timer is scheduled to fire. + /// internal const string FireAt = "durabletask.fire_at"; } } diff --git a/src/Shared/Grpc/Tracing/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs index 098b8c24..a5d04b33 100644 --- a/src/Shared/Grpc/Tracing/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/TraceHelper.cs - using System.Diagnostics; using DurableTask.Core.Command; using Google.Protobuf.WellKnownTypes; @@ -10,7 +8,13 @@ namespace Microsoft.DurableTask.Tracing; -public class TraceHelper +/// +/// Methods for starting and managing trace activities related to Durable Task operations. +/// +/// +/// Adapted from "https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/TraceHelper.cs". +/// +class TraceHelper { const string Source = "Microsoft.DurableTask"; @@ -19,11 +23,11 @@ public class TraceHelper /// /// Starts a new trace activity for scheduling an orchestration from the client. /// - /// The orchestration's execution started event. + /// The orchestration's creation request. /// /// Returns a newly started with orchestration-specific metadata. /// - internal static Activity? StartActivityForNewOrchestration(P.CreateInstanceRequest createInstanceRequest) + public static Activity? StartActivityForNewOrchestration(P.CreateInstanceRequest createInstanceRequest) { Activity? newActivity = ActivityTraceSource.StartActivity( name: CreateSpanName(TraceActivityConstants.CreateOrchestration, createInstanceRequest.Name, createInstanceRequest.Version), @@ -67,7 +71,7 @@ public class TraceHelper /// /// Returns a newly started with orchestration-specific metadata. /// - internal static Activity? StartTraceActivityForOrchestrationExecution(P.ExecutionStartedEvent? startEvent) + public static Activity? StartTraceActivityForOrchestrationExecution(P.ExecutionStartedEvent? startEvent) { if (startEvent == null) { @@ -117,10 +121,9 @@ public class TraceHelper } /// - /// Starts a new trace activity for (task) activity execution. + /// Starts a new trace activity for (task) activity execution. /// - /// The associated . - /// The associated orchestration instance metadata. + /// The associated request to start a (task) activity. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// @@ -159,15 +162,16 @@ public class TraceHelper /// Starts a new trace activity for (task) activity that represents the time between when the task message /// is enqueued and when the response message is received. /// - /// The associated . - /// The associated . + /// The ID of the associated instance. + /// The associated . + /// The associated . /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// - internal static Activity? StartTraceActivityForSchedulingTask( + static Activity? StartTraceActivityForSchedulingTask( string? instanceId, - P.HistoryEvent historyEvent, - P.TaskScheduledEvent taskScheduledEvent) + P.HistoryEvent? historyEvent, + P.TaskScheduledEvent? taskScheduledEvent) { if (taskScheduledEvent == null) { @@ -177,7 +181,7 @@ public class TraceHelper Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.Activity, taskScheduledEvent.Name, taskScheduledEvent.Version), kind: ActivityKind.Client, - startTime: historyEvent.Timestamp?.ToDateTimeOffset() ?? default, + startTime: historyEvent?.Timestamp?.ToDateTimeOffset() ?? default, parentContext: Activity.Current?.Context ?? default); if (newActivity == null) @@ -196,7 +200,7 @@ public class TraceHelper newActivity.AddTag(Schema.Task.Type, TraceActivityConstants.Activity); newActivity.AddTag(Schema.Task.Name, taskScheduledEvent.Name); newActivity.AddTag(Schema.Task.InstanceId, instanceId); - newActivity.AddTag(Schema.Task.TaskId, historyEvent.EventId); + newActivity.AddTag(Schema.Task.TaskId, historyEvent?.EventId); if (!string.IsNullOrEmpty(taskScheduledEvent.Version)) { @@ -209,12 +213,13 @@ public class TraceHelper /// /// Emits a new trace activity for a (task) activity that successfully completes. /// - /// The associated . - /// The associated . - internal static void EmitTraceActivityForTaskCompleted( + /// The ID of the associated orchestration. + /// The associated . + /// The associated . + public static void EmitTraceActivityForTaskCompleted( string? instanceId, - P.HistoryEvent historyEvent, - P.TaskScheduledEvent taskScheduledEvent) + P.HistoryEvent? historyEvent, + P.TaskScheduledEvent? taskScheduledEvent) { // The parent of this is the parent orchestration span ID. It should be the client span which started this Activity? activity = StartTraceActivityForSchedulingTask(instanceId, historyEvent, taskScheduledEvent); @@ -225,14 +230,14 @@ internal static void EmitTraceActivityForTaskCompleted( /// /// Emits a new trace activity for a (task) activity that fails. /// - /// The associated . - /// The associated . - /// The associated . - /// Specifies the method to propagate unhandled exceptions to parent orchestrations. - internal static void EmitTraceActivityForTaskFailed( + /// The ID of the associated orchestration. + /// The associated . + /// The associated . + /// The associated . + public static void EmitTraceActivityForTaskFailed( string? instanceId, - P.HistoryEvent historyEvent, - P.TaskScheduledEvent taskScheduledEvent, + P.HistoryEvent? historyEvent, + P.TaskScheduledEvent? taskScheduledEvent, P.TaskFailedEvent? failedEvent) { Activity? activity = StartTraceActivityForSchedulingTask(instanceId, historyEvent, taskScheduledEvent); @@ -255,15 +260,16 @@ internal static void EmitTraceActivityForTaskFailed( /// Starts a new trace activity for sub-orchestrations. Represents the time between enqueuing /// the sub-orchestration message and it completing. /// - /// The associated . - /// The associated . + /// The ID of the associated orchestration. + /// The associated . + /// The associated . /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// - internal static Activity? CreateTraceActivityForSchedulingSubOrchestration( + static Activity? CreateTraceActivityForSchedulingSubOrchestration( string? instanceId, - P.HistoryEvent historyEvent, - P.SubOrchestrationInstanceCreatedEvent createdEvent) + P.HistoryEvent? historyEvent, + P.SubOrchestrationInstanceCreatedEvent? createdEvent) { if (instanceId == null || createdEvent == null) { @@ -273,7 +279,7 @@ internal static void EmitTraceActivityForTaskFailed( Activity? activity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.Orchestration, createdEvent.Name, createdEvent.Version), kind: ActivityKind.Client, - startTime: historyEvent.Timestamp?.ToDateTimeOffset() ?? default, + startTime: historyEvent?.Timestamp?.ToDateTimeOffset() ?? default, parentContext: Activity.Current?.Context ?? default); if (activity == null) @@ -305,12 +311,13 @@ internal static void EmitTraceActivityForTaskFailed( /// Emits a new trace activity for sub-orchestration execution when the sub-orchestration /// completes successfully. /// - /// The associated . - /// The associated . - internal static void EmitTraceActivityForSubOrchestrationCompleted( + /// The ID of the associated orchestration. + /// The associated . + /// The associated . + public static void EmitTraceActivityForSubOrchestrationCompleted( string? instanceId, - P.HistoryEvent historyEvent, - P.SubOrchestrationInstanceCreatedEvent createdEvent) + P.HistoryEvent? historyEvent, + P.SubOrchestrationInstanceCreatedEvent? createdEvent) { // The parent of this is the parent orchestration span ID. It should be the client span which started this Activity? activity = CreateTraceActivityForSchedulingSubOrchestration(instanceId, historyEvent, createdEvent); @@ -321,17 +328,17 @@ internal static void EmitTraceActivityForSubOrchestrationCompleted( /// /// Emits a new trace activity for sub-orchestration execution when the sub-orchestration fails. /// - /// The associated . - /// The associated . - /// The associated . - /// Specifies the method to propagate unhandled exceptions to parent orchestrations. - internal static void EmitTraceActivityForSubOrchestrationFailed( - string? orchestrationInstance, - P.HistoryEvent historyEvent, - P.SubOrchestrationInstanceCreatedEvent createdEvent, + /// The ID of the associated orchestration. + /// The associated . + /// The associated . + /// The associated . + public static void EmitTraceActivityForSubOrchestrationFailed( + string? instanceId, + P.HistoryEvent? historyEvent, + P.SubOrchestrationInstanceCreatedEvent? createdEvent, P.SubOrchestrationInstanceFailedEvent? failedEvent) { - Activity? activity = CreateTraceActivityForSchedulingSubOrchestration(orchestrationInstance, historyEvent, createdEvent); + Activity? activity = CreateTraceActivityForSchedulingSubOrchestration(instanceId, historyEvent, createdEvent); if (activity is null) { @@ -347,7 +354,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( activity?.Dispose(); } - internal static Activity? StartTraceActivityForEventRaisedFromWorker( + public static Activity? StartTraceActivityForEventRaisedFromWorker( SendEventOrchestratorAction eventRaisedEvent, string? instanceId, string? executionId) @@ -378,12 +385,12 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// Creates a new trace activity for events created from the client. /// - /// The associated . - /// The associated . + /// The associated . + /// The ID of the associated orchestration. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// - internal static Activity? StartActivityForNewEventRaisedFromClient(P.RaiseEventRequest eventRaised, string instanceId) + public static Activity? StartActivityForNewEventRaisedFromClient(P.RaiseEventRequest eventRaised, string instanceId) { Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null), @@ -402,11 +409,11 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// Emits a new trace activity for timers. /// - /// The associated . + /// The ID of the associated orchestration. /// The name of the orchestration invoking the timer. /// The timer's start time. /// The associated . - internal static void EmitTraceActivityForTimer( + public static void EmitTraceActivityForTimer( string? instanceId, string orchestrationName, DateTime startTime, diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 389c6076..61ddc491 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -369,57 +369,60 @@ async Task OnRunOrchestratorAsync( Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution(executionStartedEvent); - foreach (var newEvent in request.NewEvents) + if (executionStartedEvent is not null) { - switch (newEvent.EventTypeCase) + foreach (var newEvent in request.NewEvents) { - case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCompleted: - { - var subOrchestrationEvent = request.PastEvents - .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) - .LastOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); - - TraceHelper.EmitTraceActivityForSubOrchestrationCompleted( - request.InstanceId, - subOrchestrationEvent, - subOrchestrationEvent.SubOrchestrationInstanceCreated); - break; - } - - case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceFailed: - { - var subOrchestrationEvent = request.PastEvents - .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) - .LastOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); - - TraceHelper.EmitTraceActivityForSubOrchestrationFailed( - request.InstanceId, - subOrchestrationEvent, - subOrchestrationEvent.SubOrchestrationInstanceCreated, - newEvent.SubOrchestrationInstanceFailed); - break; - } - - case P.HistoryEvent.EventTypeOneofCase.TaskCompleted: - { - var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); - TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled); - break; - } + switch (newEvent.EventTypeCase) + { + case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCompleted: + { + var subOrchestrationEvent = request.PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) + .LastOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); + + TraceHelper.EmitTraceActivityForSubOrchestrationCompleted( + request.InstanceId, + subOrchestrationEvent, + subOrchestrationEvent?.SubOrchestrationInstanceCreated); + break; + } + + case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceFailed: + { + var subOrchestrationEvent = request.PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) + .LastOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); + + TraceHelper.EmitTraceActivityForSubOrchestrationFailed( + request.InstanceId, + subOrchestrationEvent, + subOrchestrationEvent?.SubOrchestrationInstanceCreated, + newEvent.SubOrchestrationInstanceFailed); + break; + } + + case P.HistoryEvent.EventTypeOneofCase.TaskCompleted: + { + var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); + TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent?.TaskScheduled); + break; + } - case P.HistoryEvent.EventTypeOneofCase.TaskFailed: - { - var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskFailed.TaskScheduledId); - TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent.TaskScheduled, newEvent.TaskFailed); - break; - } + case P.HistoryEvent.EventTypeOneofCase.TaskFailed: + { + var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskFailed.TaskScheduledId); + TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent?.TaskScheduled, newEvent.TaskFailed); + break; + } - case P.HistoryEvent.EventTypeOneofCase.TimerFired: - { - // We immediately publish the activity span for this timer by creating the activity and immediately calling Dispose() on it. - TraceHelper.EmitTraceActivityForTimer(request.InstanceId, executionStartedEvent?.Name, newEvent.Timestamp.ToDateTime(), newEvent.TimerFired); - break; - } + case P.HistoryEvent.EventTypeOneofCase.TimerFired: + { + // We immediately publish the activity span for this timer by creating the activity and immediately calling Dispose() on it. + TraceHelper.EmitTraceActivityForTimer(request.InstanceId, executionStartedEvent.Name, newEvent.Timestamp.ToDateTime(), newEvent.TimerFired); + break; + } + } } } @@ -583,8 +586,6 @@ await this.client.AbandonTaskOrchestratorWorkItemAsync( await this.client.CompleteOrchestratorTaskAsync(response, cancellationToken: cancellationToken); } - private static readonly ActivitySource ActivitySource = new ActivitySource("Microsoft.DurableTask.Worker"); - async Task OnRunActivityAsync(P.ActivityRequest request, string completionToken, CancellationToken cancellation) { using Activity? traceActivity = TraceHelper.StartTraceActivityForTaskExecution(request); diff --git a/src/Worker/Grpc/GrpcOrchestrationRunner.cs b/src/Worker/Grpc/GrpcOrchestrationRunner.cs index 0cea8056..064b38c6 100644 --- a/src/Worker/Grpc/GrpcOrchestrationRunner.cs +++ b/src/Worker/Grpc/GrpcOrchestrationRunner.cs @@ -122,8 +122,7 @@ public static string LoadAndRun( result.Actions, completionToken: string.Empty, /* doesn't apply */ entityConversionState: null, - // TODO: Should this activity be created? - parentActivity: null); + orchestrationActivity: null); byte[] responseBytes = response.ToByteArray(); return Convert.ToBase64String(responseBytes); } From 41c44f1eb0fc9e618bb753faef288e5c467ec9bb Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 18 Jul 2025 14:57:26 -0700 Subject: [PATCH 29/36] Fixup doc and style warnings. --- .../Tracing/DiagnosticActivityExtensions.cs | 55 +++-- .../Grpc/Tracing/FieldInfoExtensionMethods.cs | 10 +- .../Grpc/Tracing/TraceActivityConstants.cs | 28 ++- src/Shared/Grpc/Tracing/TraceHelper.cs | 217 +++++++++--------- 4 files changed, 188 insertions(+), 122 deletions(-) diff --git a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs index 2eb877ca..63fdf6fe 100644 --- a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs +++ b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs @@ -1,21 +1,31 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/DiagnosticActivityExtensions.cs - using System.Diagnostics; using System.Linq.Expressions; using System.Reflection; +// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/DiagnosticActivityExtensions.cs namespace Microsoft.DurableTask.Tracing; /// -/// Replica from System.Diagnostics.DiagnosticSource >= 6.0.0 +/// Replica from System.Diagnostics.DiagnosticSource >= 6.0.0. /// enum ActivityStatusCode { + /// + /// The default value indicating the status code is not initialized. + /// Unset = 0, - OK = 1, + + /// + /// Indicates the operation has been validated and completed successfully. + /// + Ok = 1, + + /// + /// Indicates an error was encountered during the operation. + /// Error = 2, } @@ -24,39 +34,56 @@ enum ActivityStatusCode /// static class DiagnosticActivityExtensions { - static readonly Action s_setSpanId; - static readonly Action s_setStatus; + static readonly Action SetSpanIdMethod; + static readonly Action SetStatusMethod; static DiagnosticActivityExtensions() { BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance; - s_setSpanId = (typeof(Activity).GetField("_spanId", flags) ?? throw new InvalidOperationException("The field Activity._spanId was not found.")).CreateSetter(); - s_setStatus = CreateSetStatus(); + SetSpanIdMethod = (typeof(Activity).GetField("_spanId", flags) ?? throw new InvalidOperationException("The field Activity._spanId was not found.")).CreateSetter(); + SetStatusMethod = CreateSetStatus(); } + /// + /// Explicitly sets the span ID for the given activity. + /// + /// The activity on which to set the span ID. + /// The span ID to set. public static void SetSpanId(this Activity activity, string spanId) - => s_setSpanId(activity, spanId); + => SetSpanIdMethod(activity, spanId); + /// + /// Explicitly sets the status code and description for the given activity. + /// + /// The activity on which to set the span ID. + /// The status to set. + /// The description to set. public static void SetStatus(this Activity activity, ActivityStatusCode status, string description) - => s_setStatus(activity, status, description); + => SetStatusMethod(activity, status, description); static Action CreateSetStatus() { - MethodInfo method = typeof(Activity).GetMethod("SetStatus"); + MethodInfo? method = typeof(Activity).GetMethod("SetStatus"); + if (method is null) { - return (activity, status, description) => { + return (activity, status, description) => + { +#pragma warning disable CA1510 if (activity is null) { throw new ArgumentNullException(nameof(activity)); } - string str = status switch +#pragma warning restore CA1510 + + string? str = status switch { ActivityStatusCode.Unset => "UNSET", - ActivityStatusCode.OK => "OK", + ActivityStatusCode.Ok => "OK", ActivityStatusCode.Error => "ERROR", _ => null, }; + activity.SetTag("otel.status_code", str); activity.SetTag("otel.status_description", description); }; diff --git a/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs b/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs index 2d416eda..508f4dc1 100644 --- a/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs +++ b/src/Shared/Grpc/Tracing/FieldInfoExtensionMethods.cs @@ -1,11 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/FieldInfoExtensionMethods.cs - using System.Linq.Expressions; using System.Reflection; +// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/FieldInfoExtensionMethods.cs namespace Microsoft.DurableTask.Tracing; /// @@ -23,10 +22,17 @@ static class FieldInfoExtensionMethods /// A re-usable action to set the field. internal static Action CreateSetter(this FieldInfo fieldInfo) { +#pragma warning disable CA1510 if (fieldInfo == null) { throw new ArgumentNullException(nameof(fieldInfo)); } +#pragma warning restore CA1510 + + if (fieldInfo.DeclaringType is null) + { + throw new ArgumentException("FieldInfo.DeclaringType cannot be null.", nameof(fieldInfo)); + } ParameterExpression targetExp = Expression.Parameter(typeof(TTarget), "target"); Expression source = targetExp; diff --git a/src/Shared/Grpc/Tracing/TraceActivityConstants.cs b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs index 55d26715..5e83b956 100644 --- a/src/Shared/Grpc/Tracing/TraceActivityConstants.cs +++ b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs @@ -1,17 +1,41 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// NOTE: Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/TraceActivityConstants.cs - +// Modified from https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/TraceActivityConstants.cs. namespace Microsoft.DurableTask.Tracing; +/// +/// Constants for trace activity names used in Durable Task Framework. +/// class TraceActivityConstants { + /// + /// The name of the activity that represents orchestration operations. + /// public const string Orchestration = "orchestration"; + + /// + /// The name of the activity that represents activity operations. + /// public const string Activity = "activity"; + + /// + /// The name of the activity that represents entity operations. + /// public const string Event = "event"; + + /// + /// The name of the activity that represents timer operations. + /// public const string Timer = "timer"; + /// + /// The name of the activity that represents an operation to create an orchestration. + /// public const string CreateOrchestration = "create_orchestration"; + + /// + /// The name of the activity that represents an operation to raise an event. + /// public const string OrchestrationEvent = "orchestration_event"; } diff --git a/src/Shared/Grpc/Tracing/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs index a5d04b33..3d16f0f5 100644 --- a/src/Shared/Grpc/Tracing/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -127,7 +127,7 @@ class TraceHelper /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// - internal static Activity? StartTraceActivityForTaskExecution( + public static Activity? StartTraceActivityForTaskExecution( P.ActivityRequest request) { if (request.ParentTraceContext is null || !ActivityContext.TryParse(request.ParentTraceContext.TraceParent, request.ParentTraceContext.TraceState, out ActivityContext activityContext)) @@ -158,58 +158,6 @@ class TraceHelper return newActivity; } - /// - /// Starts a new trace activity for (task) activity that represents the time between when the task message - /// is enqueued and when the response message is received. - /// - /// The ID of the associated instance. - /// The associated . - /// The associated . - /// - /// Returns a newly started with (task) activity and orchestration-specific metadata. - /// - static Activity? StartTraceActivityForSchedulingTask( - string? instanceId, - P.HistoryEvent? historyEvent, - P.TaskScheduledEvent? taskScheduledEvent) - { - if (taskScheduledEvent == null) - { - return null; - } - - Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(TraceActivityConstants.Activity, taskScheduledEvent.Name, taskScheduledEvent.Version), - kind: ActivityKind.Client, - startTime: historyEvent?.Timestamp?.ToDateTimeOffset() ?? default, - parentContext: Activity.Current?.Context ?? default); - - if (newActivity == null) - { - return null; - } - - if (taskScheduledEvent.ParentTraceContext != null) - { - if (ActivityContext.TryParse(taskScheduledEvent.ParentTraceContext.TraceParent, taskScheduledEvent.ParentTraceContext?.TraceState, out ActivityContext parentContext)) - { - newActivity.SetSpanId(parentContext.SpanId.ToString()); - } - } - - newActivity.AddTag(Schema.Task.Type, TraceActivityConstants.Activity); - newActivity.AddTag(Schema.Task.Name, taskScheduledEvent.Name); - newActivity.AddTag(Schema.Task.InstanceId, instanceId); - newActivity.AddTag(Schema.Task.TaskId, historyEvent?.EventId); - - if (!string.IsNullOrEmpty(taskScheduledEvent.Version)) - { - newActivity.AddTag(Schema.Task.Version, taskScheduledEvent.Version); - } - - return newActivity; - } - /// /// Emits a new trace activity for a (task) activity that successfully completes. /// @@ -256,57 +204,6 @@ public static void EmitTraceActivityForTaskFailed( activity?.Dispose(); } - /// - /// Starts a new trace activity for sub-orchestrations. Represents the time between enqueuing - /// the sub-orchestration message and it completing. - /// - /// The ID of the associated orchestration. - /// The associated . - /// The associated . - /// - /// Returns a newly started with (task) activity and orchestration-specific metadata. - /// - static Activity? CreateTraceActivityForSchedulingSubOrchestration( - string? instanceId, - P.HistoryEvent? historyEvent, - P.SubOrchestrationInstanceCreatedEvent? createdEvent) - { - if (instanceId == null || createdEvent == null) - { - return null; - } - - Activity? activity = ActivityTraceSource.StartActivity( - CreateSpanName(TraceActivityConstants.Orchestration, createdEvent.Name, createdEvent.Version), - kind: ActivityKind.Client, - startTime: historyEvent?.Timestamp?.ToDateTimeOffset() ?? default, - parentContext: Activity.Current?.Context ?? default); - - if (activity == null) - { - return null; - } - - if (createdEvent.ParentTraceContext != null) - { - if (ActivityContext.TryParse(createdEvent.ParentTraceContext.TraceParent, createdEvent.ParentTraceContext?.TraceState, out ActivityContext parentContext)) - { - activity.SetSpanId(parentContext.SpanId.ToString()); - } - } - - activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); - activity.SetTag(Schema.Task.Name, createdEvent.Name); - activity.SetTag(Schema.Task.InstanceId, instanceId); - - if (!string.IsNullOrEmpty(createdEvent.Version)) - { - activity.SetTag(Schema.Task.Version, createdEvent.Version); - } - - return activity; - } - /// /// Emits a new trace activity for sub-orchestration execution when the sub-orchestration /// completes successfully. @@ -354,6 +251,15 @@ public static void EmitTraceActivityForSubOrchestrationFailed( activity?.Dispose(); } + /// + /// Emits a new trace activity for events raised from the worker. + /// + /// The associated . + /// The instance ID of the associated orchestration. + /// The execution ID of the associated orchestration. + /// + /// Returns a newly started with (task) activity and orchestration-specific metadata. + /// public static Activity? StartTraceActivityForEventRaisedFromWorker( SendEventOrchestratorAction eventRaisedEvent, string? instanceId, @@ -449,6 +355,109 @@ static string CreateSpanName(string spanDescription, string? taskName, string? t } } + /// + /// Starts a new trace activity for (task) activity that represents the time between when the task message + /// is enqueued and when the response message is received. + /// + /// The ID of the associated instance. + /// The associated . + /// The associated . + /// + /// Returns a newly started with (task) activity and orchestration-specific metadata. + /// + static Activity? StartTraceActivityForSchedulingTask( + string? instanceId, + P.HistoryEvent? historyEvent, + P.TaskScheduledEvent? taskScheduledEvent) + { + if (taskScheduledEvent == null) + { + return null; + } + + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.Activity, taskScheduledEvent.Name, taskScheduledEvent.Version), + kind: ActivityKind.Client, + startTime: historyEvent?.Timestamp?.ToDateTimeOffset() ?? default, + parentContext: Activity.Current?.Context ?? default); + + if (newActivity == null) + { + return null; + } + + if (taskScheduledEvent.ParentTraceContext != null) + { + if (ActivityContext.TryParse(taskScheduledEvent.ParentTraceContext.TraceParent, taskScheduledEvent.ParentTraceContext?.TraceState, out ActivityContext parentContext)) + { + newActivity.SetSpanId(parentContext.SpanId.ToString()); + } + } + + newActivity.AddTag(Schema.Task.Type, TraceActivityConstants.Activity); + newActivity.AddTag(Schema.Task.Name, taskScheduledEvent.Name); + newActivity.AddTag(Schema.Task.InstanceId, instanceId); + newActivity.AddTag(Schema.Task.TaskId, historyEvent?.EventId); + + if (!string.IsNullOrEmpty(taskScheduledEvent.Version)) + { + newActivity.AddTag(Schema.Task.Version, taskScheduledEvent.Version); + } + + return newActivity; + } + + /// + /// Starts a new trace activity for sub-orchestrations. Represents the time between enqueuing + /// the sub-orchestration message and it completing. + /// + /// The ID of the associated orchestration. + /// The associated . + /// The associated . + /// + /// Returns a newly started with (task) activity and orchestration-specific metadata. + /// + static Activity? CreateTraceActivityForSchedulingSubOrchestration( + string? instanceId, + P.HistoryEvent? historyEvent, + P.SubOrchestrationInstanceCreatedEvent? createdEvent) + { + if (instanceId == null || createdEvent == null) + { + return null; + } + + Activity? activity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.Orchestration, createdEvent.Name, createdEvent.Version), + kind: ActivityKind.Client, + startTime: historyEvent?.Timestamp?.ToDateTimeOffset() ?? default, + parentContext: Activity.Current?.Context ?? default); + + if (activity == null) + { + return null; + } + + if (createdEvent.ParentTraceContext != null) + { + if (ActivityContext.TryParse(createdEvent.ParentTraceContext.TraceParent, createdEvent.ParentTraceContext?.TraceState, out ActivityContext parentContext)) + { + activity.SetSpanId(parentContext.SpanId.ToString()); + } + } + + activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); + activity.SetTag(Schema.Task.Name, createdEvent.Name); + activity.SetTag(Schema.Task.InstanceId, instanceId); + + if (!string.IsNullOrEmpty(createdEvent.Version)) + { + activity.SetTag(Schema.Task.Version, createdEvent.Version); + } + + return activity; + } + static string CreateTimerSpanName(string orchestrationName) { return $"{TraceActivityConstants.Orchestration}:{orchestrationName}:{TraceActivityConstants.Timer}"; From f9f8ea3e275ca1361bc12e668f7a9117020ac87f Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 18 Jul 2025 15:47:05 -0700 Subject: [PATCH 30/36] Revert package props. --- Directory.Packages.props | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 51385f9c..8388030f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,6 +6,7 @@ --> true + @@ -18,16 +19,19 @@ + + - + + @@ -36,6 +40,7 @@ + @@ -48,20 +53,22 @@ + - + - + + @@ -71,4 +78,5 @@ - \ No newline at end of file + + From 0d651316c47cd65f8dd389c7ba8b4ce512071103 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 18 Jul 2025 16:31:47 -0700 Subject: [PATCH 31/36] Updates per PR feedback. --- src/Shared/Grpc/Tracing/TraceHelper.cs | 21 ++++--------- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 30 ++++++++++++++----- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/Shared/Grpc/Tracing/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs index 3d16f0f5..e508696f 100644 --- a/src/Shared/Grpc/Tracing/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -46,19 +46,11 @@ class TraceHelper } } - if (Activity.Current?.Id != null || Activity.Current?.TraceStateString != null) + if (Activity.Current is not null) { createInstanceRequest.ParentTraceContext ??= new P.TraceContext(); - - if (Activity.Current?.Id != null) - { - createInstanceRequest.ParentTraceContext.TraceParent = Activity.Current?.Id; - } - - if (Activity.Current?.TraceStateString != null) - { - createInstanceRequest.ParentTraceContext.TraceState = Activity.Current?.TraceStateString; - } + createInstanceRequest.ParentTraceContext.TraceParent = Activity.Current.Id!; + createInstanceRequest.ParentTraceContext.TraceState = Activity.Current.TraceStateString; } return newActivity; @@ -438,12 +430,9 @@ static string CreateSpanName(string spanDescription, string? taskName, string? t return null; } - if (createdEvent.ParentTraceContext != null) + if (createdEvent.ParentTraceContext != null && ActivityContext.TryParse(createdEvent.ParentTraceContext.TraceParent, createdEvent.ParentTraceContext.TraceState, out ActivityContext parentContext)) { - if (ActivityContext.TryParse(createdEvent.ParentTraceContext.TraceParent, createdEvent.ParentTraceContext?.TraceState, out ActivityContext parentContext)) - { - activity.SetSpanId(parentContext.SpanId.ToString()); - } + activity.SetSpanId(parentContext.SpanId.ToString()); } activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 61ddc491..270d3680 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -377,9 +377,11 @@ async Task OnRunOrchestratorAsync( { case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCompleted: { - var subOrchestrationEvent = request.PastEvents - .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) - .LastOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); + var subOrchestrationEvent = + request + .PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) + .FirstOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); TraceHelper.EmitTraceActivityForSubOrchestrationCompleted( request.InstanceId, @@ -390,9 +392,11 @@ async Task OnRunOrchestratorAsync( case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceFailed: { - var subOrchestrationEvent = request.PastEvents - .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) - .LastOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); + var subOrchestrationEvent = + request + .PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) + .FirstOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); TraceHelper.EmitTraceActivityForSubOrchestrationFailed( request.InstanceId, @@ -404,14 +408,24 @@ async Task OnRunOrchestratorAsync( case P.HistoryEvent.EventTypeOneofCase.TaskCompleted: { - var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); + var taskScheduledEvent = + request + .PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled) + .LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); + TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent?.TaskScheduled); break; } case P.HistoryEvent.EventTypeOneofCase.TaskFailed: { - var taskScheduledEvent = request.PastEvents.Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled).LastOrDefault(x => x.EventId == newEvent.TaskFailed.TaskScheduledId); + var taskScheduledEvent = + request + .PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled) + .LastOrDefault(x => x.EventId == newEvent.TaskFailed.TaskScheduledId); + TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent?.TaskScheduled, newEvent.TaskFailed); break; } From 1afa837c7979a382f79c40717632f1d3ea453346 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Mon, 21 Jul 2025 13:39:29 -0700 Subject: [PATCH 32/36] Add orchestration status tag on completion. --- src/Shared/Grpc/ProtoUtils.cs | 33 +++++----- src/Shared/Grpc/Tracing/Schema.cs | 21 +++--- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 66 ++++++++++--------- ...rpcSubOrchestrationInstanceCreatedEvent.cs | 2 +- .../Dispatcher/TaskOrchestrationDispatcher.cs | 2 +- .../TracingIntegrationTests.cs | 3 + 6 files changed, 69 insertions(+), 58 deletions(-) diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index f5c28dd2..934e1f73 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -304,12 +304,21 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( { var protoAction = new P.OrchestratorAction { Id = action.Id }; - ActivityContext? clientActivityContext = null; - - if (orchestrationActivity != null) + P.TraceContext? CreateTraceContext() { + if (orchestrationActivity is null) + { + return null; + } + ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); - clientActivityContext = new(orchestrationActivity.TraceId, clientSpanId, orchestrationActivity.ActivityTraceFlags, orchestrationActivity.TraceStateString); + ActivityContext clientActivityContext = new(orchestrationActivity.TraceId, clientSpanId, orchestrationActivity.ActivityTraceFlags, orchestrationActivity.TraceStateString); + + return new P.TraceContext + { + TraceParent = $"00-{clientActivityContext.TraceId}-{clientActivityContext.SpanId}-0{clientActivityContext.TraceFlags:d}", + TraceState = clientActivityContext.TraceState, + }; } switch (action.OrchestratorActionType) @@ -322,13 +331,7 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( Name = scheduleTaskAction.Name, Version = scheduleTaskAction.Version, Input = scheduleTaskAction.Input, - ParentTraceContext = clientActivityContext is not null - ? new P.TraceContext - { - TraceParent = $"00-{clientActivityContext.Value.TraceId}-{clientActivityContext.Value.SpanId}-0{clientActivityContext.Value.TraceFlags:d}", - TraceState = clientActivityContext.Value.TraceState, - } - : null, + ParentTraceContext = CreateTraceContext(), }; if (scheduleTaskAction.Tags != null) @@ -348,13 +351,7 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( InstanceId = subOrchestrationAction.InstanceId, Name = subOrchestrationAction.Name, Version = subOrchestrationAction.Version, - ParentTraceContext = clientActivityContext is not null - ? new P.TraceContext - { - TraceParent = $"00-{clientActivityContext.Value.TraceId}-{clientActivityContext.Value.SpanId}-0{clientActivityContext.Value.TraceFlags:d}", - TraceState = clientActivityContext.Value.TraceState, - } - : null, + ParentTraceContext = CreateTraceContext(), }; break; case OrchestratorActionType.CreateTimer: diff --git a/src/Shared/Grpc/Tracing/Schema.cs b/src/Shared/Grpc/Tracing/Schema.cs index ef684aea..37777817 100644 --- a/src/Shared/Grpc/Tracing/Schema.cs +++ b/src/Shared/Grpc/Tracing/Schema.cs @@ -19,41 +19,46 @@ public static class Task /// /// The type of activity being executed, such as "orchestration", "activity", or "event". /// - internal const string Type = "durabletask.type"; + public const string Type = "durabletask.type"; /// /// The name of the orchestration, activity, or event associated with the tracing event. /// - internal const string Name = "durabletask.task.name"; + public const string Name = "durabletask.task.name"; /// /// The version of the orchestration or activity being executed. /// - internal const string Version = "durabletask.task.version"; + public const string Version = "durabletask.task.version"; /// /// The ID of the orchestration instance associated with the tracing event. /// - internal const string InstanceId = "durabletask.task.instance_id"; + public const string InstanceId = "durabletask.task.instance_id"; /// /// The execution ID of the orchestration instance associated with the tracing event. /// - internal const string ExecutionId = "durabletask.task.execution_id"; + public const string ExecutionId = "durabletask.task.execution_id"; + + /// + /// The runtime status of the completed orchestration associated with the trace event. + /// + public const string Status = "durabletask.task.status"; /// /// The event ID of the task being executed. /// - internal const string TaskId = "durabletask.task.task_id"; + public const string TaskId = "durabletask.task.task_id"; /// /// The ID of the orchestration instance for which the event will be raised. /// - internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; + public const string EventTargetInstanceId = "durabletask.event.target_instance_id"; /// /// The time at which the timer is scheduled to fire. /// - internal const string FireAt = "durabletask.fire_at"; + public const string FireAt = "durabletask.fire_at"; } } diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 270d3680..1c970813 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -371,48 +371,58 @@ async Task OnRunOrchestratorAsync( if (executionStartedEvent is not null) { + P.HistoryEvent? GetSuborchestrationInstanceCreatedEvent(int eventId) + { + var subOrchestrationEvent = + request + .PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) + .FirstOrDefault(x => x.EventId == eventId); + + return subOrchestrationEvent; + } + + P.HistoryEvent? GetTaskScheduledEvent(int eventId) + { + var taskScheduledEvent = + request + .PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled) + .LastOrDefault(x => x.EventId == eventId); + + return taskScheduledEvent; + } + foreach (var newEvent in request.NewEvents) { switch (newEvent.EventTypeCase) { case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCompleted: { - var subOrchestrationEvent = - request - .PastEvents - .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) - .FirstOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); + var subOrchestrationInstanceCreatedEvent = GetSuborchestrationInstanceCreatedEvent(newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); TraceHelper.EmitTraceActivityForSubOrchestrationCompleted( request.InstanceId, - subOrchestrationEvent, - subOrchestrationEvent?.SubOrchestrationInstanceCreated); + subOrchestrationInstanceCreatedEvent, + subOrchestrationInstanceCreatedEvent?.SubOrchestrationInstanceCreated); break; } case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceFailed: { - var subOrchestrationEvent = - request - .PastEvents - .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCreated) - .FirstOrDefault(x => x.EventId == newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); + var subOrchestrationInstanceCreatedEvent = GetSuborchestrationInstanceCreatedEvent(newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); TraceHelper.EmitTraceActivityForSubOrchestrationFailed( request.InstanceId, - subOrchestrationEvent, - subOrchestrationEvent?.SubOrchestrationInstanceCreated, + subOrchestrationInstanceCreatedEvent, + subOrchestrationInstanceCreatedEvent?.SubOrchestrationInstanceCreated, newEvent.SubOrchestrationInstanceFailed); break; } case P.HistoryEvent.EventTypeOneofCase.TaskCompleted: { - var taskScheduledEvent = - request - .PastEvents - .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled) - .LastOrDefault(x => x.EventId == newEvent.TaskCompleted.TaskScheduledId); + var taskScheduledEvent = GetTaskScheduledEvent(newEvent.TaskCompleted.TaskScheduledId); TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent?.TaskScheduled); break; @@ -420,22 +430,16 @@ async Task OnRunOrchestratorAsync( case P.HistoryEvent.EventTypeOneofCase.TaskFailed: { - var taskScheduledEvent = - request - .PastEvents - .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.TaskScheduled) - .LastOrDefault(x => x.EventId == newEvent.TaskFailed.TaskScheduledId); + var taskScheduledEvent = GetTaskScheduledEvent(newEvent.TaskFailed.TaskScheduledId); TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent?.TaskScheduled, newEvent.TaskFailed); break; } case P.HistoryEvent.EventTypeOneofCase.TimerFired: - { - // We immediately publish the activity span for this timer by creating the activity and immediately calling Dispose() on it. - TraceHelper.EmitTraceActivityForTimer(request.InstanceId, executionStartedEvent.Name, newEvent.Timestamp.ToDateTime(), newEvent.TimerFired); - break; - } + // We immediately publish the activity span for this timer by creating the activity and immediately calling Dispose() on it. + TraceHelper.EmitTraceActivityForTimer(request.InstanceId, executionStartedEvent.Name, newEvent.Timestamp.ToDateTime(), newEvent.TimerFired); + break; } } } @@ -588,7 +592,9 @@ await this.client.AbandonTaskOrchestratorWorkItemAsync( traceActivity?.SetStatus(ActivityStatusCode.Error, completeOrchestrationAction.CompleteOrchestration.Result); } - traceActivity?.Stop(); + traceActivity?.SetTag(Schema.Task.Status, completeOrchestrationAction.CompleteOrchestration.OrchestrationStatus.ToString()); + + traceActivity?.Dispose(); } this.Logger.SendingOrchestratorResponse( diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs index f170b492..466f891c 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs @@ -16,4 +16,4 @@ public GrpcSubOrchestrationInstanceCreatedEvent(int eventId) [DataMember] public DistributedTraceContext? ParentTraceContext { get; set; } -} \ No newline at end of file +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs index 64074fb2..3b16d1c4 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs @@ -89,7 +89,7 @@ protected override async Task ExecuteWorkItemAsync(TaskOrchestrationWorkItem wor continue; } - ExecutionStartedEvent? executionStartedEvent = workItem.OrchestrationRuntimeState.NewEvents.OfType().FirstOrDefault(); + ExecutionStartedEvent? executionStartedEvent = workItem.OrchestrationRuntimeState.ExecutionStartedEvent; if (executionStartedEvent?.ParentTraceContext is not null) { diff --git a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs index 6bbff3df..6bd5c421 100644 --- a/test/Grpc.IntegrationTests/TracingIntegrationTests.cs +++ b/test/Grpc.IntegrationTests/TracingIntegrationTests.cs @@ -100,6 +100,7 @@ public async Task MultiTaskOrchestration() a.TagObjects.Should().ContainKey("durabletask.task.instance_id").WhoseValue.Should().Be(instanceId); a.TagObjects.Should().ContainKey("durabletask.task.name").WhoseValue.Should().Be(orchestratorName); a.TagObjects.Should().ContainKey("durabletask.type").WhoseValue.Should().Be("orchestration"); + a.TagObjects.Should().ContainKey("durabletask.task.status").WhoseValue.Should().Be("Completed"); }); var orchestrationActivity = orchestrationActivities.First(); @@ -201,6 +202,8 @@ public async Task TaskOrchestrationWithActivityFailure() a.ParentId.Should().Be(createActivity.Id); a.ParentSpanId.Should().Be(createActivity.SpanId); a.Status.Should().Be(ActivityStatusCode.Error); + + a.TagObjects.Should().ContainKey("durabletask.task.status").WhoseValue.Should().Be("Failed"); }); var orchestrationActivity = orchestrationActivities.First(); From 5a5c1998e610140043e48adf4da3e0984a46b55f Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 24 Jul 2025 23:39:34 -0700 Subject: [PATCH 33/36] Update for move of orchestration trace context to "outer" message. --- src/Grpc/orchestrator_service.proto | 11 ++++++++--- src/Shared/Grpc/ProtoUtils.cs | 8 ++++++-- src/Shared/Grpc/Tracing/TraceHelper.cs | 16 +++++++--------- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 2 +- .../GrpcSidecar/Grpc/ProtobufUtils.cs | 4 +--- .../GrpcSidecar/Grpc/TaskHubGrpcServer.cs | 15 +++++++++++++-- 6 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index c0c4d18b..95bfeedc 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -77,7 +77,6 @@ message ExecutionStartedEvent { TraceContext parentTraceContext = 7; google.protobuf.StringValue orchestrationSpanID = 8; map tags = 9; - google.protobuf.Timestamp orchestrationSpanStartTime = 10; } message ExecutionCompletedEvent { @@ -317,6 +316,11 @@ message OrchestratorAction { } } +message OrchestrationTraceContext { + google.protobuf.StringValue spanID = 1; + google.protobuf.Timestamp spanStartTime = 2; +} + message OrchestratorRequest { string instanceId = 1; google.protobuf.StringValue executionId = 2; @@ -325,6 +329,8 @@ message OrchestratorRequest { OrchestratorEntityParameters entityParameters = 5; bool requiresHistoryStreaming = 6; map properties = 7; + + OrchestrationTraceContext orchestrationTraceContext = 8; } message OrchestratorResponse { @@ -337,8 +343,7 @@ message OrchestratorResponse { // This field is optional. If not set, the service should assume that the orchestrator processed all events. google.protobuf.Int32Value numEventsProcessed = 5; - google.protobuf.StringValue orchestrationSpanID = 6; - google.protobuf.Timestamp orchestrationSpanStartTime = 7; + OrchestrationTraceContext orchestrationTraceContext = 6; } message CreateInstanceRequest { diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index 934e1f73..54957e72 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -296,8 +296,12 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( InstanceId = instanceId, CustomStatus = customStatus, CompletionToken = completionToken, - OrchestrationSpanID = orchestrationActivity?.SpanId.ToString(), - OrchestrationSpanStartTime = orchestrationActivity?.StartTimeUtc.ToTimestamp(), + OrchestrationTraceContext = + new() + { + SpanID = orchestrationActivity?.SpanId.ToString(), + SpanStartTime = orchestrationActivity?.StartTimeUtc.ToTimestamp(), + }, }; foreach (OrchestratorAction action in actions) diff --git a/src/Shared/Grpc/Tracing/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs index e508696f..b777b214 100644 --- a/src/Shared/Grpc/Tracing/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -60,10 +60,13 @@ class TraceHelper /// Starts a new trace activity for orchestration execution. /// /// The orchestration's execution started event. + /// The orchestration trace context containing span metadata. /// /// Returns a newly started with orchestration-specific metadata. /// - public static Activity? StartTraceActivityForOrchestrationExecution(P.ExecutionStartedEvent? startEvent) + public static Activity? StartTraceActivityForOrchestrationExecution( + P.ExecutionStartedEvent? startEvent, + P.OrchestrationTraceContext? orchestrationTraceContext) { if (startEvent == null) { @@ -77,7 +80,7 @@ class TraceHelper string activityName = CreateSpanName(TraceActivityConstants.Orchestration, startEvent.Name, startEvent.Version); ActivityKind activityKind = ActivityKind.Server; - DateTimeOffset startTime = startEvent.OrchestrationSpanStartTime?.ToDateTimeOffset() ?? default; + DateTimeOffset startTime = orchestrationTraceContext?.SpanStartTime?.ToDateTimeOffset() ?? default; Activity? activity = ActivityTraceSource.StartActivity( activityName, @@ -99,14 +102,9 @@ class TraceHelper activity.SetTag(Schema.Task.Version, startEvent.Version); } - if (startEvent.OrchestrationSpanID != null) + if (orchestrationTraceContext?.SpanID != null) { - activity.SetSpanId(startEvent.OrchestrationSpanID!); - } - else - { - startEvent.OrchestrationSpanID = activity.SpanId.ToString(); - startEvent.OrchestrationSpanStartTime = Timestamp.FromDateTime(activity.StartTimeUtc); + activity.SetSpanId(orchestrationTraceContext.SpanID!); } return activity; diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 1c970813..49140721 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -367,7 +367,7 @@ async Task OnRunOrchestratorAsync( .Select(e => e.ExecutionStarted) .FirstOrDefault(); - Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution(executionStartedEvent); + Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution(executionStartedEvent, request.OrchestrationTraceContext); if (executionStartedEvent is not null) { diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs index 6d6f0277..ae8c16a1 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs @@ -97,9 +97,7 @@ public static Proto.HistoryEvent ToHistoryEventProto(HistoryEvent e) { TraceParent = startedEvent.ParentTraceContext.TraceParent, TraceState = startedEvent.ParentTraceContext.TraceState, - }, - OrchestrationSpanID = startedEvent.ParentTraceContext?.SpanId, - OrchestrationSpanStartTime = startedEvent.ParentTraceContext?.ActivityStartTime is not null ? Timestamp.FromDateTimeOffset(startedEvent.ParentTraceContext.ActivityStartTime.Value) : null + } }; break; case EventType.ExecutionTerminated: diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs index 38998fa3..6e341f8c 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs @@ -372,8 +372,8 @@ static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationState state, { Actions = request.Actions.Select(ProtobufUtils.ToOrchestratorAction), CustomStatus = request.CustomStatus, - OrchestrationActivitySpanId = request.OrchestrationSpanID, - OrchestrationActivityStartTime = request.OrchestrationSpanStartTime?.ToDateTimeOffset(), + OrchestrationActivitySpanId = request.OrchestrationTraceContext.SpanID, + OrchestrationActivityStartTime = request.OrchestrationTraceContext.SpanStartTime?.ToDateTimeOffset(), }; tcs.TrySetResult(result); @@ -468,6 +468,16 @@ async Task ITaskExecutor.ExecuteOrchestrator( IEnumerable pastEvents, IEnumerable newEvents) { + var executionStartedEvent = pastEvents.OfType().FirstOrDefault(); + + P.OrchestrationTraceContext? orchestrationTraceContext = executionStartedEvent?.ParentTraceContext?.SpanId is not null + ? new P.OrchestrationTraceContext + { + SpanID = executionStartedEvent.ParentTraceContext.SpanId, + SpanStartTime = executionStartedEvent.ParentTraceContext.ActivityStartTime?.ToTimestamp(), + } + : null; + // Create a task completion source that represents the async completion of the orchestrator execution. // This must be done before we start the orchestrator execution. TaskCompletionSource tcs = @@ -482,6 +492,7 @@ await this.SendWorkItemToClientAsync(new P.WorkItem InstanceId = instance.InstanceId, ExecutionId = instance.ExecutionId, NewEvents = { newEvents.Select(ProtobufUtils.ToHistoryEventProto) }, + OrchestrationTraceContext = orchestrationTraceContext, PastEvents = { pastEvents.Select(ProtobufUtils.ToHistoryEventProto) }, } }); From 1dc3277ae0df3c80e354c2922183fac98f57bc85 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 31 Jul 2025 11:59:40 -0700 Subject: [PATCH 34/36] Fix test breaks. --- .../GrpcSidecar/Grpc/TaskHubGrpcServer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs index 09b55092..b035ed53 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs @@ -372,8 +372,8 @@ static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationState state, { Actions = request.Actions.Select(ProtobufUtils.ToOrchestratorAction), CustomStatus = request.CustomStatus, - OrchestrationActivitySpanId = request.OrchestrationTraceContext.SpanID, - OrchestrationActivityStartTime = request.OrchestrationTraceContext.SpanStartTime?.ToDateTimeOffset(), + OrchestrationActivitySpanId = request.OrchestrationTraceContext?.SpanID, + OrchestrationActivityStartTime = request.OrchestrationTraceContext?.SpanStartTime?.ToDateTimeOffset(), }; tcs.TrySetResult(result); From ae9a2fd7f4d58f850210e17c7f54ae7d8e720261 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Thu, 31 Jul 2025 13:43:02 -0700 Subject: [PATCH 35/36] Updates per more PR feedback. --- .../Tracing/DiagnosticActivityExtensions.cs | 4 +- .../Grpc/Tracing/TraceActivityConstants.cs | 2 +- src/Shared/Grpc/Tracing/TraceHelper.cs | 40 +++++++++++----- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 47 ++++++++++++++----- 4 files changed, 67 insertions(+), 26 deletions(-) diff --git a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs index 63fdf6fe..34c88e1d 100644 --- a/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs +++ b/src/Shared/Grpc/Tracing/DiagnosticActivityExtensions.cs @@ -40,7 +40,9 @@ static class DiagnosticActivityExtensions static DiagnosticActivityExtensions() { BindingFlags flags = BindingFlags.NonPublic | BindingFlags.Instance; - SetSpanIdMethod = (typeof(Activity).GetField("_spanId", flags) ?? throw new InvalidOperationException("The field Activity._spanId was not found.")).CreateSetter(); + SetSpanIdMethod = (typeof(Activity).GetField("_spanId", flags) + ?? throw new InvalidOperationException("The field Activity._spanId was not found.")) + .CreateSetter(); SetStatusMethod = CreateSetStatus(); } diff --git a/src/Shared/Grpc/Tracing/TraceActivityConstants.cs b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs index 5e83b956..cba4bf52 100644 --- a/src/Shared/Grpc/Tracing/TraceActivityConstants.cs +++ b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs @@ -7,7 +7,7 @@ namespace Microsoft.DurableTask.Tracing; /// /// Constants for trace activity names used in Durable Task Framework. /// -class TraceActivityConstants +static class TraceActivityConstants { /// /// The name of the activity that represents orchestration operations. diff --git a/src/Shared/Grpc/Tracing/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs index b777b214..1283ff12 100644 --- a/src/Shared/Grpc/Tracing/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -14,7 +14,7 @@ namespace Microsoft.DurableTask.Tracing; /// /// Adapted from "https://github.com/Azure/durabletask/blob/main/src/DurableTask.Core/Tracing/TraceHelper.cs". /// -class TraceHelper +static class TraceHelper { const string Source = "Microsoft.DurableTask"; @@ -30,7 +30,10 @@ class TraceHelper public static Activity? StartActivityForNewOrchestration(P.CreateInstanceRequest createInstanceRequest) { Activity? newActivity = ActivityTraceSource.StartActivity( - name: CreateSpanName(TraceActivityConstants.CreateOrchestration, createInstanceRequest.Name, createInstanceRequest.Version), + name: CreateSpanName( + TraceActivityConstants.CreateOrchestration, + createInstanceRequest.Name, + createInstanceRequest.Version), kind: ActivityKind.Producer); if (newActivity != null) @@ -44,13 +47,10 @@ class TraceHelper { newActivity.SetTag(Schema.Task.Version, createInstanceRequest.Version); } - } - if (Activity.Current is not null) - { createInstanceRequest.ParentTraceContext ??= new P.TraceContext(); - createInstanceRequest.ParentTraceContext.TraceParent = Activity.Current.Id!; - createInstanceRequest.ParentTraceContext.TraceState = Activity.Current.TraceStateString; + createInstanceRequest.ParentTraceContext.TraceParent = newActivity.Id!; + createInstanceRequest.ParentTraceContext.TraceState = newActivity.TraceStateString; } return newActivity; @@ -73,7 +73,11 @@ class TraceHelper return null; } - if (startEvent.ParentTraceContext is null || !ActivityContext.TryParse(startEvent.ParentTraceContext.TraceParent, startEvent.ParentTraceContext.TraceState, out ActivityContext activityContext)) + if (startEvent.ParentTraceContext is null + || !ActivityContext.TryParse( + startEvent.ParentTraceContext.TraceParent, + startEvent.ParentTraceContext.TraceState, + out ActivityContext activityContext)) { return null; } @@ -120,7 +124,11 @@ class TraceHelper public static Activity? StartTraceActivityForTaskExecution( P.ActivityRequest request) { - if (request.ParentTraceContext is null || !ActivityContext.TryParse(request.ParentTraceContext.TraceParent, request.ParentTraceContext.TraceState, out ActivityContext activityContext)) + if (request.ParentTraceContext is null + || !ActivityContext.TryParse( + request.ParentTraceContext.TraceParent, + request.ParentTraceContext.TraceState, + out ActivityContext activityContext)) { return null; } @@ -234,7 +242,8 @@ public static void EmitTraceActivityForSubOrchestrationFailed( if (failedEvent != null) { - string statusDescription = failedEvent.FailureDetails.ErrorMessage ?? "Unspecified sub-orchestration failure"; + string statusDescription = failedEvent.FailureDetails.ErrorMessage + ?? "Unspecified sub-orchestration failure"; activity?.SetStatus(ActivityStatusCode.Error, statusDescription); } @@ -378,7 +387,10 @@ static string CreateSpanName(string spanDescription, string? taskName, string? t if (taskScheduledEvent.ParentTraceContext != null) { - if (ActivityContext.TryParse(taskScheduledEvent.ParentTraceContext.TraceParent, taskScheduledEvent.ParentTraceContext?.TraceState, out ActivityContext parentContext)) + if (ActivityContext.TryParse( + taskScheduledEvent.ParentTraceContext.TraceParent, + taskScheduledEvent.ParentTraceContext?.TraceState, + out ActivityContext parentContext)) { newActivity.SetSpanId(parentContext.SpanId.ToString()); } @@ -428,7 +440,11 @@ static string CreateSpanName(string spanDescription, string? taskName, string? t return null; } - if (createdEvent.ParentTraceContext != null && ActivityContext.TryParse(createdEvent.ParentTraceContext.TraceParent, createdEvent.ParentTraceContext.TraceState, out ActivityContext parentContext)) + if (createdEvent.ParentTraceContext != null + && ActivityContext.TryParse( + createdEvent.ParentTraceContext.TraceParent, + createdEvent.ParentTraceContext.TraceState, + out ActivityContext parentContext)) { activity.SetSpanId(parentContext.SpanId.ToString()); } diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 732cbcbc..2a4610cf 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -370,7 +370,9 @@ async Task OnRunOrchestratorAsync( .Select(e => e.ExecutionStarted) .FirstOrDefault(); - Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution(executionStartedEvent, request.OrchestrationTraceContext); + Activity? traceActivity = TraceHelper.StartTraceActivityForOrchestrationExecution( + executionStartedEvent, + request.OrchestrationTraceContext); if (executionStartedEvent is not null) { @@ -402,7 +404,9 @@ async Task OnRunOrchestratorAsync( { case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceCompleted: { - var subOrchestrationInstanceCreatedEvent = GetSuborchestrationInstanceCreatedEvent(newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); + P.HistoryEvent? subOrchestrationInstanceCreatedEvent = + GetSuborchestrationInstanceCreatedEvent( + newEvent.SubOrchestrationInstanceCompleted.TaskScheduledId); TraceHelper.EmitTraceActivityForSubOrchestrationCompleted( request.InstanceId, @@ -413,7 +417,9 @@ async Task OnRunOrchestratorAsync( case P.HistoryEvent.EventTypeOneofCase.SubOrchestrationInstanceFailed: { - var subOrchestrationInstanceCreatedEvent = GetSuborchestrationInstanceCreatedEvent(newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); + P.HistoryEvent? subOrchestrationInstanceCreatedEvent = + GetSuborchestrationInstanceCreatedEvent( + newEvent.SubOrchestrationInstanceFailed.TaskScheduledId); TraceHelper.EmitTraceActivityForSubOrchestrationFailed( request.InstanceId, @@ -425,23 +431,35 @@ async Task OnRunOrchestratorAsync( case P.HistoryEvent.EventTypeOneofCase.TaskCompleted: { - var taskScheduledEvent = GetTaskScheduledEvent(newEvent.TaskCompleted.TaskScheduledId); + P.HistoryEvent? taskScheduledEvent = + GetTaskScheduledEvent(newEvent.TaskCompleted.TaskScheduledId); - TraceHelper.EmitTraceActivityForTaskCompleted(request.InstanceId, taskScheduledEvent, taskScheduledEvent?.TaskScheduled); + TraceHelper.EmitTraceActivityForTaskCompleted( + request.InstanceId, + taskScheduledEvent, + taskScheduledEvent?.TaskScheduled); break; } case P.HistoryEvent.EventTypeOneofCase.TaskFailed: { - var taskScheduledEvent = GetTaskScheduledEvent(newEvent.TaskFailed.TaskScheduledId); + P.HistoryEvent? taskScheduledEvent = + GetTaskScheduledEvent(newEvent.TaskFailed.TaskScheduledId); - TraceHelper.EmitTraceActivityForTaskFailed(request.InstanceId, taskScheduledEvent, taskScheduledEvent?.TaskScheduled, newEvent.TaskFailed); + TraceHelper.EmitTraceActivityForTaskFailed( + request.InstanceId, + taskScheduledEvent, + taskScheduledEvent?.TaskScheduled, + newEvent.TaskFailed); break; } case P.HistoryEvent.EventTypeOneofCase.TimerFired: - // We immediately publish the activity span for this timer by creating the activity and immediately calling Dispose() on it. - TraceHelper.EmitTraceActivityForTimer(request.InstanceId, executionStartedEvent.Name, newEvent.Timestamp.ToDateTime(), newEvent.TimerFired); + TraceHelper.EmitTraceActivityForTimer( + request.InstanceId, + executionStartedEvent.Name, + newEvent.Timestamp.ToDateTime(), + newEvent.TimerFired); break; } } @@ -611,16 +629,21 @@ await this.client.AbandonTaskOrchestratorWorkItemAsync( }; } - var completeOrchestrationAction = response.Actions.FirstOrDefault(a => a.CompleteOrchestration is not null); + var completeOrchestrationAction = response.Actions.FirstOrDefault( + a => a.CompleteOrchestration is not null); if (completeOrchestrationAction is not null) { if (completeOrchestrationAction.CompleteOrchestration.OrchestrationStatus == P.OrchestrationStatus.Failed) { - traceActivity?.SetStatus(ActivityStatusCode.Error, completeOrchestrationAction.CompleteOrchestration.Result); + traceActivity?.SetStatus( + ActivityStatusCode.Error, + completeOrchestrationAction.CompleteOrchestration.Result); } - traceActivity?.SetTag(Schema.Task.Status, completeOrchestrationAction.CompleteOrchestration.OrchestrationStatus.ToString()); + traceActivity?.SetTag( + Schema.Task.Status, + completeOrchestrationAction.CompleteOrchestration.OrchestrationStatus.ToString()); traceActivity?.Dispose(); } From 6254ca4fdf3b30aa2c8397bc202607a9e6e18bc1 Mon Sep 17 00:00:00 2001 From: Phillip Hoff Date: Fri, 8 Aug 2025 09:47:53 -0700 Subject: [PATCH 36/36] Update protobuf version. --- src/Grpc/versions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Grpc/versions.txt b/src/Grpc/versions.txt index 121ec0ec..5c0de357 100644 --- a/src/Grpc/versions.txt +++ b/src/Grpc/versions.txt @@ -1,2 +1,2 @@ -# The following files were downloaded from branch main at 2025-06-02 21:12:34 UTC -https://raw.githubusercontent.com/microsoft/durabletask-protobuf/fd9369c6a03d6af4e95285e432b7c4e943c06970/protos/orchestrator_service.proto +# The following files were downloaded from branch main at 2025-08-08 16:46:11 UTC +https://raw.githubusercontent.com/microsoft/durabletask-protobuf/e88acbd07ae38b499dbe8c4e333e9e3feeb2a9cc/protos/orchestrator_service.proto