Refactor telemetry & exporter setup: deprecations, noise reduction, and API updates

release_notes.md

@@ -1,14 +1,15 @@
-### Release notes
-
-<!-- Please add your release notes in the following format:
-- My change description (#PR)
--->
-- Add JitTrace Files for v4.1042
-- Updating OTel related nuget packages (#11216)
-- Moving to version 1.5.7 of Microsoft.Azure.AppService.Middleware.Functions (#11231)
-- Refactor code to move the logic to search for WorkerConfigs to a default worker configuration resolver (#11219)
-- Update Node.js Worker Version to [3.11.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.11.0)
-- Adding restart reason to the logs (#11191)
-- Fix custom handler streaming bug where `$return` output binding scenarios result in error logs (#11262)
-- Update outdated bundle message to align as per the policy (#11230)
-- Rev `Microsoft.Azure.Functions.DotNetIsolatedNativeHost` to `1.0.13` (#11269)
+### Release notes
+
+<!-- Please add your release notes in the following format:
+- My change description (#PR)
+-->
+- Add JitTrace Files for v4.1042
+- Updating OTel related nuget packages (#11216)
+- Moving to version 1.5.7 of Microsoft.Azure.AppService.Middleware.Functions (#11231)
+- Refactor code to move the logic to search for WorkerConfigs to a default worker configuration resolver (#11219)
+- Update Node.js Worker Version to [3.11.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.11.0)
+- Adding restart reason to the logs (#11191)
+- Fix custom handler streaming bug where `$return` output binding scenarios result in error logs (#11262)
+- Update outdated bundle message to align as per the policy (#11230)
+- Rev `Microsoft.Azure.Functions.DotNetIsolatedNativeHost` to `1.0.13` (#11269)
+- Refactor telemetry & exporter setup: deprecations, noise reduction, and API updates (#11260)

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -1284,7 +1284,7 @@ internal void Log(GrpcEvent msg)
                                 // TODO fix RpcException catch all https://github.com/Azure/azure-functions-dotnet-worker/issues/370
                                 var exception = new Workers.Rpc.RpcException(rpcLog.Message, rpcLog.Exception.Message, rpcLog.Exception.StackTrace);
                                 context.Logger.Log(logLevel, new EventId(0, rpcLog.EventId), rpcLog.Message, exception, (state, exc) => state);
-                                Activity.Current?.RecordException(exception);
+                                Activity.Current?.AddException(exception);
                                 Activity.Current?.SetStatus(ActivityStatusCode.Error, exception.Message);
                             }
                             else

src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/DependencyValidator.cs

@@ -55,10 +55,11 @@ private static ExpectedDependencyBuilder CreateExpectedDependencies()
                 .Optional<FunctionsSyncService>() // Conditionally registered.
                 .OptionalExternal("Microsoft.AspNetCore.DataProtection.Internal.DataProtectionHostedService", "Microsoft.AspNetCore.DataProtection", "adb9793829ddae60") // Popularly-registered by DataProtection.
                 .OptionalExternal("Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckPublisherHostedService", "Microsoft.Extensions.Diagnostics.HealthChecks", "adb9793829ddae60") // Popularly-registered by Health Check Monitor.
-                .OptionalExternal("OpenTelemetry.Extensions.Hosting.Implementation.TelemetryHostedService", "OpenTelemetry.Extensions.Hosting", "7bd6737fe5b67e3c") // Enable OpenTelemetry.Net instrumentation library
                 .OptionalExternal("Microsoft.Azure.WebJobs.Hosting.PrimaryHostCoordinator", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35")
                 .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ConcurrencyManagerService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35")
-                .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ScaleMonitorService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35");
+                .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ScaleMonitorService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35")
+                .OptionalExternal("OpenTelemetry.Extensions.Hosting.Implementation.TelemetryHostedService", "OpenTelemetry.Extensions.Hosting", "7bd6737fe5b67e3c") // Enable OpenTelemetry.Net instrumentation library
+                .OptionalExternal("Azure.Monitor.OpenTelemetry.Exporter.OpenTelemetryBuilderExtensions", "Azure.Monitor.OpenTelemetry.Exporter", "92742159e12e44c8"); // Enable live metrics
 
             expected.ExpectSubcollection<ILoggerProvider>()
                 .Expect<FunctionFileLoggerProvider>()

src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/ServiceMatch.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
@@ -218,7 +218,7 @@ private static IEnumerable<InvalidServiceDescriptor> FindCollectionDifferences(I
                 {
                     return !expectedServices.Any(e => IsMatch(e, r))
                         && !optionalServices.Any(o => IsMatch(o, r))
-                        && !optionalExternalServices.Any(o => o.IsMatch(r.ImplementationType));
+                        && !optionalExternalServices.Any(o => o.IsMatch(r));
                 })
                 .Select(p => new InvalidServiceDescriptor(p, InvalidServiceDescriptorReason.Invalid));
 
@@ -247,18 +247,26 @@ public ExternalService(string typeName, string assemblyName, string assemblyPubl
 
             public string AssemblyPublicKey { get; }
 
-            public bool IsMatch(Type serviceType)
+            public bool IsMatch(ServiceDescriptor descriptor)
             {
-                if (serviceType == null)
+                if (descriptor.ImplementationType is { } serviceType)
                 {
-                    return false;
+                    var serviceAssemblyName = serviceType.Assembly.GetName();
+
+                    return serviceType.FullName == TypeName
+                        && serviceAssemblyName.Name == AssemblyName
+                        && GetPublicKeyTokenString(serviceAssemblyName.GetPublicKeyToken()) == AssemblyPublicKey;
                 }
+                else if (descriptor.ImplementationFactory?.Target?.GetType() is { } implementationFactory)
+                {
+                    var factoryAssemblyName = implementationFactory.Assembly.GetName();
 
-                var serviceAssemblyName = serviceType.Assembly.GetName();
+                    return implementationFactory.DeclaringType.FullName == TypeName
+                        && factoryAssemblyName.Name == AssemblyName
+                        && GetPublicKeyTokenString(factoryAssemblyName.GetPublicKeyToken()) == AssemblyPublicKey;
+                }
 
-                return serviceType.FullName == TypeName
-                    && serviceAssemblyName.Name == AssemblyName
-                    && GetPublicKeyTokenString(serviceAssemblyName.GetPublicKeyToken()) == AssemblyPublicKey;
+                return false;
             }
 
             private static string GetPublicKeyTokenString(byte[] token)

src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
@@ -8,7 +8,6 @@
 using Azure.Core;
 using Azure.Identity;
 using Azure.Monitor.OpenTelemetry.Exporter;
-using Azure.Monitor.OpenTelemetry.LiveMetrics;
 using Microsoft.AspNetCore.Routing;
 using Microsoft.Azure.WebJobs.Script.Metrics;
 using Microsoft.Extensions.Configuration;
@@ -34,122 +33,106 @@ internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder,
             bool enableOtlp = !string.IsNullOrWhiteSpace(otlpEndpoint);
             bool enableAzureMonitor = !string.IsNullOrWhiteSpace(azMonConnectionString);
 
-            TokenCredential credential = enableAzureMonitor ? GetTokenCredential(context.Configuration) : null;
-
             // If placeholder mode is disabled and both OTLP and Azure Monitor are not enabled, avoid configuring OpenTelemetry.
             if (!enableOtlp && !enableAzureMonitor && telemetryMode != TelemetryMode.Placeholder)
             {
                 return;
             }
 
-            loggingBuilder.ConfigureLogging(enableOtlp, enableAzureMonitor, azMonConnectionString, credential, telemetryMode);
+            loggingBuilder.ConfigureLogging();
 
             loggingBuilder.Services
                 .AddOpenTelemetry()
+                .ConfigureExporters(context.Configuration, enableOtlp, enableAzureMonitor, azMonConnectionString, telemetryMode)
                 .ConfigureResource(r => ConfigureResource(r))
-                .ConfigureMetrics(enableOtlp, enableAzureMonitor, azMonConnectionString, credential, telemetryMode)
-                .ConfigureTracing(enableOtlp, enableAzureMonitor, azMonConnectionString, credential, telemetryMode)
+                .ConfigureMetrics()
+                .ConfigureTracing()
                 .ConfigureEventLogLevel(context.Configuration);
 
             // Azure SDK instrumentation is experimental.
             AppContext.SetSwitch("Azure.Experimental.EnableActivitySource", true);
         }
 
-        private static IOpenTelemetryBuilder ConfigureMetrics(this IOpenTelemetryBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential, TelemetryMode telemetryMode)
+        private static IOpenTelemetryBuilder ConfigureExporters(this IOpenTelemetryBuilder builder, IConfiguration configuration, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TelemetryMode telemetryMode)
+        {
+            // Avoid configuring the exporter in placeholder mode, as it will default to sending telemetry to the predefined endpoint. These transmissions will be unsuccessful and create unnecessary noise.
+            if (telemetryMode == TelemetryMode.Placeholder)
+            {
+                return builder;
+            }
+
+            if (enableOtlp)
+            {
+                builder.UseOtlpExporter();
+            }
+
+            if (enableAzureMonitor)
+            {
+                TokenCredential credential = GetTokenCredential(configuration);
+                builder.UseAzureMonitorExporter(options => ConfigureAzureMonitorOptions(options, azMonConnectionString, credential));
+            }
+
+            return builder;
+        }
+
+        private static IOpenTelemetryBuilder ConfigureMetrics(this IOpenTelemetryBuilder builder)
         {
             return builder.WithMetrics(builder =>
             {
                 builder.AddAspNetCoreInstrumentation()
-                .AddRuntimeInstrumentation()
-                .AddProcessInstrumentation()
-                .AddMeter(HostMetrics.FaasMeterName)
-                .AddView(HostMetrics.FaasInvokeDuration, new ExplicitBucketHistogramConfiguration
-                {
-                    Boundaries = new double[] { 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10 }
-                });
-
-                // Avoid configuring the exporter in placeholder mode, as it will default to sending telemetry to the predefined endpoint. These transmissions will be unsuccessful and create unnecessary noise.
-                if (telemetryMode != TelemetryMode.Placeholder)
-                {
-                    if (enableOtlp)
-                    {
-                        builder.AddOtlpExporter();
-                    }
-                    if (enableAzureMonitor)
+                    .AddRuntimeInstrumentation()
+                    .AddProcessInstrumentation()
+                    .AddMeter(HostMetrics.FaasMeterName)
+                    .AddView(HostMetrics.FaasInvokeDuration, new ExplicitBucketHistogramConfiguration
                     {
-                        builder.AddAzureMonitorMetricExporter(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
-                    }
-                }
+                        Boundaries = new double[] { 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10 },
+                        RecordMinMax = true
+                    });
             });
         }
 
-        private static IOpenTelemetryBuilder ConfigureTracing(this IOpenTelemetryBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential, TelemetryMode telemetryMode)
+        private static IOpenTelemetryBuilder ConfigureTracing(this IOpenTelemetryBuilder builder)
         {
             return builder.WithTracing(builder =>
             {
                 builder
-                .AddSource("Azure.*")
-                .AddSource("Microsoft.Azure.Webjobs.Extensions.*")
-                .AddSource("WebJobs.Extensions.DurableTask")
-                .AddSource("DurableTask.*")
-                .AddAspNetCoreInstrumentation(o =>
-                {
-                    o.EnrichWithHttpResponse = (activity, httpResponse) =>
+                    .AddSource("Azure.Messaging.ServiceBus.ServiceBusProcessor")
+                    .AddSource("Azure.Messaging.EventHubs.EventProcessor")
+                    .AddSource("Azure.Functions.Extensions.Mcp")
+                    .AddSource("Microsoft.Azure.WebJobs.Extensions.*")
+                    .AddSource("Microsoft.Azure.WebJobs")
+                    .AddSource("WebJobs.Extensions.DurableTask")
+                    .AddSource("DurableTask.*")
+                    .AddAspNetCoreInstrumentation(o =>
                     {
-                        if (Activity.Current is not null)
+                        o.EnrichWithHttpResponse = (activity, httpResponse) =>
                         {
-                            Activity.Current.AddTag(ResourceSemanticConventions.FaaSTrigger, OpenTelemetryConstants.HttpTriggerType);
-
-                            var routingFeature = httpResponse.HttpContext.Features.Get<IRoutingFeature>();
-                            if (routingFeature is null)
+                            if (Activity.Current is not null)
                             {
-                                return;
-                            }
-
-                            var template = routingFeature.RouteData.Routers.FirstOrDefault(r => r is Route) as Route;
-                            Activity.Current.DisplayName = $"{Activity.Current.DisplayName} {template?.RouteTemplate}";
-                            Activity.Current.AddTag(ResourceSemanticConventions.HttpRoute, template?.RouteTemplate);
-                        }
-                    };
-                });
-
-                // Avoid configuring the exporter in placeholder mode, as it will default to sending telemetry to the predefined endpoint. These transmissions will be unsuccessful and create unnecessary noise.
-                if (telemetryMode != TelemetryMode.Placeholder)
-                {
-                    if (enableOtlp)
-                    {
-                        builder.AddOtlpExporter();
-                    }
+                                Activity.Current.AddTag(ResourceSemanticConventions.FaaSTrigger, OpenTelemetryConstants.HttpTriggerType);
 
-                    if (enableAzureMonitor)
-                    {
-                        builder.AddAzureMonitorTraceExporter(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
-                    }
-                }
+                                var routingFeature = httpResponse.HttpContext.Features.Get<IRoutingFeature>();
+                                if (routingFeature is null)
+                                {
+                                    return;
+                                }
 
-                builder.AddProcessor(ActivitySanitizingProcessor.Instance);
+                                var template = routingFeature.RouteData.Routers.FirstOrDefault(r => r is Route) as Route;
+                                Activity.Current.DisplayName = $"{Activity.Current.DisplayName} {template?.RouteTemplate}";
+                                Activity.Current.AddTag(ResourceSemanticConventions.HttpRoute, template?.RouteTemplate);
+                            }
+                        };
+                        o.Filter = context => !context.Request.Host.Host.Equals("127.0.0.1", StringComparison.OrdinalIgnoreCase);
+                    })
+                    .AddProcessor(ActivitySanitizingProcessor.Instance);
             });
         }
 
-        private static ILoggingBuilder ConfigureLogging(this ILoggingBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential, TelemetryMode telemetryMode)
+        private static ILoggingBuilder ConfigureLogging(this ILoggingBuilder builder)
         {
             builder.AddOpenTelemetry(o =>
             {
                 o.SetResourceBuilder(ConfigureResource(ResourceBuilder.CreateDefault()));
-
-                // Avoid configuring the exporter in placeholder mode, as it will default to sending telemetry to the predefined endpoint. These transmissions will be unsuccessful and create unnecessary noise.
-                if (telemetryMode != TelemetryMode.Placeholder)
-                {
-                    if (enableOtlp)
-                    {
-                        o.AddOtlpExporter();
-                    }
-                    if (enableAzureMonitor)
-                    {
-                        o.AddAzureMonitorLogExporter(options => ConfigureAzureMonitorOptions(options, azMonConnectionString, credential));
-                    }
-                }
-
                 o.IncludeFormattedMessage = true;
                 o.IncludeScopes = false;
             });