Refactor telemetry & exporter setup: deprecations, noise reduction, and API updates

release_notes.md

@@ -1,11 +1,12 @@
-### Release notes
-
-<!-- Please add your release notes in the following format:
-- My change description (#PR)
--->
-- Add JitTrace Files for v4.1042
-- Updating OTel related nuget packages (#11216)
-- Moving to version 1.5.7 of Microsoft.Azure.AppService.Middleware.Functions (https://github.com/Azure/azure-functions-host/pull/11231)
-- Refactor code to move the logic to search for WorkerConfigs to a default worker configuration resolver (#11219)
-- Update Node.js Worker Version to [3.11.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.11.0)
-- Adding restart reason to the logs (#11191)
+### Release notes
+
+<!-- Please add your release notes in the following format:
+- My change description (#PR)
+-->
+- Add JitTrace Files for v4.1042
+- Updating OTel related nuget packages (#11216)
+- Moving to version 1.5.7 of Microsoft.Azure.AppService.Middleware.Functions (https://github.com/Azure/azure-functions-host/pull/11231)
+- Refactor code to move the logic to search for WorkerConfigs to a default worker configuration resolver (#11219)
+- Update Node.js Worker Version to [3.11.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.11.0)
+- Adding restart reason to the logs (#11191)
+- Refactor telemetry & exporter setup: deprecations, noise reduction, and API updates (#11260)

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -1284,7 +1284,7 @@ internal void Log(GrpcEvent msg)
                                 // TODO fix RpcException catch all https://github.com/Azure/azure-functions-dotnet-worker/issues/370
                                 var exception = new Workers.Rpc.RpcException(rpcLog.Message, rpcLog.Exception.Message, rpcLog.Exception.StackTrace);
                                 context.Logger.Log(logLevel, new EventId(0, rpcLog.EventId), rpcLog.Message, exception, (state, exc) => state);
-                                Activity.Current?.RecordException(exception);
+                                Activity.Current?.AddException(exception);
                                 Activity.Current?.SetStatus(ActivityStatusCode.Error, exception.Message);
                             }
                             else

src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/DependencyValidator.cs

@@ -55,10 +55,11 @@ private static ExpectedDependencyBuilder CreateExpectedDependencies()
                 .Optional<FunctionsSyncService>() // Conditionally registered.
                 .OptionalExternal("Microsoft.AspNetCore.DataProtection.Internal.DataProtectionHostedService", "Microsoft.AspNetCore.DataProtection", "adb9793829ddae60") // Popularly-registered by DataProtection.
                 .OptionalExternal("Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckPublisherHostedService", "Microsoft.Extensions.Diagnostics.HealthChecks", "adb9793829ddae60") // Popularly-registered by Health Check Monitor.
-                .OptionalExternal("OpenTelemetry.Extensions.Hosting.Implementation.TelemetryHostedService", "OpenTelemetry.Extensions.Hosting", "7bd6737fe5b67e3c") // Enable OpenTelemetry.Net instrumentation library
                 .OptionalExternal("Microsoft.Azure.WebJobs.Hosting.PrimaryHostCoordinator", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35")
                 .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ConcurrencyManagerService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35")
-                .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ScaleMonitorService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35");
+                .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ScaleMonitorService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35")
+                .OptionalExternal("OpenTelemetry.Extensions.Hosting.Implementation.TelemetryHostedService", "OpenTelemetry.Extensions.Hosting", "7bd6737fe5b67e3c") // Enable OpenTelemetry.Net instrumentation library
+                .OptionalExternal("Azure.Monitor.OpenTelemetry.Exporter.OpenTelemetryBuilderExtensions", "Azure.Monitor.OpenTelemetry.Exporter", "92742159e12e44c8"); // Enable live metrics
 
             expected.ExpectSubcollection<ILoggerProvider>()
                 .Expect<FunctionFileLoggerProvider>()

src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/ServiceMatch.cs

@@ -218,7 +218,7 @@ private static IEnumerable<InvalidServiceDescriptor> FindCollectionDifferences(I
                 {
                     return !expectedServices.Any(e => IsMatch(e, r))
                         && !optionalServices.Any(o => IsMatch(o, r))
-                        && !optionalExternalServices.Any(o => o.IsMatch(r.ImplementationType));
+                        && !optionalExternalServices.Any(o => o.IsMatch(r.ImplementationType, r.ImplementationFactory?.Target?.GetType()));
                 })
                 .Select(p => new InvalidServiceDescriptor(p, InvalidServiceDescriptorReason.Invalid));
 
@@ -247,18 +247,30 @@ public ExternalService(string typeName, string assemblyName, string assemblyPubl
 
             public string AssemblyPublicKey { get; }
 
-            public bool IsMatch(Type serviceType)
+            public bool IsMatch(Type serviceType, Type implementationFactory)
             {
-                if (serviceType == null)
+                if (serviceType == null && implementationFactory == null)
                 {
                     return false;
                 }
 
-                var serviceAssemblyName = serviceType.Assembly.GetName();
+                // If we have a service type, we can check it directly.
+                if (serviceType is not null)
+                {
+                    var serviceAssemblyName = serviceType.Assembly.GetName();
 
-                return serviceType.FullName == TypeName
-                    && serviceAssemblyName.Name == AssemblyName
-                    && GetPublicKeyTokenString(serviceAssemblyName.GetPublicKeyToken()) == AssemblyPublicKey;
+                    return serviceType.FullName == TypeName
+                        && serviceAssemblyName.Name == AssemblyName
+                        && GetPublicKeyTokenString(serviceAssemblyName.GetPublicKeyToken()) == AssemblyPublicKey;
+                }
+                else
+                {
+                    var factoryAssemblyName = implementationFactory.Assembly.GetName();
+
+                    return implementationFactory.DeclaringType.FullName == TypeName
+                        && factoryAssemblyName.Name == AssemblyName
+                        && GetPublicKeyTokenString(factoryAssemblyName.GetPublicKeyToken()) == AssemblyPublicKey;
+                }
             }
 
             private static string GetPublicKeyTokenString(byte[] token)

src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs

@@ -8,7 +8,6 @@
 using Azure.Core;
 using Azure.Identity;
 using Azure.Monitor.OpenTelemetry.Exporter;
-using Azure.Monitor.OpenTelemetry.LiveMetrics;
 using Microsoft.AspNetCore.Routing;
 using Microsoft.Azure.WebJobs.Script.Metrics;
 using Microsoft.Extensions.Configuration;
@@ -34,28 +33,49 @@ internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder,
             bool enableOtlp = !string.IsNullOrWhiteSpace(otlpEndpoint);
             bool enableAzureMonitor = !string.IsNullOrWhiteSpace(azMonConnectionString);
 
-            TokenCredential credential = enableAzureMonitor ? GetTokenCredential(context.Configuration) : null;
-
             // If placeholder mode is disabled and both OTLP and Azure Monitor are not enabled, avoid configuring OpenTelemetry.
             if (!enableOtlp && !enableAzureMonitor && telemetryMode != TelemetryMode.Placeholder)
             {
                 return;
             }
 
-            loggingBuilder.ConfigureLogging(enableOtlp, enableAzureMonitor, azMonConnectionString, credential, telemetryMode);
+            loggingBuilder.ConfigureLogging();
 
             loggingBuilder.Services
                 .AddOpenTelemetry()
+                .ConfigureExporters(context.Configuration, enableOtlp, enableAzureMonitor, azMonConnectionString, telemetryMode)
                 .ConfigureResource(r => ConfigureResource(r))
-                .ConfigureMetrics(enableOtlp, enableAzureMonitor, azMonConnectionString, credential, telemetryMode)
-                .ConfigureTracing(enableOtlp, enableAzureMonitor, azMonConnectionString, credential, telemetryMode)
+                .ConfigureMetrics()
+                .ConfigureTracing()
                 .ConfigureEventLogLevel(context.Configuration);
 
             // Azure SDK instrumentation is experimental.
             AppContext.SetSwitch("Azure.Experimental.EnableActivitySource", true);
         }
 
-        private static IOpenTelemetryBuilder ConfigureMetrics(this IOpenTelemetryBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential, TelemetryMode telemetryMode)
+        private static IOpenTelemetryBuilder ConfigureExporters(this IOpenTelemetryBuilder builder, IConfiguration configuration, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TelemetryMode telemetryMode)
+        {
+            // Avoid configuring the exporter in placeholder mode, as it will default to sending telemetry to the predefined endpoint. These transmissions will be unsuccessful and create unnecessary noise.
+            if (telemetryMode == TelemetryMode.Placeholder)
+            {
+                return builder;
+            }
+
+            if (enableOtlp)
+            {
+                builder.UseOtlpExporter();
+            }
+
+            if (enableAzureMonitor)
+            {
+                TokenCredential credential = GetTokenCredential(configuration);
+                builder.UseAzureMonitorExporter(options => ConfigureAzureMonitorOptions(options, azMonConnectionString, credential));
+            }
+
+            return builder;
+        }
+
+        private static IOpenTelemetryBuilder ConfigureMetrics(this IOpenTelemetryBuilder builder)
         {
             return builder.WithMetrics(builder =>
             {
@@ -65,30 +85,20 @@ private static IOpenTelemetryBuilder ConfigureMetrics(this IOpenTelemetryBuilder
                 .AddMeter(HostMetrics.FaasMeterName)
                 .AddView(HostMetrics.FaasInvokeDuration, new ExplicitBucketHistogramConfiguration
                 {
-                    Boundaries = new double[] { 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10 }
+                    Boundaries = new double[] { 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10 },
+                    RecordMinMax = true
                 });
-
-                // Avoid configuring the exporter in placeholder mode, as it will default to sending telemetry to the predefined endpoint. These transmissions will be unsuccessful and create unnecessary noise.
-                if (telemetryMode != TelemetryMode.Placeholder)
-                {
-                    if (enableOtlp)
-                    {
-                        builder.AddOtlpExporter();
-                    }
-                    if (enableAzureMonitor)
-                    {
-                        builder.AddAzureMonitorMetricExporter(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
-                    }
-                }
             });
         }
 
-        private static IOpenTelemetryBuilder ConfigureTracing(this IOpenTelemetryBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential, TelemetryMode telemetryMode)
+        private static IOpenTelemetryBuilder ConfigureTracing(this IOpenTelemetryBuilder builder)
         {
             return builder.WithTracing(builder =>
             {
                 builder
-                .AddSource("Azure.*")
+                .AddSource("Azure.Messaging.ServiceBus.ServiceBusProcessor")
+                .AddSource("Azure.Messaging.EventHubs.EventProcessor")
+                .AddSource("Azure.Functions.Extensions.Mcp")
                 .AddSource("Microsoft.Azure.Webjobs.Extensions.*")
                 .AddSource("WebJobs.Extensions.DurableTask")
                 .AddSource("DurableTask.*")
@@ -111,45 +121,17 @@ private static IOpenTelemetryBuilder ConfigureTracing(this IOpenTelemetryBuilder
                             Activity.Current.AddTag(ResourceSemanticConventions.HttpRoute, template?.RouteTemplate);
                         }
                     };
-                });
-
-                // Avoid configuring the exporter in placeholder mode, as it will default to sending telemetry to the predefined endpoint. These transmissions will be unsuccessful and create unnecessary noise.
-                if (telemetryMode != TelemetryMode.Placeholder)
-                {
-                    if (enableOtlp)
-                    {
-                        builder.AddOtlpExporter();
-                    }
-
-                    if (enableAzureMonitor)
-                    {
-                        builder.AddAzureMonitorTraceExporter(opt => ConfigureAzureMonitorOptions(opt, azMonConnectionString, credential));
-                    }
-                }
-
-                builder.AddProcessor(ActivitySanitizingProcessor.Instance);
+                    o.Filter = context => !context.Request.Host.Host.Equals("127.0.0.1", StringComparison.OrdinalIgnoreCase);
+                })
+                .AddProcessor(ActivitySanitizingProcessor.Instance);
             });
         }
 
-        private static ILoggingBuilder ConfigureLogging(this ILoggingBuilder builder, bool enableOtlp, bool enableAzureMonitor, string azMonConnectionString, TokenCredential credential, TelemetryMode telemetryMode)
+        private static ILoggingBuilder ConfigureLogging(this ILoggingBuilder builder)
         {
             builder.AddOpenTelemetry(o =>
             {
                 o.SetResourceBuilder(ConfigureResource(ResourceBuilder.CreateDefault()));
-
-                // Avoid configuring the exporter in placeholder mode, as it will default to sending telemetry to the predefined endpoint. These transmissions will be unsuccessful and create unnecessary noise.
-                if (telemetryMode != TelemetryMode.Placeholder)
-                {
-                    if (enableOtlp)
-                    {
-                        o.AddOtlpExporter();
-                    }
-                    if (enableAzureMonitor)
-                    {
-                        o.AddAzureMonitorLogExporter(options => ConfigureAzureMonitorOptions(options, azMonConnectionString, credential));
-                    }
-                }
-
                 o.IncludeFormattedMessage = true;
                 o.IncludeScopes = false;
             });