Open Telemetry log sampler

TraceBasedLogSamplingProcessor.cs

using Microsoft.Extensions.Logging;
using OpenTelemetry;
using OpenTelemetry.Logs;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using static System.Runtime.InteropServices.JavaScript.JSType;

public class TraceBasedLogSamplingProcessor : BaseProcessor<LogRecord>
{
    private readonly bool waitForFirstTraceEntry;
    private readonly LogLevel alwaysPassLogLevel;
    private readonly Func<LogRecord, bool> shouldLogRecord;

    private bool firstTraceEntryReceived = false;

    /// <summary>
    /// Processor that will sample log records based on the presence of a TraceId, and emit those records based on the Activity.Recorded flag.
    /// All log messages without a TraceId will be emitted
    /// </summary>
    /// <param name="AlwaysPassLogLevel">Will emit all log messages at this level or greater, regardless of the trace status</param>
    /// <returns>A TraceBasedLogSamplingProcessor instance</returns>
    public static TraceBasedLogSamplingProcessor PassAllLogRecordsWithoutTraceId(LogLevel AlwaysPassLogLevel = LogLevel.Critical)
    {
        return new TraceBasedLogSamplingProcessor(true, AlwaysPassLogLevel, (data) => true);
    }


    /// <summary>
    /// Processor that will sample log records based on the presence of a TraceId, and emit those records based on the Activity.Recorded flag.
    /// Log messages without a TraceId will be sampled at the specified rate
    /// </summary>
    /// <param name="rate">A value > 0 and < 1 to represent the sample rate. A value of 0.2 will emit 20% of the log records without a TraceId</param>
    /// <param name="WaitForFirstTraceEntry">When true, it will emit all log messages during startup until the first entry associated with a TraceId is seen</param>
    /// <param name="AlwaysPassLogLevel">Will emit all log messages at this level or greater, regardless of the trace status</param>
    /// <returns>A TraceBasedLogSamplingProcessor instance</returns>
    /// <exception cref="ArgumentOutOfRangeException">Occurs if the rate isn't between 0 and 1</exception>
    public static TraceBasedLogSamplingProcessor FixedRateLogRecordsWithoutTraceId(double rate, bool WaitForFirstTraceEntry = true, LogLevel AlwaysPassLogLevel = LogLevel.Critical)
    {
        if (rate < 0 || rate > 1) { throw new ArgumentOutOfRangeException(nameof(rate), "Value must be between 0 and 1"); }
        var random = new Random();
        return new TraceBasedLogSamplingProcessor(WaitForFirstTraceEntry, AlwaysPassLogLevel, (data) => (random.NextDouble() > 1 - rate));
    }


    /// <summary>
    /// Logger that will sample log records based on the presence of a trace id, and emit those records based on the Activity.Recorded flag.
    /// Log messages without a trace id will be rate limited to the specified rate
    /// </summary>
    /// <param name="MaxRecordsPerSecond">The max number of log messages without a Trace that should be emitted</param>
    /// <param name="WaitForFirstTraceEntry">When true, it will emit all log messages during startup until the first entry associated with a TraceId is seen</param>
    /// <param name="AlwaysPassLogLevel">Will emit all log messages at this level or greater, regardless of the trace status</param>
    /// <returns>A TraceBasedLogSamplingProcessor instance</returns>
    /// <exception cref="ArgumentOutOfRangeException">Occurs if the rate &lt; 0</exception>
    public static TraceBasedLogSamplingProcessor RateLimitedLogRecordsWithoutTraceId(double MaxRecordsPerSecond, bool WaitForFirstTraceEntry = true, LogLevel AlwaysPassLogLevel = LogLevel.Critical)
    {
        if (MaxRecordsPerSecond < 0) { throw new ArgumentOutOfRangeException(nameof(MaxRecordsPerSecond), "Value must be greater than 0"); }
        double maxBalance = MaxRecordsPerSecond < 1.0 ? 1.0 : MaxRecordsPerSecond;
        var rateLimiter = new RateLimiter(MaxRecordsPerSecond, maxBalance);
        return new TraceBasedLogSamplingProcessor(WaitForFirstTraceEntry, AlwaysPassLogLevel, (data) => (rateLimiter.TrySpend(1.0)));
    }

    public TraceBasedLogSamplingProcessor(bool WaitForFirstTraceEntry, LogLevel AlwaysPassLogLevel, Func<LogRecord, bool> ShouldLogRecord)
    {
        this.waitForFirstTraceEntry = WaitForFirstTraceEntry;
        this.alwaysPassLogLevel = AlwaysPassLogLevel;
        this.shouldLogRecord = ShouldLogRecord;
    }

    public override void OnEnd(LogRecord data)
    {
        // If the log level is None, we don't want to do anything as they should not be emitted
        if (data.LogLevel == LogLevel.None) return;
        // Don't filter logs that are at or above the alwaysPassLogLevel
        if (data.LogLevel >= alwaysPassLogLevel) return;
        // If the span id is default, we are not in a trace
        if (data.SpanId == default)
        {
            if (waitForFirstTraceEntry && !firstTraceEntryReceived)
            {
                return;
            }
            if (shouldLogRecord(data))
            {
                return;
            }
        }
        else
        {
            firstTraceEntryReceived = true;
            if (data.TraceFlags.HasFlag(ActivityTraceFlags.Recorded))
            {
                return;
            }
        }
        data.LogLevel = LogLevel.None;
    }
}

Proposal for a head-based log sampler that exhibits the following behaviors:

• Sampler that will filter log messages based on the presence of an associated TraceId
  • If there is a TraceId then follow its Recorded property
    • This would result in traces that are emitted all having their corresponding logs.
    • Logs for traces that are not being emitted would be dropped
  • For logs not associated with a TraceId then optionally:
    • Include them all
    • Sample them with a fixed rate - eg 20% get emitted
    • Sample them with a rate limit - eg 5 logs would be emitted per second, the rest would be dropped  
• Option to not sample any log messages of a specific level, such as Critical -
• Option to not sample any log messages until log messages with a TraceId are seen. This should ensure that startup messages are not sampled.

This was written for the OTel processor API, assuming that setting a log level to None will filter it. But that doesn't work as the level is assessed before calling the processors and is not looked at again.