diff --git a/.gitignore b/.gitignore
index 36508301c3..0d84b7a8bc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -87,3 +87,5 @@ docs/_cache/
.devcontainer/devcontainer-lock.json
*.lscache
+# BenchmarkDotNet output
+BenchmarkDotNet.Artifacts/
diff --git a/Exceptionless.slnx b/Exceptionless.slnx
index a7466782db..04a7c0da84 100644
--- a/Exceptionless.slnx
+++ b/Exceptionless.slnx
@@ -23,6 +23,8 @@
+
+
diff --git a/benchmarks/Directory.Build.props b/benchmarks/Directory.Build.props
new file mode 100644
index 0000000000..993e0a1792
--- /dev/null
+++ b/benchmarks/Directory.Build.props
@@ -0,0 +1,7 @@
+
+
+
+
+ false
+
+
diff --git a/benchmarks/Exceptionless.Benchmarks/Exceptionless.Benchmarks.csproj b/benchmarks/Exceptionless.Benchmarks/Exceptionless.Benchmarks.csproj
new file mode 100644
index 0000000000..e5b85aad23
--- /dev/null
+++ b/benchmarks/Exceptionless.Benchmarks/Exceptionless.Benchmarks.csproj
@@ -0,0 +1,19 @@
+
+
+ Exe
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/benchmarks/Exceptionless.Benchmarks/Processing/EventIngestionProcessingBenchmarks.cs b/benchmarks/Exceptionless.Benchmarks/Processing/EventIngestionProcessingBenchmarks.cs
new file mode 100644
index 0000000000..d05032e109
--- /dev/null
+++ b/benchmarks/Exceptionless.Benchmarks/Processing/EventIngestionProcessingBenchmarks.cs
@@ -0,0 +1,72 @@
+using BenchmarkDotNet.Attributes;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.Ingestion;
+using Exceptionless.Core.Services;
+
+namespace Exceptionless.Benchmarks.Processing;
+
+[MemoryDiagnoser]
+public class EventIngestionProcessingBenchmarks
+{
+ private const string StackTrace = """
+ at System.Threading.Tasks.Task.Run()
+ at Example.Services.OrderService.Save() in /src/OrderService.cs:line 42
+ at Example.Api.OrdersController.Post() in /src/OrdersController.cs:line 18
+ """;
+
+ private readonly StackTraceParser _parser = new();
+ private StackFingerprintService _fingerprintService = null!;
+ private EventMaterializer _materializer = null!;
+ private EventIngestionV3Event _source = null!;
+ private Organization _organization = null!;
+ private Project _project = null!;
+ private StackFingerprint _fingerprint = null!;
+
+ [GlobalSetup]
+ public void Setup()
+ {
+ _fingerprintService = new StackFingerprintService(_parser);
+ _materializer = new EventMaterializer(_parser, TimeProvider.System);
+ _organization = new Organization { Id = "507f191e810c19729de860ea", Name = "Benchmark" };
+ _project = new Project
+ {
+ Id = "507f191e810c19729de860eb",
+ OrganizationId = _organization.Id,
+ Name = "Benchmark"
+ };
+ _source = new EventIngestionV3Event
+ {
+ Id = "benchmark-event-1",
+ Type = Event.KnownTypes.Error,
+ Message = "Operation failed",
+ ExceptionType = "System.InvalidOperationException",
+ StackTrace = StackTrace,
+ Tags = ["benchmark", "v3"]
+ };
+ _fingerprint = _fingerprintService.Create(_source, _organization, _project);
+ }
+
+ [Benchmark]
+ public bool FingerprintRawStack()
+ {
+ return _parser.TryFindFrame(StackTrace, static frame => frame.DeclaringNamespace?.StartsWith("Example") is true, out _, out _, out _);
+ }
+
+ [Benchmark]
+ public StackFingerprint CreateFingerprint()
+ {
+ return _fingerprintService.Create(_source, _organization, _project);
+ }
+
+ [Benchmark]
+ public int ParseStructuredFrames()
+ {
+ return _parser.Parse(StackTrace).Count;
+ }
+
+ [Benchmark]
+ public PersistentEvent MaterializeSurvivor()
+ {
+ return _materializer.Materialize(_source, _fingerprint, _organization, _project);
+ }
+}
diff --git a/benchmarks/Exceptionless.Benchmarks/Program.cs b/benchmarks/Exceptionless.Benchmarks/Program.cs
new file mode 100644
index 0000000000..528ba5c55d
--- /dev/null
+++ b/benchmarks/Exceptionless.Benchmarks/Program.cs
@@ -0,0 +1,11 @@
+using BenchmarkDotNet.Running;
+
+namespace Exceptionless.Benchmarks;
+
+public static class Program
+{
+ public static void Main(string[] args)
+ {
+ BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+ }
+}
diff --git a/benchmarks/Exceptionless.Benchmarks/Serialization/EventIngestionDeserializationBenchmarks.cs b/benchmarks/Exceptionless.Benchmarks/Serialization/EventIngestionDeserializationBenchmarks.cs
new file mode 100644
index 0000000000..739ba8fc13
--- /dev/null
+++ b/benchmarks/Exceptionless.Benchmarks/Serialization/EventIngestionDeserializationBenchmarks.cs
@@ -0,0 +1,222 @@
+using System.IO.Pipelines;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using BenchmarkDotNet.Attributes;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.Ingestion;
+using Exceptionless.Core.Serialization;
+using Exceptionless.Web.Utility;
+
+namespace Exceptionless.Benchmarks.Serialization;
+
+[MemoryDiagnoser]
+public class EventIngestionDeserializationBenchmarks
+{
+ private const int MaximumEventSize = 512 * 1024;
+ private byte[] _v2Payload = null!;
+ private byte[] _v3StreamPayload = null!;
+ private JsonSerializerOptions _v2JsonOptions = null!;
+
+ [Params(1, 100, 1000)]
+ public int EventCount { get; set; }
+
+ [GlobalSetup]
+ public void Setup()
+ {
+ _v2JsonOptions = new JsonSerializerOptions().ConfigureExceptionlessDefaults();
+ _v2JsonOptions.RespectNullableAnnotations = false;
+ var v2Events = new V2BenchmarkEvent[EventCount];
+ var v3Events = new EventIngestionV3Event[EventCount];
+ for (int index = 0; index < v3Events.Length; index++)
+ {
+ const string message = "Operation failed";
+ const string exceptionType = "System.InvalidOperationException";
+ const string stackTrace = " at Example.Service.Run() in Service.cs:line 42";
+ v2Events[index] = new V2BenchmarkEvent(
+ Event.KnownTypes.Error,
+ DateTimeOffset.UnixEpoch.AddSeconds(index),
+ message,
+ new V2BenchmarkData(new V2BenchmarkError(exceptionType, message, stackTrace)));
+ v3Events[index] = new EventIngestionV3Event
+ {
+ Id = $"01J0000000000000000000{index:D4}",
+ Type = "error",
+ Date = DateTimeOffset.UnixEpoch.AddSeconds(index),
+ Message = message,
+ ExceptionType = exceptionType,
+ StackTrace = stackTrace
+ };
+ }
+
+ _v2Payload = EventCount == 1
+ ? JsonSerializer.SerializeToUtf8Bytes(v2Events[0])
+ : JsonSerializer.SerializeToUtf8Bytes(v2Events);
+
+ using var stream = new MemoryStream(_v2Payload.Length);
+ for (int index = 0; index < v3Events.Length; index++)
+ {
+ JsonSerializer.Serialize(
+ stream,
+ v3Events[index],
+ EventIngestionJsonContext.Default.EventIngestionV3Event);
+ stream.WriteByte((byte)'\n');
+ }
+
+ _v3StreamPayload = stream.ToArray();
+ }
+
+ [Benchmark(Baseline = true)]
+ public int DeserializeV2Payload()
+ {
+ string input = Encoding.UTF8.GetString(_v2Payload);
+ return input.GetJsonType() switch
+ {
+ JsonType.Object => JsonSerializer.Deserialize(input, _v2JsonOptions) is null ? 0 : 1,
+ JsonType.Array => JsonSerializer.Deserialize(input, _v2JsonOptions)?.Length ?? 0,
+ _ => 0
+ };
+ }
+
+ [Benchmark]
+ public Task FrameAndRouteV3NdjsonAsync() => ReadV3NdjsonAsync(materializeSurvivors: false);
+
+ [Benchmark]
+ public Task FrameRouteAndMaterializeV3SurvivorsAsync() => ReadV3NdjsonAsync(materializeSurvivors: true);
+
+ private async Task ReadV3NdjsonAsync(bool materializeSurvivors)
+ {
+ using var stream = new MemoryStream(_v3StreamPayload, writable: false);
+ var reader = PipeReader.Create(stream);
+ int count = 0;
+ try
+ {
+ while (await EventIngestionV3StreamReader.ReadAsync(reader, MaximumEventSize, CancellationToken.None) is { } record)
+ {
+ try
+ {
+ EventIngestionV3Event parsed = materializeSurvivors
+ ? record.BufferedRecord.Materialize()
+ : record.Event;
+ if (!String.IsNullOrEmpty(parsed.Id))
+ count++;
+ }
+ finally
+ {
+ record.BufferedRecord.Dispose();
+ }
+ }
+ }
+ finally
+ {
+ await reader.CompleteAsync();
+ }
+
+ return count;
+ }
+
+ private sealed record V2BenchmarkEvent(
+ [property: JsonPropertyName("type")] string Type,
+ [property: JsonPropertyName("date")] DateTimeOffset Date,
+ [property: JsonPropertyName("message")] string Message,
+ [property: JsonPropertyName("data")] V2BenchmarkData Data);
+
+ private sealed record V2BenchmarkData(
+ [property: JsonPropertyName("@simple_error")] V2BenchmarkError SimpleError);
+
+ private sealed record V2BenchmarkError(
+ [property: JsonPropertyName("type")] string Type,
+ [property: JsonPropertyName("message")] string Message,
+ [property: JsonPropertyName("stack_trace")] string StackTrace);
+}
+
+///
+/// Keeps large raw stacks visible in allocation results. One accepted error is enough to expose
+/// duplicate LOH strings without multiplying the benchmark process's retained payload by a batch.
+///
+[MemoryDiagnoser]
+public class LargeStackEventIngestionDeserializationBenchmarks
+{
+ private const int MaximumEventSize = 512 * 1024;
+ private byte[] _v2Payload = null!;
+ private byte[] _v3Payload = null!;
+ private JsonSerializerOptions _v2JsonOptions = null!;
+
+ [Params(16 * 1024, 128 * 1024)]
+ public int StackTraceLength { get; set; }
+
+ [GlobalSetup]
+ public void Setup()
+ {
+ string stackTrace = new('x', StackTraceLength);
+ _v2JsonOptions = new JsonSerializerOptions().ConfigureExceptionlessDefaults();
+ _v2JsonOptions.RespectNullableAnnotations = false;
+ _v2Payload = JsonSerializer.SerializeToUtf8Bytes(new Dictionary
+ {
+ ["type"] = "error",
+ ["message"] = "Operation failed",
+ ["data"] = new Dictionary
+ {
+ ["@simple_error"] = new
+ {
+ type = "Example.Exception",
+ message = "Operation failed",
+ stack_trace = stackTrace
+ }
+ }
+ });
+ _v3Payload = JsonSerializer.SerializeToUtf8Bytes(new
+ {
+ id = "large-stack-event",
+ type = "error",
+ message = "Operation failed",
+ exception_type = "Example.Exception",
+ stack_trace = stackTrace
+ });
+ }
+
+ [Benchmark(Baseline = true)]
+ public PersistentEvent? DeserializeV2Payload()
+ {
+ string input = Encoding.UTF8.GetString(_v2Payload);
+ return JsonSerializer.Deserialize(input, _v2JsonOptions);
+ }
+
+ [Benchmark]
+ public Task FrameAndRouteV3NdjsonAsync() => ReadV3NdjsonAsync(materializeSurvivor: false);
+
+ [Benchmark]
+ public Task FrameRouteAndMaterializeV3SurvivorAsync() => ReadV3NdjsonAsync(materializeSurvivor: true);
+
+ private async Task ReadV3NdjsonAsync(bool materializeSurvivor)
+ {
+ using var stream = new MemoryStream(_v3Payload, writable: false);
+ var reader = PipeReader.Create(stream);
+ try
+ {
+ EventIngestionV3StreamRecord? record = await EventIngestionV3StreamReader.ReadAsync(
+ reader,
+ MaximumEventSize,
+ CancellationToken.None);
+ if (record is null)
+ return 0;
+
+ try
+ {
+ EventIngestionV3Event parsed = materializeSurvivor
+ ? record.Value.BufferedRecord.Materialize()
+ : record.Value.Event;
+ return parsed.StackTrace?.Length ?? 0;
+ }
+ finally
+ {
+ record.Value.BufferedRecord.Dispose();
+ }
+ }
+ finally
+ {
+ await reader.CompleteAsync();
+ }
+ }
+}
diff --git a/benchmarks/Exceptionless.Ingestion.Load/Exceptionless.Ingestion.Load.csproj b/benchmarks/Exceptionless.Ingestion.Load/Exceptionless.Ingestion.Load.csproj
new file mode 100644
index 0000000000..f780803dce
--- /dev/null
+++ b/benchmarks/Exceptionless.Ingestion.Load/Exceptionless.Ingestion.Load.csproj
@@ -0,0 +1,13 @@
+
+
+ Exe
+
+
+
+
+
+
+
+
+
+
diff --git a/benchmarks/Exceptionless.Ingestion.Load/IngestionLoadRunner.cs b/benchmarks/Exceptionless.Ingestion.Load/IngestionLoadRunner.cs
new file mode 100644
index 0000000000..f010c6ad3b
--- /dev/null
+++ b/benchmarks/Exceptionless.Ingestion.Load/IngestionLoadRunner.cs
@@ -0,0 +1,609 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Net;
+using System.Net.Http.Headers;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Exceptionless.Core.Models.Ingestion;
+using Exceptionless.Core.Serialization;
+
+namespace Exceptionless.Ingestion.Load;
+
+internal sealed class IngestionLoadRunner
+{
+ private const string EventPostIdHeader = "X-Exceptionless-Event-Post-Id";
+ private const string TrackEventPostHeader = "X-Exceptionless-Track-Event-Post";
+ private static readonly JsonSerializerOptions _protocolJsonOptions = new(JsonSerializerDefaults.Web);
+ private readonly LoadOptions _options;
+ private readonly HttpClient _client;
+
+ public IngestionLoadRunner(LoadOptions options, HttpMessageHandler? handler = null)
+ {
+ _options = options;
+ _client = handler is null ? new HttpClient() : new HttpClient(handler, disposeHandler: false);
+ _client.Timeout = Timeout.InfiniteTimeSpan;
+ _client.DefaultRequestVersion = HttpVersion.Version20;
+ _client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower;
+ }
+
+ public async Task RunAsync()
+ {
+ Console.WriteLine($"Comparing {String.Join(" and ", _options.Protocols)}: event_type={_options.EventType.ToString().ToLowerInvariant()} stack_scenario={_options.StackScenario.ToString().ToLowerInvariant()} events={_options.EventCount} expected_persisted={_options.ExpectedPersisted} batch_size={_options.BatchSize} requests={(int)Math.Ceiling((double)_options.EventCount / _options.BatchSize)} concurrency={_options.Concurrency} trials={_options.Trials} compression={_options.Compression} completion_poll_concurrency={_options.CompletionPollConcurrency}");
+ Console.WriteLine("Submission headers and query visibility use common boundaries. Instrumented terminal processing includes observer counters because V2 tracks requests while V3 tracks persisted events; V3 durable acknowledgement is reported separately.");
+
+ if (_options.WarmupEvents > 0)
+ {
+ foreach (IngestionProtocol protocol in _options.Protocols)
+ {
+ Console.WriteLine($"Warming {protocol} with {_options.WarmupEvents} events...");
+ await ExecuteAsync(protocol, trial: -1, _options.WarmupEvents, GetExpectedPersistedCount(_options.WarmupEvents), isWarmup: true);
+ }
+ }
+
+ var results = new List();
+ for (int trial = 0; trial < _options.Trials; trial++)
+ {
+ IEnumerable order = trial % 2 == 0 ? _options.Protocols : _options.Protocols.Reverse();
+ foreach (IngestionProtocol protocol in order)
+ {
+ LoadRunResult result = await ExecuteAsync(protocol, trial, _options.EventCount, _options.ExpectedPersisted, isWarmup: false);
+ results.Add(result);
+ WriteResult(result);
+ }
+ }
+
+ WriteSummary(results);
+ if (!String.IsNullOrWhiteSpace(_options.ResultsPath))
+ await WriteEvidenceAsync(results, _options.ResultsPath);
+
+ _client.Dispose();
+ return 0;
+ }
+
+ private async Task ExecuteAsync(IngestionProtocol protocol, int trial, int eventCount, int expectedPersisted, bool isWarmup)
+ {
+ using var cancellation = new CancellationTokenSource(_options.Timeout);
+ string phase = isWarmup ? "warm" : $"t{trial + 1}";
+ string runMarker = $"load-{_options.Seed}-{protocol.ToString().ToLowerInvariant()}-{phase}-{Guid.NewGuid():N}";
+ string signatureNamespace = GetSignatureNamespace(protocol, phase, isWarmup);
+ string[] expectedV3ClientIds = protocol is IngestionProtocol.V3 && expectedPersisted > 0
+ ? GetExpectedV3ClientIds(runMarker, eventCount, expectedPersisted)
+ : [];
+ DateTimeOffset eventDate = DateTimeOffset.UtcNow;
+ int requestCount = (eventCount + _options.BatchSize - 1) / _options.BatchSize;
+ int nextRequest = -1;
+ var requestLatencies = new ConcurrentBag();
+ var processingCorrelationIds = new ConcurrentBag();
+ var totals = new LoadTotals();
+ long runStarted = Stopwatch.GetTimestamp();
+ long lastSubmissionResponse = runStarted;
+ long lastDurableAcknowledgement = runStarted;
+
+ Task[] workers = Enumerable.Range(0, Math.Min(_options.Concurrency, requestCount)).Select(_ => Task.Run(async () =>
+ {
+ while (true)
+ {
+ int requestIndex = Interlocked.Increment(ref nextRequest);
+ if (requestIndex >= requestCount)
+ return;
+
+ int start = requestIndex * _options.BatchSize;
+ int count = Math.Min(_options.BatchSize, eventCount - start);
+ using var content = new StreamingEventContent(_options, protocol, runMarker, signatureNamespace, eventDate, start, count);
+ using var request = new HttpRequestMessage(HttpMethod.Post, GetIngestionUrl(protocol)) { Content = content };
+ request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.SubmissionToken);
+ if (protocol is IngestionProtocol.V2)
+ request.Headers.Add(TrackEventPostHeader, "true");
+
+ long started = Stopwatch.GetTimestamp();
+ using HttpResponseMessage response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellation.Token);
+ long responseReceived = Stopwatch.GetTimestamp();
+ UpdateMaximum(ref lastSubmissionResponse, responseReceived);
+ requestLatencies.Add(Stopwatch.GetElapsedTime(started, responseReceived).TotalMilliseconds);
+ if (!response.IsSuccessStatusCode)
+ {
+ string detail = await response.Content.ReadAsStringAsync(cancellation.Token);
+ throw new InvalidOperationException($"{protocol} request {requestIndex + 1} returned {(int)response.StatusCode}: {Limit(detail)}");
+ }
+
+ if (protocol is IngestionProtocol.V2)
+ {
+ if (!response.Headers.TryGetValues(EventPostIdHeader, out var values) || values.SingleOrDefault() is not { Length: > 0 } processingCorrelationId)
+ throw new InvalidOperationException($"V2 request {requestIndex + 1} did not return the required {EventPostIdHeader} tracking header.");
+ processingCorrelationIds.Add(processingCorrelationId);
+ }
+ else
+ {
+ await using Stream body = await response.Content.ReadAsStreamAsync(cancellation.Token);
+ EventIngestionV3Response? terminal = await JsonSerializer.DeserializeAsync(body, EventIngestionJsonContext.Default.EventIngestionV3Response, cancellation.Token);
+ if (terminal is null)
+ throw new InvalidOperationException($"V3 request {requestIndex + 1} returned an empty terminal response.");
+ totals.Add(terminal);
+ UpdateMaximum(ref lastDurableAcknowledgement, Stopwatch.GetTimestamp());
+ }
+
+ Interlocked.Add(ref totals.UncompressedBytes, content.UncompressedBytes);
+ Interlocked.Add(ref totals.TransferredBytes, content.TransferredBytes);
+ Interlocked.Increment(ref totals.SuccessfulRequests);
+ }
+ }, cancellation.Token)).ToArray();
+
+ await Task.WhenAll(workers);
+ TimeSpan submissionElapsed = Stopwatch.GetElapsedTime(runStarted, Volatile.Read(ref lastSubmissionResponse));
+
+ if (protocol is IngestionProtocol.V3)
+ {
+ if (totals.Received != eventCount)
+ throw new InvalidOperationException($"V3 terminal responses reported {totals.Received} received events; expected {eventCount}.");
+ if (!isWarmup && totals.Persisted != expectedPersisted)
+ throw new InvalidOperationException($"V3 terminal responses reported {totals.Persisted} persisted events; expected {expectedPersisted}.");
+ }
+
+ TimeSpan? durableAcknowledgementElapsed = protocol is IngestionProtocol.V3
+ ? Stopwatch.GetElapsedTime(runStarted, Volatile.Read(ref lastDurableAcknowledgement))
+ : null;
+ Task fullProcessingTask = protocol is IngestionProtocol.V2
+ ? WaitForV2PipelineCompletionAsync(processingCorrelationIds.ToArray(), requestCount, runStarted, cancellation.Token)
+ : expectedPersisted > 0
+ ? WaitForV3FullProcessingAsync(expectedV3ClientIds, runStarted, cancellation.Token)
+ : Task.FromResult(new CompletionObservation(durableAcknowledgementElapsed!.Value, 0, 0, 0, 0));
+ Task queryVisibilityTask = !isWarmup && expectedPersisted > 0
+ ? WaitForQueryVisibilityAsync(runMarker, expectedPersisted, runStarted, cancellation.Token)
+ : Task.FromResult(null);
+
+ CompletionObservation fullProcessing = await fullProcessingTask;
+ QueryVisibilityObservation? queryVisibility = await queryVisibilityTask;
+
+ return new LoadRunResult(
+ protocol,
+ trial,
+ runMarker,
+ signatureNamespace,
+ eventCount,
+ expectedPersisted,
+ requestCount,
+ totals.SuccessfulRequests,
+ totals.UncompressedBytes,
+ totals.TransferredBytes,
+ submissionElapsed,
+ fullProcessing.Elapsed,
+ durableAcknowledgementElapsed,
+ queryVisibility?.Elapsed,
+ queryVisibility?.ObservedPersisted ?? 0,
+ protocol is IngestionProtocol.V2
+ ? CompletionIdentifierKind.EventPost
+ : expectedPersisted > 0
+ ? CompletionIdentifierKind.PersistedEvent
+ : CompletionIdentifierKind.None,
+ fullProcessing.TrackedIdentifiers,
+ fullProcessing.StatusRequests,
+ fullProcessing.IdentifierReads,
+ fullProcessing.Sweeps,
+ queryVisibility?.Requests ?? 0,
+ Percentile(requestLatencies, 0.50),
+ Percentile(requestLatencies, 0.95),
+ Percentile(requestLatencies, 0.99),
+ totals.Received,
+ totals.Persisted,
+ totals.Discarded,
+ totals.Duplicate,
+ totals.Blocked,
+ totals.Invalid);
+ }
+
+ private Task WaitForV3FullProcessingAsync(string[] clientIds, long runStarted, CancellationToken cancellationToken)
+ {
+ string[][] chunks = clientIds.Chunk(1000).Select(chunk => chunk.ToArray()).ToArray();
+ return WaitForCompletionAsync(
+ chunks,
+ runStarted,
+ GetV3ProcessingSummaryAsync,
+ static (chunk, summary) => summary.Requested == chunk.Length && summary.Completed == chunk.Length,
+ cancellationToken);
+ }
+
+ private async Task GetV3ProcessingSummaryAsync(string[] clientIds, CancellationToken cancellationToken)
+ {
+ Uri url = new(_options.BaseUrl, $"api/v3/projects/{_options.ProjectId}/events/processing/status");
+ using var request = new HttpRequestMessage(HttpMethod.Post, url);
+ request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.SubmissionToken);
+ byte[] payload = JsonSerializer.SerializeToUtf8Bytes(new EventIngestionV3ProcessingStatusRequest(clientIds), _protocolJsonOptions);
+ request.Content = new ByteArrayContent(payload);
+ request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+
+ using HttpResponseMessage response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ if (!response.IsSuccessStatusCode)
+ {
+ string detail = await response.Content.ReadAsStringAsync(cancellationToken);
+ throw new InvalidOperationException($"V3 full-processing status returned {(int)response.StatusCode}: {Limit(detail)}");
+ }
+
+ await using Stream body = await response.Content.ReadAsStreamAsync(cancellationToken);
+ return await JsonSerializer.DeserializeAsync(body, _protocolJsonOptions, cancellationToken)
+ ?? throw new InvalidOperationException("V3 full-processing status returned an empty response.");
+ }
+
+ private Task WaitForV2PipelineCompletionAsync(string[] processingCorrelationIds, int requestCount, long runStarted, CancellationToken cancellationToken)
+ {
+ if (processingCorrelationIds.Length != requestCount)
+ throw new InvalidOperationException($"V2 returned {processingCorrelationIds.Length} tracked event-post ids for {requestCount} successful requests.");
+
+ string[][] chunks = processingCorrelationIds.Chunk(1000).Select(chunk => chunk.ToArray()).ToArray();
+ return WaitForCompletionAsync(
+ chunks,
+ runStarted,
+ GetV2ProcessingSummaryAsync,
+ static (chunk, summary) => summary.Requested == chunk.Length && summary.Completed == chunk.Length,
+ cancellationToken);
+ }
+
+ private async Task WaitForCompletionAsync(
+ string[][] chunks,
+ long runStarted,
+ Func> getSummaryAsync,
+ Func isComplete,
+ CancellationToken cancellationToken)
+ {
+ var pending = chunks.ToList();
+ int statusRequests = 0;
+ long identifierReads = 0;
+ int sweeps = 0;
+ while (pending.Count > 0)
+ {
+ sweeps++;
+ var nextPending = new List();
+ foreach (string[][] page in pending.Chunk(_options.CompletionPollConcurrency))
+ {
+ TSummary[] summaries = await Task.WhenAll(page.Select(chunk => getSummaryAsync(chunk, cancellationToken)));
+ statusRequests += page.Length;
+ identifierReads += page.Sum(chunk => (long)chunk.Length);
+ for (int index = 0; index < page.Length; index++)
+ {
+ if (!isComplete(page[index], summaries[index]))
+ nextPending.Add(page[index]);
+ }
+ }
+
+ if (nextPending.Count == 0)
+ {
+ return new CompletionObservation(
+ Stopwatch.GetElapsedTime(runStarted),
+ chunks.Sum(chunk => (long)chunk.Length),
+ statusRequests,
+ identifierReads,
+ sweeps);
+ }
+
+ pending = nextPending;
+ await Task.Delay(_options.PollInterval, cancellationToken);
+ }
+
+ throw new InvalidOperationException("At least one completion identifier is required.");
+ }
+
+ private async Task GetV2ProcessingSummaryAsync(string[] processingCorrelationIds, CancellationToken cancellationToken)
+ {
+ Uri url = new(_options.BaseUrl, $"api/v2/projects/{_options.ProjectId}/events/posts/status");
+ using var request = new HttpRequestMessage(HttpMethod.Post, url);
+ request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.SubmissionToken);
+ byte[] payload = JsonSerializer.SerializeToUtf8Bytes(new EventPostProcessingStatusRequest(processingCorrelationIds), _protocolJsonOptions);
+ request.Content = new ByteArrayContent(payload);
+ request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+
+ using HttpResponseMessage response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ if (!response.IsSuccessStatusCode)
+ {
+ string detail = await response.Content.ReadAsStringAsync(cancellationToken);
+ throw new InvalidOperationException($"V2 processing status returned {(int)response.StatusCode}: {Limit(detail)}");
+ }
+
+ await using Stream body = await response.Content.ReadAsStreamAsync(cancellationToken);
+ return await JsonSerializer.DeserializeAsync(body, _protocolJsonOptions, cancellationToken)
+ ?? throw new InvalidOperationException("V2 processing status returned an empty response.");
+ }
+
+ private async Task WaitForQueryVisibilityAsync(string runMarker, int expectedPersisted, long runStarted, CancellationToken cancellationToken)
+ {
+ Uri url = new(_options.BaseUrl, $"api/v2/projects/{_options.ProjectId}/events/count?filter={Uri.EscapeDataString($"tag:{runMarker}")}");
+ int requests = 0;
+ while (true)
+ {
+ using var request = new HttpRequestMessage(HttpMethod.Get, url);
+ SetReadAuthorization(request);
+ using HttpResponseMessage response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ requests++;
+ if (!response.IsSuccessStatusCode)
+ {
+ string detail = await response.Content.ReadAsStringAsync(cancellationToken);
+ throw new InvalidOperationException($"Query-visible count returned {(int)response.StatusCode}: {Limit(detail)}");
+ }
+
+ await using Stream body = await response.Content.ReadAsStreamAsync(cancellationToken);
+ using JsonDocument json = await JsonDocument.ParseAsync(body, cancellationToken: cancellationToken);
+ long total = json.RootElement.GetProperty("total").GetInt64();
+ if (total >= expectedPersisted)
+ return new QueryVisibilityObservation(Stopwatch.GetElapsedTime(runStarted), total, requests);
+ await Task.Delay(_options.PollInterval, cancellationToken);
+ }
+ }
+
+ private string GetSignatureNamespace(IngestionProtocol protocol, string phase, bool isWarmup)
+ {
+ string prefix = $"{_options.Seed}-{protocol.ToString().ToLowerInvariant()}";
+ if (_options.StackScenario is StackScenario.Hot)
+ return $"{prefix}-hot";
+
+ return $"{prefix}-{phase}-{(isWarmup ? "warm" : "new")}-{Guid.NewGuid():N}";
+ }
+
+ private string[] GetExpectedV3ClientIds(string runMarker, int eventCount, int expectedPersisted)
+ {
+ IEnumerable expectedIndexes = Enumerable.Range(0, eventCount);
+ if (expectedPersisted != eventCount)
+ expectedIndexes = expectedIndexes.Where(index => index % 100 >= _options.DiscardPercent);
+
+ int[] indexes = expectedIndexes.ToArray();
+ if (indexes.Length != expectedPersisted)
+ {
+ throw new InvalidOperationException(
+ $"Cannot identify the {expectedPersisted} V3 events expected to persist from this corpus. " +
+ "Use the discard-derived default, zero, or the full event count for --expected-persisted.");
+ }
+
+ return indexes.Select(index => StreamingEventContent.GetV3ClientId(runMarker, index)).ToArray();
+ }
+
+ private int GetExpectedPersistedCount(int eventCount)
+ {
+ if (_options.ExpectedPersisted == _options.EventCount)
+ return eventCount;
+ if (_options.ExpectedPersisted == 0)
+ return 0;
+
+ int discarded = eventCount / 100 * _options.DiscardPercent + Math.Min(eventCount % 100, _options.DiscardPercent);
+ return eventCount - discarded;
+ }
+
+ private Uri GetIngestionUrl(IngestionProtocol protocol) => new(_options.BaseUrl, $"api/{protocol.ToString().ToLowerInvariant()}/projects/{_options.ProjectId}/events");
+
+ private void SetReadAuthorization(HttpRequestMessage request)
+ {
+ if (!String.IsNullOrWhiteSpace(_options.ReadToken))
+ {
+ request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ReadToken);
+ return;
+ }
+
+ string credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_options.ReadUser}:{_options.ReadPassword}"));
+ request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials);
+ }
+
+ private async Task WriteEvidenceAsync(IReadOnlyList results, string resultsPath)
+ {
+ string fullPath = Path.GetFullPath(resultsPath);
+ string? directory = Path.GetDirectoryName(fullPath);
+ if (!String.IsNullOrEmpty(directory))
+ Directory.CreateDirectory(directory);
+
+ var evidence = new LoadEvidence(
+ "3",
+ DateTimeOffset.UtcNow,
+ new LoadEnvironment(
+ _options.EnvironmentLabel,
+ RuntimeInformation.FrameworkDescription,
+ RuntimeInformation.OSDescription,
+ RuntimeInformation.ProcessArchitecture.ToString(),
+ System.Environment.ProcessorCount,
+ typeof(Program).Assembly.GetCustomAttribute()?.InformationalVersion),
+ new LoadConfiguration(
+ _options.BaseUrl.ToString(),
+ _options.ProjectId,
+ _options.Protocols,
+ _options.EventType,
+ _options.StackScenario,
+ _options.EventCount,
+ _options.ExpectedPersisted,
+ _options.Concurrency,
+ _options.BatchSize,
+ _options.Trials,
+ _options.WarmupEvents,
+ _options.SignatureCardinality,
+ _options.DiscardPercent,
+ _options.Compression,
+ _options.Seed,
+ _options.Message.Length,
+ _options.Timeout.TotalSeconds,
+ _options.PollInterval.TotalMilliseconds,
+ _options.CompletionPollConcurrency),
+ results);
+ var serializerOptions = new JsonSerializerOptions
+ {
+ WriteIndented = true,
+ PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
+ };
+ serializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower));
+ await using FileStream output = File.Create(fullPath);
+ await JsonSerializer.SerializeAsync(output, evidence, serializerOptions);
+ Console.WriteLine($"Evidence written to {fullPath}");
+ }
+
+ private static double Percentile(IEnumerable values, double percentile)
+ {
+ double[] ordered = values.Order().ToArray();
+ if (ordered.Length == 0)
+ return 0;
+ int index = (int)Math.Ceiling(percentile * ordered.Length) - 1;
+ return ordered[Math.Clamp(index, 0, ordered.Length - 1)];
+ }
+
+ private static string Limit(string value) => value.Length <= 500 ? value : value[..500];
+
+ private static void UpdateMaximum(ref long target, long value)
+ {
+ long current = Volatile.Read(ref target);
+ while (value > current)
+ {
+ long observed = Interlocked.CompareExchange(ref target, value, current);
+ if (observed == current)
+ return;
+ current = observed;
+ }
+ }
+
+ private static void WriteResult(LoadRunResult result)
+ {
+ double submissionRate = Rate(result.EventCount, result.SubmissionElapsed);
+ string durableAcknowledgement = result.V3DurableAcknowledgementElapsed.HasValue
+ ? $" durable_ack={Rate(result.EventCount, result.V3DurableAcknowledgementElapsed.Value):F0} events/s {result.V3DurableAcknowledgementElapsed.Value.TotalSeconds:F3}s"
+ : String.Empty;
+ string queryVisible = result.QueryVisibleElapsed.HasValue
+ ? $"{Rate(result.ExpectedPersisted, result.QueryVisibleElapsed.Value):F0} persisted/s {result.QueryVisibleElapsed.Value.TotalSeconds:F3}s"
+ : "n/a";
+ Console.WriteLine($"{result.Protocol} trial={result.Trial + 1} requests={result.SuccessfulRequests}/{result.RequestCount} submission={submissionRate:F0} events/s full_processing_observed={Rate(result.EventCount, result.ObservedFullProcessingElapsed):F0} events/s {result.ObservedFullProcessingElapsed.TotalSeconds:F3}s{durableAcknowledgement} query_visible={queryVisible} observed_persisted={result.ObservedPersisted} latency_ms_p50/p95/p99={result.P50Milliseconds:F1}/{result.P95Milliseconds:F1}/{result.P99Milliseconds:F1} bytes={result.TransferredBytes} raw_bytes={result.UncompressedBytes}");
+ string completionIdentifierKind = JsonNamingPolicy.SnakeCaseLower.ConvertName(result.CompletionIdentifierKind.ToString());
+ Console.WriteLine($" completion_observer identifier_kind={completionIdentifierKind} tracked_ids={result.CompletionTrackedIdentifiers} status_requests={result.CompletionStatusRequests} identifier_reads={result.CompletionIdentifierReads} sweeps={result.CompletionSweeps} query_requests={result.QueryVisibilityRequests}");
+ if (result.Protocol is IngestionProtocol.V3)
+ Console.WriteLine($" terminal received={result.Received} persisted={result.Persisted} discarded={result.Discarded} duplicate={result.Duplicate} blocked={result.Blocked} invalid={result.Invalid}");
+ }
+
+ private static void WriteSummary(IReadOnlyList results)
+ {
+ Console.WriteLine("Median measured results:");
+ foreach (IGrouping group in results.GroupBy(r => r.Protocol).OrderBy(g => g.Key))
+ {
+ double submissionRate = Median(group.Select(r => Rate(r.EventCount, r.SubmissionElapsed)));
+ double? queryVisibleRate = group.All(r => r.QueryVisibleElapsed.HasValue)
+ ? Median(group.Select(r => Rate(r.ExpectedPersisted, r.QueryVisibleElapsed!.Value)))
+ : null;
+ double fullProcessingRate = Median(group.Select(r => Rate(r.EventCount, r.ObservedFullProcessingElapsed)));
+ string durableAcknowledgement = group.Key is IngestionProtocol.V3
+ ? $" durable_ack={Median(group.Select(r => Rate(r.EventCount, r.V3DurableAcknowledgementElapsed!.Value))):F0} events/s"
+ : String.Empty;
+ Console.WriteLine($" {group.Key}: submission={submissionRate:F0} events/s full_processing_observed={fullProcessingRate:F0} events/s{durableAcknowledgement} query_visible={(queryVisibleRate.HasValue ? $"{queryVisibleRate:F0} persisted/s" : "n/a")}");
+ }
+ Console.WriteLine("Full-processing observations include protocol-specific tracking and polling overhead; use the recorded observer counters and server telemetry before attributing a difference to pipeline efficiency.");
+ }
+
+ private static double Rate(long count, TimeSpan elapsed) => count / Math.Max(elapsed.TotalSeconds, 0.001);
+
+ private static double Median(IEnumerable source)
+ {
+ double[] values = source.Order().ToArray();
+ int middle = values.Length / 2;
+ return values.Length % 2 == 0 ? (values[middle - 1] + values[middle]) / 2 : values[middle];
+ }
+
+ private sealed class LoadTotals
+ {
+ public long SuccessfulRequests;
+ public long UncompressedBytes;
+ public long TransferredBytes;
+ public long Received;
+ public long Persisted;
+ public long Discarded;
+ public long Duplicate;
+ public long Blocked;
+ public long Invalid;
+
+ public void Add(EventIngestionV3Response value)
+ {
+ Interlocked.Add(ref Received, value.Received);
+ Interlocked.Add(ref Persisted, value.Persisted);
+ Interlocked.Add(ref Discarded, value.Discarded);
+ Interlocked.Add(ref Duplicate, value.Duplicate);
+ Interlocked.Add(ref Blocked, value.Blocked);
+ Interlocked.Add(ref Invalid, value.Invalid);
+ }
+ }
+}
+
+internal sealed record EventPostProcessingStatusRequest(string[] Ids);
+internal sealed record EventPostProcessingSummary(int Requested, int Queued, int Completed, int Unknown);
+internal sealed record EventIngestionV3ProcessingStatusRequest(
+ [property: JsonPropertyName("client_ids")] string[] ClientIds);
+internal sealed record EventIngestionV3ProcessingSummary(int Requested, int Pending, int Completed);
+internal sealed record CompletionObservation(
+ TimeSpan Elapsed,
+ long TrackedIdentifiers,
+ int StatusRequests,
+ long IdentifierReads,
+ int Sweeps);
+internal sealed record QueryVisibilityObservation(TimeSpan Elapsed, long ObservedPersisted, int Requests);
+
+internal enum CompletionIdentifierKind
+{
+ None,
+ EventPost,
+ PersistedEvent
+}
+
+internal sealed record LoadRunResult(
+ IngestionProtocol Protocol,
+ int Trial,
+ string RunMarker,
+ string SignatureNamespace,
+ int EventCount,
+ int ExpectedPersisted,
+ int RequestCount,
+ long SuccessfulRequests,
+ long UncompressedBytes,
+ long TransferredBytes,
+ TimeSpan SubmissionElapsed,
+ TimeSpan ObservedFullProcessingElapsed,
+ TimeSpan? V3DurableAcknowledgementElapsed,
+ TimeSpan? QueryVisibleElapsed,
+ long ObservedPersisted,
+ CompletionIdentifierKind CompletionIdentifierKind,
+ long CompletionTrackedIdentifiers,
+ int CompletionStatusRequests,
+ long CompletionIdentifierReads,
+ int CompletionSweeps,
+ int QueryVisibilityRequests,
+ double P50Milliseconds,
+ double P95Milliseconds,
+ double P99Milliseconds,
+ long Received,
+ long Persisted,
+ long Discarded,
+ long Duplicate,
+ long Blocked,
+ long Invalid);
+
+internal sealed record LoadEvidence(
+ string SchemaVersion,
+ DateTimeOffset CapturedUtc,
+ LoadEnvironment Environment,
+ LoadConfiguration Configuration,
+ IReadOnlyList Results);
+
+internal sealed record LoadEnvironment(
+ string? Label,
+ string Runtime,
+ string OperatingSystem,
+ string ProcessArchitecture,
+ int ProcessorCount,
+ string? BuildVersion);
+
+internal sealed record LoadConfiguration(
+ string BaseUrl,
+ string ProjectId,
+ IReadOnlyList Protocols,
+ LoadEventType EventType,
+ StackScenario StackScenario,
+ int EventCount,
+ int ExpectedPersisted,
+ int Concurrency,
+ int BatchSize,
+ int Trials,
+ int WarmupEvents,
+ int SignatureCardinality,
+ int DiscardPercent,
+ string Compression,
+ string Seed,
+ int MessageBytes,
+ double TimeoutSeconds,
+ double PollIntervalMilliseconds,
+ int CompletionPollConcurrency);
diff --git a/benchmarks/Exceptionless.Ingestion.Load/LoadOptions.cs b/benchmarks/Exceptionless.Ingestion.Load/LoadOptions.cs
new file mode 100644
index 0000000000..65cf77a649
--- /dev/null
+++ b/benchmarks/Exceptionless.Ingestion.Load/LoadOptions.cs
@@ -0,0 +1,182 @@
+using Exceptionless.Core.Models.Ingestion;
+
+namespace Exceptionless.Ingestion.Load;
+
+internal enum IngestionProtocol
+{
+ V2,
+ V3
+}
+
+internal enum LoadEventType
+{
+ Log,
+ Error
+}
+
+internal enum StackScenario
+{
+ Hot,
+ New
+}
+
+internal sealed record LoadOptions(
+ Uri BaseUrl,
+ string ProjectId,
+ string SubmissionToken,
+ string? ReadToken,
+ string? ReadUser,
+ string? ReadPassword,
+ IReadOnlyList Protocols,
+ LoadEventType EventType,
+ StackScenario StackScenario,
+ int EventCount,
+ int ExpectedPersisted,
+ int Concurrency,
+ int BatchSize,
+ int Trials,
+ int WarmupEvents,
+ int SignatureCardinality,
+ int DiscardPercent,
+ string Compression,
+ string Seed,
+ string? ResultsPath,
+ string? EnvironmentLabel,
+ string Message,
+ TimeSpan Timeout,
+ TimeSpan PollInterval,
+ int CompletionPollConcurrency)
+{
+ public static LoadOptions Parse(string[] args)
+ {
+ var values = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ for (int index = 0; index < args.Length; index += 2)
+ {
+ if (index + 1 >= args.Length || !args[index].StartsWith("--", StringComparison.Ordinal))
+ throw new ArgumentException("Every option must use '--name value'.");
+ values[args[index][2..]] = args[index + 1];
+ }
+
+ string? rawBaseUrl = values.GetValueOrDefault("base-url");
+ if (String.IsNullOrWhiteSpace(rawBaseUrl) || !Uri.TryCreate(rawBaseUrl, UriKind.Absolute, out var baseUrl))
+ throw new ArgumentException("--base-url must be an absolute API origin.");
+
+ string projectId = values.GetValueOrDefault("project-id")
+ ?? Environment.GetEnvironmentVariable("EXCEPTIONLESS_PROJECT_ID")
+ ?? throw new ArgumentException("Set --project-id or EXCEPTIONLESS_PROJECT_ID.");
+ string submissionToken = values.GetValueOrDefault("submission-token")
+ ?? Environment.GetEnvironmentVariable("EXCEPTIONLESS_API_KEY")
+ ?? throw new ArgumentException("Set --submission-token or EXCEPTIONLESS_API_KEY.");
+ string? readToken = values.GetValueOrDefault("read-token")
+ ?? Environment.GetEnvironmentVariable("EXCEPTIONLESS_READ_TOKEN");
+ string? readUser = values.GetValueOrDefault("read-user")
+ ?? Environment.GetEnvironmentVariable("EXCEPTIONLESS_READ_USER");
+ string? readPassword = values.GetValueOrDefault("read-password")
+ ?? Environment.GetEnvironmentVariable("EXCEPTIONLESS_READ_PASSWORD");
+ IReadOnlyList protocols = ParseProtocols(values.GetValueOrDefault("protocol", "both"));
+ LoadEventType eventType = values.GetValueOrDefault("event-type", "error").ToLowerInvariant() switch
+ {
+ "log" => LoadEventType.Log,
+ "error" => LoadEventType.Error,
+ _ => throw new ArgumentException("--event-type must be log or error.")
+ };
+ StackScenario stackScenario = values.GetValueOrDefault("stack-scenario", "hot").ToLowerInvariant() switch
+ {
+ "hot" => StackScenario.Hot,
+ "new" => StackScenario.New,
+ _ => throw new ArgumentException("--stack-scenario must be hot or new.")
+ };
+ int eventCount = GetInt(values, "events", 10_000, 1, 10_000_000);
+ int concurrency = GetInt(values, "concurrency", 4, 1, 1024);
+ int batchSize = GetInt(values, "batch-size", 100, 1, 10_000);
+ int trials = GetInt(values, "trials", 3, 1, 100);
+ int warmupEvents = GetInt(values, "warmup-events", 100, 0, 100_000);
+ int cardinality = GetInt(values, "signature-cardinality", 10, 1, 1_000_000);
+ int discardPercent = GetInt(values, "discard-percent", 0, 0, 100);
+ int expectedPersisted = GetInt(values, "expected-persisted", eventCount - GetDiscardCandidateCount(eventCount, discardPercent), 0, eventCount);
+ if (expectedPersisted > 0 && String.IsNullOrWhiteSpace(readToken) && (String.IsNullOrWhiteSpace(readUser) || String.IsNullOrWhiteSpace(readPassword)))
+ throw new ArgumentException("Set --read-token, or both --read-user and --read-password, so query-visible completion can be measured for both protocols.");
+ if (discardPercent > 0 && eventType is not LoadEventType.Error)
+ throw new ArgumentException("--discard-percent requires --event-type error.");
+ if (discardPercent > 0 && stackScenario is StackScenario.New)
+ throw new ArgumentException("Discard comparisons require --stack-scenario hot so the pre-discarded stack identities remain stable.");
+ int messageBytes = GetInt(values, "message-bytes", 64, 0, EventIngestionV3Limits.MaximumMessageLength);
+ int timeoutSeconds = GetInt(values, "timeout-seconds", 300, 1, 86_400);
+ int pollIntervalMilliseconds = GetInt(values, "poll-interval-ms", 250, 10, 60_000);
+ int completionPollConcurrency = GetInt(values, "completion-poll-concurrency", 4, 1, 64);
+ string compression = values.GetValueOrDefault("compression", "none").ToLowerInvariant();
+ if (compression is not ("none" or "gzip"))
+ throw new ArgumentException("Apples-to-apples comparisons support the encodings common to both APIs: none or gzip.");
+
+ return new LoadOptions(
+ EnsureTrailingSlash(baseUrl),
+ projectId,
+ submissionToken,
+ readToken,
+ readUser,
+ readPassword,
+ protocols,
+ eventType,
+ stackScenario,
+ eventCount,
+ expectedPersisted,
+ concurrency,
+ batchSize,
+ trials,
+ warmupEvents,
+ cardinality,
+ discardPercent,
+ compression,
+ SanitizeSeed(values.GetValueOrDefault("seed", "default")),
+ values.GetValueOrDefault("results"),
+ values.GetValueOrDefault("environment-label"),
+ new string('x', messageBytes),
+ TimeSpan.FromSeconds(timeoutSeconds),
+ TimeSpan.FromMilliseconds(pollIntervalMilliseconds),
+ completionPollConcurrency);
+ }
+
+ public static void WriteUsage()
+ {
+ Console.Error.WriteLine("dotnet run -c Release --project benchmarks/Exceptionless.Ingestion.Load -- --base-url --project-id --protocol v2|v3|both [--submission-token ] [--read-token | --read-user --read-password ] [--events 10000] [--batch-size 1|1000] [--concurrency 4] [--trials 3] [--warmup-events 100] [--event-type log|error] [--stack-scenario hot|new] [--signature-cardinality 10] [--discard-percent 0] [--compression none|gzip] [--completion-poll-concurrency 4] [--results result.json] [--environment-label text]");
+ }
+
+ private static IReadOnlyList ParseProtocols(string value)
+ {
+ return value.ToLowerInvariant() switch
+ {
+ "v2" => [IngestionProtocol.V2],
+ "v3" => [IngestionProtocol.V3],
+ "both" => [IngestionProtocol.V2, IngestionProtocol.V3],
+ _ => throw new ArgumentException("--protocol must be v2, v3, or both.")
+ };
+ }
+
+ private static Uri EnsureTrailingSlash(Uri value)
+ {
+ var builder = new UriBuilder(value);
+ if (!builder.Path.EndsWith('/'))
+ builder.Path += "/";
+ return builder.Uri;
+ }
+
+ private static string SanitizeSeed(string value)
+ {
+ string sanitized = new(value.Where(c => Char.IsAsciiLetterOrDigit(c) || c == '-').Take(24).ToArray());
+ return String.IsNullOrEmpty(sanitized) ? "default" : sanitized;
+ }
+
+ private static int GetInt(IReadOnlyDictionary values, string key, int defaultValue, int minimum, int maximum)
+ {
+ if (!values.TryGetValue(key, out string? raw))
+ return defaultValue;
+ if (!Int32.TryParse(raw, out int value) || value < minimum || value > maximum)
+ throw new ArgumentException($"--{key} must be between {minimum} and {maximum}.");
+ return value;
+ }
+
+ private static int GetDiscardCandidateCount(int eventCount, int discardPercent)
+ {
+ return eventCount / 100 * discardPercent + Math.Min(eventCount % 100, discardPercent);
+ }
+}
diff --git a/benchmarks/Exceptionless.Ingestion.Load/Program.cs b/benchmarks/Exceptionless.Ingestion.Load/Program.cs
new file mode 100644
index 0000000000..73ab6a9845
--- /dev/null
+++ b/benchmarks/Exceptionless.Ingestion.Load/Program.cs
@@ -0,0 +1,34 @@
+namespace Exceptionless.Ingestion.Load;
+
+public static class Program
+{
+ public static async Task Main(string[] args)
+ {
+ LoadOptions options;
+ try
+ {
+ options = LoadOptions.Parse(args);
+ }
+ catch (ArgumentException ex)
+ {
+ Console.Error.WriteLine(ex.Message);
+ LoadOptions.WriteUsage();
+ return 2;
+ }
+
+ try
+ {
+ return await new IngestionLoadRunner(options).RunAsync();
+ }
+ catch (OperationCanceledException)
+ {
+ Console.Error.WriteLine($"The comparison exceeded the {options.Timeout:c} per-run timeout.");
+ return 1;
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine(ex.Message);
+ return 1;
+ }
+ }
+}
diff --git a/benchmarks/Exceptionless.Ingestion.Load/StreamingEventContent.cs b/benchmarks/Exceptionless.Ingestion.Load/StreamingEventContent.cs
new file mode 100644
index 0000000000..ef793f7fd0
--- /dev/null
+++ b/benchmarks/Exceptionless.Ingestion.Load/StreamingEventContent.cs
@@ -0,0 +1,169 @@
+using System.IO.Compression;
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.Ingestion;
+using Exceptionless.Core.Serialization;
+
+namespace Exceptionless.Ingestion.Load;
+
+internal sealed class StreamingEventContent : HttpContent
+{
+ private static readonly byte[] _arrayStart = [(byte)'['];
+ private static readonly byte[] _arrayEnd = [(byte)']'];
+ private static readonly byte[] _comma = [(byte)','];
+ private static readonly byte[] _newline = [(byte)'\n'];
+ private readonly LoadOptions _options;
+ private readonly IngestionProtocol _protocol;
+ private readonly string _runMarker;
+ private readonly string _corpusName;
+ private readonly DateTimeOffset _eventDate;
+ private readonly int _start;
+ private readonly int _count;
+
+ public StreamingEventContent(LoadOptions options, IngestionProtocol protocol, string runMarker, string signatureNamespace, DateTimeOffset eventDate, int start, int count)
+ {
+ _options = options;
+ _protocol = protocol;
+ _runMarker = runMarker;
+ _corpusName = signatureNamespace.Replace("-", String.Empty, StringComparison.Ordinal);
+ _eventDate = eventDate;
+ _start = start;
+ _count = count;
+ Headers.ContentType = new MediaTypeHeaderValue(protocol is IngestionProtocol.V2 ? "application/json" : "application/x-ndjson");
+ if (options.Compression is "gzip")
+ Headers.ContentEncoding.Add(options.Compression);
+ }
+
+ public long UncompressedBytes { get; private set; }
+ public long TransferredBytes { get; private set; }
+
+ protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) => WriteAsync(stream, CancellationToken.None);
+
+ protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context, CancellationToken cancellationToken) => WriteAsync(stream, cancellationToken);
+
+ protected override bool TryComputeLength(out long length)
+ {
+ length = 0;
+ return false;
+ }
+
+ private async Task WriteAsync(Stream output, CancellationToken cancellationToken)
+ {
+ var transferred = new CountingWriteStream(output);
+ await using Stream? compressor = _options.Compression is "gzip"
+ ? new GZipStream(transferred, CompressionLevel.Fastest, leaveOpen: true)
+ : null;
+ var uncompressed = new CountingWriteStream(compressor ?? transferred);
+
+ if (_protocol is IngestionProtocol.V2 && _count > 1)
+ await uncompressed.WriteAsync(_arrayStart, cancellationToken);
+
+ for (int offset = 0; offset < _count; offset++)
+ {
+ if (_protocol is IngestionProtocol.V2 && _count > 1 && offset > 0)
+ await uncompressed.WriteAsync(_comma, cancellationToken);
+
+ int index = _start + offset;
+ string referenceId = $"{_runMarker}-{index:D8}";
+ bool discardedCandidate = index % 100 < _options.DiscardPercent;
+ int signature = index % _options.SignatureCardinality;
+ string signatureKind = discardedCandidate ? "Discarded" : "Active";
+ string? exceptionType = null;
+ string? stackTrace = null;
+ if (_options.EventType is LoadEventType.Error)
+ {
+ exceptionType = $"Load.{_corpusName}.{signatureKind}Exception{signature}";
+ stackTrace = $"at Load.{_corpusName}.{signatureKind}Service{signature}.Run() in /src/Load.cs:line {signature + 1}";
+ }
+
+ if (_protocol is IngestionProtocol.V2)
+ {
+ var source = new V2LoadEvent(
+ _options.EventType is LoadEventType.Error ? Event.KnownTypes.Error : Event.KnownTypes.Log,
+ _eventDate,
+ _options.Message,
+ referenceId,
+ [_runMarker],
+ _options.EventType is LoadEventType.Error
+ ? new V2LoadEventData(new V2SimpleError(exceptionType!, _options.Message, stackTrace!))
+ : null);
+ await JsonSerializer.SerializeAsync(uncompressed, source, LoadJsonContext.Default.V2LoadEvent, cancellationToken);
+ }
+ else
+ {
+ var source = new EventIngestionV3Event
+ {
+ Id = GetV3ClientId(_runMarker, index),
+ Type = _options.EventType is LoadEventType.Error ? Event.KnownTypes.Error : Event.KnownTypes.Log,
+ Date = _eventDate,
+ Message = _options.Message,
+ ReferenceId = referenceId,
+ Tags = [_runMarker],
+ ExceptionType = exceptionType,
+ StackTrace = stackTrace
+ };
+ await JsonSerializer.SerializeAsync(uncompressed, source, EventIngestionJsonContext.Default.EventIngestionV3Event, cancellationToken);
+ await uncompressed.WriteAsync(_newline, cancellationToken);
+ }
+ }
+
+ if (_protocol is IngestionProtocol.V2 && _count > 1)
+ await uncompressed.WriteAsync(_arrayEnd, cancellationToken);
+
+ await uncompressed.FlushAsync(cancellationToken);
+ if (compressor is not null)
+ await compressor.DisposeAsync();
+ await transferred.FlushAsync(cancellationToken);
+ UncompressedBytes = uncompressed.BytesWritten;
+ TransferredBytes = transferred.BytesWritten;
+ }
+
+ internal static string GetV3ClientId(string runMarker, int index) => $"{runMarker}-{index:D8}-v3";
+}
+
+internal sealed record V2LoadEvent(
+ [property: JsonPropertyName("type")] string Type,
+ [property: JsonPropertyName("date")] DateTimeOffset Date,
+ [property: JsonPropertyName("message")] string Message,
+ [property: JsonPropertyName("reference_id")] string ReferenceId,
+ [property: JsonPropertyName("tags")] string[] Tags,
+ [property: JsonPropertyName("data"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] V2LoadEventData? Data);
+
+internal sealed record V2LoadEventData(
+ [property: JsonPropertyName("@simple_error")] V2SimpleError SimpleError);
+
+internal sealed record V2SimpleError(
+ [property: JsonPropertyName("type")] string Type,
+ [property: JsonPropertyName("message")] string Message,
+ [property: JsonPropertyName("stack_trace")] string StackTrace);
+
+[JsonSerializable(typeof(V2LoadEvent))]
+internal sealed partial class LoadJsonContext : JsonSerializerContext;
+
+internal sealed class CountingWriteStream(Stream inner) : Stream
+{
+ public long BytesWritten { get; private set; }
+ public override bool CanRead => false;
+ public override bool CanSeek => false;
+ public override bool CanWrite => true;
+ public override long Length => throw new NotSupportedException();
+ public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
+ public override void Flush() => inner.Flush();
+ public override Task FlushAsync(CancellationToken cancellationToken) => inner.FlushAsync(cancellationToken);
+ public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+ public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+ public override void SetLength(long value) => throw new NotSupportedException();
+ public override void Write(byte[] buffer, int offset, int count)
+ {
+ inner.Write(buffer, offset, count);
+ BytesWritten += count;
+ }
+ public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default)
+ {
+ await inner.WriteAsync(buffer, cancellationToken);
+ BytesWritten += buffer.Length;
+ }
+}
diff --git a/benchmarks/README.md b/benchmarks/README.md
new file mode 100644
index 0000000000..58862409c8
--- /dev/null
+++ b/benchmarks/README.md
@@ -0,0 +1,141 @@
+# Event ingestion benchmarks
+
+`Exceptionless.Benchmarks` contains allocation-aware microbenchmarks for the V2
+parser core (UTF-8 bytes to string, JSON-shape detection, and
+`PersistentEvent`/`PersistentEvent[]` deserialization) and the exact production
+V3 `EventIngestionV3StreamReader`. The V3 results separately measure bounded
+NDJSON framing plus the routing-only projection used by early discard, and the
+same path followed by full DTO materialization for survivors. These parsing
+microbenchmarks do not include HTTP, decompression, validation, routing I/O,
+stack creation, persistence, queues, or side effects; they cannot support an
+end-to-end throughput or cost claim by themselves.
+
+Run a short comparison with:
+
+```bash
+dotnet run -c Release --project benchmarks/Exceptionless.Benchmarks -- --job short
+```
+
+## Apples-to-apples V2 and V3 load comparison
+
+`Exceptionless.Ingestion.Load` submits the same conceptual event corpus through
+the real V2 and V3 HTTP endpoints. It records four boundaries:
+
+- **Submission** starts before the first request and stops when the final
+ successful response headers arrive. V2's response is only a `202 Accepted`;
+ V3's response contains terminal primary-write counts.
+- **Observed full processing** is the common semantic terminal boundary, but its
+ benchmark instrumentation is intentionally visible in every result. V2
+ requests opt into tracking, capture the queue-entry correlation id returned in
+ `X-Exceptionless-Event-Post-Id`, and follow any per-event child retries through
+ the legacy pipeline and final queue/archive cleanup. V3 polls the existing
+ final `notifications` side-effect marker for every persisted survivor, after
+ archive, enrichment, statistics, and notification/webhook enqueueing have
+ succeeded. A 100%-discard V3 run has no survivor side effects, so its durable
+ acknowledgement is also its full processing boundary.
+- **V3 durable acknowledgement** is V3-only. It is the successful response after
+ the primary event and durable side-effect intent are written. V2 has no
+ equivalent client-visible durable-event acknowledgement.
+- **Query visibility** uses the same project event-count query and run tag for
+ both protocols. It is reported as persisted survivors per second and includes
+ the read/index visibility delay. It is `n/a` for a 100%-discard corpus because
+ there are intentionally no documents to query.
+
+The V2 request path returns the existing enqueue id and performs no synchronous
+tracking-cache I/O, so the submission boundary does not include a tracking-cache
+write. Its worker does perform opt-in Redis tracking writes while processing.
+V3 adds no completion writes: the status endpoint reads the final idempotency
+markers that normal side-effect execution already records. Thus the two
+full-processing observers establish the same terminal meaning but do not have
+equal instrumentation cost: V2 tracks one identifier per HTTP request while V3
+tracks one identifier per persisted event. The harness limits status polling to
+`--completion-poll-concurrency` (default 4), removes completed 1,000-identifier
+chunks from later sweeps, and records tracked identifiers, status requests,
+identifier reads, and sweeps in every trial. A V3 result with many events per
+request can still have substantially more observer work and observation lag than
+V2; do not attribute an observed-full-processing difference to pipeline
+efficiency without these counters and server telemetry.
+
+The completion endpoints are benchmark-oriented, require
+`EventIngestionV3:EnableProcessingStatus=true`, and markers expire with
+`EventIngestionV3:IdempotencyWindow`, so polling must occur as part of the run.
+Submission, observed full processing, and query visibility are reported on the
+same corpus. Submission and query visibility have the closest apples-to-apples
+observers. Observed full processing is a useful terminal semantic check with the
+instrumentation caveat above; V3 durable acknowledgement is reported separately
+because V2 has no equivalent contract.
+
+Run separate one-event and large-batch scenarios. `--batch-size 1` sends one JSON
+object per V2 request and one NDJSON line per V3 request. A larger batch sends
+one V2 JSON array or a V3 stream with exactly one object per NDJSON line. The
+load client writes both forms incrementally and does not construct a giant
+in-memory array.
+
+Each generated event has a unique `reference_id`, matching production duplicate
+detection semantics. A unique tag shared by the run is used only to poll durable
+query visibility.
+
+```bash
+export EXCEPTIONLESS_API_KEY=...
+export EXCEPTIONLESS_PROJECT_ID=...
+export EXCEPTIONLESS_READ_TOKEN=...
+
+# One event per request.
+dotnet run -c Release --project benchmarks/Exceptionless.Ingestion.Load -- \
+ --base-url https://api-ex.dev.localhost:7111/ --protocol both \
+ --events 10000 --batch-size 1 --concurrency 32 --trials 5 \
+ --event-type error --stack-scenario hot --signature-cardinality 100 \
+ --compression gzip --seed single-error \
+ --results artifacts/ingestion-single-error.json \
+ --environment-label "1 API; 1 job; local Elasticsearch and Redis"
+
+# Large real-world client batches.
+dotnet run -c Release --project benchmarks/Exceptionless.Ingestion.Load -- \
+ --base-url https://api-ex.dev.localhost:7111/ --protocol both \
+ --events 100000 --batch-size 1000 --concurrency 8 --trials 5 \
+ --event-type error --stack-scenario hot --signature-cardinality 100 \
+ --compression gzip --seed batch-error \
+ --results artifacts/ingestion-batch-error.json \
+ --environment-label "1 API; 1 job; local Elasticsearch and Redis"
+```
+
+Instead of `EXCEPTIONLESS_READ_TOKEN`, a local comparison can use
+`--read-user --read-password `. The read credential is needed
+only to poll the project count endpoint and is never printed.
+
+The default warmup sends 100 events through each selected protocol and waits for
+observed full processing before measurement; odd trials reverse the protocol
+order. `--stack-scenario hot` uses stable, protocol-specific signature
+namespaces for warmup and every trial.
+`--stack-scenario new` uses a fresh namespace for each protocol and trial while
+still allowing a separate warmup namespace to warm JIT, HTTP, and datastore
+paths. This prevents one protocol from creating or warming the other protocol's
+measured stacks. Use `--event-type log` to isolate general pipeline overhead and
+`--event-type error` to exercise real stack processing. `none` and `gzip` are
+the encodings common to both APIs and therefore the only comparison choices.
+
+Discard comparisons require `--stack-scenario hot`. First run both protocols
+with the intended seed and discard percentage but override
+`--expected-persisted` to the full event count, then mark the generated
+`Load.hot.DiscardedException*` stacks discarded. Run the measured
+comparison with the same seed and omit `--expected-persisted`; the harness
+derives it from `--discard-percent`. V3 reports terminal outcome counts, while
+the observed-full-processing boundary proves that discard-only requests finished
+without waiting for markers that intentionally do not exist for discarded
+events.
+
+Every result includes request count, payload bytes, common submission and
+observed-full-processing events/second, query-visible persisted events/second,
+request latency p50/p95/p99, completion-observer counters, query poll count, V3
+durable acknowledgement, and V3 terminal counts. Pass
+`--results artifacts/ingestion-v3.json` to write a machine-readable, secret-free
+artifact containing every trial, sanitized configuration, runtime/OS/CPU
+metadata, and the build informational version. Use `--environment-label` for
+the Elasticsearch/Redis topology and API/job instance counts. Publish the JSON
+artifact with the exact commit and do not report a comparison from warmup output
+or from a single trial.
+
+The load client records payload bytes and client-observed timings, not server
+CPU, allocation rate, GC collections, Redis/Elasticsearch operation counts, or
+cost. Collect those from the API, job, Redis, and Elasticsearch telemetry for
+the same run before making efficiency or cost-per-million claims.
diff --git a/docs/event-ingestion-v3-architecture.md b/docs/event-ingestion-v3-architecture.md
new file mode 100644
index 0000000000..1b7a49692f
--- /dev/null
+++ b/docs/event-ingestion-v3-architecture.md
@@ -0,0 +1,514 @@
+# Event ingestion V3 architecture
+
+This document records the implementation decisions and performance baseline
+methodology for [issue #2368](https://github.com/exceptionless/Exceptionless/issues/2368).
+It is deliberately separate from the public API documentation until the V3
+contract is ready for rollout.
+
+The companion [client protocol and ecosystem plan](/event-ingestion-v3-client-protocol/)
+defines the minimum sender, capability profiles, Sentry and Raygun comparison,
+and compatibility rules for future event and non-event telemetry.
+
+## Objectives
+
+Event ingestion V3 is a breaking, performance-first API with these invariants:
+
+- ASP.NET Core Minimal APIs own the HTTP boundary. V3 does not use an MVC
+ controller or MVC input formatter.
+- Clients send independent JSON objects as they capture events. Each nonblank
+ NDJSON line contains exactly one object; adjacent objects on one line are
+ invalid, and only the final line may omit its newline. Clients never construct
+ a top-level JSON array.
+- The request body is consumed incrementally from a `PipeReader` with
+ source-generated System.Text.Json metadata.
+- The API processes bounded microbatches inline. A successful response means
+ that every acknowledged event is durable or reached another terminal success
+ state such as discarded or duplicate.
+- The primary event path does not write the request to object storage and does
+ not enqueue it for another process to deserialize.
+- Clients send raw stack traces. The server computes the grouping fingerprint
+ and builds the persisted structured error.
+- A new language can implement the core sender using only its standard HTTPS
+ and JSON facilities. Advanced reliability and framework features are
+ independently testable capability profiles, not ingestion prerequisites.
+- Events assigned to discarded stacks terminate before billable admission,
+ full error materialization, event persistence, stack statistics, or side
+ effects.
+- V3 correctness is scale-out safe and never depends on sticky sessions or
+ process-local authoritative state.
+- V2 remains contract-compatible. V3 does not pay a compatibility tax to share
+ an implementation with V2.
+
+## Current V2 flow
+
+The V2 path is intentionally durable before processing, but that durability
+requires the payload to cross several boundaries:
+
+1. `EventController.PostAsync` resolves the project and wraps the request body
+ with `EventPostRequestBodyStream`.
+2. `EventPostService.SaveAndEnqueueAsync` writes the payload to file storage and
+ enqueues an `EventPost` pointer.
+3. `EventPostsJob` downloads the complete payload into a byte array.
+4. Compressed payloads are expanded into a second complete byte array.
+5. The job parses the complete body into a collection of `PersistentEvent`
+ objects.
+6. `EventPipeline` runs a reflection-discovered action list.
+7. `PipelineBase` creates a filtered context list for every action.
+8. `AssignToStackAction` resolves or creates stacks and only then detects the
+ discarded status.
+9. `SaveEventAction` persists events, followed by statistics, notifications,
+ counters, and processed plugins.
+
+Plan admission occurs before stack assignment in both `OverageMiddleware` and
+`EventPostsJob`. A request can therefore be rejected before the system learns
+that its events belong to a free discarded stack.
+
+## V3 processing boundary
+
+The V3 endpoint is a transport adapter over an explicit, batch-oriented Core
+processor. For each bounded microbatch, the processor performs these steps in
+order:
+
+1. Validate only the small identity and grouping envelope.
+2. Compute canonical stack fingerprints.
+3. Resolve unique stack routes in bulk.
+4. Terminate known-discarded events and record an aggregate discard count.
+5. Validate optional context for survivors and detect durable duplicates.
+6. Apply retention and fixed-version nonbilling rules to new events.
+7. Reserve billable quota for remaining new events only.
+8. Materialize complete events and structured errors.
+9. Resolve or create missing stacks.
+10. Persist events and durable side-effect intent in bulk.
+11. Commit or release billable reservations.
+
+Each API replica applies two independent concurrency boundaries. A relatively
+high active-stream guard bounds open HTTP requests for the replica and for each
+organization. An idle stream holds only that stream permit; it does not consume
+scarce processing capacity. When a microbatch is ready, the endpoint stops
+reading and waits for the lower per-organization and global processing limits
+before it calls the Core processor. This bounds memory and lets HTTP/TCP flow
+control apply backpressure without letting slow producers starve ready work.
+Active-stream admission is deliberately two-stage: a global permit is acquired
+before the raw request-body limit is relaxed, then the endpoint acquires an
+organization permit keyed by the organization of the authorized, routed project.
+The second stage does not consume another global permit, and explicit-project
+requests are never attributed to an unrelated default organization.
+
+Both processing queues are bounded and ordered oldest first, with the
+organization boundary applied before global admission. A full queue returns
+`429`; if earlier microbatches in the same request are already durable, the
+normal problem response includes their partial result and replay guidance.
+Reading and processing may overlap through a channel with capacity one only if
+benchmarks demonstrate a material benefit.
+
+## Decision: inline acknowledgement
+
+V3 acknowledges after the primary event write, not after a queue write. This
+removes file-storage I/O, queue I/O, worker scheduling, a payload download, and
+a second deserialization from the normal path.
+
+If durable storage is unavailable, V3 fails quickly with a retryable response.
+It does not enqueue after processing begins because a partially successful
+write makes that fallback ambiguous. A separate, explicit buffered mode is
+required if the product must acknowledge while the event store is unavailable.
+
+## Decision: bounded stream segments
+
+An indefinitely open upload cannot provide a reliable acknowledgement boundary.
+Official clients therefore stream events immediately into a request segment and
+close the segment when a configured count, byte, or elapsed-time limit is
+reached. The client retains only the unacknowledged segment and replays it after
+a connection loss or retryable response.
+
+This is streaming rather than array batching: each event is serialized once,
+written immediately as one NDJSON line, and followed by a newline before the
+next event. The final newline is optional; concatenating adjacent JSON objects
+without a line delimiter is invalid.
+
+## Decision: bounded idempotency with date-routable identities
+
+Every V3 event carries a stable client event identifier. The server derives a
+date-routable storage identity from the project, client identifier, and supplied
+event date, then uses create-only persistence semantics. This normal SDK path is
+stateless and adds no per-event cache write. When `date` is omitted (or must be
+clamped because it is in the future), the server instead atomically claims the
+identity in a distributed mapping. Each mapping expires independently at the
+configured idempotency window, keeping storage bounded, and preserves the exact
+canonical event date and first receipt time so a retry across midnight cannot
+route the write and lookup to different indexes. Benchmark processing-status
+mode also claims mappings because its read-only endpoint accepts client ids,
+not complete events. A duplicate create is a terminal success.
+
+If a response is lost after persistence, the client replays the complete segment
+with the same identifiers. Previously stored events become duplicate successes;
+new events continue normally. Side-effect intent uses the same deterministic
+identity. Durable queue markers suppress normal notification and webhook replay;
+delivery remains at-least-once in the narrow case where a queue accepts an item
+but its deduplication marker cannot be recorded.
+
+## Decision: discarded-stack fast path
+
+Discard classification requires the same canonical fingerprint used by stack
+assignment. V3 computes that fingerprint from a minimal event representation
+before building the complete `PersistentEvent`, `Error`, and `StackFrame`
+object graph.
+
+Stack routing uses a lightweight value containing stack identifier, status, and
+a cache schema version. Resolution is deduplicated within the microbatch, read
+from the distributed cache in bulk, and falls back to a projected repository
+query for cache misses. Cold fills and authoritative mutations take one
+project-generation lock and compare/write all affected entries in bulk, avoiding
+a distributed lock and cache command chain for every distinct signature while
+still preventing an older repository result from overwriting a newer status.
+
+The distributed cache is authoritative initially. A process-local positive
+discard cache is unsafe without proven versioning and invalidation because a
+stale entry would silently discard events after a stack is reopened.
+
+A discarded event performs none of the following:
+
+- Billable quota reservation.
+- Full error or stack-frame materialization.
+- Event indexing.
+- Stack occurrence updates.
+- Notification, webhook, archive, enrichment, or outbox writes.
+
+Discarded events still count toward protective byte, request, and event-rate
+limits. Those controls prevent abuse and are not customer billing.
+
+## Decision: quota reservation and settlement
+
+The existing read-then-process usage check is not an adequate scale-out
+reservation primitive. V3 separates protective admission from billable usage
+and uses three logical billable operations:
+
+- Reserve capacity for non-discarded survivors.
+- Commit capacity for successfully persisted events.
+- Release capacity for invalid events, duplicates, cancellations, or failed
+ writes.
+
+The distributed operation must be atomic across API instances. A customer at
+the billable limit can still submit an event assigned to an already-discarded
+stack, while active and new-stack events are blocked.
+
+Availability calculation and the Redis lease are serialized by a short
+distributed lock per organization. This prevents a delayed caller from holding
+an old availability snapshot until an earlier request has committed usage and
+released its lease. Within that decision, Redis admission uses one server-side
+atomic operation per microbatch: it purges a bounded number of expired leases,
+computes remaining capacity, and records the new lease without scanning active
+reservations. Active leases remain organization-scoped across bucket and finite
+plan-limit changes until commit, release, or expiry. This can conservatively
+delay admission for work crossing a boundary, but it prevents that in-flight
+work from disappearing and reusing the same monthly capacity. Unrelated
+organizations and replicas continue concurrently; the in-memory implementation
+provides the same lease semantics for single-process development and tests.
+The leases coordinate V3 admissions. Legacy V2 workers retain their existing
+read-then-process quota check during rollout, so a mixed V2/V3 deployment also
+retains V2's pre-existing concurrent oversubscription window until that worker
+is deliberately migrated to the reservation primitive.
+
+Billable settlement is deliberately at-most-once. The batch writer that gets a
+successful result from the create-only event write owns the settlement; a
+persisted duplicate only repairs idempotent side effects and is never charged
+again. Usage is added to scalar five-minute organization and project counters,
+so admission reads and settlement writes remain O(1) and Redis does not retain
+one key or set member per event.
+
+An ambiguous Elasticsearch bulk failure cannot prove which racing request
+created a document. V3 therefore skips settlement for every ambiguous result
+rather than risk charging twice. This narrow crash window can undercount but
+cannot overcharge. `ex.ingestion.v3.usage.committed` and
+`ex.ingestion.v3.usage.ambiguous_skipped` expose the tradeoff; a future offline
+Elasticsearch reconciliation job may repair undercounts without adding a
+per-event ledger to the ingestion hot path.
+
+The periodic usage saver acknowledges organization and project discovery ids
+individually. It persists the usage document before recording a processed
+bucket marker, then removes the scalar counters and finally removes that one
+discovery id. A failure leaves the failed id and every unvisited id available
+for retry. If persistence and the marker succeeded but cleanup failed, the
+retry observes the marker and completes cleanup without applying the bucket a
+second time. The last applied bucket is also persisted atomically with the
+organization or project totals. A process failure after the repository commit
+but before the Redis marker write therefore retries only the Redis
+acknowledgement and cleanup; it cannot apply that bucket twice. Legacy documents
+without the field apply their first pending bucket normally and establish the
+durable marker on that save.
+
+## Decision: side-effect durability
+
+The primary event is authoritative. Notifications, webhooks, archive export,
+nonessential geolocation, and repairable derived statistics remain outside the
+request critical path.
+
+The inline writer persists deterministic side-effect or repair intent before
+acknowledgement. Derived statistics and internal stages are idempotent;
+notification and webhook enqueueing uses durable duplicate suppression while
+retaining at-least-once failure semantics. Partial bulk outcomes are reconciled
+by deterministic identifiers rather than by placing the original event payload
+back on the ingestion queue.
+
+The retry-state budget is explicit: statistics and terminal notification
+completion share one integer bitmask key per persisted event for the configured
+idempotency window. Stack statistics use one project-partitioned atomic Redis
+settlement per microbatch: the script claims previously unseen event identities,
+updates every affected stack aggregate, and sets the statistics bits together.
+Projects hash independently across Redis Cluster slots. Overlapping or
+retried batches therefore cannot increment an occurrence twice, and the handler
+does not issue a serial command chain per stack. Notification and webhook queue
+markers exist only for events that actually trigger those effects. Thus one
+million persisted events retain one million expiring side-effect state keys, not
+one key per stage. Rollout measurements must include Redis bytes per state key,
+script duration, lock commands, and key-expiration churn; a compact sharded-hash
+store is the next design step if that measured footprint misses the
+cost-per-million target.
+
+## V2 compatibility boundary
+
+V2 retains its routes, payloads, 202 response, controller, and durable queue.
+Shared services are extracted below the transport boundary:
+
+- Canonical fingerprinting.
+- Stack route resolution.
+- Server-side stack parsing.
+- Batch stack resolution and creation.
+- Batch event persistence.
+- Usage accounting primitives.
+
+The V3 request and acknowledgement path must not instantiate V2 queue models,
+plugin contexts, or compatibility converters. The asynchronous side-effect
+worker may temporarily adapt persisted events to the existing enrichment and
+notification actions while those actions are extracted into transport-neutral
+services; it does not run the legacy preprocessing or post-processing plugin
+pipeline. V2 may adopt a shared service only when its compatibility fixtures
+remain unchanged and the change is neutral or positive in measurements.
+
+## Baseline methodology
+
+All comparisons use the same commit, machine, runtime configuration,
+Elasticsearch topology, Redis topology, payload corpus, compression setting,
+and client concurrency. Record the following environment information with every
+result:
+
+- Commit SHA and configuration.
+- .NET SDK and runtime versions.
+- Operating system and processor topology.
+- Elasticsearch and Redis versions/resources.
+- Number of API and job instances.
+- Payload size distribution and stack-signature cardinality.
+- Client concurrency and connection reuse.
+
+Measure V2 and V3 separately for these scenarios:
+
+- One hot active stack.
+- High-cardinality active stacks.
+- New stacks.
+- 0%, 10%, 50%, 90%, and 100% discarded events.
+- Small, median, large, and maximum events.
+- Compressed and uncompressed requests.
+- Healthy, slow, and unavailable Elasticsearch.
+- One, two, four, and eight API instances.
+
+Capture:
+
+- Events per second and bytes per second.
+- CPU time per event.
+- Allocated bytes per event and Gen 0/1/2 collections.
+- Working set and managed heap under a long stream.
+- p50, p95, and p99 time from final event byte to durable acknowledgement.
+- File-storage, queue, cache, and Elasticsearch operations per event.
+- Stack-route cache hit and miss rates.
+- Cost per million persisted and discarded events.
+
+## Performance gates
+
+- No request-sized byte array, string, `MemoryStream`, `JsonDocument`, or
+ top-level event collection on V3.
+- No large-object-heap allocation caused by normal stream framing or microbatch
+ growth.
+- Memory remains bounded by the active-stream guard and microbatch limits during
+ long or slow streams. Idle streams do not reserve processing permits.
+- A discarded event creates no complete error/frame graph and performs no event
+ index write.
+- At most one distributed route lookup per unique signature per microbatch.
+- The normal V3 path performs no object-storage payload write/read and no
+ primary ingestion queue operation.
+- Scaling from one to four API instances reaches at least 80% efficiency until
+ the shared datastore is the measured bottleneck.
+- Rollout evidence must demonstrate lower V3 CPU, allocation rate, and
+ time-to-durable-event relative to V2 before those improvements are claimed.
+ Numeric reduction targets are set from the repeatable baseline rather than an
+ unmeasured estimate.
+
+## Verification requirements
+
+Completion requires all of the following evidence:
+
+- Unit tests for contracts, fingerprinting, parsing, routing, quota settlement,
+ idempotency, and result aggregation.
+- Integration tests for chunked NDJSON, compression, authentication, limits,
+ discard behavior, persistence, replay, and failure responses.
+- Multi-instance tests for route status changes and quota reservation.
+- V2 serialization audit results before and after shared-service adoption.
+- OpenAPI V3 baseline and HTTP samples.
+- Language-neutral client specification, golden fixtures, black-box conformance
+ tests, and timed new-language implementation evidence meeting the client
+ effort gates.
+- Microbenchmark reports and end-to-end load results for the scenario matrix.
+- A staged rollout through the internal project and selected projects with a
+ documented rollback switch.
+
+## Implemented API and client behavior
+
+V3 is exposed through `POST /api/v3/events` and
+`POST /api/v3/projects/{projectId}/events`. Both are Minimal API routes and
+require a client-scoped bearer token. The request content type is
+`application/x-ndjson`; each nonblank line contains exactly one complete event
+object. Lines may span arbitrary network writes, CRLF is accepted, and the final
+newline is optional. Adjacent objects on one line and top-level arrays are
+rejected. The routes accept identity, gzip, and Brotli content encoding.
+
+Clients must generate one stable `id` per captured event and reuse it whenever
+the containing segment is replayed. On first receipt, the server atomically
+derives a project-scoped, date-routable persisted identifier from `id` and the
+original event `date`. This path is stateless, so clients must retain both fields
+unchanged for retries. When `date` is omitted, a bounded distributed mapping
+preserves the first receipt time for the configured idempotency window,
+including retries that cross a UTC day boundary. `reference_id` remains a
+separate business identifier and is not transport idempotency.
+
+Clients write events to the HTTP request as they occur, but close and reopen the
+segment at a bounded count, byte size, elapsed time, flush, or shutdown
+boundary. They retain only the unacknowledged segment. A connection loss, 5xx,
+or timeout replays the complete segment with the same event ids. A 200 response
+returns terminal counts for `persisted`, `discarded`, `duplicate`, `blocked`,
+and `invalid`. Earlier microbatches may already be durable when a later part of
+the segment produces a request-level failure; replay remains safe.
+
+Error clients send `exception_type` and the original `stack_trace`. The server
+parses .NET, Java, JavaScript, and Python frame forms, preserves caused-by
+chains, and stores structured frames. Unsupported formats use a normalized,
+hashed raw-trace fallback and increment a parser-fallback metric without logging
+the trace. Optional `stacking.signature_data` and `stacking.title` provide
+explicit manual stacking. Manual signature data is canonicalized by ordinal key
+and hashes an unambiguous encoding of both keys and values, making grouping
+independent of JSON property order across client languages.
+
+Only `id` and `type` are required. Maintained SDKs should also identify
+themselves with `client.name` and `client.version`; applications can send
+first-class `version` and `level` values without knowing internal event data
+keys. Unknown event properties are ignored so additions remain compatible.
+Top-level custom `data` keys beginning with `@`, plus legacy `sessionend` and
+`haserror` keys, are reserved and rejected. This prevents custom JSON from
+bypassing the typed request redaction and project PII policy or overriding
+first-class grouping and metadata. Runtime validation uses the same durable
+limits advertised by OpenAPI: 2,000-character messages, 100-character tags,
+1,000-character manual stack titles, and reference identifiers containing 8-100
+letters, digits, or hyphens.
+V3 initially accepts error, log, usage, and custom types. It explicitly rejects
+the legacy-only 404 and session lifecycle types until their stateful V2
+preprocessing has a native V3 design, avoiding silent semantic drift.
+
+`/api/v3/events` remains an event-only NDJSON stream. It will not acquire a
+per-line kind/payload wrapper or a stream header. A future optional,
+length-delimited advanced transport can carry the exact same event JSON beside
+attachments, minidumps, profiles, or other typed items. This keeps the minimum
+client smaller than Sentry's envelope implementation without forcing binary or
+heterogeneous data into the event object.
+
+The executable reference client and repeatable load harness live in
+`benchmarks/Exceptionless.Ingestion.Load`; usage is documented in
+`benchmarks/README.md`. It submits the same conceptual corpus through the real
+V2 and V3 endpoints and records four explicitly named boundaries: submission
+response, observed full processing, V3 durable acknowledgement, and query
+visibility of persisted survivors. Observed full processing follows V2 through
+its legacy pipeline, correlated child retries, and final queue/archive cleanup.
+For V3 it waits for the existing final `notifications` marker, which is recorded
+only after archive, enrichment, statistics, and notification/webhook enqueueing
+have succeeded. Query visibility remains a second common downstream comparison
+for persisted survivors.
+
+V2 tracking uses per-post correlation rather than global queue depth or sentinel
+events, so mixed and 100%-discard runs are measurable. The controller returns
+the existing V2 queue entry id without synchronous tracking-cache writes; the
+worker lazily creates and updates the opt-in status while processing. V3 status
+resolves persisted ids from the existing distributed idempotency mapping and
+only reads markers that normal side-effect idempotency already writes, adding no
+ingestion write or extending the mapping lifetime. These read-only status
+contracts are benchmark-oriented and are disabled unless
+`EventIngestionV3:EnableProcessingStatus` is explicitly enabled; this prevents
+untrusted V2 clients from creating tracking keys in normal deployments. V3
+markers expire with the configured idempotency window. The status routes are
+also excluded from public OpenAPI documents so generated SDKs do not treat
+benchmark instrumentation as product API. Warmup also waits for full processing
+so its side-effect backlog cannot contaminate a measured trial. Both protocols
+use the same read-count query. Status polling is bounded, completed chunks are
+not polled again, and each trial records the exact observer load. The observer
+cardinality is nevertheless different—V2 has one status identifier per request
+and V3 has one per persisted event—so observed full-processing time is a
+conservative terminal check, not an unqualified measure of pipeline CPU or cost.
+Submission and query visibility have the closest apples-to-apples observers;
+efficiency claims additionally require correlated server, Redis, and
+Elasticsearch telemetry.
+The harness supports one-event requests and large client batches, isolates hot
+and new stack identities by protocol, alternates protocol order, reports median
+multi-trial results, and can emit a secret-free JSON evidence artifact.
+Request examples live in `tests/http/events-v3.http`, and the isolated OpenAPI
+document is available at `/docs/v3/openapi.json`.
+
+## Configuration and rollout
+
+`EventIngestionV3:Enabled` defaults to `false`. Rollout can be constrained with
+`AllowedProjectIds` and `AllowedOrganizationIds`; empty sets allow every
+authenticated project. Other controls are:
+
+- `MicroBatchSize`, `MaximumMicroBatchBytes`, and `MaximumEventSize`.
+- `MaximumCompressedBodySize`, `MaximumDecompressedBodySize`, and
+ `MaximumEventsPerRequest`.
+- `MaximumActiveStreams` and `MaximumActiveStreamsPerOrganization` bound open
+ streaming requests. Their defaults are the greater of eight times the
+ corresponding processing limit or 128 globally and 32 per organization.
+ `ActiveStreamQueueLimit` and `ActiveStreamQueueLimitPerOrganization` are
+ configurable and default to zero so excess slow connections fail promptly.
+- `MaximumConcurrentRequests` and `ConcurrencyQueueLimit` bound simultaneous
+ microbatch processor calls globally. `MaximumConcurrentRequestsPerOrganization`
+ and `ConcurrencyQueueLimitPerOrganization` provide fair organization admission.
+ These processing permits are acquired only while a full microbatch executes.
+- `MaximumStackCreationConcurrency`.
+- `RequestTimeout` and `IdempotencyWindow`.
+- `EnableProcessingStatus` for opt-in benchmark completion endpoints and V2 tracking.
+- `StackRouteCacheDuration` and `NegativeStackRouteCacheDuration`.
+
+All stream and processing concurrency limits are per API replica, not cluster
+wide. Scaling out therefore adds API-side processing capacity with each replica
+while retaining bounded request and queued work on every node; Redis,
+Elasticsearch, and downstream workers remain shared bottlenecks that must be
+verified with the multi-instance performance gate above.
+
+Start with the internal Exceptionless project in the allowlist, compare V2 and
+V3 stack signatures, terminal counts, usage, route-cache metrics, allocation
+profiles, and datastore saturation, then expand the allowlists gradually.
+Rollback is immediate: disable `EventIngestionV3:Enabled` and have clients use
+the unchanged V2 endpoint. V2 retains its controller, queued payload handoff,
+payload formats, and 202 response. Shared distributed quota primitives and
+stack-route cache lifecycle improvements do not change that contract.
+
+## Durable side effects
+
+The primary event and deterministic side-effect work item are required before
+acknowledgement. The background handler performs stack-usage repair,
+request/environment/geolocation enrichment, notification and webhook enqueueing,
+and deterministic archive export when archival is enabled. Work-item,
+notification, and webhook identities are stable for the configured idempotency
+window. Stack-statistics completion and aggregation are one atomic operation;
+there is no interval in which the count has changed but its event identity is
+still unclaimed. Other explicit idempotency stages acquire distributed
+per-event locks, recheck completion after acquiring them, run the pending
+effects, and record completion only after the stage succeeds. A failed or
+abandoned worker cannot poison a retry with a premature claim. Externally
+visible notification delivery remains at least once and its handlers must be
+idempotent. V3 deliberately does not run the legacy post-processing plugin
+pipeline: those plugins were written against V2 semantics and are not a safe
+compatibility boundary for the new materializer. Retries repair pending work
+from the authoritative event documents.
diff --git a/docs/event-ingestion-v3-client-protocol.md b/docs/event-ingestion-v3-client-protocol.md
new file mode 100644
index 0000000000..717830eade
--- /dev/null
+++ b/docs/event-ingestion-v3-client-protocol.md
@@ -0,0 +1,411 @@
+# Event ingestion V3 client protocol and ecosystem plan
+
+This document defines the client-authoring contract for event ingestion V3. It
+also records the Sentry and Raygun research that shaped the protocol. The
+primary design metric is not only server throughput: a competent developer in a
+new language must be able to build a correct basic sender without copying an
+existing Exceptionless SDK or implementing an observability framework.
+
+## Success criteria
+
+A basic client must need only standard platform facilities for HTTPS, JSON,
+UTF-8, time, and random identifiers. It must not need to:
+
+- Parse a stack trace into frames.
+- Construct an error/inner-error object graph.
+- Implement an envelope or multipart grammar.
+- Buffer a JSON array before sending.
+- Download project configuration before its first event.
+- Implement tracing, scopes, breadcrumbs, sampling, symbolication, offline
+ storage, or framework integrations before it can report an error.
+- Know Exceptionless persistence data keys such as `@error` or `@environment`.
+
+The minimum useful error event is therefore:
+
+```json
+{"id":"018f5f5e-8f6d-7a30-bf5b-9a10b0c4d6e7","type":"error","message":"checkout failed","exception_type":"PaymentError","stack_trace":"PaymentError: checkout failed\n at charge (/app/payments.js:42:7)"}
+```
+
+The client writes a newline before writing the next event. Every nonblank line
+contains exactly one JSON object; adjacent objects on the same line are invalid.
+The final event may omit its trailing newline. The server owns stack parsing,
+canonical grouping, discarded-stack detection, quota admission, and persistence.
+
+Client effort is a release gate. Before the public contract is frozen, run a
+timed implementation exercise with developers who have not worked on V3:
+
+- First accepted log from the written specification in 30 minutes or less.
+- First accepted handled error with the native raw stack in 60 minutes or less.
+- A Profile 0 reference sender in 200 non-generated source lines or fewer in a
+ language with a standard HTTP and JSON library, excluding tests and examples.
+- No third-party runtime dependency required for Profile 0.
+- No Exceptionless-specific concept beyond endpoint, token, event, segment, and
+ terminal result required for Profile 0.
+- The same event serializer is reused unchanged when reliable buffering or the
+ future advanced transport is added.
+
+Failure of one of these gates is a protocol usability defect, not merely a
+documentation issue.
+
+## Minimum implementation surface
+
+A dependency-free client has five mandatory responsibilities:
+
+1. Accept a server URL and project client token.
+2. Generate a stable event `id` and preserve it for retries.
+3. Serialize an object containing `id` and `type`; error clients add the raw
+ `stack_trace` and normally `message` and `exception_type`.
+4. POST UTF-8 JSON objects with exactly one object per nonblank line and
+ `Content-Type: application/x-ndjson` and `Authorization: Bearer `.
+5. Close the bounded request segment, read the terminal response, and replay
+ the same ids after a timeout, connection failure, or retryable HTTP status.
+
+No batching abstraction is required. A first implementation may open a request
+for one event at a time. A production client can reuse a connection and keep a
+request segment open while events arrive, closing it on the first configured
+count, byte, age, flush, or shutdown limit.
+
+The API defaults omitted `date` to server receipt time. This keeps the first
+sender small, although production clients should send the capture time. The API
+does not require `client`, `version`, `level`, request, environment, user, or
+custom data.
+
+## Capability profiles
+
+The ecosystem should describe clients by capabilities instead of treating a
+large official SDK as the minimum viable implementation.
+
+### Profile 0: core sender
+
+- Required fields, raw stack traces, token authentication, and NDJSON.
+- One-event requests are valid.
+- Stable ids and safe whole-segment replay.
+- Terminal response handling and bounded event size.
+
+This profile is enough for a community integration to be listed as an
+Exceptionless-compatible sender.
+
+### Profile 1: reliable transport
+
+- Connection reuse and bounded streaming segments.
+- Count, byte, age, explicit flush, and shutdown segment boundaries.
+- Bounded in-memory queue with a documented overflow policy.
+- Exponential backoff with jitter and `Retry-After` support.
+- Optional gzip or Brotli request compression.
+- Optional bounded disk persistence for unacknowledged segments.
+- Flush with a caller-provided deadline.
+
+Reliability is deliberately independent from event construction so the same
+transport can be reused by framework-specific packages.
+
+### Profile 2: platform SDK
+
+- Unhandled exception integration appropriate to the platform.
+- First-class application `version`, severity `level`, and SDK `client`
+ metadata.
+- Safe request, user, and environment capture with PII controls.
+- Optional breadcrumbs, trace correlation, source context, and platform-aware
+ diagnostics as those contracts become available.
+- A before-send hook that can mutate or discard an event.
+
+### Profile 3: framework integrations
+
+- Thin adapters for web frameworks, logging systems, job processors, desktop
+ UI frameworks, and mobile lifecycle APIs.
+- Framework packages depend on the platform SDK; they do not reimplement the
+ wire protocol or retry queue.
+
+## First-class event fields
+
+Only `id` and `type` are required. Optional fields are additive and a server
+must ignore unknown fields so an upgraded client can continue sending to an
+older V3 server.
+
+The launch contract accepts `error`, `log`, `usage`, and stable custom event
+types. The legacy stateful types `404`, `session`, `sessionend`, and `heartbeat`
+must continue using V2 until their preprocessing semantics have native V3
+contracts; V3 rejects them per event instead of silently changing grouping or
+session behavior.
+
+| Concern | V3 field | Client work |
+| --- | --- | --- |
+| Idempotency | `id` | Generate once and retain through retries; replay the same `date` too when supplied. |
+| Classification | `type` | Use a known type or a stable custom type. |
+| Capture time | `date` | RFC 3339 timestamp; server time is the fallback. |
+| Display | `source`, `message`, `value`, `tags` | Optional scalars and string tags. |
+| Application release | `version` | Optional release/build version. |
+| Severity | `level` | Optional platform severity name. |
+| SDK identity | `client.name`, `client.version` | Recommended for maintained SDKs. |
+| Error | `exception_type`, `stack_trace` | Send the runtime's original text. |
+| Grouping override | `stacking` | Advanced opt-in only. |
+| Context | `user`, `request`, `environment` | Optional typed objects. |
+| Custom extension | `data` | JSON object for user-defined values within documented limits. |
+
+The launch limits match the values that can be stored without truncation or
+filtering:
+
+| Field | Limit |
+| --- | --- |
+| `id` | 1-100 characters. |
+| `type` | 1-100 characters. |
+| `source`, `message` | 1-2,000 characters when present. |
+| `reference_id` | 8-100 letters, digits, or `-`. |
+| `tags` | At most 50 nonblank values, each at most 100 characters. |
+| `stacking.title` | 1-1,000 characters when present. |
+| `stacking.signature_data` | 1-100 entries; keys are 1-255 characters and values must be non-null and at most 16 KiB. |
+| Object metadata | At most 2,000 JSON values, 32 levels deep, with property names at most 255 characters and string values at most 64 KiB. |
+
+When `stacking` is supplied, `stacking.signature_data` must contain at least one
+key/value pair. The server canonicalizes pairs by ordinal key and hashes both
+length-delimited keys and values, so JSON property order never changes grouping
+and distinct keys cannot collide merely because their values match.
+
+`client.name` should be stable and ecosystem-wide, for example
+`exceptionless.go`, and `client.version` should be the SDK package version, not
+the application version. The server maps these fields to submission-client
+metadata. This is necessary for compatibility analysis, rollout targeting, and
+support without requiring every language to emulate a historical User-Agent.
+
+`data` is an escape hatch for user data and experiments, not a second route to
+first-class event metadata. Top-level keys beginning with `@` are reserved by
+Exceptionless, as are the legacy state keys `sessionend` and `haserror`; V3
+rejects them case-insensitively. This prevents custom data from bypassing typed
+request redaction, project PII settings, grouping, or other first-class
+semantics. These names remain valid inside a nested custom object because only
+the event data object's top level can collide with persisted event metadata.
+Frequently used, searchable, billable, grouping-sensitive, or
+security-sensitive semantics graduate to documented optional fields.
+
+## Comparison with Sentry
+
+Sentry's current event transport packs event JSON into an Envelope sent to
+`/api/{project_id}/envelope/`. An envelope contains a JSON header, then one or
+more items, each with another JSON header and a payload. Payloads containing
+newlines require byte lengths. The protocol can combine events with binary
+attachments, batch selected item types, persist envelopes offline, and carry
+new item types. Implementations must handle unknown item types and attributes.
+See Sentry's official [Envelope specification](https://develop.sentry.dev/sdk/foundations/transport/envelopes/).
+
+Sentry's event itself is a broad, evolving schema. Exceptions are a list of
+structured values with type/value, mechanism, thread association, and a
+structured stack trace; chained exceptions have defined ordering and tree
+metadata. See the official [event payload](https://develop.sentry.dev/sdk/foundations/transport/event-payloads/),
+[exception](https://develop.sentry.dev/sdk/foundations/transport/event-payloads/exception/),
+and [stack trace](https://develop.sentry.dev/sdk/foundations/transport/event-payloads/stacktrace/)
+specifications.
+
+That design gives Sentry a strong heterogeneous transport, but a new SDK has
+substantially more surface area:
+
+- Parse a DSN into endpoint, project id, and authentication values.
+- Generate Sentry-specific ids and event metadata.
+- Turn runtime exceptions and causes into the structured exception model.
+- Turn runtime stacks into correctly ordered structured frames.
+- Serialize envelope and item headers and compute payload byte lengths.
+- Categorize item types for rate limits and process Sentry rate-limit headers.
+- Queue envelopes, flush a background worker, apply backpressure, report client
+ drops, and optionally persist offline.
+- Add platform integrations, scopes, breadcrumbs, request data, release data,
+ in-app frames, source context, and tracing for full SDK parity.
+
+This is visible in Sentry's official [SDK expected-features list](https://develop.sentry.dev/sdk/expected-features/)
+and in representative open-source transports such as
+[sentry-go](https://github.com/getsentry/sentry-go). The comparison is not that
+these features are undesirable. It is that Sentry's definition of a complete
+SDK couples a much larger product surface to the client-authoring task.
+
+Exceptionless V3 is easier for the first event because it has a normal URL and
+bearer token, a two-field minimum object, native NDJSON framing, and server-side
+stack parsing. Sentry is more future-proof for binary and non-event telemetry
+because its envelope has an explicit item-type and length boundary.
+
+## Comparison with Raygun
+
+Raygun Crash Reporting uses a conceptually simple HTTPS transport:
+`POST https://api.raygun.com/entries`, JSON content, and an `X-ApiKey` header.
+However, its documented payload requires `occurredOn`, `details`, an error, and
+a stack trace with a line number. The error contains structured stack frames,
+and the complete details shape includes client, environment, request, response,
+user, tags, custom data, breadcrumbs, version, and grouping data. The maximum
+documented single payload is 128 KB. See Raygun's official
+[Crash Reporting API](https://raygun.com/documentation/product-guides/crash-reporting/api/).
+
+The open-source clients show where that work lands. The
+[Node client](https://github.com/MindscapeHQ/raygun4node) depends on a stack
+parser, converts call sites into line/column/class/file/method frames, recursively
+builds inner errors, gathers environment and request context, filters circular
+custom data, and then serializes the nested message. Its optional batch
+transport joins fully serialized messages into a large JSON array for
+`/entries/bulk`. The [Android client](https://github.com/MindscapeHQ/raygun4android)
+adds uncaught exception handling, offline caching, device identity, network
+state, and obfuscation mapping concerns. The
+[.NET client](https://github.com/MindscapeHQ/raygun4net) walks runtime stack
+frames and portable executable debug information before submission.
+
+Raygun is easier than Sentry at the HTTP framing layer, but harder than
+Exceptionless V3 for a new error sender because the client must understand and
+materialize Raygun's error graph. Raygun's crash schema has evolved through
+optional fields and custom data, while other products and browser report types
+use separate endpoints. That separation avoids destabilizing crash ingestion,
+but it does not provide Sentry's general multi-item transport.
+
+## Apples-to-apples implementation comparison
+
+The comparison below is for a new language's first correct handled-error
+sender, not for installing an existing SDK and not for full automatic
+instrumentation.
+
+| Responsibility | Exceptionless V3 | Raygun Crash API | Sentry Envelope API |
+| --- | --- | --- | --- |
+| Endpoint configuration | URL plus token | Fixed/default URL plus API key | Parse DSN and derive endpoint/authentication |
+| Minimum event shape | `id`, `type` | `occurredOn`, `details`, error, stack frame line | Envelope header, item header, event id, event JSON |
+| Stack input | Original string | Structured frames | Structured frames |
+| Chained errors | Server parses common raw forms | Client builds inner-error graph | Client builds ordered exception values/tree |
+| Multiple events | Stream one JSON object per NDJSON line | Single documents or buffered JSON array bulk | Multiple envelope items only where item rules allow |
+| Binary attachment | Future advanced transport | Not part of crash JSON contract | Native length-delimited attachment item |
+| Unknown future fields | Ignored; additive | Optional/custom data | Preserved/ignored according to envelope scope |
+| Minimum retry identity | Required stable event id | No equivalent id in documented crash body | Event id |
+| Minimum dependencies | Standard HTTPS/JSON | HTTPS/JSON plus stack inspection/parser | HTTPS/JSON plus stack model and envelope framing |
+
+For a production-grade SDK, all three eventually need bounded buffering,
+timeouts, retries, redaction, unhandled-error hooks, and platform integrations.
+Exceptionless should publish those as higher profiles instead of blocking the
+first client on them.
+
+## Decision: keep the event endpoint permanently simple
+
+`POST /api/v3/events` remains a homogeneous stream of event objects. We will
+not add a wrapper such as `{ "kind": "event", "payload": ... }` to every line,
+and the first line will not become a stream header. Those designs tax every
+event and every client for features most clients never use.
+
+The event object evolves additively:
+
+- Existing field meanings do not change.
+- New fields are optional.
+- Unknown fields are ignored.
+- Fields may be promoted from experimental `data` conventions to first-class
+ fields, with a server transition period that accepts both.
+- A semantic change that cannot follow these rules requires a new endpoint or
+ media type, not a `schema_version` switch inside every event.
+- Limits and validation errors are documented and machine-testable.
+
+This gives ordinary event clients the smallest contract while allowing the
+event model to grow with optional release, severity, SDK, context, breadcrumb,
+trace, thread, module, and advanced error data.
+
+## Decision: reserve an advanced heterogeneous transport
+
+Attachments, minidumps, profiles, replays, symbol files, and other binary or
+non-event records must not be base64 fields added to the event object. A future
+advanced endpoint will use explicit item headers containing at least item type,
+content type, and byte length. Unknown item types must be safely skippable.
+
+The advanced transport must embed the exact V3 event JSON as its `event` item
+payload. A client upgrades by adding framing around the serializer it already
+has; it does not maintain a second event model. The endpoint can then support
+atomic event-plus-attachment submissions and independent limits or billing
+categories without changing `/api/v3/events`.
+
+The advanced transport is intentionally not required for Profiles 0 through 2
+until a client needs a feature that cannot be represented as an event. Its
+design should reuse the proven parts of Sentry's envelope—typed items,
+byte-length framing, unknown-item handling, and common metadata—without copying
+Sentry-specific DSNs, authentication, event schema, or SDK feature requirements.
+
+## Reserved evolution paths
+
+The following additions must remain possible without breaking the simple
+error path:
+
+- Advanced exceptions: an optional structured exception collection for
+ runtimes that can supply cause trees, mechanisms, handled state, native
+ codes, or pre-parsed frames. `exception_type` and `stack_trace` remain the
+ preferred simple form; precedence and mutual-exclusion rules must be explicit.
+- Breadcrumbs: a bounded optional collection with timestamp, category, level,
+ message, and JSON data.
+- Trace correlation: optional W3C-compatible trace and span identifiers. Full
+ transaction/span ingestion may use its own item type.
+- Threads and modules: optional diagnostic collections, never required to send
+ an error.
+- Deployment identity: additive environment/release/distribution fields that
+ do not conflict with the existing machine `environment` object.
+- Key/value attributes: an indexed bounded scalar map if string `tags` and
+ arbitrary `data` prove insufficient.
+- Client outcomes: an optional aggregate item for locally discarded events,
+ queue overflow, serialization failure, and rate limiting.
+
+Names for these fields are reserved in the public specification before SDKs
+invent incompatible custom-data conventions. They are implemented only with a
+server consumer and cross-language fixtures.
+
+## Client conformance kit
+
+The public launch is not complete with OpenAPI alone. NDJSON streaming and
+retry semantics need executable, language-neutral evidence:
+
+1. Publish a short normative wire specification with exact request, response,
+ status, retry, size, compression, and field-precedence rules.
+2. Publish JSON Schema for one event and keep the isolated V3 OpenAPI document.
+ Explain that OpenAPI's single request schema represents each NDJSON line,
+ not an array.
+3. Publish golden request and response fixtures for minimal log, minimal error,
+ Unicode, multiline stacks, multiple lines, optional final newline,
+ same-line adjacent-object rejection, compression, partial invalid
+ results, duplicates, discarded events, and unknown future fields.
+4. Ship a black-box conformance runner. It starts a local receiver, invokes a
+ client adapter, deliberately drops a response, splits JSON across writes,
+ returns 429/503, and verifies stable-id replay and bounded flush behavior.
+5. Ship a tiny reference transport in pseudocode plus dependency-free examples
+ in at least JavaScript, Python, Go, Java/Kotlin, Rust, and Swift. Examples
+ demonstrate the core sender, not a framework-sized official SDK.
+6. Give ecosystem packages a machine-readable capability manifest and display
+ Profile 0/1/2/3 badges in the client directory.
+7. Run every maintained client against the same fixtures in CI. A contract
+ addition lands only after old-client, new-client, and unknown-field cases
+ pass.
+
+## Client ecosystem implementation sequence
+
+1. Freeze the Profile 0 field semantics, status/retry matrix, limits, and
+ additive-evolution rules. Keep the server's unknown-field test and isolated
+ OpenAPI baseline as compatibility gates.
+2. Extract the current load generator's V3 writer into a small reference
+ transport whose public surface is event serialization, segment policy,
+ send, and flush. Do not expose server persistence models.
+3. Publish JSON Schema, golden fixtures, and the black-box conformance runner.
+ Make the runner usable before an SDK repository or package is accepted.
+4. Build two pilot clients in unrelated ecosystems, recommended Python and Go,
+ from the specification rather than by porting the .NET client. Record time,
+ source lines, dependencies, ambiguities, allocations, and retry defects.
+5. Resolve every cross-language ambiguity in the normative specification, then
+ repeat the timed exercise with Java/Kotlin and Swift or Rust. Do not freeze
+ the contract based only on .NET and JavaScript ergonomics.
+6. Publish the Profile 0 examples and Profile 1 transport guidance. Create a
+ client directory showing owner, package, supported runtime versions,
+ capability profile, conformance version, and last successful conformance run.
+7. Add official platform integrations only after the shared transport passes
+ conformance. Keep framework packages thin so transport fixes flow to the
+ whole ecosystem.
+8. Design the optional advanced item transport when the first concrete binary
+ or heterogeneous feature needs it. Validate that an existing Profile 0 event
+ serializer can be embedded unchanged before accepting the design.
+
+## Review gates for future protocol changes
+
+Every proposal that changes ingestion must answer:
+
+- Does a Profile 0 sender need to change? If yes, use a new endpoint/media type.
+- Can the server compute or default the value more reliably than every client?
+- Is the field needed for grouping, discard classification, billing, search,
+ redaction, or display? If so, it should not remain an undocumented data key.
+- Can old servers ignore it and can old clients omit it?
+- Does it require a binary payload or a different retention/billing category?
+ If so, use the advanced item transport.
+- What are its byte, count, depth, and privacy limits?
+- Are field precedence and duplicate/retry behavior deterministic?
+- Do golden fixtures cover at least two unrelated language implementations?
+
+These gates preserve the strongest aspect of V3—the server does expensive,
+product-specific interpretation once—while retaining a clean path to richer
+telemetry.
diff --git a/src/Exceptionless.Core/Attributes/ApiIgnoreAttribute.cs b/src/Exceptionless.Core/Attributes/ApiIgnoreAttribute.cs
new file mode 100644
index 0000000000..24d6f60381
--- /dev/null
+++ b/src/Exceptionless.Core/Attributes/ApiIgnoreAttribute.cs
@@ -0,0 +1,8 @@
+namespace Exceptionless.Core.Attributes;
+
+///
+/// Excludes a persisted model member from HTTP JSON contracts while retaining it for storage
+/// serialization. Prefer dedicated API models when a larger contract is being designed.
+///
+[AttributeUsage(AttributeTargets.Property)]
+public sealed class ApiIgnoreAttribute : Attribute;
diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs
index efcfae9efb..af87e5f70c 100644
--- a/src/Exceptionless.Core/Bootstrapper.cs
+++ b/src/Exceptionless.Core/Bootstrapper.cs
@@ -19,6 +19,7 @@
using Exceptionless.Core.Plugins.Formatting;
using Exceptionless.Core.Plugins.WebHook;
using Exceptionless.Core.Queries.Validation;
+using Exceptionless.Core.Queues;
using Exceptionless.Core.Queues.Models;
using Exceptionless.Core.Repositories;
using Exceptionless.Core.Repositories.Configuration;
@@ -100,6 +101,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
+ handlers.Register(s.GetRequiredService);
return handlers;
});
@@ -111,6 +113,12 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
services.AddSingleton(s => CreateQueue(s, TimeSpan.FromHours(1)));
services.TryAddEnumerable(ServiceDescriptor.Singleton, WorkItemDuplicateDetectionQueueBehavior>());
+ // V2 keeps Foundatio-compatible dequeue-scoped duplicate detection. V3 bypasses that
+ // claim and uses only the durable pending/completed behavior so enqueue failures recover.
+ services.TryAddEnumerable(ServiceDescriptor.Singleton, EventNotificationDuplicateDetectionQueueBehavior>());
+ services.TryAddEnumerable(ServiceDescriptor.Singleton, DurableEventNotificationDuplicateDetectionQueueBehavior>());
+ services.TryAddEnumerable(ServiceDescriptor.Singleton, WebHookNotificationDuplicateDetectionQueueBehavior>());
+ services.TryAddEnumerable(ServiceDescriptor.Singleton, DurableWebHookNotificationDuplicateDetectionQueueBehavior>());
services.AddSingleton();
services.AddSingleton();
@@ -197,6 +205,21 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
services.AddSingleton();
services.AddSingleton();
services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton(s => s.GetRequiredService());
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton(s => s.GetRequiredService());
+ services.AddSingleton();
+ services.AddSingleton(s => s.GetRequiredService());
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
services.AddTransient();
}
@@ -208,7 +231,9 @@ private static async ValueTask ConnectToPublicAddressAsync(SocketsHttpCo
foreach (var address in addresses)
{
if (!OAuthClientMetadataService.IsPublicAddress(address))
+ {
continue;
+ }
var socket = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) { NoDelay = true };
try
@@ -229,53 +254,81 @@ private static async ValueTask ConnectToPublicAddressAsync(SocketsHttpCo
public static void LogConfiguration(IServiceProvider serviceProvider, AppOptions appOptions, ILogger logger)
{
if (!logger.IsEnabled(LogLevel.Warning))
+ {
return;
+ }
if (String.IsNullOrEmpty(appOptions.CacheOptions.Provider))
+ {
logger.LogWarning("Distributed cache is NOT enabled on {MachineName}", Environment.MachineName);
+ }
if (String.IsNullOrEmpty(appOptions.MessageBusOptions.Provider))
+ {
logger.LogWarning("Distributed message bus is NOT enabled on {MachineName}", Environment.MachineName);
+ }
if (String.IsNullOrEmpty(appOptions.QueueOptions.Provider))
+ {
logger.LogWarning("Distributed queue is NOT enabled on {MachineName}", Environment.MachineName);
+ }
if (String.IsNullOrEmpty(appOptions.StorageOptions.Provider))
+ {
logger.LogWarning("Distributed storage is NOT enabled on {MachineName}", Environment.MachineName);
+ }
if (!appOptions.EnableWebSockets)
+ {
logger.LogWarning("Web Sockets is NOT enabled on {MachineName}", Environment.MachineName);
+ }
if (String.IsNullOrEmpty(appOptions.EmailOptions.SmtpHost))
+ {
logger.LogWarning("Emails will NOT be sent until the SmtpHost is configured on {MachineName}", Environment.MachineName);
+ }
var fileStorage = serviceProvider.GetService();
if (fileStorage is InMemoryFileStorage)
+ {
logger.LogWarning("Using in memory file storage on {MachineName}", Environment.MachineName);
+ }
if (appOptions.ElasticsearchOptions.DisableIndexConfiguration)
+ {
logger.LogWarning("Index Configuration is NOT enabled on {MachineName}", Environment.MachineName);
+ }
if (appOptions.EventSubmissionDisabled)
+ {
logger.LogWarning("Event Submission is NOT enabled on {MachineName}", Environment.MachineName);
+ }
if (!appOptions.AuthOptions.EnableAccountCreation)
+ {
logger.LogWarning("Account Creation is NOT enabled on {MachineName}", Environment.MachineName);
+ }
}
private static async Task CreateSampleDataAsync(IServiceProvider container)
{
var options = container.GetRequiredService();
if (!options.EnableSampleData)
+ {
return;
+ }
var elasticsearchOptions = container.GetRequiredService();
if (elasticsearchOptions.DisableIndexConfiguration)
+ {
return;
+ }
var userRepository = container.GetRequiredService();
if (await userRepository.CountAsync() != 0)
+ {
return;
+ }
var dataHelper = container.GetRequiredService();
await dataHelper.CreateDataAsync();
@@ -292,6 +345,7 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log
services.AddJob(o => o.WaitForStartupActions());
services.AddJob(o => o.WaitForStartupActions());
services.AddJob(o => o.WaitForStartupActions());
+ services.AddJob(o => o.WaitForStartupActions());
services.AddJob(o => o.WaitForStartupActions());
services.AddJob(o => o.WaitForStartupActions());
@@ -320,6 +374,18 @@ private static IQueue CreateQueue(IServiceProvider container, TimeSpan? wo
});
}
- private sealed class WorkItemDuplicateDetectionQueueBehavior(ICacheClient cacheClient, ILoggerFactory loggerFactory)
- : DuplicateDetectionQueueBehavior(cacheClient, loggerFactory, TimeSpan.FromHours(24));
+ private sealed class WorkItemDuplicateDetectionQueueBehavior(ICacheClient cacheClient, AppOptions options, ILoggerFactory loggerFactory)
+ : DuplicateDetectionQueueBehavior(cacheClient, loggerFactory, options.EventIngestionV3.IdempotencyWindow);
+
+ private sealed class EventNotificationDuplicateDetectionQueueBehavior(ICacheClient cacheClient, AppOptions options, ILoggerFactory loggerFactory)
+ : ConditionalDuplicateDetectionQueueBehavior(cacheClient, loggerFactory, options.EventIngestionV3.IdempotencyWindow);
+
+ private sealed class DurableEventNotificationDuplicateDetectionQueueBehavior(ICacheClient cacheClient, AppOptions options, ILoggerFactory loggerFactory)
+ : DurableDuplicateDetectionQueueBehavior(cacheClient, loggerFactory, options.EventIngestionV3.IdempotencyWindow);
+
+ private sealed class WebHookNotificationDuplicateDetectionQueueBehavior(ICacheClient cacheClient, AppOptions options, ILoggerFactory loggerFactory)
+ : ConditionalDuplicateDetectionQueueBehavior(cacheClient, loggerFactory, options.EventIngestionV3.IdempotencyWindow);
+
+ private sealed class DurableWebHookNotificationDuplicateDetectionQueueBehavior(ICacheClient cacheClient, AppOptions options, ILoggerFactory loggerFactory)
+ : DurableDuplicateDetectionQueueBehavior(cacheClient, loggerFactory, options.EventIngestionV3.IdempotencyWindow);
}
diff --git a/src/Exceptionless.Core/Configuration/AppOptions.cs b/src/Exceptionless.Core/Configuration/AppOptions.cs
index ff6217829a..8b434d8545 100644
--- a/src/Exceptionless.Core/Configuration/AppOptions.cs
+++ b/src/Exceptionless.Core/Configuration/AppOptions.cs
@@ -80,6 +80,7 @@ public class AppOptions
public StripeOptions StripeOptions { get; internal set; } = null!;
public AuthOptions AuthOptions { get; internal set; } = null!;
public OAuthServerOptions OAuthServerOptions { get; internal set; } = null!;
+ public EventIngestionV3Options EventIngestionV3 { get; internal set; } = null!;
public static AppOptions ReadFromConfiguration(IConfiguration config)
{
@@ -133,11 +134,84 @@ public static AppOptions ReadFromConfiguration(IConfiguration config)
options.StripeOptions = StripeOptions.ReadFromConfiguration(config);
options.AuthOptions = AuthOptions.ReadFromConfiguration(config);
options.OAuthServerOptions = OAuthServerOptions.ReadFromConfiguration(config);
+ options.EventIngestionV3 = EventIngestionV3Options.ReadFromConfiguration(config);
return options;
}
}
+public sealed class EventIngestionV3Options
+{
+ public bool Enabled { get; internal set; }
+ public bool EnableProcessingStatus { get; internal set; }
+ public int MicroBatchSize { get; internal set; }
+ public long MaximumMicroBatchBytes { get; internal set; }
+ public long MaximumEventSize { get; internal set; }
+ public long MaximumCompressedBodySize { get; internal set; }
+ public long MaximumDecompressedBodySize { get; internal set; }
+ public TimeSpan RequestTimeout { get; internal set; }
+ public TimeSpan IdempotencyWindow { get; internal set; }
+ public TimeSpan StackRouteCacheDuration { get; internal set; }
+ public TimeSpan NegativeStackRouteCacheDuration { get; internal set; }
+ public int MaximumEventsPerRequest { get; internal set; }
+ public int MaximumActiveStreams { get; internal set; }
+ public int ActiveStreamQueueLimit { get; internal set; }
+ public int MaximumActiveStreamsPerOrganization { get; internal set; }
+ public int ActiveStreamQueueLimitPerOrganization { get; internal set; }
+ public int MaximumConcurrentRequests { get; internal set; }
+ public int ConcurrencyQueueLimit { get; internal set; }
+ public int MaximumConcurrentRequestsPerOrganization { get; internal set; }
+ public int ConcurrencyQueueLimitPerOrganization { get; internal set; }
+ public int MaximumStackCreationConcurrency { get; internal set; }
+ public int MaximumStackUsageConcurrency { get; internal set; }
+ public TimeSpan StackUsageClaimLease { get; internal set; }
+ public IReadOnlySet AllowedProjectIds { get; internal set; } = new HashSet();
+ public IReadOnlySet AllowedOrganizationIds { get; internal set; } = new HashSet();
+
+ internal static EventIngestionV3Options ReadFromConfiguration(IConfiguration config)
+ {
+ IConfigurationSection section = config.GetSection(nameof(AppOptions.EventIngestionV3));
+ int maximumConcurrentRequests = Math.Max(section.GetValue(nameof(MaximumConcurrentRequests), Math.Max(Environment.ProcessorCount * 4, 16)), 1);
+ int concurrencyQueueLimit = Math.Max(section.GetValue(nameof(ConcurrencyQueueLimit), Math.Max(Environment.ProcessorCount * 16, 64)), 0);
+ int maximumConcurrentRequestsPerOrganization = Math.Clamp(section.GetValue(nameof(MaximumConcurrentRequestsPerOrganization), Math.Max(maximumConcurrentRequests / 4, 1)), 1, maximumConcurrentRequests);
+ int defaultMaximumActiveStreams = (int)Math.Min(Math.Max((long)maximumConcurrentRequests * 8, 128), Int32.MaxValue);
+ int defaultMaximumActiveStreamsPerOrganization = (int)Math.Min(Math.Max((long)maximumConcurrentRequestsPerOrganization * 8, 32), Int32.MaxValue);
+ int maximumActiveStreams = Math.Max(section.GetValue(nameof(MaximumActiveStreams), defaultMaximumActiveStreams), 1);
+ int activeStreamQueueLimit = Math.Max(section.GetValue(nameof(ActiveStreamQueueLimit), 0), 0);
+ return new EventIngestionV3Options
+ {
+ Enabled = section.GetValue(nameof(Enabled), false),
+ EnableProcessingStatus = section.GetValue(nameof(EnableProcessingStatus), false),
+ MicroBatchSize = Math.Clamp(section.GetValue(nameof(MicroBatchSize), 100), 1, 1000),
+ MaximumMicroBatchBytes = Math.Max(section.GetValue(nameof(MaximumMicroBatchBytes), 1024L * 1024), 1),
+ MaximumEventSize = Math.Max(section.GetValue(nameof(MaximumEventSize), 512L * 1024), 1),
+ MaximumCompressedBodySize = Math.Max(section.GetValue(nameof(MaximumCompressedBodySize), 10L * 1024 * 1024), 1),
+ MaximumDecompressedBodySize = Math.Max(section.GetValue(nameof(MaximumDecompressedBodySize), 50L * 1024 * 1024), 1),
+ RequestTimeout = section.GetValue(nameof(RequestTimeout), TimeSpan.FromMinutes(2)),
+ IdempotencyWindow = section.GetValue(nameof(IdempotencyWindow), TimeSpan.FromDays(7)),
+ StackRouteCacheDuration = section.GetValue(nameof(StackRouteCacheDuration), TimeSpan.FromHours(1)),
+ NegativeStackRouteCacheDuration = section.GetValue(nameof(NegativeStackRouteCacheDuration), TimeSpan.FromSeconds(30)),
+ MaximumEventsPerRequest = Math.Clamp(section.GetValue(nameof(MaximumEventsPerRequest), 10000), 1, 100000),
+ MaximumActiveStreams = maximumActiveStreams,
+ ActiveStreamQueueLimit = activeStreamQueueLimit,
+ MaximumActiveStreamsPerOrganization = Math.Clamp(section.GetValue(nameof(MaximumActiveStreamsPerOrganization), defaultMaximumActiveStreamsPerOrganization), 1, maximumActiveStreams),
+ ActiveStreamQueueLimitPerOrganization = Math.Clamp(section.GetValue(nameof(ActiveStreamQueueLimitPerOrganization), 0), 0, activeStreamQueueLimit),
+ MaximumConcurrentRequests = maximumConcurrentRequests,
+ ConcurrencyQueueLimit = concurrencyQueueLimit,
+ MaximumConcurrentRequestsPerOrganization = maximumConcurrentRequestsPerOrganization,
+ ConcurrencyQueueLimitPerOrganization = Math.Clamp(section.GetValue(nameof(ConcurrencyQueueLimitPerOrganization), Math.Max(concurrencyQueueLimit / 4, 0)), 0, concurrencyQueueLimit),
+ MaximumStackCreationConcurrency = Math.Clamp(section.GetValue(nameof(MaximumStackCreationConcurrency), 8), 1, 64),
+ MaximumStackUsageConcurrency = Math.Clamp(section.GetValue(nameof(MaximumStackUsageConcurrency), 16), 1, 64),
+ StackUsageClaimLease = TimeSpan.FromMilliseconds(Math.Clamp(
+ section.GetValue(nameof(StackUsageClaimLease), TimeSpan.FromMinutes(1)).TotalMilliseconds,
+ TimeSpan.FromSeconds(10).TotalMilliseconds,
+ TimeSpan.FromMinutes(10).TotalMilliseconds)),
+ AllowedProjectIds = section.GetSection(nameof(AllowedProjectIds)).Get()?.ToHashSet(StringComparer.Ordinal) ?? new HashSet(StringComparer.Ordinal),
+ AllowedOrganizationIds = section.GetSection(nameof(AllowedOrganizationIds)).Get()?.ToHashSet(StringComparer.Ordinal) ?? new HashSet(StringComparer.Ordinal)
+ };
+ }
+}
+
public enum AppMode
{
Development,
diff --git a/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs b/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs
index 8019f6a9ae..dc3ee64825 100644
--- a/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs
+++ b/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs
@@ -6,6 +6,20 @@ namespace Exceptionless.Core.Extensions;
public static class RequestInfoExtensions
{
+ public const int MaximumDataValueLength = 1000;
+
+ public static readonly IReadOnlyList DefaultDataExclusions =
+ [
+ "*VIEWSTATE*",
+ "*EVENTVALIDATION*",
+ "*ASPX*",
+ "__RequestVerificationToken",
+ "ASP.NET_SessionId",
+ "__LastErrorId",
+ "WAWebSiteID",
+ "ARRAffinity"
+ ];
+
public static RequestInfo ApplyDataExclusions(this RequestInfo request, ITextSerializer serializer, IList exclusions, int maxLength = 1000)
{
request.Cookies = ApplyExclusions(request.Cookies, exclusions, maxLength);
@@ -15,16 +29,32 @@ public static RequestInfo ApplyDataExclusions(this RequestInfo request, ITextSer
return request;
}
+ ///
+ /// Applies request exclusions to already materialized JSON-compatible post data. Unlike the
+ /// legacy string overload, this handles nested V3 object and array values before persistence.
+ ///
+ public static RequestInfo ApplyDataExclusions(this RequestInfo request, IList exclusions, int maxLength = 1000)
+ {
+ request.Cookies = ApplyExclusions(request.Cookies, exclusions, maxLength);
+ request.QueryString = ApplyExclusions(request.QueryString, exclusions, maxLength);
+ request.PostData = ApplyObjectExclusions(request.PostData, exclusions, maxLength);
+ return request;
+ }
+
private static object? ApplyPostDataExclusions(object? data, ITextSerializer serializer, IEnumerable exclusions, int maxLength)
{
if (data is null)
+ {
return null;
+ }
var dictionary = data as Dictionary;
if (dictionary is null && data is string json)
{
if (!json.IsJson())
+ {
return data;
+ }
try
{
@@ -36,16 +66,60 @@ public static RequestInfo ApplyDataExclusions(this RequestInfo request, ITextSer
return dictionary is not null ? ApplyExclusions(dictionary, exclusions, maxLength) : data;
}
+ private static object? ApplyObjectExclusions(object? value, IList exclusions, int maxLength)
+ {
+ if (value is IDictionary objectDictionary)
+ {
+ foreach (string key in objectDictionary.Keys.ToArray())
+ {
+ if (String.IsNullOrEmpty(key) || key.AnyWildcardMatches(exclusions, true))
+ {
+ objectDictionary.Remove(key);
+ continue;
+ }
+
+ objectDictionary[key] = ApplyObjectExclusions(objectDictionary[key], exclusions, maxLength);
+ }
+
+ return value;
+ }
+
+ if (value is IDictionary stringDictionary)
+ {
+ return ApplyExclusions(new Dictionary(stringDictionary), exclusions, maxLength);
+ }
+
+ if (value is IList values)
+ {
+ for (int index = 0; index < values.Count; index++)
+ {
+ values[index] = ApplyObjectExclusions(values[index], exclusions, maxLength);
+ }
+
+ return value;
+ }
+
+ return value is string text && text.Length > maxLength
+ ? "Value is too large to be included."
+ : value;
+ }
+
private static Dictionary? ApplyExclusions(Dictionary? dictionary, IEnumerable exclusions, int maxLength)
{
if (dictionary is null || dictionary.Count == 0)
+ {
return dictionary;
+ }
foreach (string key in dictionary.Keys.Where(k => String.IsNullOrEmpty(k) || StringExtensions.AnyWildcardMatches(k, exclusions, true)).ToList())
+ {
dictionary.Remove(key);
+ }
foreach (string key in dictionary.Where(kvp => kvp.Value is not null && kvp.Value.Length > maxLength).Select(kvp => kvp.Key).ToList())
+ {
dictionary[key] = String.Format("Value is too large to be included.");
+ }
return dictionary;
}
@@ -57,26 +131,34 @@ public static string GetFullPath(this RequestInfo requestInfo, bool includeHttpM
{
var sb = new StringBuilder();
if (includeHttpMethod && !String.IsNullOrEmpty(requestInfo.HttpMethod))
+ {
sb.Append(requestInfo.HttpMethod).Append(" ");
+ }
if (includeHost && !String.IsNullOrEmpty(requestInfo.Host))
{
sb.Append(requestInfo.IsSecure.GetValueOrDefault() ? "https://" : "http://");
sb.Append(requestInfo.Host);
if (requestInfo.Port.HasValue && requestInfo.Port != 80 && requestInfo.Port != 443)
+ {
sb.Append(":").Append(requestInfo.Port);
+ }
}
if (requestInfo.Path is not null)
{
if (!requestInfo.Path.StartsWith("/"))
+ {
sb.Append("/");
+ }
sb.Append(requestInfo.Path);
}
if (includeQueryString && requestInfo.QueryString is not null && requestInfo.QueryString.Count > 0)
+ {
sb.Append("?").Append(CreateQueryString(requestInfo.QueryString));
+ }
return sb.ToString();
}
@@ -84,17 +166,23 @@ public static string GetFullPath(this RequestInfo requestInfo, bool includeHttpM
private static string CreateQueryString(IEnumerable> args)
{
if (args is null)
+ {
return String.Empty;
+ }
if (!args.Any())
+ {
return String.Empty;
+ }
var sb = new StringBuilder(args.Count() * 10);
foreach (var p in args)
{
if (String.IsNullOrEmpty(p.Key) && p.Value is null)
+ {
continue;
+ }
if (!String.IsNullOrEmpty(p.Key))
{
@@ -102,7 +190,10 @@ private static string CreateQueryString(IEnumerable
sb.Append('=');
}
if (p.Value is not null)
+ {
sb.Append(p.Value);
+ }
+
sb.Append('&');
}
sb.Length--; // remove trailing &
diff --git a/src/Exceptionless.Core/Extensions/StackExtensions.cs b/src/Exceptionless.Core/Extensions/StackExtensions.cs
index 0f11c42880..c0750095a1 100644
--- a/src/Exceptionless.Core/Extensions/StackExtensions.cs
+++ b/src/Exceptionless.Core/Extensions/StackExtensions.cs
@@ -10,6 +10,7 @@ public static void MarkFixed(this Stack stack, SemanticVersion? version, TimePro
stack.Status = StackStatus.Fixed;
stack.DateFixed = timeProvider.GetUtcNow().UtcDateTime;
stack.FixedInVersion = version?.ToString();
+ stack.RegressionEventId = null;
stack.SnoozeUntilUtc = null;
}
@@ -18,19 +19,26 @@ public static void MarkOpen(this Stack stack)
stack.Status = StackStatus.Open;
stack.DateFixed = null;
stack.FixedInVersion = null;
+ stack.RegressionEventId = null;
stack.SnoozeUntilUtc = null;
}
public static Stack ApplyOffset(this Stack stack, TimeSpan offset)
{
if (stack.DateFixed.HasValue)
+ {
stack.DateFixed = stack.DateFixed.Value.Add(offset);
+ }
if (stack.FirstOccurrence != DateTime.MinValue)
+ {
stack.FirstOccurrence = stack.FirstOccurrence.Add(offset);
+ }
if (stack.LastOccurrence != DateTime.MinValue)
+ {
stack.LastOccurrence = stack.LastOccurrence.Add(offset);
+ }
return stack;
}
@@ -38,7 +46,9 @@ public static Stack ApplyOffset(this Stack stack, TimeSpan offset)
public static string? GetTypeName(this Stack stack)
{
if (stack.SignatureInfo.TryGetValue("ExceptionType", out string? type) && !String.IsNullOrEmpty(type))
+ {
return type.TypeName();
+ }
return type;
}
diff --git a/src/Exceptionless.Core/Jobs/EventPostsJob.cs b/src/Exceptionless.Core/Jobs/EventPostsJob.cs
index 0b7aaffec3..8c0ff5e552 100644
--- a/src/Exceptionless.Core/Jobs/EventPostsJob.cs
+++ b/src/Exceptionless.Core/Jobs/EventPostsJob.cs
@@ -55,6 +55,18 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
var entry = context.QueueEntry;
var ep = entry.Value;
+ // The controller returns the queue id without performing cache I/O so opt-in benchmark
+ // instrumentation does not distort V2 submission latency. The worker lazily creates the
+ // correlation state when processing begins; until then the status endpoint reports unknown.
+ if (ep.TrackProcessing && String.IsNullOrEmpty(ep.ProcessingCorrelationId))
+ {
+ bool initialized = await _eventPostService.InitializeProcessingTrackingAsync(entry.Id, ep.ProjectId);
+ if (initialized)
+ {
+ ep = ep with { ProcessingCorrelationId = entry.Id };
+ }
+ }
+
using var _ = _logger.BeginScope(new ExceptionlessState().Organization(ep.OrganizationId).Project(ep.ProjectId));
string payloadPath = Path.ChangeExtension(ep.FilePath, ".payload");
@@ -72,7 +84,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
AppDiagnostics.PostsMessageSize.Record(payload.LongLength);
if (payload.LongLength > _maximumEventPostFileSize)
{
- await Task.WhenAll(AppDiagnostics.PostsCompleteTime.TimeAsync(() => entry.CompleteAsync()), projectTask, organizationTask);
+ await Task.WhenAll(CompleteEntryAsync(entry, ep, _timeProvider.GetUtcNow().UtcDateTime), projectTask, organizationTask);
return JobResult.FailedWithMessage($"Unable to process payload '{payloadPath}' ({payload.LongLength} bytes): Maximum event post size limit ({_appOptions.MaximumEventPostSize} bytes) reached.");
}
@@ -83,13 +95,19 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
if (!isInternalProject && _logger.IsEnabled(LogLevel.Information))
{
using (_logger.BeginScope(new ExceptionlessState().Tag("processing").Tag("compressed").Tag(ep.ContentEncoding).Value(payload.Length)))
+ {
_logger.LogInformation("Processing post: id={QueueEntryId} path={FilePath} project={ProjectId} ip={IpAddress} v={ApiVersion} agent={UserAgent}", entry.Id, payloadPath, ep.ProjectId, ep.IpAddress, ep.ApiVersion, ep.UserAgent);
+ }
}
var project = await projectTask;
if (project is null)
{
- if (!isInternalProject) _logger.LogError("Unable to process EventPost {FilePath}: Unable to load project: {ProjectId}", payloadPath, ep.ProjectId);
+ if (!isInternalProject)
+ {
+ _logger.LogError("Unable to process EventPost {FilePath}: Unable to load project: {ProjectId}", payloadPath, ep.ProjectId);
+ }
+
await Task.WhenAll(CompleteEntryAsync(entry, ep, _timeProvider.GetUtcNow().UtcDateTime), organizationTask);
return JobResult.Success;
}
@@ -101,7 +119,9 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
if (!isInternalProject && isDebugLogLevelEnabled)
{
using (_logger.BeginScope(new ExceptionlessState().Tag("decompressing").Tag(ep.ContentEncoding)))
+ {
_logger.LogDebug("Decompressing EventPost: {QueueEntryId} ({CompressedBytes} bytes)", entry.Id, payload.Length);
+ }
}
maxEventPostSize = _maximumUncompressedEventPostSize;
@@ -125,9 +145,14 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
{
var org = await organizationTask;
if (org is not null)
+ {
await _usageService.IncrementTooBigAsync(org.Id, project.Id);
+ }
else
+ {
_logger.LogWarning("Organization {OrganizationId} not found, skipping too-big usage increment for event post {EventPostId}", ep.OrganizationId, entry.Id);
+ }
+
await CompleteEntryAsync(entry, ep, _timeProvider.GetUtcNow().UtcDateTime);
return JobResult.FailedWithMessage($"Unable to process decompressed EventPost data '{payloadPath}' ({payload.Length} bytes compressed, {uncompressedData.Length} bytes): Maximum uncompressed event post size limit ({maxEventPostSize} bytes) reached.");
}
@@ -135,7 +160,9 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
if (!isInternalProject && isDebugLogLevelEnabled)
{
using (_logger.BeginScope(new ExceptionlessState().Tag("uncompressed").Value(uncompressedData.Length)))
+ {
_logger.LogDebug("Processing uncompressed EventPost: {QueueEntryId} ({UncompressedBytes} bytes)", entry.Id, uncompressedData.Length);
+ }
}
var createdUtc = _timeProvider.GetUtcNow().UtcDateTime;
@@ -156,7 +183,9 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
if (organization is null)
{
if (!isInternalProject)
+ {
_logger.LogError("Unable to process EventPost {FilePath}: Unable to load organization: {OrganizationId}", payloadPath, project.OrganizationId);
+ }
await CompleteEntryAsync(entry, ep, _timeProvider.GetUtcNow().UtcDateTime);
return JobResult.Success;
@@ -167,7 +196,9 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
if (eventsToProcess < 1)
{
if (!isInternalProject)
+ {
_logger.LogDebug("Unable to process EventPost {FilePath}: Over plan limits", payloadPath);
+ }
await _usageService.IncrementBlockedAsync(organization.Id, project.Id, events.Count);
@@ -195,7 +226,9 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
if (!isInternalProject && isDebugLogLevelEnabled)
{
using (_logger.BeginScope(new ExceptionlessState().Value(contexts.Count)))
+ {
_logger.LogDebug("Ran {@Value} events through the pipeline: id={QueueEntryId} success={SuccessCount} error={ErrorCount}", contexts.Count, entry.Id, contexts.Count(r => r.IsProcessed), contexts.Count(r => r.HasError));
+ }
}
// increment the plan usage counters (note: OverageHandler already incremented usage by 1)
@@ -208,14 +241,24 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
foreach (var ctx in contexts)
{
if (ctx.IsCancelled)
+ {
continue;
+ }
if (!ctx.HasError)
+ {
continue;
+ }
+
+ if (!isInternalProject)
+ {
+ _logger.LogError(ctx.Exception, "Error processing EventPost {QueueEntryId} {FilePath}: {Message}", entry.Id, payloadPath, ctx.ErrorMessage);
+ }
- if (!isInternalProject) _logger.LogError(ctx.Exception, "Error processing EventPost {QueueEntryId} {FilePath}: {Message}", entry.Id, payloadPath, ctx.ErrorMessage);
if (ctx.Exception is MiniValidatorException)
+ {
continue;
+ }
errorCount++;
if (!isSingleEvent)
@@ -227,7 +270,11 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
}
catch (Exception ex)
{
- if (!isInternalProject) _logger.LogError(ex, "Error processing EventPost {QueueEntryId} {FilePath}: {Message}", entry.Id, payloadPath, ex.Message);
+ if (!isInternalProject)
+ {
+ _logger.LogError(ex, "Error processing EventPost {QueueEntryId} {FilePath}: {Message}", entry.Id, payloadPath, ex.Message);
+ }
+
if (ex is ArgumentException || ex is DocumentNotFoundException)
{
await CompleteEntryAsync(entry, ep, createdUtc);
@@ -236,16 +283,27 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
errorCount++;
if (!isSingleEvent)
+ {
eventsToRetry.AddRange(events);
+ }
}
+ bool completeTracking = true;
if (eventsToRetry.Count > 0)
- await AppDiagnostics.PostsRetryTime.TimeAsync(() => RetryEventsAsync(eventsToRetry, ep, entry, project, isInternalProject));
+ {
+ bool propagateTracking = await _eventPostService.AddPendingProcessingUnitsAsync(ep, eventsToRetry.Count);
+ await AppDiagnostics.PostsRetryTime.TimeAsync(() => RetryEventsAsync(eventsToRetry, ep, entry, project, isInternalProject, propagateTracking));
+ completeTracking = propagateTracking;
+ }
if (isSingleEvent && errorCount > 0)
+ {
await AbandonEntryAsync(entry);
+ }
else
- await CompleteEntryAsync(entry, ep, createdUtc);
+ {
+ await CompleteEntryAsync(entry, ep, createdUtc, completeTracking);
+ }
return JobResult.Success;
}
@@ -255,14 +313,18 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
using (_logger.BeginScope(new ExceptionlessState().Tag("parsing")))
{
if (!isInternalProject && _logger.IsEnabled(LogLevel.Debug))
+ {
_logger.LogDebug("Parsing EventPost: {QueueEntryId}", queueEntryId);
+ }
var events = new List();
try
{
var encoding = Encoding.UTF8;
if (!String.IsNullOrEmpty(ep.CharSet))
+ {
encoding = Encoding.GetEncoding(ep.CharSet);
+ }
AppDiagnostics.PostsParsingTime.Time(() =>
{
@@ -276,7 +338,9 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
// set the reference id to the event id if one was defined.
if (!String.IsNullOrEmpty(ev.Id) && String.IsNullOrEmpty(ev.ReferenceId))
+ {
ev.ReferenceId = ev.Id;
+ }
// the event id and stack id should never be set for posted events
ev.Id = ev.StackId = null!;
@@ -288,17 +352,22 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex
catch (Exception ex)
{
AppDiagnostics.PostsParseErrors.Add(1);
- if (!isInternalProject) _logger.LogError(ex, "An error occurred while processing the EventPost {QueueEntryId}: {Message}", queueEntryId, ex.Message);
+ if (!isInternalProject)
+ {
+ _logger.LogError(ex, "An error occurred while processing the EventPost {QueueEntryId}: {Message}", queueEntryId, ex.Message);
+ }
}
if (!isInternalProject && _logger.IsEnabled(LogLevel.Debug))
+ {
_logger.LogDebug("Parsed {ParsedCount} events from EventPost: {QueueEntryId}", events?.Count ?? 0, queueEntryId);
+ }
return events;
}
}
- private async Task RetryEventsAsync(List eventsToRetry, EventPostInfo ep, IQueueEntry queueEntry, Project project, bool isInternalProject)
+ private async Task RetryEventsAsync(List eventsToRetry, EventPost ep, IQueueEntry queueEntry, Project project, bool isInternalProject, bool propagateTracking)
{
AppDiagnostics.EventsRetryCount.Add(eventsToRetry.Count);
foreach (var ev in eventsToRetry)
@@ -316,6 +385,7 @@ await _eventPostService.EnqueueAsync(new EventPost(false)
IpAddress = ep.IpAddress,
MediaType = ep.MediaType,
OrganizationId = ep.OrganizationId ?? project.OrganizationId,
+ ProcessingCorrelationId = propagateTracking ? ep.ProcessingCorrelationId : null,
ProjectId = ep.ProjectId ?? project.Id,
UserAgent = ep.UserAgent
}, stream);
@@ -325,7 +395,9 @@ await _eventPostService.EnqueueAsync(new EventPost(false)
if (!isInternalProject && _logger.IsEnabled(LogLevel.Critical))
{
using (_logger.BeginScope(new ExceptionlessState().Property("Event", new { ev.Date, ev.StackId, ev.Type, ev.Source, ev.Message, ev.Value, ev.Geo, ev.ReferenceId, ev.Tags })))
+ {
_logger.LogCritical(ex, "Error while requeuing event post {QueueEntryId} {FilePath}: {Message}", queueEntry.Id, queueEntry.Value.FilePath, ex.Message);
+ }
}
AppDiagnostics.EventsRetryErrors.Add(1);
@@ -338,12 +410,16 @@ private Task AbandonEntryAsync(IQueueEntry queueEntry)
return AppDiagnostics.PostsAbandonTime.TimeAsync(queueEntry.AbandonAsync);
}
- private Task CompleteEntryAsync(IQueueEntry entry, EventPost eventPost, DateTime created)
+ private Task CompleteEntryAsync(IQueueEntry entry, EventPost eventPost, DateTime created, bool completeTracking = true)
{
return AppDiagnostics.PostsCompleteTime.TimeAsync(async () =>
{
await entry.CompleteAsync();
- await _eventPostService.CompleteEventPostAsync(eventPost.FilePath, eventPost.ProjectId, created, eventPost.ShouldArchive);
+ bool eventPostCompleted = await _eventPostService.CompleteEventPostAsync(eventPost.FilePath, eventPost.ProjectId, created, eventPost.ShouldArchive);
+ if (eventPostCompleted)
+ {
+ await _eventPostService.MarkProcessingCompletedAsync(entry.Id, eventPost, completeTracking);
+ }
});
}
diff --git a/src/Exceptionless.Core/Jobs/IngestionStackEventCountJob.cs b/src/Exceptionless.Core/Jobs/IngestionStackEventCountJob.cs
new file mode 100644
index 0000000000..c05114615d
--- /dev/null
+++ b/src/Exceptionless.Core/Jobs/IngestionStackEventCountJob.cs
@@ -0,0 +1,52 @@
+using Exceptionless.Core.Services;
+using Foundatio.Jobs;
+using Foundatio.Resilience;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs;
+
+///
+/// Drains V3 stack-usage settlements without a cluster-wide lock. Redis leases partition work
+/// across every job instance, and Elasticsearch settlement sequences make retries idempotent.
+///
+[Job(Description = "Apply V3 event occurrence counts to stacks.", InitialDelay = "2s", Interval = "1s")]
+public sealed class IngestionStackEventCountJob : JobBase, IHealthCheck
+{
+ private readonly StackService _stackService;
+ private DateTime? _lastRun;
+
+ public IngestionStackEventCountJob(
+ StackService stackService,
+ TimeProvider timeProvider,
+ IResiliencePolicyProvider resiliencePolicyProvider,
+ ILoggerFactory loggerFactory)
+ : base(timeProvider, resiliencePolicyProvider, loggerFactory)
+ {
+ _stackService = stackService;
+ }
+
+ protected override async Task RunInternalAsync(JobContext context)
+ {
+ _lastRun = _timeProvider.GetUtcNow().UtcDateTime;
+ _logger.LogTrace("Start applying V3 stack event counts");
+ await _stackService.SaveIngestionStackUsagesAsync(cancellationToken: context.CancellationToken);
+ _logger.LogTrace("Finished applying V3 stack event counts");
+ return JobResult.Success;
+ }
+
+ public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+ {
+ if (!_lastRun.HasValue)
+ {
+ return Task.FromResult(HealthCheckResult.Healthy("Job has not been run yet."));
+ }
+
+ if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(_lastRun.Value) > TimeSpan.FromSeconds(15))
+ {
+ return Task.FromResult(HealthCheckResult.Unhealthy("Job has not run in the last 15 seconds."));
+ }
+
+ return Task.FromResult(HealthCheckResult.Healthy("Job has run in the last 15 seconds."));
+ }
+}
diff --git a/src/Exceptionless.Core/Jobs/StackEventCountJob.cs b/src/Exceptionless.Core/Jobs/StackEventCountJob.cs
index 3e2dea7ca1..0a4d23000f 100644
--- a/src/Exceptionless.Core/Jobs/StackEventCountJob.cs
+++ b/src/Exceptionless.Core/Jobs/StackEventCountJob.cs
@@ -34,7 +34,7 @@ protected override async Task RunInternalAsync(JobContext context)
{
_lastRun = _timeProvider.GetUtcNow().UtcDateTime;
_logger.LogTrace("Start save stack event counts");
- await _stackService.SaveStackUsagesAsync(cancellationToken: context.CancellationToken);
+ await _stackService.SaveLegacyStackUsagesAsync(cancellationToken: context.CancellationToken);
_logger.LogTrace("Finished save stack event counts");
return JobResult.Success;
}
@@ -42,10 +42,14 @@ protected override async Task RunInternalAsync(JobContext context)
public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
if (!_lastRun.HasValue)
+ {
return Task.FromResult(HealthCheckResult.Healthy("Job has not been run yet."));
+ }
if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(_lastRun.Value) > TimeSpan.FromSeconds(15))
+ {
return Task.FromResult(HealthCheckResult.Unhealthy("Job has not run in the last 15 seconds."));
+ }
return Task.FromResult(HealthCheckResult.Healthy("Job has run in the last 15 seconds."));
}
diff --git a/src/Exceptionless.Core/Jobs/StackStatusJob.cs b/src/Exceptionless.Core/Jobs/StackStatusJob.cs
index 0f0c8d3d2f..9f309311df 100644
--- a/src/Exceptionless.Core/Jobs/StackStatusJob.cs
+++ b/src/Exceptionless.Core/Jobs/StackStatusJob.cs
@@ -1,5 +1,7 @@
using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Models.Ingestion;
using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Services;
using Foundatio.Caching;
using Foundatio.Jobs;
using Foundatio.Lock;
@@ -15,15 +17,17 @@ public class StackStatusJob : JobWithLockBase, IHealthCheck
{
private readonly IStackRepository _stackRepository;
private readonly ILockProvider _lockProvider;
+ private readonly IStackRouteResolver _stackRouteResolver;
private DateTime? _lastRun;
- public StackStatusJob(IStackRepository stackRepository, ICacheClient cacheClient,
+ public StackStatusJob(IStackRepository stackRepository, IStackRouteResolver stackRouteResolver, ICacheClient cacheClient,
TimeProvider timeProvider,
IResiliencePolicyProvider resiliencePolicyProvider,
ILoggerFactory loggerFactory
) : base(timeProvider, resiliencePolicyProvider, loggerFactory)
{
_stackRepository = stackRepository;
+ _stackRouteResolver = stackRouteResolver;
_lockProvider = new ThrottlingLockProvider(cacheClient, 1, TimeSpan.FromSeconds(10), timeProvider, resiliencePolicyProvider, loggerFactory);
}
@@ -43,18 +47,26 @@ protected override async Task RunInternalAsync(JobContext context)
while (results.Documents.Count > 0 && !context.CancellationToken.IsCancellationRequested)
{
foreach (var stack in results.Documents)
+ {
stack.MarkOpen();
+ }
await _stackRepository.SaveAsync(results.Documents);
+ await Task.WhenAll(results.Documents.Select(stack =>
+ _stackRouteResolver.UpdateAsync(stack.ProjectId, stack.SignatureHash, StackRouteResolver.CreateRoute(stack))));
// Sleep so we are not hammering the backend.
await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider);
if (context.CancellationToken.IsCancellationRequested || !await results.NextPageAsync())
+ {
break;
+ }
if (results.Documents.Count > 0)
+ {
await context.RenewLockAsync();
+ }
}
_logger.LogTrace("Finished save stack event counts");
@@ -64,10 +76,14 @@ protected override async Task RunInternalAsync(JobContext context)
public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
if (!_lastRun.HasValue)
+ {
return Task.FromResult(HealthCheckResult.Healthy("Job has not been run yet."));
+ }
if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(_lastRun.Value) > TimeSpan.FromMinutes(1))
+ {
return Task.FromResult(HealthCheckResult.Unhealthy("Job has not run in the last minute."));
+ }
return Task.FromResult(HealthCheckResult.Healthy("Job has run in the last minute."));
}
diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/EventIngestionSideEffectsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/EventIngestionSideEffectsWorkItemHandler.cs
new file mode 100644
index 0000000000..2d45ace859
--- /dev/null
+++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/EventIngestionSideEffectsWorkItemHandler.cs
@@ -0,0 +1,173 @@
+using System.Security.Cryptography;
+using System.Text;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.WorkItems;
+using Exceptionless.Core.Pipeline;
+using Exceptionless.Core.Plugins.EventProcessor;
+using Exceptionless.Core.Plugins.EventProcessor.Default;
+using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Services;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Foundatio.Queues;
+using Foundatio.Repositories;
+using Foundatio.Storage;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs.WorkItemHandlers;
+
+public sealed class EventIngestionSideEffectsWorkItemHandler(
+ IOrganizationRepository organizationRepository,
+ IProjectRepository projectRepository,
+ IEventRepository eventRepository,
+ IStackRepository stackRepository,
+ IIngestionStackUsageStore ingestionStackUsageStore,
+ RequestInfoPlugin requestInfoPlugin,
+ EnvironmentInfoPlugin environmentInfoPlugin,
+ GeoPlugin geoPlugin,
+ QueueNotificationAction queueNotificationAction,
+ IngestionSideEffectExecutor sideEffectExecutor,
+ IQueue workItemQueue,
+ IFileStorage storage,
+ AppOptions options,
+ ILockProvider lockProvider,
+ ILoggerFactory loggerFactory) : WorkItemHandlerBase(loggerFactory)
+{
+ public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default)
+ {
+ var sideEffects = (EventIngestionSideEffectsWorkItem)workItem;
+ return lockProvider.TryAcquireAsync(sideEffects.UniqueIdentifier, TimeSpan.FromMinutes(5), cancellationToken);
+ }
+
+ public override async Task HandleItemAsync(WorkItemContext context)
+ {
+ var workItem = context.GetData()!;
+ var organization = await organizationRepository.GetByIdAsync(workItem.OrganizationId, o => o.Cache());
+ var project = await projectRepository.GetByIdAsync(workItem.ProjectId, o => o.Cache());
+ if (organization is null || project is null)
+ {
+ return;
+ }
+
+ var events = await eventRepository.GetByIdsAsync(workItem.EventIds);
+ if (events.Count == 0)
+ {
+ return;
+ }
+
+ if (options.EnableArchive)
+ {
+ DateTime archiveDate = events.Min(ev => ev.CreatedUtc);
+ string archivePath = GetArchivePath(workItem, archiveDate);
+ bool archived = await storage.SaveObjectAsync(archivePath, events.OrderBy(ev => ev.Id).ToArray(), context.CancellationToken);
+ if (!archived)
+ {
+ throw new InvalidOperationException($"Unable to archive V3 ingestion side-effect batch '{workItem.UniqueIdentifier}'.");
+ }
+ }
+
+ var stacks = await stackRepository.GetByIdsAsync(events.Select(e => e.StackId).Distinct().ToArray());
+ var stacksById = stacks.ToDictionary(stack => stack.Id);
+ var contexts = new List(events.Count);
+ foreach (var ev in events)
+ {
+ if (!stacksById.TryGetValue(ev.StackId, out var stack))
+ {
+ continue;
+ }
+
+ contexts.Add(new EventContext(ev, organization, project)
+ {
+ Stack = stack,
+ IsNew = ev.IsFirstOccurrence,
+ IsRegression = ev.IsRegression,
+ IsIngestionV3 = true,
+ IsProcessed = true
+ });
+ }
+
+ if (contexts.Count == 0)
+ {
+ return;
+ }
+
+ if (!project.IsConfigured.GetValueOrDefault())
+ {
+ await sideEffectExecutor.ExecuteAsync(IngestionSideEffectExecutor.ProjectConfiguredStage, project.Id, [project.Id], async _ =>
+ {
+ await workItemQueue.EnqueueAsync(new SetProjectIsConfiguredWorkItem
+ {
+ ProjectId = project.Id,
+ IsConfigured = true
+ });
+ }, context.CancellationToken);
+ }
+
+ var enrichmentContexts = contexts
+ .Where(eventContext => eventContext.Event.Data?.ContainsKey(Event.KnownDataKeys.RequestInfo) is true
+ || eventContext.Event.Data?.ContainsKey(Event.KnownDataKeys.EnvironmentInfo) is true)
+ .ToList();
+ if (enrichmentContexts.Count > 0)
+ {
+ await requestInfoPlugin.EventBatchProcessingAsync(enrichmentContexts);
+ foreach (var eventContext in enrichmentContexts)
+ {
+ await environmentInfoPlugin.EventProcessingAsync(eventContext);
+ }
+
+ await geoPlugin.EventBatchProcessingAsync(enrichmentContexts);
+ await eventRepository.SaveAsync(enrichmentContexts.Select(eventContext => eventContext.Event), o => o.Notifications(false));
+ }
+
+ var settledStackUsages = await ingestionStackUsageStore.SettleAsync(
+ contexts.Select(eventContext => new IngestionStackUsage(
+ eventContext.Event.Id,
+ organization.Id,
+ project.Id,
+ eventContext.Event.StackId,
+ eventContext.Event.Date.UtcDateTime)).ToArray(),
+ context.CancellationToken);
+ foreach (var usage in settledStackUsages)
+ {
+ if (stacksById.TryGetValue(usage.StackId, out var stack))
+ {
+ stack.TotalOccurrences += usage.Count;
+ if (stack.FirstOccurrence > usage.MinimumOccurrenceDateUtc)
+ {
+ stack.FirstOccurrence = usage.MinimumOccurrenceDateUtc;
+ }
+
+ if (stack.LastOccurrence < usage.MaximumOccurrenceDateUtc)
+ {
+ stack.LastOccurrence = usage.MaximumOccurrenceDateUtc;
+ }
+ }
+ }
+
+ var notificationContextsById = contexts.ToDictionary(eventContext => eventContext.Event.Id, StringComparer.Ordinal);
+ await sideEffectExecutor.ExecuteAsync(IngestionSideEffectExecutor.TerminalStage, project.Id, notificationContextsById.Keys.ToArray(), async pendingIds =>
+ {
+ var pendingContexts = pendingIds.Select(id => notificationContextsById[id]).ToArray();
+ await queueNotificationAction.ProcessIngestionV3BatchAsync(pendingContexts);
+ }, context.CancellationToken);
+ }
+
+ private static string GetArchivePath(EventIngestionSideEffectsWorkItem workItem, DateTime createdUtc)
+ {
+ return Path.Combine(
+ "archive",
+ "v3",
+ createdUtc.ToString("yy"),
+ createdUtc.ToString("MM"),
+ createdUtc.ToString("dd"),
+ createdUtc.ToString("HH"),
+ createdUtc.ToString("mm"),
+ workItem.ProjectId,
+ String.Concat(GetWorkItemHash(workItem), ".json"));
+ }
+
+ private static string GetWorkItemHash(EventIngestionSideEffectsWorkItem workItem)
+ {
+ return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(workItem.UniqueIdentifier))).ToLowerInvariant();
+ }
+}
diff --git a/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3BufferedRecord.cs b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3BufferedRecord.cs
new file mode 100644
index 0000000000..b5e7171ae3
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3BufferedRecord.cs
@@ -0,0 +1,117 @@
+using System.Buffers;
+using System.Text.Json;
+using Exceptionless.Core.Serialization;
+
+namespace Exceptionless.Core.Models.Ingestion;
+
+///
+/// Owns one bounded NDJSON record and its routing-only projection. The complete event is
+/// materialized only after stack routing proves that the record will continue through ingestion.
+///
+internal sealed class EventIngestionV3BufferedRecord : IDisposable
+{
+ private IMemoryOwner? _owner;
+ private EventIngestionV3Event? _materializedEvent;
+
+ public EventIngestionV3BufferedRecord(
+ IMemoryOwner owner,
+ int length,
+ EventIngestionV3Event routingEvent)
+ {
+ ArgumentNullException.ThrowIfNull(owner);
+ ArgumentNullException.ThrowIfNull(routingEvent);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(length);
+ if (length > owner.Memory.Length)
+ {
+ throw new ArgumentOutOfRangeException(nameof(length));
+ }
+
+ _owner = owner;
+ Length = length;
+ RoutingEvent = routingEvent;
+ }
+
+ public int Length { get; }
+
+ public EventIngestionV3Event RoutingEvent { get; }
+
+ internal bool IsMaterialized => _materializedEvent is not null;
+
+ public EventIngestionV3Event Materialize()
+ {
+ if (_materializedEvent is not null)
+ {
+ return _materializedEvent;
+ }
+
+ ObjectDisposedException.ThrowIf(_owner is null, this);
+ var payload = JsonSerializer.Deserialize(
+ _owner.Memory.Span[..Length],
+ EventIngestionV3SurvivorJsonContext.Default.EventIngestionV3SurvivorPayload)
+ ?? throw new JsonException("The stream cannot contain null events.");
+
+ // Routing strings can dominate an error event, especially the raw stack trace. They were
+ // already decoded by the cheap routing projection, so deserialize only the survivor-only
+ // fields and reuse those references instead of allocating a second LOH-sized stack string.
+ _materializedEvent = new EventIngestionV3Event
+ {
+ Id = RoutingEvent.Id,
+ Type = RoutingEvent.Type,
+ Date = payload.Date,
+ Source = RoutingEvent.Source,
+ Message = payload.Message,
+ ReferenceId = payload.ReferenceId,
+ Value = payload.Value,
+ Tags = payload.Tags,
+ Version = payload.Version,
+ Level = payload.Level,
+ Client = payload.Client,
+ ExceptionType = RoutingEvent.ExceptionType,
+ StackTrace = RoutingEvent.StackTrace,
+ Stacking = RoutingEvent.Stacking is null
+ ? null
+ : new EventIngestionV3Stacking
+ {
+ Title = payload.Stacking?.Title,
+ SignatureData = RoutingEvent.Stacking.SignatureData
+ },
+ User = payload.User,
+ Request = payload.Request,
+ Environment = payload.Environment,
+ Data = payload.Data
+ };
+ return _materializedEvent;
+ }
+
+ public void Dispose()
+ {
+ _owner?.Dispose();
+ _owner = null;
+ }
+}
+
+///
+/// The fields that are needed only after discard routing. Routing properties are intentionally
+/// absent so System.Text.Json skips their values without decoding duplicate strings.
+///
+internal sealed record EventIngestionV3SurvivorPayload
+{
+ public DateTimeOffset? Date { get; init; }
+ public string? Message { get; init; }
+ public string? ReferenceId { get; init; }
+ public decimal? Value { get; init; }
+ public string[]? Tags { get; init; }
+ public string? Version { get; init; }
+ public string? Level { get; init; }
+ public EventIngestionV3Client? Client { get; init; }
+ public EventIngestionV3SurvivorStacking? Stacking { get; init; }
+ public EventIngestionV3User? User { get; init; }
+ public EventIngestionV3Request? Request { get; init; }
+ public EventIngestionV3Environment? Environment { get; init; }
+ public JsonElement? Data { get; init; }
+}
+
+internal sealed record EventIngestionV3SurvivorStacking
+{
+ public string? Title { get; init; }
+}
diff --git a/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Event.cs b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Event.cs
new file mode 100644
index 0000000000..2646371490
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Event.cs
@@ -0,0 +1,125 @@
+using System.ComponentModel.DataAnnotations;
+using System.Text.Json;
+
+namespace Exceptionless.Core.Models.Ingestion;
+
+///
+/// A compact V3 ingestion event. Organization and project ownership come from
+/// the authenticated request rather than the event payload.
+///
+public sealed record EventIngestionV3Event
+{
+ [Required]
+ [StringLength(EventIngestionV3Limits.MaximumEventIdLength, MinimumLength = 1)]
+ public string Id { get; init; } = null!;
+
+ [Required]
+ [StringLength(EventIngestionV3Limits.MaximumTypeLength, MinimumLength = 1)]
+ public string Type { get; init; } = null!;
+
+ public DateTimeOffset? Date { get; init; }
+
+ [StringLength(EventIngestionV3Limits.MaximumSourceLength, MinimumLength = 1)]
+ public string? Source { get; init; }
+
+ [StringLength(EventIngestionV3Limits.MaximumMessageLength, MinimumLength = 1)]
+ public string? Message { get; init; }
+
+ [StringLength(EventIngestionV3Limits.MaximumReferenceIdLength, MinimumLength = EventIngestionV3Limits.MinimumReferenceIdLength)]
+ public string? ReferenceId { get; init; }
+
+ public decimal? Value { get; init; }
+
+ [MaxLength(EventIngestionV3Limits.MaximumTags)]
+ public string[]? Tags { get; init; }
+
+ [StringLength(EventIngestionV3Limits.MaximumVersionLength, MinimumLength = 1)]
+ public string? Version { get; init; }
+
+ [StringLength(EventIngestionV3Limits.MaximumLevelLength, MinimumLength = 1)]
+ public string? Level { get; init; }
+
+ public EventIngestionV3Client? Client { get; init; }
+
+ [StringLength(EventIngestionV3Limits.MaximumExceptionTypeLength, MinimumLength = 1)]
+ public string? ExceptionType { get; init; }
+
+ [StringLength(EventIngestionV3Limits.MaximumStackTraceLength, MinimumLength = 1)]
+ public string? StackTrace { get; init; }
+
+ public EventIngestionV3Stacking? Stacking { get; init; }
+
+ public EventIngestionV3User? User { get; init; }
+
+ public EventIngestionV3Request? Request { get; init; }
+
+ public EventIngestionV3Environment? Environment { get; init; }
+
+ public JsonElement? Data { get; init; }
+}
+
+public sealed record EventIngestionV3Client
+{
+ [Required]
+ [StringLength(EventIngestionV3Limits.MaximumClientNameLength, MinimumLength = 1)]
+ public string Name { get; init; } = null!;
+
+ [Required]
+ [StringLength(EventIngestionV3Limits.MaximumClientVersionLength, MinimumLength = 1)]
+ public string Version { get; init; } = null!;
+}
+
+public sealed record EventIngestionV3Stacking
+{
+ [StringLength(EventIngestionV3Limits.MaximumStackTitleLength, MinimumLength = 1)]
+ public string? Title { get; init; }
+
+ [Required]
+ public Dictionary SignatureData { get; init; } = null!;
+}
+
+public sealed record EventIngestionV3User
+{
+ [StringLength(EventIngestionV3Limits.MaximumUserIdentityLength, MinimumLength = 1)]
+ public string? Identity { get; init; }
+
+ [StringLength(EventIngestionV3Limits.MaximumUserNameLength, MinimumLength = 1)]
+ public string? Name { get; init; }
+
+ public JsonElement? Data { get; init; }
+}
+
+public sealed record EventIngestionV3Request
+{
+ public string? UserAgent { get; init; }
+ public string? HttpMethod { get; init; }
+ public bool? IsSecure { get; init; }
+ public string? Host { get; init; }
+ public int? Port { get; init; }
+ public string? Path { get; init; }
+ public string? Referrer { get; init; }
+ public string? ClientIpAddress { get; init; }
+ public Dictionary? Headers { get; init; }
+ public Dictionary? Cookies { get; init; }
+ public Dictionary? QueryString { get; init; }
+ public JsonElement? PostData { get; init; }
+ public JsonElement? Data { get; init; }
+}
+
+public sealed record EventIngestionV3Environment
+{
+ public string? Architecture { get; init; }
+ public string? OSName { get; init; }
+ public string? OSVersion { get; init; }
+ public string? MachineName { get; init; }
+ public string? RuntimeVersion { get; init; }
+ public string? ProcessName { get; init; }
+ public string? ProcessId { get; init; }
+ public string? ThreadName { get; init; }
+ public string? ThreadId { get; init; }
+ public int? ProcessorCount { get; init; }
+ public long? TotalPhysicalMemory { get; init; }
+ public long? AvailablePhysicalMemory { get; init; }
+ public long? ProcessMemorySize { get; init; }
+ public JsonElement? Data { get; init; }
+}
diff --git a/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3EventSizer.cs b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3EventSizer.cs
new file mode 100644
index 0000000000..fdedfe630a
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3EventSizer.cs
@@ -0,0 +1,142 @@
+using System.Text;
+using System.Text.Json;
+
+namespace Exceptionless.Core.Models.Ingestion;
+
+public static class EventIngestionV3EventSizer
+{
+ public static long GetEstimatedSize(EventIngestionV3Event source)
+ {
+ long size = 128;
+ size += GetSize(source.Id) + GetSize(source.Type) + GetSize(source.Source) + GetSize(source.Message);
+ size += GetSize(source.ReferenceId) + GetSize(source.ExceptionType) + GetSize(source.StackTrace);
+ size += GetSize(source.Version) + GetSize(source.Level);
+ size += GetSize(source.Tags);
+ size += GetSize(source.Data);
+ if (source.Client is not null)
+ {
+ size += 32 + GetSize(source.Client.Name) + GetSize(source.Client.Version);
+ }
+
+ if (source.Stacking is not null)
+ {
+ size += 32 + GetSize(source.Stacking.Title) + GetSize(source.Stacking.SignatureData);
+ }
+
+ if (source.User is not null)
+ {
+ size += 32 + GetSize(source.User.Identity) + GetSize(source.User.Name) + GetSize(source.User.Data);
+ }
+
+ if (source.Request is not null)
+ {
+ size += 96 + GetSize(source.Request.UserAgent) + GetSize(source.Request.HttpMethod) + GetSize(source.Request.Host);
+ size += GetSize(source.Request.Path) + GetSize(source.Request.Referrer) + GetSize(source.Request.ClientIpAddress);
+ size += GetSize(source.Request.Headers) + GetSize(source.Request.Cookies) + GetSize(source.Request.QueryString);
+ size += GetSize(source.Request.PostData) + GetSize(source.Request.Data);
+ }
+ if (source.Environment is not null)
+ {
+ size += 96 + GetSize(source.Environment.Architecture) + GetSize(source.Environment.OSName);
+ size += GetSize(source.Environment.OSVersion) + GetSize(source.Environment.MachineName);
+ size += GetSize(source.Environment.RuntimeVersion) + GetSize(source.Environment.ProcessName);
+ size += GetSize(source.Environment.ProcessId) + GetSize(source.Environment.ThreadName);
+ size += GetSize(source.Environment.ThreadId) + GetSize(source.Environment.Data);
+ }
+
+ return size;
+ }
+
+ private static int GetSize(string? value) => value is null ? 0 : Encoding.UTF8.GetByteCount(value);
+
+ private static long GetSize(IEnumerable? values)
+ {
+ if (values is null)
+ {
+ return 0;
+ }
+
+ long size = 0;
+ foreach (string value in values)
+ {
+ size += GetSize(value);
+ }
+
+ return size;
+ }
+
+ private static long GetSize(IReadOnlyDictionary? values)
+ {
+ if (values is null)
+ {
+ return 0;
+ }
+
+ long size = 0;
+ foreach (var pair in values)
+ {
+ size += GetSize(pair.Key) + GetSize(pair.Value);
+ }
+
+ return size;
+ }
+
+ private static long GetSize(IReadOnlyDictionary? values)
+ {
+ if (values is null)
+ {
+ return 0;
+ }
+
+ long size = 0;
+ foreach (var pair in values)
+ {
+ size += GetSize(pair.Key) + GetSize(pair.Value);
+ }
+
+ return size;
+ }
+
+ private static long GetSize(JsonElement? element)
+ {
+ if (element is null)
+ {
+ return 0;
+ }
+
+ return GetSize(element.Value);
+ }
+
+ private static long GetSize(JsonElement element)
+ {
+ if (element.ValueKind == JsonValueKind.Object)
+ {
+ long size = 0;
+ foreach (JsonProperty property in element.EnumerateObject())
+ {
+ size += GetSize(property.Name) + GetSize(property.Value) + 4;
+ }
+
+ return size;
+ }
+ if (element.ValueKind == JsonValueKind.Array)
+ {
+ long size = 0;
+ foreach (JsonElement item in element.EnumerateArray())
+ {
+ size += GetSize(item) + 1;
+ }
+
+ return size;
+ }
+
+ return element.ValueKind switch
+ {
+ JsonValueKind.String => GetSize(element.GetString()),
+ JsonValueKind.Number => 32,
+ JsonValueKind.True or JsonValueKind.False => 5,
+ JsonValueKind.Null or JsonValueKind.Undefined => 4,
+ _ => 0
+ };
+ }
+}
diff --git a/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Limits.cs b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Limits.cs
new file mode 100644
index 0000000000..3de2932fca
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Limits.cs
@@ -0,0 +1,28 @@
+namespace Exceptionless.Core.Models.Ingestion;
+
+public static class EventIngestionV3Limits
+{
+ public const int MaximumEventIdLength = 100;
+ public const int MaximumTypeLength = 100;
+ public const int MaximumSourceLength = 2000;
+ public const int MaximumMessageLength = 2000;
+ public const int MinimumReferenceIdLength = 8;
+ public const int MaximumReferenceIdLength = 100;
+ public const int MaximumExceptionTypeLength = 2000;
+ public const int MaximumStackTraceLength = 256 * 1024;
+ public const int MaximumUserIdentityLength = 255;
+ public const int MaximumUserNameLength = 255;
+ public const int MaximumVersionLength = 255;
+ public const int MaximumLevelLength = 32;
+ public const int MaximumClientNameLength = 255;
+ public const int MaximumClientVersionLength = 255;
+ public const int MaximumStackTitleLength = 1000;
+ public const int MaximumTags = 50;
+ public const int MaximumTagLength = 100;
+ public const int MaximumMetadataEntries = 100;
+ public const int MaximumMetadataKeyLength = 255;
+ public const int MaximumMetadataValueLength = 16 * 1024;
+ public const int MaximumDataTokens = 2000;
+ public const int MaximumDataStringLength = 64 * 1024;
+ public const int MaximumJsonDepth = 32;
+}
diff --git a/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Response.cs b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Response.cs
new file mode 100644
index 0000000000..3ac510bc30
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Response.cs
@@ -0,0 +1,30 @@
+namespace Exceptionless.Core.Models.Ingestion;
+
+public sealed record EventIngestionV3Response
+{
+ public int Received { get; set; }
+ public int Persisted { get; set; }
+ public int Discarded { get; set; }
+ public int Duplicate { get; set; }
+ public int Blocked { get; set; }
+ public int Invalid { get; set; }
+ public List Errors { get; init; } = [];
+
+ public void Add(EventIngestionV3Response other)
+ {
+ Received += other.Received;
+ Persisted += other.Persisted;
+ Discarded += other.Discarded;
+ Duplicate += other.Duplicate;
+ Blocked += other.Blocked;
+ Invalid += other.Invalid;
+
+ int remaining = 100 - Errors.Count;
+ if (remaining > 0 && other.Errors.Count > 0)
+ {
+ Errors.AddRange(other.Errors.Take(remaining));
+ }
+ }
+}
+
+public sealed record EventIngestionV3Error(string Id, string Code, string Message);
diff --git a/src/Exceptionless.Core/Models/Ingestion/EventIngestionWrite.cs b/src/Exceptionless.Core/Models/Ingestion/EventIngestionWrite.cs
new file mode 100644
index 0000000000..dc6ae8d0a0
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Ingestion/EventIngestionWrite.cs
@@ -0,0 +1,47 @@
+using Exceptionless.Core.Models;
+
+namespace Exceptionless.Core.Models.Ingestion;
+
+public sealed record EventIngestionWrite(
+ string ClientId,
+ PersistentEvent Event,
+ StackFingerprint Fingerprint,
+ StackRoute? Route,
+ bool IsRegressionCandidate = false);
+
+public sealed record EventIngestionIdentity(
+ string ClientId,
+ string EventId,
+ bool IsDuplicate,
+ bool IsPersisted,
+ string? PersistedStackId,
+ StackStatus? PersistedStackStatus,
+ bool IsRecoveryEligible,
+ DateTime CreatedUtc,
+ DateTimeOffset EventDate);
+
+public sealed record EventIngestionReconciliation(
+ string EventId,
+ string StackId);
+
+public sealed record EventUsageSettlement(string EventId, DateTime CreatedUtc);
+
+public sealed record EventBatchWriteResult(
+ int Persisted,
+ int Duplicate,
+ IReadOnlyCollection Settlements);
+
+public sealed record EventIngestionReservation(
+ string Id,
+ string OrganizationId,
+ int Count,
+ bool IsUnlimited = false)
+{
+ public static EventIngestionReservation Unlimited(string organizationId, int count) => new(String.Empty, organizationId, count, true);
+}
+
+public sealed class EventBatchWriteException(Exception innerException, IReadOnlyCollection settlements)
+ : Exception("Event persistence completed only partially.", innerException)
+{
+ public IReadOnlyCollection Settlements { get; } = settlements;
+}
diff --git a/src/Exceptionless.Core/Models/Ingestion/StackFingerprint.cs b/src/Exceptionless.Core/Models/Ingestion/StackFingerprint.cs
new file mode 100644
index 0000000000..b198b30616
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Ingestion/StackFingerprint.cs
@@ -0,0 +1,3 @@
+namespace Exceptionless.Core.Models.Ingestion;
+
+public sealed record StackFingerprint(string SignatureHash, IReadOnlyDictionary SignatureData, string? Title = null);
diff --git a/src/Exceptionless.Core/Models/Ingestion/StackRoute.cs b/src/Exceptionless.Core/Models/Ingestion/StackRoute.cs
new file mode 100644
index 0000000000..3e1f74c5cb
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Ingestion/StackRoute.cs
@@ -0,0 +1,43 @@
+using Exceptionless.Core.Models;
+
+namespace Exceptionless.Core.Models.Ingestion;
+
+public sealed record StackRoute(
+ string StackId,
+ StackStatus Status,
+ long Version,
+ string? FixedInVersion = null,
+ DateTime? DateFixed = null,
+ bool OccurrencesAreCritical = false,
+ string? RegressionEventId = null,
+ string? IngestionFirstEventId = null)
+{
+ public bool IsDiscarded => Status is StackStatus.Discarded;
+}
+
+public sealed record StackRouteCacheEntry(
+ bool Exists,
+ long Version,
+ string? StackId = null,
+ StackStatus Status = StackStatus.Open,
+ string? FixedInVersion = null,
+ DateTime? DateFixed = null,
+ bool OccurrencesAreCritical = false,
+ string? RegressionEventId = null,
+ string? IngestionFirstEventId = null)
+{
+ public StackRoute? ToRoute() => Exists && StackId is not null
+ ? new StackRoute(StackId, Status, Version, FixedInVersion, DateFixed, OccurrencesAreCritical, RegressionEventId, IngestionFirstEventId)
+ : null;
+
+ public static StackRouteCacheEntry FromRoute(StackRoute route) => new(
+ true,
+ route.Version,
+ route.StackId,
+ route.Status,
+ route.FixedInVersion,
+ route.DateFixed,
+ route.OccurrencesAreCritical,
+ route.RegressionEventId,
+ route.IngestionFirstEventId);
+}
diff --git a/src/Exceptionless.Core/Models/Organization.cs b/src/Exceptionless.Core/Models/Organization.cs
index 0a92a03660..34319b2434 100644
--- a/src/Exceptionless.Core/Models/Organization.cs
+++ b/src/Exceptionless.Core/Models/Organization.cs
@@ -169,6 +169,11 @@ public Organization()
public ICollection Usage { get; set; }
public DateTime? LastEventDateUtc { get; set; }
+ ///
+ /// Most recent five-minute usage bucket durably applied to this organization.
+ ///
+ public DateTime? LastAppliedUsageBucketUtc { get; set; }
+
///
/// Optional data entries that contain additional configuration information for this organization.
///
@@ -280,4 +285,3 @@ public enum BillingStatus
Unpaid = 4
}
-
diff --git a/src/Exceptionless.Core/Models/PersistentEvent.cs b/src/Exceptionless.Core/Models/PersistentEvent.cs
index 968a4acd37..6c1b88a33a 100644
--- a/src/Exceptionless.Core/Models/PersistentEvent.cs
+++ b/src/Exceptionless.Core/Models/PersistentEvent.cs
@@ -1,5 +1,7 @@
+using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
using System.Diagnostics;
+using System.Text.Json.Serialization;
using Exceptionless.Core.Attributes;
using Exceptionless.Core.Extensions;
using Foundatio.Repositories.Models;
@@ -41,6 +43,32 @@ public class PersistentEvent : Event, IOwnedByOrganizationAndProjectAndStackWith
///
public bool IsFirstOccurrence { get; set; }
+ ///
+ /// Whether this occurrence caused a fixed stack to regress.
+ ///
+ [ReadOnly(true)]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+ public bool IsRegression { get; set; }
+
+ ///
+ /// Records that this occurrence was selected to regress the fixed stack generation
+ /// observed during ingestion. This makes a retry independent of the replay payload.
+ ///
+ [ReadOnly(true)]
+ [ApiIgnore]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+ public bool IngestionIsRegressionCandidate { get; set; }
+
+ [ReadOnly(true)]
+ [ApiIgnore]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+ public string? IngestionRegressionFixedInVersion { get; set; }
+
+ [ReadOnly(true)]
+ [ApiIgnore]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+ public DateTime IngestionRegressionDateFixed { get; set; }
+
///
/// The date that the event was created in the system.
///
diff --git a/src/Exceptionless.Core/Models/Project.cs b/src/Exceptionless.Core/Models/Project.cs
index 4ac783e893..794bc496f1 100644
--- a/src/Exceptionless.Core/Models/Project.cs
+++ b/src/Exceptionless.Core/Models/Project.cs
@@ -52,6 +52,11 @@ public Project()
public ICollection Usage { get; set; }
public DateTime? LastEventDateUtc { get; set; }
+ ///
+ /// Most recent five-minute usage bucket durably applied to this project.
+ ///
+ public DateTime? LastAppliedUsageBucketUtc { get; set; }
+
///
/// Optional data entries that contain additional configuration information for this project.
///
diff --git a/src/Exceptionless.Core/Models/Queues/EventNotification.cs b/src/Exceptionless.Core/Models/Queues/EventNotification.cs
index f5d732f7ad..6d3d7425ba 100644
--- a/src/Exceptionless.Core/Models/Queues/EventNotification.cs
+++ b/src/Exceptionless.Core/Models/Queues/EventNotification.cs
@@ -1,9 +1,16 @@
-namespace Exceptionless.Core.Queues.Models;
+using System.Text.Json.Serialization;
+using Exceptionless.Core.Queues;
-public record EventNotification
+namespace Exceptionless.Core.Queues.Models;
+
+public record EventNotification : IHaveDurableUniqueIdentifier
{
public required string EventId { get; set; }
public required bool IsNew { get; set; }
public required bool IsRegression { get; set; }
public required int TotalOccurrences { get; set; }
+ public string DeduplicationId { get; set; } = Guid.NewGuid().ToString("N");
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+ public bool UseDurableDeduplication { get; set; }
+ public string UniqueIdentifier => DeduplicationId;
}
diff --git a/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs b/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs
index 5ba9ecb819..4930ee2085 100644
--- a/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs
+++ b/src/Exceptionless.Core/Models/Queues/EventPostInfo.cs
@@ -1,4 +1,6 @@
-namespace Exceptionless.Core.Queues.Models;
+using System.Text.Json.Serialization;
+
+namespace Exceptionless.Core.Queues.Models;
public record EventPostInfo
{
@@ -20,5 +22,9 @@ public EventPost(bool shouldArchive)
}
public bool ShouldArchive { get; init; }
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+ public bool TrackProcessing { get; init; }
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+ public string? ProcessingCorrelationId { get; init; }
public string FilePath { get; set; } = null!;
}
diff --git a/src/Exceptionless.Core/Models/Queues/WebHookNotification.cs b/src/Exceptionless.Core/Models/Queues/WebHookNotification.cs
index ee0e849518..6a4f9e8bee 100644
--- a/src/Exceptionless.Core/Models/Queues/WebHookNotification.cs
+++ b/src/Exceptionless.Core/Models/Queues/WebHookNotification.cs
@@ -1,6 +1,9 @@
-namespace Exceptionless.Core.Queues.Models;
+using System.Text.Json.Serialization;
+using Exceptionless.Core.Queues;
-public record WebHookNotification
+namespace Exceptionless.Core.Queues.Models;
+
+public record WebHookNotification : IHaveDurableUniqueIdentifier
{
public required string OrganizationId { get; set; }
public required string ProjectId { get; set; }
@@ -8,6 +11,10 @@ public record WebHookNotification
public required WebHookType Type { get; set; } = WebHookType.General;
public required string Url { get; set; }
public required object? Data { get; set; }
+ public string DeduplicationId { get; set; } = Guid.NewGuid().ToString("N");
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+ public bool UseDurableDeduplication { get; set; }
+ public string UniqueIdentifier => DeduplicationId;
}
public enum WebHookType
diff --git a/src/Exceptionless.Core/Models/Stack.cs b/src/Exceptionless.Core/Models/Stack.cs
index 911835df31..cb451f283d 100644
--- a/src/Exceptionless.Core/Models/Stack.cs
+++ b/src/Exceptionless.Core/Models/Stack.cs
@@ -1,4 +1,5 @@
using System.Collections.ObjectModel;
+using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
using System.Diagnostics;
using System.Runtime.Serialization;
@@ -70,6 +71,23 @@ public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISu
///
public DateTime? DateFixed { get; set; }
+ ///
+ /// The event that most recently caused this stack to regress.
+ ///
+ [ObjectId]
+ public string? RegressionEventId { get; set; }
+
+ ///
+ /// Durable identity of the event selected as the first occurrence while V3 creates a stack.
+ /// A retry can therefore recover first-occurrence side effects if the process stops after the
+ /// stack write but before the event write.
+ ///
+ [ObjectId]
+ [ReadOnly(true)]
+ [ApiIgnore]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public string? IngestionFirstEventId { get; set; }
+
///
/// The stack title.
///
@@ -81,6 +99,13 @@ public class Stack : IOwnedByOrganizationAndProjectWithIdentity, IHaveDates, ISu
///
public int TotalOccurrences { get; set; }
+ ///
+ /// Monotonic fence for idempotent V3 aggregate settlements. This is persisted with the stack
+ /// so full-document saves cannot erase the last applied settlement identity.
+ ///
+ [ApiIgnore]
+ public long IngestionStackUsageSequence { get; set; }
+
///
/// The date of the 1st occurrence of this stack in UTC time.
///
diff --git a/src/Exceptionless.Core/Models/WorkItems/EventIngestionSideEffectsWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/EventIngestionSideEffectsWorkItem.cs
new file mode 100644
index 0000000000..bd7de060c1
--- /dev/null
+++ b/src/Exceptionless.Core/Models/WorkItems/EventIngestionSideEffectsWorkItem.cs
@@ -0,0 +1,12 @@
+using Foundatio.Queues;
+
+namespace Exceptionless.Core.Models.WorkItems;
+
+public sealed record EventIngestionSideEffectsWorkItem : IHaveUniqueIdentifier
+{
+ public required string OrganizationId { get; init; }
+ public required string ProjectId { get; init; }
+ public required string BatchId { get; init; }
+ public required string[] EventIds { get; init; }
+ public string UniqueIdentifier => String.Concat("event-ingestion-v3:", ProjectId, ":", BatchId);
+}
diff --git a/src/Exceptionless.Core/Pipeline/070_QueueNotificationAction.cs b/src/Exceptionless.Core/Pipeline/070_QueueNotificationAction.cs
index 9c9a6b8fa4..cfbd490a20 100644
--- a/src/Exceptionless.Core/Pipeline/070_QueueNotificationAction.cs
+++ b/src/Exceptionless.Core/Pipeline/070_QueueNotificationAction.cs
@@ -4,6 +4,7 @@
using Exceptionless.Core.Queues.Models;
using Exceptionless.Core.Repositories;
using Foundatio.Queues;
+using Foundatio.Repositories.Models;
using Microsoft.Extensions.Logging;
namespace Exceptionless.Core.Pipeline;
@@ -29,27 +30,80 @@ public override async Task ProcessAsync(EventContext ctx)
{
// if they don't have premium features, then we don't need to queue notifications
if (!ctx.Organization.HasPremiumFeatures)
+ {
return;
+ }
if (ctx.Stack is null || !ctx.Stack.AllowNotifications)
+ {
return;
+ }
if (ShouldQueueNotification(ctx))
- await _notificationQueue.EnqueueAsync(new EventNotification
+ {
+ await QueueEventNotificationAsync(ctx);
+ }
+
+ var webHooks = await GetWebHooksAsync(ctx.Event.OrganizationId, ctx.Event.ProjectId);
+ await QueueWebHooksAsync(ctx, webHooks.Documents);
+ }
+
+ ///
+ /// Queues V3 notifications as a bounded, sequential batch. Webhooks are loaded once for each
+ /// organization/project pair and queue failures are allowed to escape so the side-effect worker
+ /// can retry the batch. Deterministic queue identifiers suppress already-enqueued work on retry.
+ ///
+ public async Task ProcessIngestionV3BatchAsync(IReadOnlyCollection contexts)
+ {
+ ArgumentNullException.ThrowIfNull(contexts);
+
+ var eligibleContexts = contexts
+ .Where(ctx => ctx.Organization.HasPremiumFeatures && ctx.Stack?.AllowNotifications is true)
+ .ToArray();
+
+ foreach (var group in eligibleContexts.GroupBy(ctx => (ctx.Event.OrganizationId, ctx.Event.ProjectId)))
+ {
+ var webHooks = await GetWebHooksAsync(group.Key.OrganizationId, group.Key.ProjectId);
+ foreach (var ctx in group)
{
- EventId = ctx.Event.Id,
- IsNew = ctx.IsNew,
- IsRegression = ctx.IsRegression,
- TotalOccurrences = ctx.Stack.TotalOccurrences
- });
+ if (ShouldQueueNotification(ctx))
+ {
+ await QueueEventNotificationAsync(ctx);
+ }
- var webHooks = await _webHookRepository.GetByOrganizationIdOrProjectIdAsync(ctx.Event.OrganizationId, ctx.Event.ProjectId);
- foreach (var hook in webHooks.Documents)
+ await QueueWebHooksAsync(ctx, webHooks.Documents);
+ }
+ }
+ }
+
+ private Task> GetWebHooksAsync(string organizationId, string projectId)
+ {
+ return _webHookRepository.GetByOrganizationIdOrProjectIdAsync(organizationId, projectId);
+ }
+
+ private Task QueueEventNotificationAsync(EventContext ctx)
+ {
+ return _notificationQueue.EnqueueAsync(new EventNotification
+ {
+ EventId = ctx.Event.Id,
+ IsNew = ctx.IsNew,
+ IsRegression = ctx.IsRegression,
+ TotalOccurrences = ctx.Stack!.TotalOccurrences,
+ DeduplicationId = String.Concat("event-notification:", ctx.Event.Id),
+ UseDurableDeduplication = ctx.IsIngestionV3
+ });
+ }
+
+ private async Task QueueWebHooksAsync(EventContext ctx, IReadOnlyCollection webHooks)
+ {
+ foreach (var hook in webHooks)
{
if (!ShouldCallWebHook(hook, ctx))
+ {
continue;
+ }
- var context = new WebHookDataContext(hook, ctx.Organization, ctx.Project, ctx.Stack, ctx.Event, ctx.IsNew, ctx.IsRegression);
+ var context = new WebHookDataContext(hook, ctx.Organization, ctx.Project, ctx.Stack!, ctx.Event, ctx.IsNew, ctx.IsRegression);
var notification = new WebHookNotification
{
OrganizationId = ctx.Event.OrganizationId,
@@ -57,7 +111,9 @@ await _notificationQueue.EnqueueAsync(new EventNotification
WebHookId = hook.Id,
Url = hook.Url,
Type = WebHookType.General,
- Data = await _webHookDataPluginManager.CreateFromEventAsync(context)
+ Data = await _webHookDataPluginManager.CreateFromEventAsync(context),
+ DeduplicationId = String.Concat("event-webhook:", ctx.Event.Id, ":", hook.Id, ":", WebHookType.General),
+ UseDurableDeduplication = ctx.IsIngestionV3
};
if (notification.Data is null)
@@ -74,25 +130,39 @@ await _notificationQueue.EnqueueAsync(new EventNotification
private bool ShouldCallWebHook(WebHook hook, EventContext ctx)
{
if (!hook.IsEnabled)
+ {
return false;
+ }
if (!String.IsNullOrEmpty(hook.ProjectId) && !String.Equals(ctx.Project.Id, hook.ProjectId))
+ {
return false;
+ }
if (ctx.IsNew && ctx.Event.IsError() && hook.EventTypes.Contains(WebHook.KnownEventTypes.NewError))
+ {
return true;
+ }
if (ctx.Event.IsCritical() && ctx.Event.IsError() && hook.EventTypes.Contains(WebHook.KnownEventTypes.CriticalError))
+ {
return true;
+ }
if (ctx.IsRegression && hook.EventTypes.Contains(WebHook.KnownEventTypes.StackRegression))
+ {
return true;
+ }
if (ctx.IsNew && hook.EventTypes.Contains(WebHook.KnownEventTypes.NewEvent))
+ {
return true;
+ }
if (ctx.Event.IsCritical() && hook.EventTypes.Contains(WebHook.KnownEventTypes.CriticalEvent))
+ {
return true;
+ }
return false;
}
@@ -100,22 +170,34 @@ private bool ShouldCallWebHook(WebHook hook, EventContext ctx)
private bool ShouldQueueNotification(EventContext ctx)
{
if (ctx.Project.NotificationSettings.Count == 0)
+ {
return false;
+ }
if (ctx.IsNew && ctx.Event.IsError() && ctx.Project.NotificationSettings.Any(n => n.Value.ReportNewErrors))
+ {
return true;
+ }
if (ctx.Event.IsCritical() && ctx.Event.IsError() && ctx.Project.NotificationSettings.Any(n => n.Value.ReportCriticalErrors))
+ {
return true;
+ }
if (ctx.IsRegression && ctx.Project.NotificationSettings.Any(n => n.Value.ReportEventRegressions))
+ {
return true;
+ }
if (ctx.IsNew && ctx.Project.NotificationSettings.Any(n => n.Value.ReportNewEvents))
+ {
return true;
+ }
if (ctx.Event.IsCritical() && ctx.Project.NotificationSettings.Any(n => n.Value.ReportCriticalEvents))
+ {
return true;
+ }
return false;
}
diff --git a/src/Exceptionless.Core/Pipeline/100_RunEventProcessedPluginsAction.cs b/src/Exceptionless.Core/Pipeline/100_RunEventProcessedPluginsAction.cs
index 2b763bc176..856ce8c494 100644
--- a/src/Exceptionless.Core/Pipeline/100_RunEventProcessedPluginsAction.cs
+++ b/src/Exceptionless.Core/Pipeline/100_RunEventProcessedPluginsAction.cs
@@ -18,4 +18,5 @@ public override Task ProcessBatchAsync(ICollection contexts)
{
return _pluginManager.EventBatchProcessedAsync(contexts);
}
+
}
diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs
index 7b9ecc2b14..c62de37821 100644
--- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs
+++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs
@@ -11,18 +11,8 @@ namespace Exceptionless.Core.Plugins.EventProcessor;
[Priority(40)]
public sealed class RequestInfoPlugin : EventProcessorPluginBase
{
- public const int MAX_VALUE_LENGTH = 1000;
- public static readonly List DefaultExclusions =
- [
- "*VIEWSTATE*",
- "*EVENTVALIDATION*",
- "*ASPX*",
- "__RequestVerificationToken",
- "ASP.NET_SessionId",
- "__LastErrorId",
- "WAWebSiteID",
- "ARRAffinity"
- ];
+ public const int MAX_VALUE_LENGTH = RequestInfoExtensions.MaximumDataValueLength;
+ public static readonly List DefaultExclusions = [.. RequestInfoExtensions.DefaultDataExclusions];
private readonly UserAgentParser _parser;
private readonly ITextSerializer _serializer;
@@ -41,7 +31,9 @@ public override async Task EventBatchProcessingAsync(ICollection c
{
var request = context.Event.GetRequestInfo(_serializer, _logger);
if (request is null)
+ {
continue;
+ }
if (context.IncludePrivateInformation)
{
@@ -72,7 +64,9 @@ private void AddClientIpAddress(RequestInfo request, SubmissionClient? submissio
{
bool requestIpIsLocal = submissionClient.IpAddress.IsLocalHost();
if (ips.Count == 0 || !requestIpIsLocal && ips.Count(ip => !ip.IsLocalHost()) == 0)
+ {
ips.Add(submissionClient.IpAddress);
+ }
}
request.ClientIpAddress = ips.Distinct().ToDelimitedString();
@@ -81,11 +75,15 @@ private void AddClientIpAddress(RequestInfo request, SubmissionClient? submissio
private async Task SetBrowserOsAndDeviceFromUserAgent(RequestInfo request, EventContext context)
{
if (String.IsNullOrEmpty(request.UserAgent))
+ {
return;
+ }
var info = await _parser.ParseAsync(request.UserAgent);
if (info is null)
+ {
return;
+ }
request.Data ??= new DataDictionary();
if (!String.Equals(info.UA.Family, "Other"))
@@ -99,7 +97,9 @@ private async Task SetBrowserOsAndDeviceFromUserAgent(RequestInfo request, Event
}
if (!String.Equals(info.Device.Family, "Other"))
+ {
request.Data[RequestInfo.KnownDataKeys.Device] = info.Device.Family;
+ }
if (!String.Equals(info.OS.Family, "Other"))
{
diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/EventContext.cs b/src/Exceptionless.Core/Plugins/EventProcessor/EventContext.cs
index 5b348b016c..1f54920437 100644
--- a/src/Exceptionless.Core/Plugins/EventProcessor/EventContext.cs
+++ b/src/Exceptionless.Core/Plugins/EventProcessor/EventContext.cs
@@ -27,6 +27,7 @@ public EventContext(PersistentEvent ev, Organization organization, Project proje
public bool IsDiscarded { get; set; }
public bool IsNew { get; set; }
public bool IsRegression { get; set; }
+ public bool IsIngestionV3 { get; set; }
public bool IncludePrivateInformation { get; set; }
public bool AllowExtendedEventDateRange { get; set; }
public string? SignatureHash { get; set; }
diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/EventPluginManager.cs b/src/Exceptionless.Core/Plugins/EventProcessor/EventPluginManager.cs
index 08d91136d8..0e86c93cbc 100644
--- a/src/Exceptionless.Core/Plugins/EventProcessor/EventPluginManager.cs
+++ b/src/Exceptionless.Core/Plugins/EventProcessor/EventPluginManager.cs
@@ -34,7 +34,9 @@ public async Task EventBatchProcessingAsync(ICollection contexts)
{
var contextsToProcess = contexts.Where(c => c is { IsCancelled: false, HasError: false }).ToList();
if (contextsToProcess.Count == 0)
+ {
break;
+ }
string metricName = String.Concat(_metricPrefix, plugin.Name.ToLower());
try
@@ -42,7 +44,9 @@ public async Task EventBatchProcessingAsync(ICollection contexts)
await AppDiagnostics.TimeAsync(() => plugin.EventBatchProcessingAsync(contextsToProcess), metricName);
if (contextsToProcess.All(c => c.IsCancelled || c.HasError))
+ {
break;
+ }
}
catch (Exception ex)
{
@@ -60,7 +64,9 @@ public async Task EventBatchProcessedAsync(ICollection contexts)
{
var contextsToProcess = contexts.Where(c => c is { IsCancelled: false, HasError: false }).ToList();
if (contextsToProcess.Count == 0)
+ {
break;
+ }
string metricName = String.Concat(_metricPrefix, plugin.Name.ToLower());
try
@@ -68,7 +74,9 @@ public async Task EventBatchProcessedAsync(ICollection contexts)
await AppDiagnostics.TimeAsync(() => plugin.EventBatchProcessedAsync(contextsToProcess), metricName);
if (contextsToProcess.All(c => c.IsCancelled || c.HasError))
+ {
break;
+ }
}
catch (Exception ex)
{
@@ -76,4 +84,5 @@ public async Task EventBatchProcessedAsync(ICollection contexts)
}
}
}
+
}
diff --git a/src/Exceptionless.Core/Properties/AssemblyInfo.cs b/src/Exceptionless.Core/Properties/AssemblyInfo.cs
index 2941889e70..5dc06e1737 100644
--- a/src/Exceptionless.Core/Properties/AssemblyInfo.cs
+++ b/src/Exceptionless.Core/Properties/AssemblyInfo.cs
@@ -1,5 +1,6 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Exceptionless.Tests")]
+[assembly: InternalsVisibleTo("Exceptionless.Benchmarks")]
[assembly: InternalsVisibleTo("Exceptionless.Web")]
[assembly: InternalsVisibleTo("Exceptionless.Insulation")]
diff --git a/src/Exceptionless.Core/Queues/ConditionalDuplicateDetectionQueueBehavior.cs b/src/Exceptionless.Core/Queues/ConditionalDuplicateDetectionQueueBehavior.cs
new file mode 100644
index 0000000000..2e0fe58ece
--- /dev/null
+++ b/src/Exceptionless.Core/Queues/ConditionalDuplicateDetectionQueueBehavior.cs
@@ -0,0 +1,48 @@
+using Foundatio.Caching;
+using Foundatio.Queues;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Queues;
+
+///
+/// Preserves Foundatio's dequeue-scoped duplicate detection for legacy messages while allowing
+/// durable messages to be claimed exclusively by .
+///
+public class ConditionalDuplicateDetectionQueueBehavior(
+ ICacheClient cache,
+ ILoggerFactory loggerFactory,
+ TimeSpan detectionWindow) : QueueBehaviorBase where T : class
+{
+ private readonly ILogger _logger = loggerFactory.CreateLogger();
+
+ protected override async Task OnEnqueuing(object sender, EnqueuingEventArgs args)
+ {
+ string? identifier = GetIdentifier(args.Data);
+ if (String.IsNullOrEmpty(identifier) || await cache.AddAsync(identifier, true, detectionWindow))
+ {
+ return;
+ }
+
+ _logger.LogInformation("Discarding queue entry due to duplicate {UniqueIdentifier}", identifier);
+ args.Cancel = true;
+ }
+
+ protected override async Task OnDequeued(object sender, DequeuedEventArgs args)
+ {
+ string? identifier = GetIdentifier(args.Entry.Value);
+ if (!String.IsNullOrEmpty(identifier))
+ {
+ await cache.RemoveAsync(identifier);
+ }
+ }
+
+ private static string? GetIdentifier(T data)
+ {
+ if (data is IHaveDurableUniqueIdentifier { UseDurableDeduplication: true })
+ {
+ return null;
+ }
+
+ return (data as IHaveUniqueIdentifier)?.UniqueIdentifier;
+ }
+}
diff --git a/src/Exceptionless.Core/Queues/DurableDuplicateDetectionQueueBehavior.cs b/src/Exceptionless.Core/Queues/DurableDuplicateDetectionQueueBehavior.cs
new file mode 100644
index 0000000000..014ac64961
--- /dev/null
+++ b/src/Exceptionless.Core/Queues/DurableDuplicateDetectionQueueBehavior.cs
@@ -0,0 +1,68 @@
+using Foundatio.Caching;
+using Foundatio.Queues;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Queues;
+
+public interface IHaveDurableUniqueIdentifier : IHaveUniqueIdentifier
+{
+ bool UseDurableDeduplication { get; }
+}
+
+///
+/// Keeps successful queue deduplication markers after dequeue. A short pending lease protects
+/// against a producer dying after claiming an identifier but before the queue accepts the item.
+///
+public class DurableDuplicateDetectionQueueBehavior(
+ ICacheClient cache,
+ ILoggerFactory loggerFactory,
+ TimeSpan detectionWindow) : QueueBehaviorBase where T : class
+{
+ private static readonly TimeSpan PendingWindow = TimeSpan.FromMinutes(1);
+ private readonly ILogger _logger = loggerFactory.CreateLogger>();
+
+ protected override async Task OnEnqueuing(object sender, EnqueuingEventArgs args)
+ {
+ string? identifier = GetIdentifier(args.Data);
+ if (String.IsNullOrEmpty(identifier))
+ {
+ return;
+ }
+
+ string key = GetCacheKey(identifier);
+ if (await cache.AddAsync(key, "pending", PendingWindow))
+ {
+ return;
+ }
+
+ var existing = await cache.GetAsync(key);
+ if (existing.HasValue && String.Equals(existing.Value, "completed", StringComparison.Ordinal))
+ {
+ _logger.LogDebug("Discarding durable duplicate queue entry {UniqueIdentifier}", identifier);
+ args.Cancel = true;
+ return;
+ }
+
+ // Do not turn an abandoned producer lease into a permanent false success. The caller can
+ // retry after the short lease expires, preserving at-least-once delivery.
+ throw new InvalidOperationException($"Queue identifier '{identifier}' is currently being enqueued.");
+ }
+
+ protected override async Task OnEnqueued(object sender, EnqueuedEventArgs args)
+ {
+ string? identifier = GetIdentifier(args.Entry.Value);
+ if (String.IsNullOrEmpty(identifier))
+ {
+ return;
+ }
+
+ await cache.SetAsync(GetCacheKey(identifier), "completed", detectionWindow);
+ }
+
+ internal static string GetCacheKey(string identifier) =>
+ String.Concat("queue:durable-deduplication:", typeof(T).FullName, ":", identifier);
+
+ private static string? GetIdentifier(T data) => data is IHaveDurableUniqueIdentifier { UseDurableDeduplication: true } durable
+ ? durable.UniqueIdentifier
+ : null;
+}
diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs
index 9c7b77b4cd..97bcd588d1 100644
--- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs
+++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs
@@ -32,7 +32,9 @@ public EventIndex(ExceptionlessElasticConfiguration configuration, IServiceProvi
_serviceProvider = serviceProvider;
if (appOptions.MaximumRetentionDays > 0)
+ {
MaxIndexAge = TimeSpan.FromDays(appOptions.MaximumRetentionDays);
+ }
AddAlias($"{Name}-today", TimeSpan.FromDays(1));
AddAlias($"{Name}-last3days", TimeSpan.FromDays(7));
@@ -80,6 +82,10 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor e.Count)
.Boolean(e => e.IsFirstOccurrence)
.FieldAlias(Alias.IsFirstOccurrence, a => a.Path(f => f.IsFirstOccurrence))
+ .Boolean(e => e.IsRegression)
+ .Boolean(e => e.IngestionIsRegressionCandidate)
+ .Keyword(e => e.IngestionRegressionFixedInVersion)
+ .Date(e => e.IngestionRegressionDateFixed)
.Object(e => e.Idx, o => o.Dynamic(DynamicMapping.True))
.Object(e => e.Data, o => o.Properties(p2 => p2
.AddVersionMapping()
@@ -98,7 +104,9 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor s.Enabled(true));
+ }
}
public override void ConfigureIndex(CreateIndexRequestDescriptor idx)
@@ -472,7 +480,9 @@ private static string Field(Expression> property)
foreach (JsonPropertyInfo jsonProperty in typeInfo.Properties)
{
if (jsonProperty.AttributeProvider is PropertyInfo modelProperty && String.Equals(modelProperty.Name, propertyName, StringComparison.Ordinal))
+ {
return jsonProperty.Name;
+ }
}
throw new InvalidOperationException($"Unable to resolve JSON field name for {typeof(TModel).FullName}.{propertyName}.");
@@ -485,7 +495,9 @@ private static PropertyInfo GetPropertyInfo(Expression m
.Keyword("token")
.Text("email_address", t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER))))
.Date(e => e.LastEventDateUtc)
+ .Date(e => e.LastAppliedUsageBucketUtc)
.AddUsageMappings());
}
diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs
index 8daf613e2a..74ee90830b 100644
--- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs
+++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs
@@ -27,6 +27,7 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor map)
.Text(e => e.Name, t => t.AddKeywordField())
.LongNumber(e => e.NextSummaryEndOfDayTicks)
.Date(e => e.LastEventDateUtc)
+ .Date(e => e.LastAppliedUsageBucketUtc)
.AddUsageMappings()
);
}
diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs
index 02b15bdd89..c3d3ec80c4 100644
--- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs
+++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs
@@ -63,10 +63,13 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor map)
.Boolean(Alias.IsFixed)
.Keyword(e => e.FixedInVersion, k => k.IgnoreAbove(1024))
.FieldAlias(Alias.FixedInVersion, a => a.Path(f => f.FixedInVersion))
+ .Keyword(e => e.RegressionEventId)
+ .Keyword(e => e.IngestionFirstEventId)
.Boolean(e => e.OccurrencesAreCritical)
.FieldAlias(Alias.OccurrencesAreCritical, a => a.Path(f => f.OccurrencesAreCritical))
.IntegerNumber(e => e.TotalOccurrences)
.FieldAlias(Alias.TotalOccurrences, a => a.Path(f => f.TotalOccurrences))
+ .LongNumber(e => e.IngestionStackUsageSequence)
);
}
diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs
index 13199d28c6..8d821fe0d5 100644
--- a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs
+++ b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs
@@ -1,4 +1,5 @@
using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.Ingestion;
using Foundatio.Repositories;
using Foundatio.Repositories.Models;
@@ -7,10 +8,12 @@ namespace Exceptionless.Core.Repositories;
public interface IStackRepository : IRepositoryOwnedByOrganizationAndProject
{
Task GetStackBySignatureHashAsync(string projectId, string signatureHash);
+ Task> GetStackRoutesBySignatureHashAsync(string projectId, IReadOnlyCollection signatureHashes);
Task> GetIdsByQueryAsync(RepositoryQueryDescriptor query, CommandOptionsDescriptor? options = null);
Task> GetExpiredSnoozedStatuses(DateTime utcNow, CommandOptionsDescriptor? options = null);
Task MarkAsRegressedAsync(string stackId);
Task IncrementEventCounterAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count, bool sendNotifications = true);
+ Task ApplyIngestionStackUsageAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count, long settlementSequence, bool sendNotifications = true);
Task SetEventCounterAsync(string stackId, DateTime firstOccurrenceUtc, DateTime lastOccurrenceUtc, long totalOccurrences, bool sendNotifications = true);
Task> GetStacksForCleanupAsync(string organizationId, DateTime cutoff);
Task> GetSoftDeleted();
diff --git a/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs b/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs
index 1d893d11e6..fb5087bb53 100644
--- a/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs
+++ b/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs
@@ -14,6 +14,7 @@ public static class StackQueryExtensions
{
internal const string StacksKey = "@Stacks";
internal const string ExcludedStacksKey = "@ExcludedStacks";
+ internal const string SignatureHashesKey = "@SignatureHashes";
public static T Stack(this T query, string stackId) where T : IRepositoryQuery
{
@@ -34,6 +35,11 @@ public static T ExcludeStack(this T query, IEnumerable stackIds) wher
{
return query.AddCollectionOptionValue(ExcludedStacksKey, stackIds);
}
+
+ public static T SignatureHash(this T query, IEnumerable signatureHashes) where T : IRepositoryQuery
+ {
+ return query.AddCollectionOptionValue(SignatureHashesKey, signatureHashes.Distinct());
+ }
}
}
@@ -50,6 +56,12 @@ public static ICollection GetExcludedStacks(this IRepositoryQuery query)
{
return query.SafeGetCollection(StackQueryExtensions.ExcludedStacksKey);
}
+
+
+ public static ICollection GetSignatureHashes(this IRepositoryQuery query)
+ {
+ return query.SafeGetCollection(StackQueryExtensions.SignatureHashesKey);
+ }
}
}
@@ -58,20 +70,39 @@ namespace Exceptionless.Core.Repositories.Queries
public class StackQueryBuilder : IElasticQueryBuilder
{
private static readonly Field StackIdField = nameof(IOwnedByStack.StackId).ToLowerUnderscoredWords();
+ private static readonly Field SignatureHashField = nameof(Stack.SignatureHash).ToLowerUnderscoredWords();
public Task BuildAsync(QueryBuilderContext ctx) where T : class, new()
{
var stackIds = ctx.Source.GetStacks();
if (stackIds.Count == 1)
+ {
ctx.Filter &= new TermQuery { Field = StackIdField, Value = stackIds.Single() };
+ }
else if (stackIds.Count > 1)
+ {
ctx.Filter &= new TermsQuery { Field = StackIdField, Terms = new TermsQueryField(stackIds.Select(FieldValueHelper.ToFieldValue).ToList()) };
+ }
var excludedStackIds = ctx.Source.GetExcludedStacks();
if (excludedStackIds.Count == 1)
+ {
ctx.Filter &= new BoolQuery { MustNot = [new TermQuery { Field = StackIdField, Value = excludedStackIds.Single() }] };
+ }
else if (excludedStackIds.Count > 1)
+ {
ctx.Filter &= new BoolQuery { MustNot = [new TermsQuery { Field = StackIdField, Terms = new TermsQueryField(excludedStackIds.Select(FieldValueHelper.ToFieldValue).ToList()) }] };
+ }
+
+ var signatureHashes = ctx.Source.GetSignatureHashes();
+ if (signatureHashes.Count == 1)
+ {
+ ctx.Filter &= new TermQuery { Field = SignatureHashField, Value = signatureHashes.Single() };
+ }
+ else if (signatureHashes.Count > 1)
+ {
+ ctx.Filter &= new TermsQuery { Field = SignatureHashField, Terms = new TermsQueryField(signatureHashes.Select(FieldValueHelper.ToFieldValue).ToList()) };
+ }
return Task.CompletedTask;
}
diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs
index 29b0c3407e..c24f899da5 100644
--- a/src/Exceptionless.Core/Repositories/StackRepository.cs
+++ b/src/Exceptionless.Core/Repositories/StackRepository.cs
@@ -1,7 +1,10 @@
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.Ingestion;
using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Services;
using Exceptionless.Core.Validation;
+using Foundatio.Caching;
using Foundatio.Repositories;
using Foundatio.Repositories.Exceptions;
using Foundatio.Repositories.Models;
@@ -13,12 +16,19 @@ namespace Exceptionless.Core.Repositories;
public class StackRepository : RepositoryOwnedByOrganizationAndProject, IStackRepository
{
private readonly TimeProvider _timeProvider;
+ private readonly AppOptions _appOptions;
+ private readonly IStackRouteCache _stackRouteCache;
private const string STACKING_VERSION = "v2";
- public StackRepository(ExceptionlessElasticConfiguration configuration, MiniValidationValidator validator, AppOptions options)
+ public StackRepository(ExceptionlessElasticConfiguration configuration, MiniValidationValidator validator, AppOptions options, IStackRouteCache stackRouteCache)
: base(configuration.Stacks, validator, options)
{
_timeProvider = configuration.TimeProvider;
+ _appOptions = options;
+ _stackRouteCache = stackRouteCache;
+ // Both the legacy signature cache and the V3 route cache must invalidate the previous
+ // key when a stack's signature changes or it is soft-deleted.
+ OriginalsEnabled = true;
AddRequiredField(s => s.SignatureHash);
}
@@ -89,7 +99,9 @@ Instant parseDate(def dt) {
{
bool modified = await PatchAsync(stackId, operation, o => o.Notifications(false));
if (!modified)
+ {
return false;
+ }
}
catch (DocumentNotFoundException)
{
@@ -97,11 +109,86 @@ Instant parseDate(def dt) {
}
if (sendNotifications)
+ {
await PublishMessageAsync(CreateEntityChanged(ChangeType.Saved, organizationId, projectId, null, stackId));
+ }
return true;
}
+ public async Task ApplyIngestionStackUsageAsync(
+ string organizationId,
+ string projectId,
+ string stackId,
+ DateTime minOccurrenceDateUtc,
+ DateTime maxOccurrenceDateUtc,
+ int count,
+ long settlementSequence,
+ bool sendNotifications = true)
+ {
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(count);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(settlementSequence);
+
+ const string script = @"
+Instant parseDate(def dt) {
+ if (dt != null) {
+ try {
+ return Instant.parse(dt);
+ } catch(DateTimeParseException e) {}
+ }
+ return Instant.MIN;
+}
+
+long appliedSequence = ctx._source.ingestion_stack_usage_sequence == null
+ ? 0
+ : ctx._source.ingestion_stack_usage_sequence;
+if (appliedSequence >= params.settlementSequence) {
+ ctx.op = 'noop';
+} else {
+ if (ctx._source.total_occurrences == 0 || parseDate(ctx._source.first_occurrence).isAfter(parseDate(params.minOccurrenceDateUtc))) {
+ ctx._source.first_occurrence = params.minOccurrenceDateUtc;
+ }
+
+ if (parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.maxOccurrenceDateUtc))) {
+ ctx._source.last_occurrence = params.maxOccurrenceDateUtc;
+ }
+
+ if (parseDate(ctx._source.updated_utc).isBefore(parseDate(params.updatedUtc))) {
+ ctx._source.updated_utc = params.updatedUtc;
+ }
+
+ ctx._source.total_occurrences += params.count;
+ ctx._source.ingestion_stack_usage_sequence = params.settlementSequence;
+}";
+
+ var operation = new ScriptPatch(script.TrimScript())
+ {
+ Params = new Dictionary(5)
+ {
+ { "minOccurrenceDateUtc", minOccurrenceDateUtc },
+ { "maxOccurrenceDateUtc", maxOccurrenceDateUtc },
+ { "count", count },
+ { "settlementSequence", settlementSequence },
+ { "updatedUtc", _timeProvider.GetUtcNow().UtcDateTime }
+ }
+ };
+
+ bool modified;
+ try
+ {
+ modified = await PatchAsync(stackId, operation, o => o.Notifications(false));
+ }
+ catch (DocumentNotFoundException)
+ {
+ return;
+ }
+
+ if (modified && sendNotifications)
+ {
+ await PublishMessageAsync(CreateEntityChanged(ChangeType.Saved, organizationId, projectId, null, stackId));
+ }
+ }
+
public async Task SetEventCounterAsync(string stackId, DateTime firstOccurrenceUtc, DateTime lastOccurrenceUtc, long totalOccurrences, bool sendNotifications = true)
{
const string script = @"
@@ -153,11 +240,44 @@ Instant parseDate(def dt) {
public async Task GetStackBySignatureHashAsync(string projectId, string signatureHash)
{
- string key = GetStackSignatureCacheKey(projectId, signatureHash);
+ long projectGeneration = await _stackRouteCache.GetProjectGenerationAsync(projectId);
+ string key = GetStackSignatureCacheKey(projectId, signatureHash, projectGeneration);
var hit = await FindOneAsync(q => q.Project(projectId).FieldEquals(s => s.SignatureHash, signatureHash), o => o.Cache(key));
return hit?.Document;
}
+ public async Task> GetStackRoutesBySignatureHashAsync(string projectId, IReadOnlyCollection