From 22a2fcf6d20a56c143dbeb1840937c35d879a488 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 13 Jul 2026 00:54:51 -0500 Subject: [PATCH 01/10] Document V3 ingestion architecture --- docs/event-ingestion-v3-architecture.md | 250 ++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 docs/event-ingestion-v3-architecture.md diff --git a/docs/event-ingestion-v3-architecture.md b/docs/event-ingestion-v3-architecture.md new file mode 100644 index 0000000000..3761f03377 --- /dev/null +++ b/docs/event-ingestion-v3-architecture.md @@ -0,0 +1,250 @@ +# Event ingestion V3 architecture + +This document records the implementation decisions and performance baseline +methodology for [issue #2368](https://github.com/exceptionless/Exceptionless/issues/2368). +It is deliberately separate from the public API documentation until the V3 +contract is ready for rollout. + +## Objectives + +Event ingestion V3 is a breaking, performance-first API with these invariants: + +- ASP.NET Core Minimal APIs own the HTTP boundary. V3 does not use an MVC + controller or MVC input formatter. +- Clients send independent JSON objects as they capture events. They never + construct a top-level JSON array. +- The request body is consumed incrementally from a `PipeReader` with + source-generated System.Text.Json metadata. +- The API processes bounded microbatches inline. A successful response means + that every acknowledged event is durable or reached another terminal success + state such as discarded or duplicate. +- The primary event path does not write the request to object storage and does + not enqueue it for another process to deserialize. +- Clients send raw stack traces. The server computes the grouping fingerprint + and builds the persisted structured error. +- Events assigned to discarded stacks terminate before billable admission, + full error materialization, event persistence, stack statistics, or side + effects. +- V3 correctness is scale-out safe and never depends on sticky sessions or + process-local authoritative state. +- V2 remains contract-compatible. V3 does not pay a compatibility tax to share + an implementation with V2. + +## Current V2 flow + +The V2 path is intentionally durable before processing, but that durability +requires the payload to cross several boundaries: + +1. `EventController.PostAsync` resolves the project and wraps the request body + with `EventPostRequestBodyStream`. +2. `EventPostService.SaveAndEnqueueAsync` writes the payload to file storage and + enqueues an `EventPost` pointer. +3. `EventPostsJob` downloads the complete payload into a byte array. +4. Compressed payloads are expanded into a second complete byte array. +5. The job parses the complete body into a collection of `PersistentEvent` + objects. +6. `EventPipeline` runs a reflection-discovered action list. +7. `PipelineBase` creates a filtered context list for every action. +8. `AssignToStackAction` resolves or creates stacks and only then detects the + discarded status. +9. `SaveEventAction` persists events, followed by statistics, notifications, + counters, and processed plugins. + +Plan admission occurs before stack assignment in both `OverageMiddleware` and +`EventPostsJob`. A request can therefore be rejected before the system learns +that its events belong to a free discarded stack. + +## V3 processing boundary + +The V3 endpoint is a transport adapter over an explicit, batch-oriented Core +processor. For each bounded microbatch, the processor performs these steps in +order: + +1. Validate the minimal event envelope. +2. Apply retention and nonbilling safety rules. +3. Compute canonical stack fingerprints. +4. Resolve unique stack routes in bulk. +5. Terminate discarded events and record an aggregate discard count. +6. Reserve billable quota for survivors only. +7. Materialize complete events and structured errors. +8. Resolve or create missing stacks. +9. Persist events and durable side-effect intent in bulk. +10. Commit or release billable reservations. + +The endpoint stops reading while a full microbatch is processed. This bounds +memory and lets HTTP/TCP flow control apply backpressure. Reading and processing +may overlap through a channel with capacity one only if benchmarks demonstrate +a material benefit. + +## Decision: inline acknowledgement + +V3 acknowledges after the primary event write, not after a queue write. This +removes file-storage I/O, queue I/O, worker scheduling, a payload download, and +a second deserialization from the normal path. + +If durable storage is unavailable, V3 fails quickly with a retryable response. +It does not enqueue after processing begins because a partially successful +write makes that fallback ambiguous. A separate, explicit buffered mode is +required if the product must acknowledge while the event store is unavailable. + +## Decision: bounded stream segments + +An indefinitely open upload cannot provide a reliable acknowledgement boundary. +Official clients therefore stream events immediately into a request segment and +close the segment when a configured count, byte, or elapsed-time limit is +reached. The client retains only the unacknowledged segment and replays it after +a connection loss or retryable response. + +This is streaming rather than array batching: each event is serialized once and +written immediately as an independent top-level JSON value. + +## Decision: deterministic idempotency + +Every V3 event carries a stable client event identifier. The server derives a +deterministic storage identity scoped to the project and uses create-only +semantics. A duplicate create is a terminal success. + +If a response is lost after persistence, the client replays the complete segment +with the same identifiers. Previously stored events become duplicate successes; +new events continue normally. Side-effect intent uses the same deterministic +identity so replay cannot duplicate notifications or webhooks. + +## Decision: discarded-stack fast path + +Discard classification requires the same canonical fingerprint used by stack +assignment. V3 computes that fingerprint from a minimal event representation +before building the complete `PersistentEvent`, `Error`, and `StackFrame` +object graph. + +Stack routing uses a lightweight value containing stack identifier, status, and +a cache schema version. Resolution is deduplicated within the microbatch, read +from the distributed cache in bulk, and falls back to a projected repository +query for cache misses. + +The distributed cache is authoritative initially. A process-local positive +discard cache is unsafe without proven versioning and invalidation because a +stale entry would silently discard events after a stack is reopened. + +A discarded event performs none of the following: + +- Billable quota reservation. +- Full error or stack-frame materialization. +- Event indexing. +- Stack occurrence updates. +- Notification, webhook, archive, enrichment, or outbox writes. + +Discarded events still count toward protective byte, request, and event-rate +limits. Those controls prevent abuse and are not customer billing. + +## Decision: quota reservation and settlement + +The existing read-then-process usage check is not an adequate scale-out +reservation primitive. V3 separates protective admission from billable usage +and uses three billable operations: + +- Reserve capacity for non-discarded survivors. +- Commit capacity for successfully persisted events. +- Release capacity for invalid events, duplicates, cancellations, or failed + writes. + +The distributed operation must be atomic across API instances. A customer at +the billable limit can still submit an event assigned to an already-discarded +stack, while active and new-stack events are blocked. + +## Decision: side-effect durability + +The primary event is authoritative. Notifications, webhooks, archive export, +nonessential geolocation, and repairable derived statistics remain outside the +request critical path. + +The inline writer persists deterministic side-effect or repair intent before +acknowledgement. Consumers are idempotent. Partial bulk outcomes are reconciled +by deterministic identifiers rather than by placing the original event payload +back on the ingestion queue. + +## V2 compatibility boundary + +V2 retains its routes, payloads, 202 response, controller, and durable queue. +Shared services are extracted below the transport boundary: + +- Canonical fingerprinting. +- Stack route resolution. +- Server-side stack parsing. +- Batch stack resolution and creation. +- Batch event persistence. +- Usage accounting primitives. + +V3 must not instantiate V2 queue models, plugin contexts, or compatibility +converters. V2 may adopt a shared service only when its compatibility fixtures +remain unchanged and the change is neutral or positive in measurements. + +## Baseline methodology + +All comparisons use the same commit, machine, runtime configuration, +Elasticsearch topology, Redis topology, payload corpus, compression setting, +and client concurrency. Record the following environment information with every +result: + +- Commit SHA and configuration. +- .NET SDK and runtime versions. +- Operating system and processor topology. +- Elasticsearch and Redis versions/resources. +- Number of API and job instances. +- Payload size distribution and stack-signature cardinality. +- Client concurrency and connection reuse. + +Measure V2 and V3 separately for these scenarios: + +- One hot active stack. +- High-cardinality active stacks. +- New stacks. +- 0%, 10%, 50%, 90%, and 100% discarded events. +- Small, median, large, and maximum events. +- Compressed and uncompressed requests. +- Healthy, slow, and unavailable Elasticsearch. +- One, two, four, and eight API instances. + +Capture: + +- Events per second and bytes per second. +- CPU time per event. +- Allocated bytes per event and Gen 0/1/2 collections. +- Working set and managed heap under a long stream. +- p50, p95, and p99 time from final event byte to durable acknowledgement. +- File-storage, queue, cache, and Elasticsearch operations per event. +- Stack-route cache hit and miss rates. +- Cost per million persisted and discarded events. + +## Performance gates + +- No request-sized byte array, string, `MemoryStream`, `JsonDocument`, or + top-level event collection on V3. +- No large-object-heap allocation caused by normal stream framing or microbatch + growth. +- Memory remains bounded by configured in-flight requests and microbatch limits + during a long stream. +- A discarded event creates no complete error/frame graph and performs no event + index write. +- At most one distributed route lookup per unique signature per microbatch. +- The normal V3 path performs no object-storage payload write/read and no + primary ingestion queue operation. +- Scaling from one to four API instances reaches at least 80% efficiency until + the shared datastore is the measured bottleneck. +- V3 reduces CPU, allocation rate, and time-to-durable-event relative to V2. + Numeric reduction targets are set from the repeatable baseline rather than an + unmeasured estimate. + +## Verification requirements + +Completion requires all of the following evidence: + +- Unit tests for contracts, fingerprinting, parsing, routing, quota settlement, + idempotency, and result aggregation. +- Integration tests for chunked NDJSON, compression, authentication, limits, + discard behavior, persistence, replay, and failure responses. +- Multi-instance tests for route status changes and quota reservation. +- V2 serialization audit results before and after shared-service adoption. +- OpenAPI V3 baseline and HTTP samples. +- Microbenchmark reports and end-to-end load results for the scenario matrix. +- A staged rollout through the internal project and selected projects with a + documented rollback switch. From dd4a593bd6e67d7d88b2b2a344bdb5d233e43adf Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 13 Jul 2026 01:02:37 -0500 Subject: [PATCH 02/10] Add V3 ingestion contracts and benchmarks --- .gitignore | 2 + Exceptionless.slnx | 1 + benchmarks/Directory.Build.props | 7 ++ .../Exceptionless.Benchmarks.csproj | 13 +++ .../Exceptionless.Benchmarks/Program.cs | 11 ++ ...EventIngestionDeserializationBenchmarks.cs | 81 +++++++++++++ .../Models/Ingestion/EventIngestionV3Event.cs | 94 +++++++++++++++ .../Ingestion/EventIngestionV3Limits.cs | 17 +++ .../EventIngestionJsonContext.cs | 18 +++ .../EventIngestionJsonContextTests.cs | 109 ++++++++++++++++++ 10 files changed, 353 insertions(+) create mode 100644 benchmarks/Directory.Build.props create mode 100644 benchmarks/Exceptionless.Benchmarks/Exceptionless.Benchmarks.csproj create mode 100644 benchmarks/Exceptionless.Benchmarks/Program.cs create mode 100644 benchmarks/Exceptionless.Benchmarks/Serialization/EventIngestionDeserializationBenchmarks.cs create mode 100644 src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Event.cs create mode 100644 src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Limits.cs create mode 100644 src/Exceptionless.Core/Serialization/EventIngestionJsonContext.cs create mode 100644 tests/Exceptionless.Tests/Serializer/EventIngestionJsonContextTests.cs diff --git a/.gitignore b/.gitignore index 36508301c3..0d84b7a8bc 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,5 @@ docs/_cache/ .devcontainer/devcontainer-lock.json *.lscache +# BenchmarkDotNet output +BenchmarkDotNet.Artifacts/ diff --git a/Exceptionless.slnx b/Exceptionless.slnx index a7466782db..3eb53a3172 100644 --- a/Exceptionless.slnx +++ b/Exceptionless.slnx @@ -23,6 +23,7 @@ + diff --git a/benchmarks/Directory.Build.props b/benchmarks/Directory.Build.props new file mode 100644 index 0000000000..993e0a1792 --- /dev/null +++ b/benchmarks/Directory.Build.props @@ -0,0 +1,7 @@ + + + + + false + + diff --git a/benchmarks/Exceptionless.Benchmarks/Exceptionless.Benchmarks.csproj b/benchmarks/Exceptionless.Benchmarks/Exceptionless.Benchmarks.csproj new file mode 100644 index 0000000000..1c7f290a04 --- /dev/null +++ b/benchmarks/Exceptionless.Benchmarks/Exceptionless.Benchmarks.csproj @@ -0,0 +1,13 @@ + + + Exe + + + + + + + + + + diff --git a/benchmarks/Exceptionless.Benchmarks/Program.cs b/benchmarks/Exceptionless.Benchmarks/Program.cs new file mode 100644 index 0000000000..528ba5c55d --- /dev/null +++ b/benchmarks/Exceptionless.Benchmarks/Program.cs @@ -0,0 +1,11 @@ +using BenchmarkDotNet.Running; + +namespace Exceptionless.Benchmarks; + +public static class Program +{ + public static void Main(string[] args) + { + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + } +} diff --git a/benchmarks/Exceptionless.Benchmarks/Serialization/EventIngestionDeserializationBenchmarks.cs b/benchmarks/Exceptionless.Benchmarks/Serialization/EventIngestionDeserializationBenchmarks.cs new file mode 100644 index 0000000000..c113a993d4 --- /dev/null +++ b/benchmarks/Exceptionless.Benchmarks/Serialization/EventIngestionDeserializationBenchmarks.cs @@ -0,0 +1,81 @@ +using System.IO.Pipelines; +using System.Text; +using System.Text.Json; +using BenchmarkDotNet.Attributes; +using Exceptionless.Core.Models.Ingestion; +using Exceptionless.Core.Serialization; + +namespace Exceptionless.Benchmarks.Serialization; + +[MemoryDiagnoser] +public class EventIngestionDeserializationBenchmarks +{ + private byte[] _arrayPayload = null!; + private byte[] _streamPayload = null!; + + [Params(1, 100, 1000)] + public int EventCount { get; set; } + + [GlobalSetup] + public void Setup() + { + var events = new EventIngestionV3Event[EventCount]; + for (int index = 0; index < events.Length; index++) + { + events[index] = new EventIngestionV3Event + { + Id = $"01J0000000000000000000{index:D4}", + Type = "error", + Date = DateTimeOffset.UnixEpoch.AddSeconds(index), + Message = "Operation failed", + ExceptionType = "System.InvalidOperationException", + StackTrace = " at Example.Service.Run() in Service.cs:line 42" + }; + } + + _arrayPayload = JsonSerializer.SerializeToUtf8Bytes( + events, + EventIngestionJsonContext.Default.EventIngestionV3EventArray); + + using var stream = new MemoryStream(_arrayPayload.Length); + for (int index = 0; index < events.Length; index++) + { + JsonSerializer.Serialize( + stream, + events[index], + EventIngestionJsonContext.Default.EventIngestionV3Event); + stream.WriteByte((byte)'\n'); + } + + _streamPayload = stream.ToArray(); + } + + [Benchmark(Baseline = true)] + public int DeserializeArray() + { + var events = JsonSerializer.Deserialize( + _arrayPayload, + EventIngestionJsonContext.Default.EventIngestionV3EventArray); + + return events?.Length ?? 0; + } + + [Benchmark] + public async Task DeserializeTopLevelValuesAsync() + { + using var stream = new MemoryStream(_streamPayload, writable: false); + var reader = PipeReader.Create(stream); + int count = 0; + + await foreach (var _ in JsonSerializer.DeserializeAsyncEnumerable( + reader, + EventIngestionJsonContext.Default.EventIngestionV3Event, + topLevelValues: true)) + { + count++; + } + + await reader.CompleteAsync(); + return count; + } +} diff --git a/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Event.cs b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Event.cs new file mode 100644 index 0000000000..3015f552fe --- /dev/null +++ b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Event.cs @@ -0,0 +1,94 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json; + +namespace Exceptionless.Core.Models.Ingestion; + +/// +/// A compact V3 ingestion event. Organization and project ownership come from +/// the authenticated request rather than the event payload. +/// +public sealed record EventIngestionV3Event +{ + [Required] + [StringLength(EventIngestionV3Limits.MaximumEventIdLength, MinimumLength = 1)] + public required string Id { get; init; } + + [Required] + [StringLength(EventIngestionV3Limits.MaximumTypeLength, MinimumLength = 1)] + public required string Type { get; init; } + + public DateTimeOffset? Date { get; init; } + + [StringLength(EventIngestionV3Limits.MaximumSourceLength, MinimumLength = 1)] + public string? Source { get; init; } + + [StringLength(EventIngestionV3Limits.MaximumMessageLength, MinimumLength = 1)] + public string? Message { get; init; } + + [StringLength(EventIngestionV3Limits.MaximumReferenceIdLength, MinimumLength = 1)] + public string? ReferenceId { get; init; } + + public decimal? Value { get; init; } + + public string[]? Tags { get; init; } + + [StringLength(EventIngestionV3Limits.MaximumExceptionTypeLength, MinimumLength = 1)] + public string? ExceptionType { get; init; } + + [StringLength(EventIngestionV3Limits.MaximumStackTraceLength, MinimumLength = 1)] + public string? StackTrace { get; init; } + + public EventIngestionV3User? User { get; init; } + + public EventIngestionV3Request? Request { get; init; } + + public EventIngestionV3Environment? Environment { get; init; } + + public JsonElement? Data { get; init; } +} + +public sealed record EventIngestionV3User +{ + [StringLength(EventIngestionV3Limits.MaximumUserIdentityLength, MinimumLength = 1)] + public string? Identity { get; init; } + + [StringLength(EventIngestionV3Limits.MaximumUserNameLength, MinimumLength = 1)] + public string? Name { get; init; } + + public JsonElement? Data { get; init; } +} + +public sealed record EventIngestionV3Request +{ + public string? UserAgent { get; init; } + public string? HttpMethod { get; init; } + public bool? IsSecure { get; init; } + public string? Host { get; init; } + public int? Port { get; init; } + public string? Path { get; init; } + public string? Referrer { get; init; } + public string? ClientIpAddress { get; init; } + public Dictionary? Headers { get; init; } + public Dictionary? Cookies { get; init; } + public Dictionary? QueryString { get; init; } + public JsonElement? PostData { get; init; } + public JsonElement? Data { get; init; } +} + +public sealed record EventIngestionV3Environment +{ + public string? Architecture { get; init; } + public string? OSName { get; init; } + public string? OSVersion { get; init; } + public string? MachineName { get; init; } + public string? RuntimeVersion { get; init; } + public string? ProcessName { get; init; } + public string? ProcessId { get; init; } + public string? ThreadName { get; init; } + public string? ThreadId { get; init; } + public int? ProcessorCount { get; init; } + public long? TotalPhysicalMemory { get; init; } + public long? AvailablePhysicalMemory { get; init; } + public long? ProcessMemorySize { get; init; } + public JsonElement? Data { get; init; } +} diff --git a/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Limits.cs b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Limits.cs new file mode 100644 index 0000000000..56ad93a6b2 --- /dev/null +++ b/src/Exceptionless.Core/Models/Ingestion/EventIngestionV3Limits.cs @@ -0,0 +1,17 @@ +namespace Exceptionless.Core.Models.Ingestion; + +public static class EventIngestionV3Limits +{ + public const int MaximumEventIdLength = 128; + public const int MaximumTypeLength = 100; + public const int MaximumSourceLength = 2000; + public const int MaximumMessageLength = 4000; + public const int MaximumReferenceIdLength = 100; + public const int MaximumExceptionTypeLength = 2000; + public const int MaximumStackTraceLength = 256 * 1024; + public const int MaximumUserIdentityLength = 255; + public const int MaximumUserNameLength = 255; + public const int MaximumTags = 50; + public const int MaximumTagLength = 255; + public const int MaximumJsonDepth = 32; +} diff --git a/src/Exceptionless.Core/Serialization/EventIngestionJsonContext.cs b/src/Exceptionless.Core/Serialization/EventIngestionJsonContext.cs new file mode 100644 index 0000000000..67a1e2cb6f --- /dev/null +++ b/src/Exceptionless.Core/Serialization/EventIngestionJsonContext.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; +using Exceptionless.Core.Models.Ingestion; + +namespace Exceptionless.Core.Serialization; + +[JsonSourceGenerationOptions( + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + GenerationMode = JsonSourceGenerationMode.Metadata, + MaxDepth = EventIngestionV3Limits.MaximumJsonDepth, + PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, + RespectNullableAnnotations = true, + UseStringEnumConverter = false)] +[JsonSerializable(typeof(EventIngestionV3Event))] +[JsonSerializable(typeof(EventIngestionV3Event[]))] +[JsonSerializable(typeof(EventIngestionV3User))] +[JsonSerializable(typeof(EventIngestionV3Request))] +[JsonSerializable(typeof(EventIngestionV3Environment))] +public sealed partial class EventIngestionJsonContext : JsonSerializerContext; diff --git a/tests/Exceptionless.Tests/Serializer/EventIngestionJsonContextTests.cs b/tests/Exceptionless.Tests/Serializer/EventIngestionJsonContextTests.cs new file mode 100644 index 0000000000..ecd828761d --- /dev/null +++ b/tests/Exceptionless.Tests/Serializer/EventIngestionJsonContextTests.cs @@ -0,0 +1,109 @@ +using System.IO.Pipelines; +using System.Text; +using System.Text.Json; +using Exceptionless.Core.Models.Ingestion; +using Exceptionless.Core.Serialization; +using Xunit; + +namespace Exceptionless.Tests.Serializer; + +public sealed class EventIngestionJsonContextTests +{ + [Fact] + public async Task DeserializeAsyncEnumerable_MultipleTopLevelValues_ReturnsEveryEvent() + { + const string payload = """ + {"id":"event-1","type":"error","stack_trace":"at Example.Run()"} + {"id":"event-2","type":"log","message":"completed"} + """; + + var events = await DeserializeAsync(payload); + + Assert.Collection(events, + first => + { + Assert.Equal("event-1", first.Id); + Assert.Equal("error", first.Type); + Assert.Equal("at Example.Run()", first.StackTrace); + }, + second => + { + Assert.Equal("event-2", second.Id); + Assert.Equal("log", second.Type); + Assert.Equal("completed", second.Message); + }); + } + + [Fact] + public Task DeserializeAsyncEnumerable_MissingRequiredId_ThrowsJsonException() + { + const string payload = """{"type":"log","message":"completed"}"""; + + return Assert.ThrowsAsync(() => DeserializeAsync(payload)); + } + + [Fact] + public async Task DeserializeAsyncEnumerable_UnknownProperty_IgnoresProperty() + { + const string payload = """{"id":"event-1","type":"log","future_value":42}"""; + + var events = await DeserializeAsync(payload); + + Assert.Single(events); + Assert.Equal("event-1", events[0].Id); + } + + [Fact] + public Task DeserializeAsyncEnumerable_TopLevelArray_ThrowsJsonException() + { + const string payload = """[{"id":"event-1","type":"log"}]"""; + + return Assert.ThrowsAsync(() => DeserializeAsync(payload)); + } + + [Fact] + public void Serialize_HtmlSensitiveCharacters_UsesSafeEncoding() + { + var value = new EventIngestionV3Event + { + Id = "event-1", + Type = "log", + Message = "