From a23d50236ea8dc9098486c8c2e9b84403340208f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 31 May 2026 23:58:12 +0000 Subject: [PATCH 1/6] Initial plan From 794182e5589ee08c239faa7e8a2b458015bbe014 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Jun 2026 00:22:57 +0000 Subject: [PATCH 2/6] Scope orphan cleanup event scans to recent data Co-authored-by: niemyjski <1020579+niemyjski@users.noreply.github.com> --- .../Jobs/CleanupOrphanedDataJob.cs | 64 +++++++++++++------ .../Jobs/CleanupDataJobTests.cs | 28 ++++++++ 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 6d74dee791..9e26e00f3e 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -19,6 +19,7 @@ namespace Exceptionless.Core.Jobs; [Job(Description = "Deletes orphaned data.", IsContinuous = false)] public class CleanupOrphanedDataJob : JobWithLockBase, IHealthCheck { + private static readonly TimeSpan OrphanedEventLookback = TimeSpan.FromDays(3); private readonly ExceptionlessElasticConfiguration _config; private readonly IElasticClient _elasticClient; private readonly IStackRepository _stackRepository; @@ -60,9 +61,13 @@ protected override async Task RunInternalAsync(JobContext context) public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { + var orphanedEventCutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); + // get approximate number of unique stack ids - var stackCardinality = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Cardinality("cardinality_stack_id", c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); + var stackCardinality = await _elasticClient.SearchAsync(s => s + .Size(0) + .Query(q => q.DateRange(r => r.Field(f => f.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc))) + .Aggregations(a => a.Cardinality("cardinality_stack_id", c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); double? uniqueStackIdCount = stackCardinality.Aggregations.Cardinality("cardinality_stack_id")?.Value; if (!uniqueStackIdCount.HasValue || uniqueStackIdCount.Value <= 0) @@ -79,8 +84,10 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { await RenewLockAsync(context); - var stackIdTerms = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Terms("terms_stack_id", c => c.Field(f => f.StackId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var stackIdTerms = await _elasticClient.SearchAsync(s => s + .Size(0) + .Query(q => q.DateRange(r => r.Field(f => f.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc))) + .Aggregations(a => a.Terms("terms_stack_id", c => c.Field(f => f.StackId).Include(batchNumber, buckets).Size(batchSize * 2)))); string[] stackIds = stackIdTerms.Aggregations.Terms("terms_stack_id").Buckets.Select(b => b.Key).ToArray(); if (stackIds.Length == 0) @@ -100,17 +107,24 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) totalOrphanedEventCount += missingStackIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing stacks {MissingStackIds} out of {StackIdCount}", batchNumber, buckets, missingStackIds.Length, missingStackIds, stackIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.StackId).Terms(missingStackIds)))); + await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Bool(b => b + .Filter( + f => f.Terms(t => t.Field(e => e.StackId).Terms(missingStackIds)), + f => f.DateRange(d => d.Field(e => e.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc)))))); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing stacks out of {StackIdCount}", totalOrphanedEventCount, totalStackIds); + _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing stacks out of {StackIdCount} since {OrphanedEventCutoffUtc}", totalOrphanedEventCount, totalStackIds, orphanedEventCutoffUtc); } public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { + var orphanedEventCutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); + // get approximate number of unique project ids - var projectCardinality = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Cardinality("cardinality_project_id", c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); + var projectCardinality = await _elasticClient.SearchAsync(s => s + .Size(0) + .Query(q => q.DateRange(r => r.Field(f => f.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc))) + .Aggregations(a => a.Cardinality("cardinality_project_id", c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); double? uniqueProjectIdCount = projectCardinality.Aggregations.Cardinality("cardinality_project_id")?.Value; if (!uniqueProjectIdCount.HasValue || uniqueProjectIdCount.Value <= 0) @@ -127,8 +141,10 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { await RenewLockAsync(context); - var projectIdTerms = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Terms("terms_project_id", c => c.Field(f => f.ProjectId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var projectIdTerms = await _elasticClient.SearchAsync(s => s + .Size(0) + .Query(q => q.DateRange(r => r.Field(f => f.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc))) + .Aggregations(a => a.Terms("terms_project_id", c => c.Field(f => f.ProjectId).Include(batchNumber, buckets).Size(batchSize * 2)))); string[] projectIds = projectIdTerms.Aggregations.Terms("terms_project_id").Buckets.Select(b => b.Key).ToArray(); if (projectIds.Length == 0) @@ -146,17 +162,24 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) } _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing projects {MissingProjectIds} out of {ProjectIdCount}", batchNumber, buckets, missingProjectIds.Length, missingProjectIds, projectIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.ProjectId).Terms(missingProjectIds)))); + await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Bool(b => b + .Filter( + f => f.Terms(t => t.Field(e => e.ProjectId).Terms(missingProjectIds)), + f => f.DateRange(d => d.Field(e => e.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc)))))); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing projects out of {ProjectIdCount}", totalOrphanedEventCount, totalProjectIds); + _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing projects out of {ProjectIdCount} since {OrphanedEventCutoffUtc}", totalOrphanedEventCount, totalProjectIds, orphanedEventCutoffUtc); } public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { + var orphanedEventCutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); + // get approximate number of unique organization ids - var organizationCardinality = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Cardinality("cardinality_organization_id", c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); + var organizationCardinality = await _elasticClient.SearchAsync(s => s + .Size(0) + .Query(q => q.DateRange(r => r.Field(f => f.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc))) + .Aggregations(a => a.Cardinality("cardinality_organization_id", c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); double? uniqueOrganizationIdCount = organizationCardinality.Aggregations.Cardinality("cardinality_organization_id")?.Value; if (!uniqueOrganizationIdCount.HasValue || uniqueOrganizationIdCount.Value <= 0) @@ -173,8 +196,10 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { await RenewLockAsync(context); - var organizationIdTerms = await _elasticClient.SearchAsync(s => s.Size(0).Aggregations(a => a - .Terms("terms_organization_id", c => c.Field(f => f.OrganizationId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var organizationIdTerms = await _elasticClient.SearchAsync(s => s + .Size(0) + .Query(q => q.DateRange(r => r.Field(f => f.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc))) + .Aggregations(a => a.Terms("terms_organization_id", c => c.Field(f => f.OrganizationId).Include(batchNumber, buckets).Size(batchSize * 2)))); string[] organizationIds = organizationIdTerms.Aggregations.Terms("terms_organization_id").Buckets.Select(b => b.Key).ToArray(); if (organizationIds.Length == 0) @@ -192,10 +217,13 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) } _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing organizations {MissingOrganizationIds} out of {OrganizationIdCount}", batchNumber, buckets, missingOrganizationIds.Length, missingOrganizationIds, organizationIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.OrganizationId).Terms(missingOrganizationIds)))); + await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Bool(b => b + .Filter( + f => f.Terms(t => t.Field(e => e.OrganizationId).Terms(missingOrganizationIds)), + f => f.DateRange(d => d.Field(e => e.CreatedUtc).GreaterThanOrEquals(orphanedEventCutoffUtc)))))); } - _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing organizations out of {OrganizationIdCount}", totalOrphanedEventCount, totalOrganizationIds); + _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing organizations out of {OrganizationIdCount} since {OrphanedEventCutoffUtc}", totalOrphanedEventCount, totalOrganizationIds, orphanedEventCutoffUtc); } public async Task FixDuplicateStacks(JobContext context) diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index 3322012e9b..8448737a8b 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -173,6 +173,34 @@ public async Task CanDeleteOrphanedEventsByStack() Assert.Equal(5000, eventCount); } + [Fact] + public async Task CanDeleteOnlyRecentOrphanedEventsByStack() + { + var organization = await _organizationRepository.AddAsync(_organizationData.GenerateSampleOrganization(_billingManager, _plans), o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + + var validEvent = _eventData.GenerateEvent(organization.Id, project.Id, stack.Id); + var recentOrphanedEvent = _eventData.GenerateEvent(organization.Id, project.Id); + var olderOrphanedEvent = _eventData.GenerateEvent(organization.Id, project.Id); + + string missingStackId = ObjectId.GenerateNewId().ToString(); + recentOrphanedEvent.StackId = missingStackId; + recentOrphanedEvent.CreatedUtc = DateTime.UtcNow; + olderOrphanedEvent.StackId = missingStackId; + olderOrphanedEvent.CreatedUtc = DateTime.UtcNow.AddDays(-4); + + await _eventRepository.AddAsync([validEvent, recentOrphanedEvent, olderOrphanedEvent], o => o.ImmediateConsistency()); + + await GetService().RunAsync(TestCancellationToken); + + var events = await _eventRepository.GetAllAsync(o => o.PageLimit(10).ImmediateConsistency()); + Assert.Equal(2, events.Total); + Assert.Contains(events.Documents, e => e.Id == validEvent.Id); + Assert.Contains(events.Documents, e => e.Id == olderOrphanedEvent.Id); + Assert.DoesNotContain(events.Documents, e => e.Id == recentOrphanedEvent.Id); + } + [Fact] public async Task RemoveProjectsAsync_SoftDeletedProjectWithEvents_IncrementsDeletedUsage() { From 7f50a3a70ca633f870552bd56f29257aa5636c2f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Jun 2026 00:27:54 +0000 Subject: [PATCH 3/6] Use injected time provider for orphan cleanup cutoff and tests Co-authored-by: niemyjski <1020579+niemyjski@users.noreply.github.com> --- src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs | 6 +++--- tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 9e26e00f3e..083de3e02c 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -61,7 +61,7 @@ protected override async Task RunInternalAsync(JobContext context) public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { - var orphanedEventCutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); + var orphanedEventCutoffUtc = _timeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); // get approximate number of unique stack ids var stackCardinality = await _elasticClient.SearchAsync(s => s @@ -118,7 +118,7 @@ await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Boo public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { - var orphanedEventCutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); + var orphanedEventCutoffUtc = _timeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); // get approximate number of unique project ids var projectCardinality = await _elasticClient.SearchAsync(s => s @@ -173,7 +173,7 @@ await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Boo public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { - var orphanedEventCutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); + var orphanedEventCutoffUtc = _timeProvider.GetUtcNow().UtcDateTime.Subtract(OrphanedEventLookback); // get approximate number of unique organization ids var organizationCardinality = await _elasticClient.SearchAsync(s => s diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index 8448737a8b..3a6163b9c4 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -186,9 +186,9 @@ public async Task CanDeleteOnlyRecentOrphanedEventsByStack() string missingStackId = ObjectId.GenerateNewId().ToString(); recentOrphanedEvent.StackId = missingStackId; - recentOrphanedEvent.CreatedUtc = DateTime.UtcNow; + recentOrphanedEvent.CreatedUtc = TimeProvider.GetUtcNow().UtcDateTime; olderOrphanedEvent.StackId = missingStackId; - olderOrphanedEvent.CreatedUtc = DateTime.UtcNow.AddDays(-4); + olderOrphanedEvent.CreatedUtc = TimeProvider.GetUtcNow().SubtractDays(4).UtcDateTime; await _eventRepository.AddAsync([validEvent, recentOrphanedEvent, olderOrphanedEvent], o => o.ImmediateConsistency()); From c9531879f6920beed7700a8b0b4637f93cb79e14 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 15 Jul 2026 21:40:10 -0500 Subject: [PATCH 4/6] Harden recent orphan event cleanup --- .../Jobs/CleanupOrphanedDataJob.cs | 23 ++++--- .../Jobs/CleanupDataJobTests.cs | 60 ------------------- .../Jobs/CleanupOrphanedDataJobTests.cs | 60 +++++++++++++++++++ 3 files changed, 74 insertions(+), 69 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 15ba3e0fdc..bfce7e6c39 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -78,7 +78,7 @@ private async Task DeleteOrphanedEventsByStackAsync(JobContext context, DateTime { // get approximate number of unique stack ids var stackCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("cardinality_stack_id", a => a.Cardinality(c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); @@ -99,7 +99,7 @@ private async Task DeleteOrphanedEventsByStackAsync(JobContext context, DateTime await RenewLockAsync(context); var stackIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("terms_stack_id", a => a.Terms(c => c.Field(f => f.StackId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); @@ -123,7 +123,7 @@ private async Task DeleteOrphanedEventsByStackAsync(JobContext context, DateTime totalOrphanedEventCount += missingStackIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing stacks {MissingStackIds} out of {StackIdCount}", batchNumber, buckets, missingStackIds.Length, missingStackIds, stackIds.Length); await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Query(q => q.Bool(b => b.Filter( f => f.Terms(t => t.Field(e => e.StackId).Terms(new TermsQueryField(missingStackIds.Select(FieldValueHelper.ToFieldValue).ToList()))), f => RecentEventQuery(f, orphanedEventCutoffUtc))))); @@ -141,7 +141,7 @@ private async Task DeleteOrphanedEventsByProjectAsync(JobContext context, DateTi { // get approximate number of unique project ids var projectCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("cardinality_project_id", a => a.Cardinality(c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); @@ -162,7 +162,7 @@ private async Task DeleteOrphanedEventsByProjectAsync(JobContext context, DateTi await RenewLockAsync(context); var projectIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("terms_project_id", a => a.Terms(c => c.Field(f => f.ProjectId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); @@ -186,7 +186,7 @@ private async Task DeleteOrphanedEventsByProjectAsync(JobContext context, DateTi totalOrphanedEventCount += missingProjectIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing projects {MissingProjectIds} out of {ProjectIdCount}", batchNumber, buckets, missingProjectIds.Length, missingProjectIds, projectIds.Length); await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Query(q => q.Bool(b => b.Filter( f => f.Terms(t => t.Field(e => e.ProjectId).Terms(new TermsQueryField(missingProjectIds.Select(FieldValueHelper.ToFieldValue).ToList()))), f => RecentEventQuery(f, orphanedEventCutoffUtc))))); @@ -204,7 +204,7 @@ private async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context, D { // get approximate number of unique organization ids var organizationCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("cardinality_organization_id", a => a.Cardinality(c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); @@ -225,7 +225,7 @@ private async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context, D await RenewLockAsync(context); var organizationIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("terms_organization_id", a => a.Terms(c => c.Field(f => f.OrganizationId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); @@ -249,7 +249,7 @@ private async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context, D totalOrphanedEventCount += missingOrganizationIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing organizations {MissingOrganizationIds} out of {OrganizationIdCount}", batchNumber, buckets, missingOrganizationIds.Length, missingOrganizationIds, organizationIds.Length); await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetEventIndexPattern()) + .Indices(GetRecentEventIndex()) .Query(q => q.Bool(b => b.Filter( f => f.Terms(t => t.Field(e => e.OrganizationId).Terms(new TermsQueryField(missingOrganizationIds.Select(FieldValueHelper.ToFieldValue).ToList()))), f => RecentEventQuery(f, orphanedEventCutoffUtc))))); @@ -440,6 +440,11 @@ private static QueryDescriptor RecentEventQuery(QueryDescriptor return query.Range(r => r.Date(d => d.Field(e => e.CreatedUtc).Gte(orphanedEventCutoffUtc))); } + private string GetRecentEventIndex() + { + return $"{_config.Events.Name}-last3days"; + } + private string GetEventIndexPattern() { return $"{_config.Events.VersionedName}-*"; diff --git a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs index 7c921cdcf7..c7300641a2 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupDataJobTests.cs @@ -323,66 +323,6 @@ public async Task CanDeleteOrphanedEventsByStack() Assert.Equal(5000, eventCount); } - [Fact] - public async Task RunAsync_OrphanedEventsAtAndBeforeLookbackBoundary_DeletesOnlyEventsWithinLookback() - { - TimeProvider.SetUtcNow(DateTimeOffset.UtcNow); - - var organization = await _organizationRepository.AddAsync( - _organizationData.GenerateSampleOrganization(_billingManager, _plans), - o => o.ImmediateConsistency()); - var project = await _projectRepository.AddAsync( - _projectData.GenerateSampleProject(), - o => o.ImmediateConsistency()); - var stack = await _stackRepository.AddAsync( - _stackData.GenerateSampleStack(), - o => o.ImmediateConsistency()); - - var cutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(CleanupOrphanedDataJob.OrphanedEventLookback); - var beforeCutoffUtc = cutoffUtc.AddMilliseconds(-1); - var validEvent = _eventData.GenerateEvent(organization.Id, project.Id, stack.Id); - - string missingStackId = ObjectId.GenerateNewId().ToString(); - var stackOrphanAtCutoff = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId); - stackOrphanAtCutoff.CreatedUtc = cutoffUtc; - var stackOrphanBeforeCutoff = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId); - stackOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; - - string missingProjectId = ObjectId.GenerateNewId().ToString(); - var projectOrphanAtCutoff = _eventData.GenerateEvent(organization.Id, missingProjectId, stack.Id); - projectOrphanAtCutoff.CreatedUtc = cutoffUtc; - var projectOrphanBeforeCutoff = _eventData.GenerateEvent(organization.Id, missingProjectId, stack.Id); - projectOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; - - string missingOrganizationId = ObjectId.GenerateNewId().ToString(); - var organizationOrphanAtCutoff = _eventData.GenerateEvent(missingOrganizationId, project.Id, stack.Id); - organizationOrphanAtCutoff.CreatedUtc = cutoffUtc; - var organizationOrphanBeforeCutoff = _eventData.GenerateEvent(missingOrganizationId, project.Id, stack.Id); - organizationOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; - - await _eventRepository.AddAsync([ - validEvent, - stackOrphanAtCutoff, - stackOrphanBeforeCutoff, - projectOrphanAtCutoff, - projectOrphanBeforeCutoff, - organizationOrphanAtCutoff, - organizationOrphanBeforeCutoff - ], o => o.ImmediateConsistency()); - - await GetService().RunAsync(TestCancellationToken); - - var remainingEvents = await _eventRepository.GetAllAsync(o => o.PageLimit(10).ImmediateConsistency()); - Assert.Equal(4, remainingEvents.Total); - Assert.Contains(remainingEvents.Documents, e => e.Id == validEvent.Id); - Assert.Contains(remainingEvents.Documents, e => e.Id == stackOrphanBeforeCutoff.Id); - Assert.Contains(remainingEvents.Documents, e => e.Id == projectOrphanBeforeCutoff.Id); - Assert.Contains(remainingEvents.Documents, e => e.Id == organizationOrphanBeforeCutoff.Id); - Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == stackOrphanAtCutoff.Id); - Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == projectOrphanAtCutoff.Id); - Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == organizationOrphanAtCutoff.Id); - } - [Fact] public async Task CanCleanupSuspendedTokens_MultiTenant_OnlySuspendedOrganizationTokensAffected() { diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs index af3f405b16..c09bd1b3cc 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -390,6 +390,66 @@ public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDe Assert.Equal(120, totalAfter); } + [Fact] + public async Task RunAsync_OrphanedEventsAtAndBeforeLookbackBoundary_DeletesOnlyEventsWithinLookback() + { + TimeProvider.SetUtcNow(DateTimeOffset.UtcNow); + + var organization = await _organizationRepository.AddAsync( + _organizationData.GenerateSampleOrganization(_billingManager, _plans), + o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync( + _projectData.GenerateSampleProject(), + o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync( + _stackData.GenerateSampleStack(), + o => o.ImmediateConsistency()); + + var cutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(CleanupOrphanedDataJob.OrphanedEventLookback); + var beforeCutoffUtc = cutoffUtc.AddMilliseconds(-1); + var validEvent = _eventData.GenerateEvent(organization.Id, project.Id, stack.Id); + + string missingStackId = ObjectId.GenerateNewId().ToString(); + var stackOrphanAtCutoff = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId); + stackOrphanAtCutoff.CreatedUtc = cutoffUtc; + var stackOrphanBeforeCutoff = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId); + stackOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; + + string missingProjectId = ObjectId.GenerateNewId().ToString(); + var projectOrphanAtCutoff = _eventData.GenerateEvent(organization.Id, missingProjectId, stack.Id); + projectOrphanAtCutoff.CreatedUtc = cutoffUtc; + var projectOrphanBeforeCutoff = _eventData.GenerateEvent(organization.Id, missingProjectId, stack.Id); + projectOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; + + string missingOrganizationId = ObjectId.GenerateNewId().ToString(); + var organizationOrphanAtCutoff = _eventData.GenerateEvent(missingOrganizationId, project.Id, stack.Id); + organizationOrphanAtCutoff.CreatedUtc = cutoffUtc; + var organizationOrphanBeforeCutoff = _eventData.GenerateEvent(missingOrganizationId, project.Id, stack.Id); + organizationOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; + + await _eventRepository.AddAsync([ + validEvent, + stackOrphanAtCutoff, + stackOrphanBeforeCutoff, + projectOrphanAtCutoff, + projectOrphanBeforeCutoff, + organizationOrphanAtCutoff, + organizationOrphanBeforeCutoff + ], o => o.ImmediateConsistency()); + + await _job.RunAsync(TestCancellationToken); + + var remainingEvents = await _eventRepository.GetAllAsync(o => o.PageLimit(10).ImmediateConsistency()); + Assert.Equal(4, remainingEvents.Total); + Assert.Contains(remainingEvents.Documents, e => e.Id == validEvent.Id); + Assert.Contains(remainingEvents.Documents, e => e.Id == stackOrphanBeforeCutoff.Id); + Assert.Contains(remainingEvents.Documents, e => e.Id == projectOrphanBeforeCutoff.Id); + Assert.Contains(remainingEvents.Documents, e => e.Id == organizationOrphanBeforeCutoff.Id); + Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == stackOrphanAtCutoff.Id); + Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == projectOrphanAtCutoff.Id); + Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == organizationOrphanAtCutoff.Id); + } + [Fact] public async Task FixDuplicateStacks_WithDuplicatesAcrossTenants_MergesCorrectly() { From b0910ef8ad6c3f8930e413f97f177864a2dac8e2 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 15 Jul 2026 22:02:33 -0500 Subject: [PATCH 5/6] Fix orphan cleanup index coverage --- .../Jobs/CleanupOrphanedDataJob.cs | 23 ++++++++----------- .../Jobs/CleanupOrphanedDataJobTests.cs | 23 +++++++++++-------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index bfce7e6c39..15ba3e0fdc 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -78,7 +78,7 @@ private async Task DeleteOrphanedEventsByStackAsync(JobContext context, DateTime { // get approximate number of unique stack ids var stackCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("cardinality_stack_id", a => a.Cardinality(c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); @@ -99,7 +99,7 @@ private async Task DeleteOrphanedEventsByStackAsync(JobContext context, DateTime await RenewLockAsync(context); var stackIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("terms_stack_id", a => a.Terms(c => c.Field(f => f.StackId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); @@ -123,7 +123,7 @@ private async Task DeleteOrphanedEventsByStackAsync(JobContext context, DateTime totalOrphanedEventCount += missingStackIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing stacks {MissingStackIds} out of {StackIdCount}", batchNumber, buckets, missingStackIds.Length, missingStackIds, stackIds.Length); await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Query(q => q.Bool(b => b.Filter( f => f.Terms(t => t.Field(e => e.StackId).Terms(new TermsQueryField(missingStackIds.Select(FieldValueHelper.ToFieldValue).ToList()))), f => RecentEventQuery(f, orphanedEventCutoffUtc))))); @@ -141,7 +141,7 @@ private async Task DeleteOrphanedEventsByProjectAsync(JobContext context, DateTi { // get approximate number of unique project ids var projectCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("cardinality_project_id", a => a.Cardinality(c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); @@ -162,7 +162,7 @@ private async Task DeleteOrphanedEventsByProjectAsync(JobContext context, DateTi await RenewLockAsync(context); var projectIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("terms_project_id", a => a.Terms(c => c.Field(f => f.ProjectId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); @@ -186,7 +186,7 @@ private async Task DeleteOrphanedEventsByProjectAsync(JobContext context, DateTi totalOrphanedEventCount += missingProjectIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing projects {MissingProjectIds} out of {ProjectIdCount}", batchNumber, buckets, missingProjectIds.Length, missingProjectIds, projectIds.Length); await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Query(q => q.Bool(b => b.Filter( f => f.Terms(t => t.Field(e => e.ProjectId).Terms(new TermsQueryField(missingProjectIds.Select(FieldValueHelper.ToFieldValue).ToList()))), f => RecentEventQuery(f, orphanedEventCutoffUtc))))); @@ -204,7 +204,7 @@ private async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context, D { // get approximate number of unique organization ids var organizationCardinality = await _elasticClient.SearchAsync(s => s - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("cardinality_organization_id", a => a.Cardinality(c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); @@ -225,7 +225,7 @@ private async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context, D await RenewLockAsync(context); var organizationIdTerms = await _elasticClient.SearchAsync(s => s - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Size(0) .Query(q => RecentEventQuery(q, orphanedEventCutoffUtc)) .AddAggregation("terms_organization_id", a => a.Terms(c => c.Field(f => f.OrganizationId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); @@ -249,7 +249,7 @@ private async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context, D totalOrphanedEventCount += missingOrganizationIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing organizations {MissingOrganizationIds} out of {OrganizationIdCount}", batchNumber, buckets, missingOrganizationIds.Length, missingOrganizationIds, organizationIds.Length); await _elasticClient.DeleteByQueryAsync(r => r - .Indices(GetRecentEventIndex()) + .Indices(GetEventIndexPattern()) .Query(q => q.Bool(b => b.Filter( f => f.Terms(t => t.Field(e => e.OrganizationId).Terms(new TermsQueryField(missingOrganizationIds.Select(FieldValueHelper.ToFieldValue).ToList()))), f => RecentEventQuery(f, orphanedEventCutoffUtc))))); @@ -440,11 +440,6 @@ private static QueryDescriptor RecentEventQuery(QueryDescriptor return query.Range(r => r.Date(d => d.Field(e => e.CreatedUtc).Gte(orphanedEventCutoffUtc))); } - private string GetRecentEventIndex() - { - return $"{_config.Events.Name}-last3days"; - } - private string GetEventIndexPattern() { return $"{_config.Events.VersionedName}-*"; diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs index c09bd1b3cc..da9f0eb053 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -391,9 +391,10 @@ public async Task DeleteOrphanedEventsByOrganization_TwoTenantsOneDeleted_OnlyDe } [Fact] - public async Task RunAsync_OrphanedEventsAtAndBeforeLookbackBoundary_DeletesOnlyEventsWithinLookback() + public async Task RunAsync_OrphanedEventsAcrossOccurrenceDates_DeletesOnlyEventsWithinCreatedUtcLookback() { - TimeProvider.SetUtcNow(DateTimeOffset.UtcNow); + var now = DateTimeOffset.UtcNow; + TimeProvider.SetUtcNow(now); var organization = await _organizationRepository.AddAsync( _organizationData.GenerateSampleOrganization(_billingManager, _plans), @@ -407,30 +408,33 @@ public async Task RunAsync_OrphanedEventsAtAndBeforeLookbackBoundary_DeletesOnly var cutoffUtc = TimeProvider.GetUtcNow().UtcDateTime.Subtract(CleanupOrphanedDataJob.OrphanedEventLookback); var beforeCutoffUtc = cutoffUtc.AddMilliseconds(-1); - var validEvent = _eventData.GenerateEvent(organization.Id, project.Id, stack.Id); + var validEvent = _eventData.GenerateEvent(organization.Id, project.Id, stack.Id, occurrenceDate: now); string missingStackId = ObjectId.GenerateNewId().ToString(); - var stackOrphanAtCutoff = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId); + var stackOrphanAtCutoff = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId, occurrenceDate: now); stackOrphanAtCutoff.CreatedUtc = cutoffUtc; - var stackOrphanBeforeCutoff = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId); + var stackOrphanBeforeCutoff = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId, occurrenceDate: now); stackOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; + var stackOrphanWithOldOccurrence = _eventData.GenerateEvent(organization.Id, project.Id, missingStackId, occurrenceDate: now.Subtract(TimeSpan.FromDays(30))); + stackOrphanWithOldOccurrence.CreatedUtc = now.UtcDateTime; string missingProjectId = ObjectId.GenerateNewId().ToString(); - var projectOrphanAtCutoff = _eventData.GenerateEvent(organization.Id, missingProjectId, stack.Id); + var projectOrphanAtCutoff = _eventData.GenerateEvent(organization.Id, missingProjectId, stack.Id, occurrenceDate: now); projectOrphanAtCutoff.CreatedUtc = cutoffUtc; - var projectOrphanBeforeCutoff = _eventData.GenerateEvent(organization.Id, missingProjectId, stack.Id); + var projectOrphanBeforeCutoff = _eventData.GenerateEvent(organization.Id, missingProjectId, stack.Id, occurrenceDate: now); projectOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; string missingOrganizationId = ObjectId.GenerateNewId().ToString(); - var organizationOrphanAtCutoff = _eventData.GenerateEvent(missingOrganizationId, project.Id, stack.Id); + var organizationOrphanAtCutoff = _eventData.GenerateEvent(missingOrganizationId, project.Id, stack.Id, occurrenceDate: now); organizationOrphanAtCutoff.CreatedUtc = cutoffUtc; - var organizationOrphanBeforeCutoff = _eventData.GenerateEvent(missingOrganizationId, project.Id, stack.Id); + var organizationOrphanBeforeCutoff = _eventData.GenerateEvent(missingOrganizationId, project.Id, stack.Id, occurrenceDate: now); organizationOrphanBeforeCutoff.CreatedUtc = beforeCutoffUtc; await _eventRepository.AddAsync([ validEvent, stackOrphanAtCutoff, stackOrphanBeforeCutoff, + stackOrphanWithOldOccurrence, projectOrphanAtCutoff, projectOrphanBeforeCutoff, organizationOrphanAtCutoff, @@ -446,6 +450,7 @@ await _eventRepository.AddAsync([ Assert.Contains(remainingEvents.Documents, e => e.Id == projectOrphanBeforeCutoff.Id); Assert.Contains(remainingEvents.Documents, e => e.Id == organizationOrphanBeforeCutoff.Id); Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == stackOrphanAtCutoff.Id); + Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == stackOrphanWithOldOccurrence.Id); Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == projectOrphanAtCutoff.Id); Assert.DoesNotContain(remainingEvents.Documents, e => e.Id == organizationOrphanAtCutoff.Id); } From d11819a3e0d5caf846b879ed1574329409844573 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 15 Jul 2026 22:22:02 -0500 Subject: [PATCH 6/6] Keep orphan cleanup health current --- .../Jobs/CleanupOrphanedDataJob.cs | 2 ++ .../Jobs/CleanupOrphanedDataJobTests.cs | 34 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 15ba3e0fdc..d6967266d7 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -59,6 +59,8 @@ ILoggerFactory loggerFactory protected override async Task RunInternalAsync(JobContext context) { + _lastRun = _timeProvider.GetUtcNow().UtcDateTime; + var orphanedEventCutoffUtc = GetOrphanedEventCutoffUtc(); await DeleteOrphanedEventsByStackAsync(context, orphanedEventCutoffUtc); await DeleteOrphanedEventsByProjectAsync(context, orphanedEventCutoffUtc); diff --git a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs index da9f0eb053..f8c6368eb2 100644 --- a/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs +++ b/tests/Exceptionless.Tests/Jobs/CleanupOrphanedDataJobTests.cs @@ -6,6 +6,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Utility; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Xunit; namespace Exceptionless.Tests.Jobs; @@ -601,7 +602,7 @@ public async Task RunAsync_NoOrphans_PreservesEverything() } [Fact] - public async Task RunAsync_EmptyDatabase_CompletesWithoutError() + public async Task RunAsync_EmptyDatabase_UpdatesHealth() { // Arrange - nothing @@ -610,5 +611,36 @@ public async Task RunAsync_EmptyDatabase_CompletesWithoutError() var totalAfter = await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency()); Assert.Equal(0, totalAfter); + + var health = await _job.CheckHealthAsync(new HealthCheckContext(), TestCancellationToken); + Assert.Equal(HealthStatus.Healthy, health.Status); + Assert.Equal("Job has run in the last 65 minutes.", health.Description); + } + + [Fact] + public async Task RunAsync_NoRecentEventsOrDuplicateStacks_UpdatesHealth() + { + var now = DateTimeOffset.UtcNow; + TimeProvider.SetUtcNow(now); + + var organization = await _organizationRepository.AddAsync( + _organizationData.GenerateSampleOrganization(_billingManager, _plans), + o => o.ImmediateConsistency()); + var project = await _projectRepository.AddAsync( + _projectData.GenerateProject(organizationId: organization.Id), + o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync( + _stackData.GenerateStack(projectId: project.Id, organizationId: organization.Id), + o => o.ImmediateConsistency()); + var historicalEvent = _eventData.GenerateEvent(organization.Id, project.Id, stack.Id, occurrenceDate: now.Subtract(TimeSpan.FromDays(30))); + historicalEvent.CreatedUtc = now.UtcDateTime.Subtract(CleanupOrphanedDataJob.OrphanedEventLookback).AddMilliseconds(-1); + await _eventRepository.AddAsync(historicalEvent, o => o.ImmediateConsistency()); + + await _job.RunAsync(TestCancellationToken); + + Assert.Equal(1, await _eventRepository.CountAsync(o => o.IncludeSoftDeletes().ImmediateConsistency())); + var health = await _job.CheckHealthAsync(new HealthCheckContext(), TestCancellationToken); + Assert.Equal(HealthStatus.Healthy, health.Status); + Assert.Equal("Job has run in the last 65 minutes.", health.Description); } }