From 2757ebc1cd06806bcd54ecd00584722a480b0016 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 12:44:26 +0200 Subject: [PATCH 01/15] feat(email): durable email outbox with push-based retry consumer Replace inline transactional email sends with a database-backed outbox so unsent mail is never lost and can be retried/resent. - Add EmailOutboxMessage (Common): jsonb payload, email_type/email_status pg enums, three states (Sending/Sent/Failed), attempt/lease/backoff fields. - EmailOutboxWorker (API hosted service): push-triggered via Redis pub/sub with a 15s poll fallback; claims rows with FOR UPDATE SKIP LOCKED so it is safe in every replica; exponential backoff + jitter, retries on 5xx/429/ transport errors. EmailOutboxRetryPolicy holds the pure, tested state machine. - IEmailOutboxDispatcher: single send path; mints each token lazily at send time (re-minted on resend) so the queue never stores a usable link. - IEmailService now returns EmailSendResult (Sent/Transient/Permanent) and no longer throws on provider failures; Mailjet/SMTP/None updated. - AccountService: activation, OAuth activation, password reset and email change now enqueue durable outbox rows (token hash seeded at insert, re-minted at send) and notify the consumer instead of sending inline. - Migration AddEmailOutbox (tooling-generated); existing token tables unchanged. - Tests: EmailOutboxRetryPolicyTests (10), EmailOutboxMessageTests (2). --- .../Tests/EmailOutboxRetryPolicyTests.cs | 129 ++ API/Services/Account/AccountService.cs | 105 +- API/Services/Email/EmailSendResult.cs | 23 + API/Services/Email/EmailServiceExtension.cs | 8 +- API/Services/Email/IEmailService.cs | 26 +- .../Email/Mailjet/MailjetEmailService.cs | 52 +- API/Services/Email/NoneEmailService.cs | 18 +- .../Email/Outbox/EmailDispatchResult.cs | 38 + .../Email/Outbox/EmailOutboxDispatcher.cs | 149 ++ .../Email/Outbox/EmailOutboxRetryPolicy.cs | 90 + .../Email/Outbox/EmailOutboxWorker.cs | 189 ++ .../Email/Outbox/IEmailOutboxDispatcher.cs | 24 + API/Services/Email/Smtp/SmtpEmailService.cs | 58 +- .../OpenShockDb/EmailOutboxMessageTests.cs | 39 + Common/Constants/Constants.cs | 14 + Common/Constants/HardLimits.cs | 2 + .../20260630103404_AddEmailOutbox.Designer.cs | 1593 +++++++++++++++++ .../20260630103404_AddEmailOutbox.cs | 110 ++ .../OpenShockContextModelSnapshot.cs | 77 +- Common/Models/EmailStatus.cs | 32 + Common/Models/EmailType.cs | 22 + Common/OpenShockDb/EmailOutboxMessage.cs | 144 ++ Common/OpenShockDb/EmailOutboxPayloadKeys.cs | 23 + Common/OpenShockDb/OpenShockContext.cs | 51 + Common/OpenShockDb/UserActivationRequest.cs | 5 + Common/OpenShockDb/UserEmailChange.cs | 6 +- Common/OpenShockDb/UserPasswordReset.cs | 6 +- .../Services/RedisPubSub/IRedisPubService.cs | 8 + Common/Services/RedisPubSub/RedisChannels.cs | 6 + .../Services/RedisPubSub/RedisPubService.cs | 6 + 30 files changed, 2941 insertions(+), 112 deletions(-) create mode 100644 API.IntegrationTests/Tests/EmailOutboxRetryPolicyTests.cs create mode 100644 API/Services/Email/EmailSendResult.cs create mode 100644 API/Services/Email/Outbox/EmailDispatchResult.cs create mode 100644 API/Services/Email/Outbox/EmailOutboxDispatcher.cs create mode 100644 API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs create mode 100644 API/Services/Email/Outbox/EmailOutboxWorker.cs create mode 100644 API/Services/Email/Outbox/IEmailOutboxDispatcher.cs create mode 100644 Common.Tests/OpenShockDb/EmailOutboxMessageTests.cs create mode 100644 Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs create mode 100644 Common/Migrations/20260630103404_AddEmailOutbox.cs create mode 100644 Common/Models/EmailStatus.cs create mode 100644 Common/Models/EmailType.cs create mode 100644 Common/OpenShockDb/EmailOutboxMessage.cs create mode 100644 Common/OpenShockDb/EmailOutboxPayloadKeys.cs diff --git a/API.IntegrationTests/Tests/EmailOutboxRetryPolicyTests.cs b/API.IntegrationTests/Tests/EmailOutboxRetryPolicyTests.cs new file mode 100644 index 00000000..5b5633c7 --- /dev/null +++ b/API.IntegrationTests/Tests/EmailOutboxRetryPolicyTests.cs @@ -0,0 +1,129 @@ +using OpenShock.API.Services.Email.Outbox; +using OpenShock.Common.Constants; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.API.IntegrationTests.Tests; + +/// +/// Pure unit tests for the email outbox retry/state machine. These don't touch the database or the +/// test host (no WebApplicationFactory data source), so they run without Docker. +/// +public class EmailOutboxRetryPolicyTests +{ + private static EmailOutboxMessage NewMessage(int attemptCount) + { + var message = EmailOutboxMessage.Create(EmailType.PasswordReset, "user@example.com", "User", + new Dictionary()); + message.AttemptCount = attemptCount; + return message; + } + + [Test] + public async Task GetBaseDelay_GrowsExponentiallyFromBaseDelay() + { + await Assert.That(EmailOutboxRetryPolicy.GetBaseDelay(1)).IsEqualTo(Duration.EmailOutboxRetryBaseDelay); + await Assert.That(EmailOutboxRetryPolicy.GetBaseDelay(2)).IsEqualTo(Duration.EmailOutboxRetryBaseDelay * 2); + await Assert.That(EmailOutboxRetryPolicy.GetBaseDelay(3)).IsEqualTo(Duration.EmailOutboxRetryBaseDelay * 4); + } + + [Test] + public async Task GetBaseDelay_IsCappedAtMaxDelay() + { + await Assert.That(EmailOutboxRetryPolicy.GetBaseDelay(1000)).IsEqualTo(Duration.EmailOutboxRetryMaxDelay); + } + + [Test] + public async Task GetRetryDelay_StaysWithinJitterBounds() + { + var baseDelay = EmailOutboxRetryPolicy.GetBaseDelay(2); + var upper = baseDelay + TimeSpan.FromSeconds(EmailOutboxRetryPolicy.MaxJitterSeconds); + + for (var i = 0; i < 200; i++) + { + var delay = EmailOutboxRetryPolicy.GetRetryDelay(2); + await Assert.That(delay).IsGreaterThanOrEqualTo(baseDelay); + await Assert.That(delay).IsLessThanOrEqualTo(upper); + } + } + + [Test] + public async Task Apply_Sent_MarksSentAndClearsState() + { + var message = NewMessage(1); + var now = DateTime.UtcNow; + + EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Sent, now); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Sent); + await Assert.That(message.SentAt).IsEqualTo(now); + await Assert.That(message.AttemptStartedAt).IsNull(); + await Assert.That(message.LastError).IsNull(); + } + + [Test] + public async Task Apply_Skipped_MarksFailedWithReason() + { + var message = NewMessage(1); + + EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Skip("password reset expired"), DateTime.UtcNow); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); + await Assert.That(message.FailedAt).IsNotNull(); + await Assert.That(message.LastError!).Contains("password reset expired"); + } + + [Test] + public async Task Apply_PermanentFailure_MarksFailed() + { + var message = NewMessage(1); + + EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Permanent("rejected recipient"), DateTime.UtcNow); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); + await Assert.That(message.FailedAt).IsNotNull(); + await Assert.That(message.LastError).IsEqualTo("rejected recipient"); + } + + [Test] + public async Task Apply_TransientFailure_BelowMaxAttempts_SchedulesRetry() + { + var message = NewMessage(1); + var now = DateTime.UtcNow; + + EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Transient("smtp timeout"), now); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Sending); + await Assert.That(message.NextAttemptAt).IsNotNull(); + await Assert.That(message.NextAttemptAt!.Value).IsGreaterThan(now); + await Assert.That(message.AttemptStartedAt).IsNull(); + await Assert.That(message.LastError).IsEqualTo("smtp timeout"); + } + + [Test] + public async Task Apply_TransientFailure_AtMaxAttempts_MarksFailed() + { + var message = NewMessage(EmailOutboxRetryPolicy.MaxAttempts); + + EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Transient("smtp timeout"), DateTime.UtcNow); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); + await Assert.That(message.FailedAt).IsNotNull(); + } + + [Test] + public async Task Truncate_LimitsToMaxLength() + { + var longText = new string('x', HardLimits.EmailOutboxLastErrorMaxLength + 50); + + var result = EmailOutboxRetryPolicy.Truncate(longText); + + await Assert.That(result!.Length).IsEqualTo(HardLimits.EmailOutboxLastErrorMaxLength); + } + + [Test] + public async Task Truncate_Null_ReturnsNull() + { + await Assert.That(EmailOutboxRetryPolicy.Truncate(null)).IsNull(); + } +} diff --git a/API/Services/Account/AccountService.cs b/API/Services/Account/AccountService.cs index e92926b0..d4675100 100644 --- a/API/Services/Account/AccountService.cs +++ b/API/Services/Account/AccountService.cs @@ -3,12 +3,10 @@ using Npgsql; using OneOf; using OneOf.Types; -using OpenShock.API.Services.Email; -using OpenShock.API.Services.Email.Mailjet.Mail; using OpenShock.Common.Constants; using OpenShock.Common.Models; using OpenShock.Common.OpenShockDb; -using OpenShock.Common.Options; +using OpenShock.Common.Services.RedisPubSub; using OpenShock.Common.Services.Session; using OpenShock.Common.Utils; using OpenShock.Common.Validation; @@ -21,29 +19,34 @@ namespace OpenShock.API.Services.Account; public sealed class AccountService : IAccountService { private readonly OpenShockContext _db; - private readonly IEmailService _emailService; + private readonly IRedisPubService _redisPubService; private readonly ISessionService _sessionService; private readonly ILogger _logger; - private readonly FrontendOptions _frontendConfig; /// /// DI Constructor /// /// - /// + /// Used to notify the email outbox consumer that mail was enqueued. /// /// - /// - public AccountService(OpenShockContext db, IEmailService emailService, - ISessionService sessionService, ILogger logger, FrontendOptions options) + public AccountService(OpenShockContext db, IRedisPubService redisPubService, + ISessionService sessionService, ILogger logger) { _db = db; - _emailService = emailService; + _redisPubService = redisPubService; _logger = logger; - _frontendConfig = options; _sessionService = sessionService; } + /// + /// Seeds a random token hash for a freshly created request row. The plaintext is discarded + /// immediately — the email outbox consumer mints the real token (and overwrites this hash) when it + /// sends, so this value is never the one delivered. It exists only to keep the column populated. + /// + private static string SeedTokenHash() + => HashingUtils.HashToken(CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength)); + private async Task IsUserNameBlacklisted(string username) { await foreach (var entry in _db.UserNameBlacklists.AsNoTracking().AsAsyncEnumerable()) @@ -101,18 +104,20 @@ public async Task, AccountWithEmailOrUsernameExists>> Create var user = accountCreate.AsT0.Value; - var token = CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength); - + // The real activation token is minted by the outbox consumer at send time; here we record the + // request (with a seeded hash) and durably enqueue the email. user.UserActivationRequest = new UserActivationRequest { UserId = user.Id, - TokenHash = HashingUtils.HashToken(token) + TokenHash = SeedTokenHash() }; + _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.AccountActivation, email, username, + new Dictionary { [EmailOutboxPayloadKeys.UserId] = user.Id.ToString() })); + await _db.SaveChangesAsync(); + await _redisPubService.SendEmailOutboxPending(); - await _emailService.ActivateAccount(new Contact(email, username), - new Uri(_frontendConfig.BaseUrl, $"/activate?token={token}")); return new Success(user); } @@ -143,8 +148,6 @@ public async Task, AccountWithEmailOrUsernameExists>> Create await using var tx = await _db.Database.BeginTransactionAsync(); - string? activationToken = null; - try { var creationTime = DateTime.UtcNow; @@ -162,18 +165,20 @@ public async Task, AccountWithEmailOrUsernameExists>> Create _db.Users.Add(user); await _db.SaveChangesAsync(); - // If email isn't trusted, create an activation request (email verification) + // If email isn't trusted, create an activation request and durably enqueue the activation + // email. The token itself is minted by the outbox consumer at send time. if (!isEmailTrusted) { - activationToken = CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength); - user.UserActivationRequest = new UserActivationRequest { UserId = user.Id, - TokenHash = HashingUtils.HashToken(activationToken), + TokenHash = SeedTokenHash(), CreatedAt = creationTime }; + _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.AccountActivation, email, username, + new Dictionary { [EmailOutboxPayloadKeys.UserId] = user.Id.ToString() })); + await _db.SaveChangesAsync(); } @@ -191,13 +196,10 @@ public async Task, AccountWithEmailOrUsernameExists>> Create await tx.CommitAsync(); - // Send verification email only after a successful commit - if (!isEmailTrusted && activationToken is not null) + // Notify the outbox consumer only after a successful commit. + if (!isEmailTrusted) { - await _emailService.ActivateAccount( - new Contact(email, username), - new Uri(_frontendConfig.BaseUrl, $"/activate?token={activationToken}") - ); + await _redisPubService.SendEmailOutboxPending(); } return new Success(user); @@ -412,19 +414,21 @@ public async Task= 3) return new TooManyPasswordResets(); - var token = CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength); + // The reset token is minted lazily by the outbox consumer at send time (and re-minted on any + // resend), so no usable reset link is ever stored. Here we only record the request and + // durably enqueue the email. var passwordReset = new UserPasswordReset { Id = Guid.CreateVersion7(), UserId = user.User.Id, - TokenHash = HashingUtils.HashToken(token), + TokenHash = SeedTokenHash(), SecurityStampAtCreate = user.User.SecurityStamp }; _db.UserPasswordResets.Add(passwordReset); + _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.PasswordReset, user.User.Email, user.User.Name, + new Dictionary { [EmailOutboxPayloadKeys.PasswordResetId] = passwordReset.Id.ToString() })); await _db.SaveChangesAsync(); - - await _emailService.PasswordReset(new Contact(user.User.Email, user.User.Name), - new Uri(_frontendConfig.BaseUrl, $"/#/account/password/recover/{passwordReset.Id}/{token}")); + await _redisPubService.SendEmailOutboxPending(); return new Success(); } @@ -574,38 +578,31 @@ public async Task x.Email == lowerCaseEmail)) return new EmailAlreadyInUse(); - var token = CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength); + // The verification token is minted lazily by the outbox consumer at send time, so no usable + // verification link is ever stored. var emailChange = new UserEmailChange { Id = Guid.CreateVersion7(), UserId = data.User.Id, OldEmail = data.User.Email, NewEmail = lowerCaseEmail, - TokenHash = HashingUtils.HashToken(token), + TokenHash = SeedTokenHash(), SecurityStampAtCreate = data.User.SecurityStamp }; + _db.UserEmailChanges.Add(emailChange); - // Dispatch the verification email *before* committing the row. If the mail service throws - // (provider outage, transient network failure), the exception propagates and the row is - // never inserted, the user can simply retry without burning a pending-count slot. - await _emailService.VerifyEmail(new Contact(lowerCaseEmail, data.User.Name), - new Uri(_frontendConfig.BaseUrl, $"/verify-email?token={token}")); + // Durably enqueue both emails: the verification link to the new address, and an informational + // notice to the previous address so the legitimate owner sees the change request even if the + // session/password used to start it was compromised. The outbox guarantees delivery with + // retries; this replaces the previous best-effort inline sends. Committing the row even if + // mail later fails is intentional — the request is durable and the consumer keeps retrying. + _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.EmailVerification, lowerCaseEmail, data.User.Name, + new Dictionary { [EmailOutboxPayloadKeys.EmailChangeId] = emailChange.Id.ToString() })); + _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.EmailChangeNotice, data.User.Email, data.User.Name, + new Dictionary { [EmailOutboxPayloadKeys.NewEmail] = lowerCaseEmail })); - _db.UserEmailChanges.Add(emailChange); await _db.SaveChangesAsync(); - - // Notify the previous address so the legitimate owner sees the change request even if - // the session/password used to start it was compromised. Best-effort: the verification - // email has already been dispatched and the row is committed, so a failure here must not - // unwind the request. - try - { - await _emailService.EmailChangeNotice(new Contact(data.User.Email, data.User.Name), lowerCaseEmail); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to send email-change notice to previous address for user {UserId}", data.User.Id); - } + await _redisPubService.SendEmailOutboxPending(); return new Success(); } diff --git a/API/Services/Email/EmailSendResult.cs b/API/Services/Email/EmailSendResult.cs new file mode 100644 index 00000000..53d2e11f --- /dev/null +++ b/API/Services/Email/EmailSendResult.cs @@ -0,0 +1,23 @@ +namespace OpenShock.API.Services.Email; + +/// +/// Outcome of a single attempt to hand an email to the provider. Drives the outbox consumer's +/// retry decision, so implementations must classify provider failures rather than swallowing them. +/// +public enum EmailSendResult +{ + /// The provider accepted the message. Terminal success. + Sent, + + /// + /// A temporary failure (timeout, network error, provider 5xx, rate-limit 429). The same send + /// should be retried later. + /// + TransientFailure, + + /// + /// A failure that retrying will not fix (e.g. provider 4xx other than 429, malformed/rejected + /// recipient). The message should not be retried. + /// + PermanentFailure +} diff --git a/API/Services/Email/EmailServiceExtension.cs b/API/Services/Email/EmailServiceExtension.cs index 86dd85de..bcc92d9e 100644 --- a/API/Services/Email/EmailServiceExtension.cs +++ b/API/Services/Email/EmailServiceExtension.cs @@ -1,5 +1,6 @@ using OpenShock.API.Options; using OpenShock.API.Services.Email.Mailjet; +using OpenShock.API.Services.Email.Outbox; using OpenShock.API.Services.Email.Smtp; namespace OpenShock.API.Services.Email; @@ -9,7 +10,12 @@ public static class EmailServiceExtension public static async Task AddEmailService(this WebApplicationBuilder builder) { var mailOptions = builder.Configuration.GetRequiredSection(MailOptions.SectionName).Get() ?? throw new NullReferenceException(); - + + // The outbox dispatcher + consumer drive all transactional email regardless of provider; even + // with mail disabled the consumer runs and marks messages terminal (the no-op provider). + builder.Services.AddSingleton(); + builder.Services.AddHostedService(); + if (mailOptions.Type == MailOptions.MailType.None) { builder.Services.AddSingleton(); // Add a dummy email service diff --git a/API/Services/Email/IEmailService.cs b/API/Services/Email/IEmailService.cs index 12aee102..4f8b1234 100644 --- a/API/Services/Email/IEmailService.cs +++ b/API/Services/Email/IEmailService.cs @@ -2,6 +2,13 @@ namespace OpenShock.API.Services.Email; +/// +/// Low-level email provider abstraction: renders a template and hands the message to the provider. +/// Implementations return an instead of throwing, so the email outbox +/// consumer (the only caller) can classify failures and decide whether to retry. Callers must not +/// invoke this directly to send transactional mail — enqueue an +/// instead so the send is durable. +/// public interface IEmailService { /// @@ -10,26 +17,26 @@ public interface IEmailService /// /// /// - /// - public Task ActivateAccount(Contact to, Uri activationLink, CancellationToken cancellationToken = default); - + /// The outcome of the send attempt. + public Task ActivateAccount(Contact to, Uri activationLink, CancellationToken cancellationToken = default); + /// /// Send a password reset email /// /// /// /// - /// - public Task PasswordReset(Contact to, Uri resetLink, CancellationToken cancellationToken = default); - + /// The outcome of the send attempt. + public Task PasswordReset(Contact to, Uri resetLink, CancellationToken cancellationToken = default); + /// /// When a user uses changes their email, we send them this email to let them verify it /// /// /// /// - /// - public Task VerifyEmail(Contact to, Uri verificationLink, CancellationToken cancellationToken = default); + /// The outcome of the send attempt. + public Task VerifyEmail(Contact to, Uri verificationLink, CancellationToken cancellationToken = default); /// /// Informational notice sent to the user's previous email address when an email change is @@ -39,5 +46,6 @@ public interface IEmailService /// The old email address being notified. /// The new email address that was requested. /// - public Task EmailChangeNotice(Contact to, string newEmail, CancellationToken cancellationToken = default); + /// The outcome of the send attempt. + public Task EmailChangeNotice(Contact to, string newEmail, CancellationToken cancellationToken = default); } diff --git a/API/Services/Email/Mailjet/MailjetEmailService.cs b/API/Services/Email/Mailjet/MailjetEmailService.cs index 90063fe0..4a49d69a 100644 --- a/API/Services/Email/Mailjet/MailjetEmailService.cs +++ b/API/Services/Email/Mailjet/MailjetEmailService.cs @@ -29,52 +29,72 @@ ILogger logger #region Interface methods - public async Task ActivateAccount(Contact to, Uri activationLink, CancellationToken cancellationToken = default) + public async Task ActivateAccount(Contact to, Uri activationLink, CancellationToken cancellationToken = default) { var (subject, htmlBody) = await _templates.AccountActivation.RenderAsync(new { To = to, ActivationLink = activationLink }); - await SendMail(to, subject, htmlBody, cancellationToken); + return await SendMail(to, subject, htmlBody, cancellationToken); } /// - public async Task PasswordReset(Contact to, Uri resetLink, CancellationToken cancellationToken = default) + public async Task PasswordReset(Contact to, Uri resetLink, CancellationToken cancellationToken = default) { var (subject, htmlBody) = await _templates.PasswordReset.RenderAsync(new { To = to, ResetLink = resetLink }); - await SendMail(to, subject, htmlBody, cancellationToken); + return await SendMail(to, subject, htmlBody, cancellationToken); } /// - public async Task VerifyEmail(Contact to, Uri verificationLink, CancellationToken cancellationToken = default) + public async Task VerifyEmail(Contact to, Uri verificationLink, CancellationToken cancellationToken = default) { var (subject, htmlBody) = await _templates.EmailVerification.RenderAsync(new { To = to, VerifyLink = verificationLink }); - await SendMail(to, subject, htmlBody, cancellationToken); + return await SendMail(to, subject, htmlBody, cancellationToken); } /// - public async Task EmailChangeNotice(Contact to, string newEmail, CancellationToken cancellationToken = default) + public async Task EmailChangeNotice(Contact to, string newEmail, CancellationToken cancellationToken = default) { var (subject, htmlBody) = await _templates.EmailChangeNotice.RenderAsync(new { To = to, NewEmail = newEmail }); - await SendMail(to, subject, htmlBody, cancellationToken); + return await SendMail(to, subject, htmlBody, cancellationToken); } #endregion - private Task SendMail(Contact to, string subject, string htmlBody, CancellationToken cancellationToken = default) + private Task SendMail(Contact to, string subject, string htmlBody, CancellationToken cancellationToken = default) => SendMails([new DirectMail { From = _sender, To = [to], Subject = subject, HTMLPart = htmlBody }], cancellationToken); - private async Task SendMails(DirectMail[] mails, CancellationToken cancellationToken = default) + private async Task SendMails(DirectMail[] mails, CancellationToken cancellationToken = default) { if (_logger.IsEnabled(LogLevel.Debug)) _logger.LogDebug("Sending mails {@Mails}", mails); var json = JsonSerializer.Serialize(new MailsWrap { Messages = mails }, JsonOptions.Default); - var response = await _httpClient.PostAsync("send", - new StringContent(json, Encoding.UTF8, MediaTypeNames.Application.Json), cancellationToken); - if (!response.IsSuccessStatusCode) + HttpResponseMessage response; + try { - _logger.LogError("Error sending mails. Got unsuccessful status code {StatusCode} for mails {@Mails} with error body {Body}", - response.StatusCode, mails, await response.Content.ReadAsStringAsync(cancellationToken)); + response = await _httpClient.PostAsync("send", + new StringContent(json, Encoding.UTF8, MediaTypeNames.Application.Json), cancellationToken); } - else _logger.LogDebug("Successfully sent mail"); + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException && !cancellationToken.IsCancellationRequested) + { + // Network failure or request timeout — worth retrying. + _logger.LogWarning(ex, "Transient failure sending mails {@Mails}", mails); + return EmailSendResult.TransientFailure; + } + + if (response.IsSuccessStatusCode) + { + _logger.LogDebug("Successfully sent mail"); + return EmailSendResult.Sent; + } + + var statusCode = (int)response.StatusCode; + var body = await response.Content.ReadAsStringAsync(cancellationToken); + _logger.LogError("Error sending mails. Got unsuccessful status code {StatusCode} for mails {@Mails} with error body {Body}", + response.StatusCode, mails, body); + + // 429 (rate limited) and 5xx (provider-side) are temporary; other 4xx are permanent. + return statusCode is 429 or >= 500 and <= 599 + ? EmailSendResult.TransientFailure + : EmailSendResult.PermanentFailure; } public void Dispose() diff --git a/API/Services/Email/NoneEmailService.cs b/API/Services/Email/NoneEmailService.cs index e1908562..e7656d7c 100644 --- a/API/Services/Email/NoneEmailService.cs +++ b/API/Services/Email/NoneEmailService.cs @@ -16,27 +16,29 @@ public NoneEmailService(ILogger logger) _logger = logger; } - public Task ActivateAccount(Contact to, Uri activationLink, CancellationToken cancellationToken = default) + // Returns Sent (not a failure) on purpose: in a deployment with mail disabled the outbox should + // mark messages terminal rather than retrying them forever. + public Task ActivateAccount(Contact to, Uri activationLink, CancellationToken cancellationToken = default) { _logger.LogError("Account activation email not sent, this is a noop implementation of the email service"); - return Task.CompletedTask; + return Task.FromResult(EmailSendResult.Sent); } - public Task PasswordReset(Contact to, Uri resetLink, CancellationToken cancellationToken = default) + public Task PasswordReset(Contact to, Uri resetLink, CancellationToken cancellationToken = default) { _logger.LogError("Password reset email not sent, this is a noop implementation of the email service"); - return Task.CompletedTask; + return Task.FromResult(EmailSendResult.Sent); } - public Task VerifyEmail(Contact to, Uri verificationLink, CancellationToken cancellationToken = default) + public Task VerifyEmail(Contact to, Uri verificationLink, CancellationToken cancellationToken = default) { _logger.LogError("Email verification email not sent, this is a noop implementation of the email service"); - return Task.CompletedTask; + return Task.FromResult(EmailSendResult.Sent); } - public Task EmailChangeNotice(Contact to, string newEmail, CancellationToken cancellationToken = default) + public Task EmailChangeNotice(Contact to, string newEmail, CancellationToken cancellationToken = default) { _logger.LogError("Email change notice not sent, this is a noop implementation of the email service"); - return Task.CompletedTask; + return Task.FromResult(EmailSendResult.Sent); } } \ No newline at end of file diff --git a/API/Services/Email/Outbox/EmailDispatchResult.cs b/API/Services/Email/Outbox/EmailDispatchResult.cs new file mode 100644 index 00000000..969957d8 --- /dev/null +++ b/API/Services/Email/Outbox/EmailDispatchResult.cs @@ -0,0 +1,38 @@ +namespace OpenShock.API.Services.Email.Outbox; + +/// What happened when the consumer tried to dispatch an outbox message. +public enum EmailDispatchOutcome +{ + /// Handed to the provider successfully. Terminal. + Sent, + + /// Temporary failure; the message should be retried later. + TransientFailure, + + /// Permanent failure; retrying will not help. + PermanentFailure, + + /// + /// Nothing was sent because the underlying request no longer needs the email (expired, already + /// used, superseded, or gone). Terminal — there is nothing to retry. + /// + Skipped +} + +/// Outcome of dispatching one , with a human-readable detail for diagnostics. +/// What happened. +/// Optional explanation, surfaced into the message's last-error field. +public readonly record struct EmailDispatchResult(EmailDispatchOutcome Outcome, string? Detail) +{ + /// A successful send. + public static readonly EmailDispatchResult Sent = new(EmailDispatchOutcome.Sent, null); + + /// A transient failure that should be retried. + public static EmailDispatchResult Transient(string detail) => new(EmailDispatchOutcome.TransientFailure, detail); + + /// A permanent failure that should not be retried. + public static EmailDispatchResult Permanent(string detail) => new(EmailDispatchOutcome.PermanentFailure, detail); + + /// The email was intentionally not sent because it is no longer needed. + public static EmailDispatchResult Skip(string detail) => new(EmailDispatchOutcome.Skipped, detail); +} diff --git a/API/Services/Email/Outbox/EmailOutboxDispatcher.cs b/API/Services/Email/Outbox/EmailOutboxDispatcher.cs new file mode 100644 index 00000000..d2e813f1 --- /dev/null +++ b/API/Services/Email/Outbox/EmailOutboxDispatcher.cs @@ -0,0 +1,149 @@ +using Microsoft.EntityFrameworkCore; +using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Common.Constants; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; +using OpenShock.Common.Options; +using OpenShock.Common.Utils; + +namespace OpenShock.API.Services.Email.Outbox; + +/// +public sealed class EmailOutboxDispatcher : IEmailOutboxDispatcher +{ + private readonly IEmailService _emailService; + private readonly FrontendOptions _frontendOptions; + private readonly ILogger _logger; + + /// DI constructor. + public EmailOutboxDispatcher(IEmailService emailService, FrontendOptions frontendOptions, ILogger logger) + { + _emailService = emailService; + _frontendOptions = frontendOptions; + _logger = logger; + } + + /// + public async Task SendAsync(EmailOutboxMessage message, OpenShockContext db, CancellationToken cancellationToken = default) + { + try + { + return message.Type switch + { + EmailType.AccountActivation => await SendAccountActivation(message, db, cancellationToken), + EmailType.PasswordReset => await SendPasswordReset(message, db, cancellationToken), + EmailType.EmailVerification => await SendEmailVerification(message, db, cancellationToken), + EmailType.EmailChangeNotice => await SendEmailChangeNotice(message, cancellationToken), + _ => EmailDispatchResult.Permanent($"Unknown email type {message.Type}") + }; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // Unexpected failure (e.g. database hiccup while loading the request row). Treat as + // transient so the message is retried rather than dropped. + _logger.LogError(ex, "Unexpected error dispatching email outbox message {MessageId}", message.Id); + return EmailDispatchResult.Transient(ex.Message); + } + } + + private async Task SendAccountActivation(EmailOutboxMessage message, OpenShockContext db, CancellationToken ct) + { + if (!TryGetGuid(message, EmailOutboxPayloadKeys.UserId, out var userId)) + return EmailDispatchResult.Permanent($"Payload missing {EmailOutboxPayloadKeys.UserId}"); + + var request = await db.UserActivationRequests + .Include(r => r.User) + .FirstOrDefaultAsync(r => r.UserId == userId, ct); + if (request is null) return EmailDispatchResult.Skip("Activation request no longer exists"); + if (request.User.ActivatedAt is not null) return EmailDispatchResult.Skip("Account already activated"); + + var token = await MintTokenAsync(db, h => request.TokenHash = h, ct); + var link = new Uri(_frontendOptions.BaseUrl, $"/activate?token={token}"); + return FromSend(await _emailService.ActivateAccount(ToContact(message), link, ct)); + } + + private async Task SendPasswordReset(EmailOutboxMessage message, OpenShockContext db, CancellationToken ct) + { + if (!TryGetGuid(message, EmailOutboxPayloadKeys.PasswordResetId, out var resetId)) + return EmailDispatchResult.Permanent($"Payload missing {EmailOutboxPayloadKeys.PasswordResetId}"); + + var reset = await db.UserPasswordResets + .Include(r => r.User) + .FirstOrDefaultAsync(r => r.Id == resetId, ct); + if (reset is null) return EmailDispatchResult.Skip("Password reset no longer exists"); + if (reset.UsedAt is not null) return EmailDispatchResult.Skip("Password reset already used"); + if (reset.CreatedAt < DateTime.UtcNow - Duration.PasswordResetRequestLifetime) + return EmailDispatchResult.Skip("Password reset expired"); + if (reset.SecurityStampAtCreate != reset.User.SecurityStamp) + return EmailDispatchResult.Skip("Password reset superseded by a newer credential change"); + + var token = await MintTokenAsync(db, h => reset.TokenHash = h, ct); + var link = new Uri(_frontendOptions.BaseUrl, $"/#/account/password/recover/{reset.Id}/{token}"); + return FromSend(await _emailService.PasswordReset(ToContact(message), link, ct)); + } + + private async Task SendEmailVerification(EmailOutboxMessage message, OpenShockContext db, CancellationToken ct) + { + if (!TryGetGuid(message, EmailOutboxPayloadKeys.EmailChangeId, out var changeId)) + return EmailDispatchResult.Permanent($"Payload missing {EmailOutboxPayloadKeys.EmailChangeId}"); + + var change = await db.UserEmailChanges + .Include(c => c.User) + .FirstOrDefaultAsync(c => c.Id == changeId, ct); + if (change is null) return EmailDispatchResult.Skip("Email change no longer exists"); + if (change.UsedAt is not null) return EmailDispatchResult.Skip("Email change already used"); + if (change.CreatedAt < DateTime.UtcNow - Duration.EmailChangeRequestLifetime) + return EmailDispatchResult.Skip("Email change expired"); + if (change.SecurityStampAtCreate != change.User.SecurityStamp) + return EmailDispatchResult.Skip("Email change superseded by a newer credential change"); + + var token = await MintTokenAsync(db, h => change.TokenHash = h, ct); + var link = new Uri(_frontendOptions.BaseUrl, $"/verify-email?token={token}"); + return FromSend(await _emailService.VerifyEmail(ToContact(message), link, ct)); + } + + private async Task SendEmailChangeNotice(EmailOutboxMessage message, CancellationToken ct) + { + var newEmail = GetString(message, EmailOutboxPayloadKeys.NewEmail); + if (string.IsNullOrEmpty(newEmail)) + return EmailDispatchResult.Permanent($"Payload missing {EmailOutboxPayloadKeys.NewEmail}"); + + // No token: this is a pure informational notice to the previous address. + return FromSend(await _emailService.EmailChangeNotice(ToContact(message), newEmail, ct)); + } + + /// + /// Mints a fresh secret token, applies its hash to the request row via , + /// and persists it before the email is sent so the emailed link always matches a stored hash. + /// Returns the plaintext token to embed in the link (never stored). + /// + private static async Task MintTokenAsync(OpenShockContext db, Action applyHash, CancellationToken ct) + { + var token = CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength); + applyHash(HashingUtils.HashToken(token)); + await db.SaveChangesAsync(ct); + return token; + } + + private static EmailDispatchResult FromSend(EmailSendResult result) => result switch + { + EmailSendResult.Sent => EmailDispatchResult.Sent, + EmailSendResult.TransientFailure => EmailDispatchResult.Transient("Provider reported a transient failure"), + _ => EmailDispatchResult.Permanent("Provider rejected the message") + }; + + private static Contact ToContact(EmailOutboxMessage message) => new(message.Recipient, message.RecipientName ?? message.Recipient); + + private static bool TryGetGuid(EmailOutboxMessage message, string key, out Guid value) + { + value = default; + return message.Payload.TryGetValue(key, out var raw) && Guid.TryParse(raw, out value); + } + + private static string? GetString(EmailOutboxMessage message, string key) + => message.Payload.TryGetValue(key, out var raw) ? raw : null; +} diff --git a/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs b/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs new file mode 100644 index 00000000..d1a3ef1d --- /dev/null +++ b/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs @@ -0,0 +1,90 @@ +using OpenShock.Common.Constants; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.API.Services.Email.Outbox; + +/// +/// Pure retry/state-machine logic for the email outbox, factored out of the consumer so it can be +/// unit-tested without a database or a running host. +/// +public static class EmailOutboxRetryPolicy +{ + /// Maximum number of attempts before a transiently-failing message is given up as failed. + public const int MaxAttempts = 10; + + /// Upper bound (seconds) of the random jitter added to each retry delay. + public const int MaxJitterSeconds = 30; + + /// + /// Deterministic part of the back-off: exponential in the attempt count, capped at + /// . Exposed (without jitter) for testing. + /// + public static TimeSpan GetBaseDelay(int attemptCount) + { + // Compute in seconds (double) and cap before converting, so a large attempt count can't + // overflow TimeSpan — Math.Pow can reach infinity, which simply compares above the cap. + var maxDelay = Duration.EmailOutboxRetryMaxDelay; + var seconds = Duration.EmailOutboxRetryBaseDelay.TotalSeconds * Math.Pow(2, Math.Max(0, attemptCount - 1)); + return seconds >= maxDelay.TotalSeconds ? maxDelay : TimeSpan.FromSeconds(seconds); + } + + /// Back-off delay for the next retry: plus a random jitter. + public static TimeSpan GetRetryDelay(int attemptCount) + => GetBaseDelay(attemptCount) + TimeSpan.FromSeconds(Random.Shared.Next(0, MaxJitterSeconds + 1)); + + /// + /// Applies a dispatch outcome to a message, advancing it to its next state: delivered, failed + /// (permanent / skipped / exhausted), or scheduled for another retry. Pure — only mutates + /// . + /// + public static void Apply(EmailOutboxMessage message, EmailDispatchResult result, DateTime nowUtc) + { + switch (result.Outcome) + { + case EmailDispatchOutcome.Sent: + message.Status = EmailStatus.Sent; + message.SentAt = nowUtc; + message.AttemptStartedAt = null; + message.LastError = null; + break; + + case EmailDispatchOutcome.Skipped: + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + message.AttemptStartedAt = null; + message.LastError = Truncate($"Skipped: {result.Detail}"); + break; + + case EmailDispatchOutcome.PermanentFailure: + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + message.AttemptStartedAt = null; + message.LastError = Truncate(result.Detail); + break; + + case EmailDispatchOutcome.TransientFailure: + message.LastError = Truncate(result.Detail); + message.AttemptStartedAt = null; + if (message.AttemptCount >= MaxAttempts) + { + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + } + else + { + message.NextAttemptAt = nowUtc + GetRetryDelay(message.AttemptCount); + } + break; + } + } + + /// Clamps an error string to the column's maximum length. + public static string? Truncate(string? value) + { + if (value is null) return null; + return value.Length <= HardLimits.EmailOutboxLastErrorMaxLength + ? value + : value[..HardLimits.EmailOutboxLastErrorMaxLength]; + } +} diff --git a/API/Services/Email/Outbox/EmailOutboxWorker.cs b/API/Services/Email/Outbox/EmailOutboxWorker.cs new file mode 100644 index 00000000..1479054f --- /dev/null +++ b/API/Services/Email/Outbox/EmailOutboxWorker.cs @@ -0,0 +1,189 @@ +using Microsoft.EntityFrameworkCore; +using OpenShock.Common.Constants; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; +using OpenShock.Common.Services.RedisPubSub; +using StackExchange.Redis; + +namespace OpenShock.API.Services.Email.Outbox; + +/// +/// The email outbox consumer: the single component that delivers enqueued +/// rows, retrying until they succeed or are exhausted. +/// +/// +/// +/// Push first, poll as a safety net. Delivery is triggered by a Redis notification published +/// the instant a message is enqueued (), so first-send +/// latency is sub-second rather than tied to a polling interval. A periodic tick +/// () still runs as a fallback: it picks up due retries +/// (whose NextAttemptAt is in the future), re-claims leases abandoned by a crashed instance, +/// and covers the rare case of a dropped notification. The notification can therefore be best-effort +/// — losing one only delays delivery to the next tick, it can never lose the email. +/// +/// +/// Runs in every API instance, safely. Rows are claimed with FOR UPDATE SKIP LOCKED, so +/// when several instances react to the same notification each grabs a disjoint set of rows and they +/// drain in parallel without ever sending the same message twice in a batch. Claiming also stamps a +/// lease (AttemptStartedAt); a row whose lease is older than +/// is considered abandoned and re-claimed, which is how +/// a send interrupted by a crash recovers. +/// +/// +/// This lives in the API host rather than the cron/Hangfire host on purpose: it needs +/// (which the API host owns) and sub-second latency, which a +/// minute-granularity scheduler cannot provide. Periodic maintenance of this table (pruning +/// old delivered rows) is the appropriate cron job, and is separate from sending. +/// +/// +public sealed class EmailOutboxWorker : BackgroundService +{ + private const int BatchSize = 20; + + private readonly IDbContextFactory _dbContextFactory; + private readonly IEmailOutboxDispatcher _dispatcher; + private readonly ISubscriber _subscriber; + private readonly ILogger _logger; + + // Coalescing wake signal: capacity 1, so any number of notifications between drains collapse into + // a single "there is work" wake-up. + private readonly SemaphoreSlim _wakeSignal = new(0, 1); + + /// DI constructor. + public EmailOutboxWorker( + IDbContextFactory dbContextFactory, + IEmailOutboxDispatcher dispatcher, + IConnectionMultiplexer connectionMultiplexer, + ILogger logger) + { + _dbContextFactory = dbContextFactory; + _dispatcher = dispatcher; + _subscriber = connectionMultiplexer.GetSubscriber(); + _logger = logger; + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await _subscriber.SubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); + + try + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await DrainAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Email outbox drain failed"); + } + + // Wait for a push notification or the fallback poll interval, whichever comes first. + try + { + await _wakeSignal.WaitAsync(Duration.EmailOutboxPollInterval, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + } + finally + { + await _subscriber.UnsubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); + } + } + + private void OnPendingNotification(RedisChannel channel, RedisValue value) + { + // Release the coalescing signal; ignore if already signalled. + try + { + _wakeSignal.Release(); + } + catch (SemaphoreFullException) + { + // Already pending a wake-up — nothing to do. + } + } + + private async Task DrainAsync(CancellationToken cancellationToken) + { + int processed; + do + { + processed = await ProcessBatchAsync(cancellationToken); + } + while (processed >= BatchSize && !cancellationToken.IsCancellationRequested); + } + + private async Task ProcessBatchAsync(CancellationToken cancellationToken) + { + await using var db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + + var nowUtc = DateTime.UtcNow; + var leaseCutoff = nowUtc - Duration.EmailOutboxLeaseTimeout; + + // Atomically claim a batch of due rows. FOR UPDATE SKIP LOCKED lets concurrent instances take + // disjoint sets; stamping the lease + bumping the attempt count is what reserves them. + List claimed; + await using (var transaction = await db.Database.BeginTransactionAsync(cancellationToken)) + { + claimed = await db.EmailOutbox.FromSql( + $""" + SELECT * FROM email_outbox + WHERE status = {EmailStatus.Sending} + AND (next_attempt_at IS NULL OR next_attempt_at <= {nowUtc}) + AND (attempt_started_at IS NULL OR attempt_started_at < {leaseCutoff}) + ORDER BY created_at + LIMIT {BatchSize} + FOR UPDATE SKIP LOCKED + """).ToListAsync(cancellationToken); + + if (claimed.Count == 0) + { + await transaction.CommitAsync(cancellationToken); + return 0; + } + + foreach (var message in claimed) + { + message.AttemptStartedAt = nowUtc; + message.AttemptCount++; + } + + await db.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + + // Send outside the claim transaction so a slow provider never holds row locks. Each message is + // marked durably right after its attempt, so a crash mid-batch only re-sends the ones whose + // lease has not yet been resolved. + foreach (var message in claimed) + { + var result = await _dispatcher.SendAsync(message, db, cancellationToken); + EmailOutboxRetryPolicy.Apply(message, result, DateTime.UtcNow); + await db.SaveChangesAsync(cancellationToken); + + if (message.Status == EmailStatus.Failed) + _logger.LogWarning("Email outbox message {MessageId} ({Type}) failed after {Attempts} attempt(s): {Detail}", + message.Id, message.Type, message.AttemptCount, result.Detail); + } + + return claimed.Count; + } + + /// + public override void Dispose() + { + _wakeSignal.Dispose(); + base.Dispose(); + } +} diff --git a/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs b/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs new file mode 100644 index 00000000..db25ece4 --- /dev/null +++ b/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs @@ -0,0 +1,24 @@ +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.API.Services.Email.Outbox; + +/// +/// Renders and sends a single . This is the one place email is +/// actually sent: it maps the message's type and dynamic payload to a template, lazily mints (and +/// persists the hash of) any secret token the email needs, hands the message to +/// , and reports the outcome. It does not own the message's lifecycle — +/// the consumer that calls it records the resulting state transition. +/// +public interface IEmailOutboxDispatcher +{ + /// + /// Attempts to deliver . Any token row touched (e.g. a + /// ) is updated via and saved before the + /// send, so the link in the email always matches a stored hash. Never throws for ordinary send + /// failures — they are returned as . + /// + /// The message to send. Its status is not modified here. + /// The context used to load/update related request rows. + /// + Task SendAsync(EmailOutboxMessage message, OpenShockContext db, CancellationToken cancellationToken = default); +} diff --git a/API/Services/Email/Smtp/SmtpEmailService.cs b/API/Services/Email/Smtp/SmtpEmailService.cs index bcce832b..86101987 100644 --- a/API/Services/Email/Smtp/SmtpEmailService.cs +++ b/API/Services/Email/Smtp/SmtpEmailService.cs @@ -26,22 +26,22 @@ ILogger logger _logger = logger; } - public Task ActivateAccount(Contact to, Uri activationLink, CancellationToken cancellationToken = default) + public Task ActivateAccount(Contact to, Uri activationLink, CancellationToken cancellationToken = default) => SendMail(to, _templates.AccountActivation, new { To = to, ActivationLink = activationLink }, cancellationToken); /// - public Task PasswordReset(Contact to, Uri resetLink, CancellationToken cancellationToken = default) + public Task PasswordReset(Contact to, Uri resetLink, CancellationToken cancellationToken = default) => SendMail(to, _templates.PasswordReset, new { To = to, ResetLink = resetLink }, cancellationToken); /// - public Task VerifyEmail(Contact to, Uri verificationLink, CancellationToken cancellationToken = default) + public Task VerifyEmail(Contact to, Uri verificationLink, CancellationToken cancellationToken = default) => SendMail(to, _templates.EmailVerification, new { To = to, VerifyLink = verificationLink }, cancellationToken); /// - public Task EmailChangeNotice(Contact to, string newEmail, CancellationToken cancellationToken = default) + public Task EmailChangeNotice(Contact to, string newEmail, CancellationToken cancellationToken = default) => SendMail(to, _templates.EmailChangeNotice, new { To = to, NewEmail = newEmail }, cancellationToken); - private async Task SendMail(Contact to, EmailTemplate template, T data, CancellationToken cancellationToken = default) + private async Task SendMail(Contact to, EmailTemplate template, T data, CancellationToken cancellationToken = default) { _logger.LogDebug("Sending email"); var (subject, htmlBody) = await template.RenderAsync(data); @@ -55,23 +55,43 @@ private async Task SendMail(Contact to, EmailTemplate template, T data, Cance Body = new TextPart(TextFormat.Html) { Text = htmlBody } }; - _logger.LogTrace("Creating smtp client and connecting..."); - using var smtpClient = new SmtpClient(); - if (!_options.VerifyCertificate) + try { - smtpClient.ServerCertificateValidationCallback = (sender, certificate, chain, errors) => true; - smtpClient.CheckCertificateRevocation = false; - } + _logger.LogTrace("Creating smtp client and connecting..."); + using var smtpClient = new SmtpClient(); + if (!_options.VerifyCertificate) + { + smtpClient.ServerCertificateValidationCallback = (sender, certificate, chain, errors) => true; + smtpClient.CheckCertificateRevocation = false; + } + + await smtpClient.ConnectAsync(_options.Host, _options.Port, _options.EnableSsl, cancellationToken); + _logger.LogTrace("Authenticating..."); + if (smtpClient.Capabilities.HasFlag(SmtpCapabilities.Authentication)) + await smtpClient.AuthenticateAsync(_options.Username, _options.Password, cancellationToken); - await smtpClient.ConnectAsync(_options.Host, _options.Port, _options.EnableSsl, cancellationToken); - _logger.LogTrace("Authenticating..."); - if (smtpClient.Capabilities.HasFlag(SmtpCapabilities.Authentication)) - await smtpClient.AuthenticateAsync(_options.Username, _options.Password, cancellationToken); + _logger.LogTrace("Smtp client connected, sending email..."); - _logger.LogTrace("Smtp client connected, sending email..."); + await smtpClient.SendAsync(message, cancellationToken); + await smtpClient.DisconnectAsync(true, cancellationToken); + _logger.LogTrace("Sent email"); - await smtpClient.SendAsync(message, cancellationToken); - await smtpClient.DisconnectAsync(true, cancellationToken); - _logger.LogTrace("Sent email"); + return EmailSendResult.Sent; + } + catch (SmtpCommandException ex) + { + // A 5xx reply (e.g. mailbox unavailable, message rejected) won't be fixed by retrying; + // 4xx replies are temporary. + var permanent = (int)ex.StatusCode is >= 500 and <= 599; + _logger.LogError(ex, "SMTP command failed with status {StatusCode} sending to {Recipient}", ex.StatusCode, to.Email); + return permanent ? EmailSendResult.PermanentFailure : EmailSendResult.TransientFailure; + } + catch (Exception ex) + { + // Connection, TLS, auth, protocol and timeout failures are all treated as temporary; the + // retry budget bounds how long a genuinely broken configuration keeps being attempted. + _logger.LogError(ex, "Transient SMTP failure sending to {Recipient}", to.Email); + return EmailSendResult.TransientFailure; + } } } \ No newline at end of file diff --git a/Common.Tests/OpenShockDb/EmailOutboxMessageTests.cs b/Common.Tests/OpenShockDb/EmailOutboxMessageTests.cs new file mode 100644 index 00000000..e164fefc --- /dev/null +++ b/Common.Tests/OpenShockDb/EmailOutboxMessageTests.cs @@ -0,0 +1,39 @@ +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.Common.Tests.OpenShockDb; + +public class EmailOutboxMessageTests +{ + [Test] + public async Task Create_InitializesAsSendingWithGivenFields() + { + var payload = new Dictionary + { + [EmailOutboxPayloadKeys.PasswordResetId] = "11111111-1111-1111-1111-111111111111" + }; + + var message = EmailOutboxMessage.Create(EmailType.PasswordReset, "user@example.com", "User", payload); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Sending); + await Assert.That(message.Type).IsEqualTo(EmailType.PasswordReset); + await Assert.That(message.Recipient).IsEqualTo("user@example.com"); + await Assert.That(message.RecipientName).IsEqualTo("User"); + await Assert.That(message.AttemptCount).IsEqualTo(0); + await Assert.That(message.SentAt).IsNull(); + await Assert.That(message.FailedAt).IsNull(); + await Assert.That(message.Id).IsNotEqualTo(Guid.Empty); + await Assert.That(message.Payload[EmailOutboxPayloadKeys.PasswordResetId]) + .IsEqualTo("11111111-1111-1111-1111-111111111111"); + } + + [Test] + public async Task Create_AllowsNullRecipientName() + { + var message = EmailOutboxMessage.Create(EmailType.EmailChangeNotice, "old@example.com", null, + new Dictionary { [EmailOutboxPayloadKeys.NewEmail] = "new@example.com" }); + + await Assert.That(message.RecipientName).IsNull(); + await Assert.That(message.Payload[EmailOutboxPayloadKeys.NewEmail]).IsEqualTo("new@example.com"); + } +} diff --git a/Common/Constants/Constants.cs b/Common/Constants/Constants.cs index e8ffbbcd..bbafc408 100644 --- a/Common/Constants/Constants.cs +++ b/Common/Constants/Constants.cs @@ -20,4 +20,18 @@ public static class Duration public static readonly TimeSpan DeviceKeepAliveTimeout = TimeSpan.FromSeconds(35); public static readonly TimeSpan DeactivatedAccountRetentionTime = TimeSpan.FromDays(14); + + /// How long a claimed email-outbox row may be in flight before it is treated as + /// abandoned (worker crashed mid-send) and eligible to be re-claimed. + public static readonly TimeSpan EmailOutboxLeaseTimeout = TimeSpan.FromMinutes(5); + + /// Safety-net poll interval for the email-outbox consumer. Normal delivery is push-driven + /// (Redis notification on enqueue); this only catches due retries and any missed notification. + public static readonly TimeSpan EmailOutboxPollInterval = TimeSpan.FromSeconds(15); + + /// Base delay for the first email-outbox retry; doubles each subsequent attempt. + public static readonly TimeSpan EmailOutboxRetryBaseDelay = TimeSpan.FromSeconds(30); + + /// Upper bound on the exponential email-outbox retry back-off. + public static readonly TimeSpan EmailOutboxRetryMaxDelay = TimeSpan.FromHours(2); } diff --git a/Common/Constants/HardLimits.cs b/Common/Constants/HardLimits.cs index 6688536c..716727cd 100644 --- a/Common/Constants/HardLimits.cs +++ b/Common/Constants/HardLimits.cs @@ -41,6 +41,8 @@ public static class HardLimits public const int PasswordHashMaxLength = 100; + public const int EmailOutboxLastErrorMaxLength = 1024; + public const int UserEmailChangeSecretMaxLength = 128; public const int UserActivationRequestSecretMaxLength = 128; public const int PasswordResetSecretMaxLength = 100; diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs new file mode 100644 index 00000000..bdff376f --- /dev/null +++ b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs @@ -0,0 +1,1593 @@ +// +using System; +using System.Collections.Generic; +using System.Net; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; + +#nullable disable + +namespace OpenShock.Common.Migrations +{ + [DbContext(typeof(MigrationOpenShockContext))] + [Migration("20260630103404_AddEmailOutbox")] + partial class AddEmailOutbox + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .HasAnnotation("ProductVersion", "10.0.9") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "configuration_value_type", new[] { "string", "bool", "int", "float", "json" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "sound", "vibrate", "shock", "stop" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "sending", "sent", "failed" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "accountActivation", "passwordReset", "emailVerification", "emailChangeNotice" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "password_encryption_type", new[] { "pbkdf2", "bcrypt_enhanced" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "permission_type", new[] { "shockers.use", "shockers.edit", "shockers.pause", "devices.edit", "devices.auth" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "role_type", new[] { "support", "staff", "admin", "system" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "shocker_model_type", new[] { "caiXianlin", "petTrainer", "petrainer998DR", "wellturnT330" }); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("FriendlyName") + .HasColumnType("text"); + + b.Property("Xml") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.AdminUsersView", b => + { + b.Property("ActivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("activated_at"); + + b.Property("ApiTokenCount") + .HasColumnType("integer") + .HasColumnName("api_token_count"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DeactivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deactivated_at"); + + b.Property("DeactivatedByUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_by_user_id"); + + b.Property("DeviceCount") + .HasColumnType("integer") + .HasColumnName("device_count"); + + b.Property("Email") + .IsRequired() + .HasColumnType("character varying") + .HasColumnName("email"); + + b.Property("EmailChangeRequestCount") + .HasColumnType("integer") + .HasColumnName("email_change_request_count"); + + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Name") + .IsRequired() + .HasColumnType("character varying") + .HasColumnName("name"); + + b.Property("NameChangeRequestCount") + .HasColumnType("integer") + .HasColumnName("name_change_request_count"); + + b.Property("PasswordHashType") + .HasColumnType("character varying") + .HasColumnName("password_hash_type"); + + b.Property("PasswordResetCount") + .HasColumnType("integer") + .HasColumnName("password_reset_count"); + + b.Property("Roles") + .IsRequired() + .HasColumnType("role_type[]") + .HasColumnName("roles"); + + b.Property("ShockerControlLogCount") + .HasColumnType("integer") + .HasColumnName("shocker_control_log_count"); + + b.Property("ShockerCount") + .HasColumnType("integer") + .HasColumnName("shocker_count"); + + b.Property("ShockerPublicShareCount") + .HasColumnType("integer") + .HasColumnName("shocker_public_share_count"); + + b.Property("ShockerUserShareCount") + .HasColumnType("integer") + .HasColumnName("shocker_user_share_count"); + + b.ToTable((string)null); + + b.ToView("admin_users_view", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiToken", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("CreatedByIp") + .IsRequired() + .HasColumnType("inet") + .HasColumnName("created_by_ip"); + + b.Property("LastUsed") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_used"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.PrimitiveCollection>("Permissions") + .IsRequired() + .HasColumnType("permission_type[]") + .HasColumnName("permissions"); + + b.Property("ShockerControlDurationMax") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(65535) + .HasColumnName("shocker_control_duration_max"); + + b.Property("ShockerControlDurationMin") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(300) + .HasColumnName("shocker_control_duration_min"); + + b.Property("ShockerControlDurationMode") + .ValueGeneratedOnAdd() + .HasColumnType("control_limit_mode") + .HasDefaultValue(ControlLimitMode.Clamp) + .HasColumnName("shocker_control_duration_mode"); + + b.Property("ShockerControlIntensityMax") + .ValueGeneratedOnAdd() + .HasColumnType("smallint") + .HasDefaultValue((byte)100) + .HasColumnName("shocker_control_intensity_max"); + + b.Property("ShockerControlIntensityMin") + .ValueGeneratedOnAdd() + .HasColumnType("smallint") + .HasDefaultValue((byte)0) + .HasColumnName("shocker_control_intensity_min"); + + b.Property("ShockerControlIntensityMode") + .ValueGeneratedOnAdd() + .HasColumnType("control_limit_mode") + .HasDefaultValue(ControlLimitMode.Clamp) + .HasColumnName("shocker_control_intensity_mode"); + + b.Property("ShockerControlPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("shocker_control_paused"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("ValidUntil") + .HasColumnType("timestamp with time zone") + .HasColumnName("valid_until"); + + b.HasKey("Id") + .HasName("api_tokens_pkey"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.HasIndex("UserId"); + + b.HasIndex("ValidUntil"); + + b.ToTable("api_tokens", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiTokenReport", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AffectedCount") + .HasColumnType("integer") + .HasColumnName("affected_count"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IpAddress") + .IsRequired() + .HasColumnType("inet") + .HasColumnName("ip_address"); + + b.Property("IpCountry") + .HasColumnType("text") + .HasColumnName("ip_country"); + + b.Property("SubmittedCount") + .HasColumnType("integer") + .HasColumnName("submitted_count"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("api_token_reports_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("api_token_reports", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ConfigurationItem", b => + { + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name") + .UseCollation("C"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("Type") + .HasColumnType("configuration_value_type") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Name") + .HasName("configuration_pkey"); + + b.ToTable("configuration", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.Property("Token") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("character varying(256)") + .HasColumnName("token") + .UseCollation("C"); + + b.HasKey("Id") + .HasName("devices_pkey"); + + b.HasIndex("OwnerId"); + + b.HasIndex("Token") + .IsUnique(); + + b.ToTable("devices", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DeviceOtaUpdate", b => + { + b.Property("DeviceId") + .HasColumnType("uuid") + .HasColumnName("device_id"); + + b.Property("UpdateId") + .HasColumnType("integer") + .HasColumnName("update_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Message") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("message"); + + b.Property("Status") + .HasColumnType("ota_update_status") + .HasColumnName("status"); + + b.Property("Version") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("version"); + + b.HasKey("DeviceId", "UpdateId") + .HasName("device_ota_updates_pkey"); + + b.HasIndex(new[] { "CreatedAt" }, "device_ota_updates_created_at_idx"); + + b.ToTable("device_ota_updates", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DiscordWebhook", b => + { + b.Property("Name") + .HasColumnType("text") + .HasColumnName("name"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("WebhookId") + .HasColumnType("bigint") + .HasColumnName("webhook_id"); + + b.Property("WebhookToken") + .IsRequired() + .HasColumnType("text") + .HasColumnName("webhook_token"); + + b.HasKey("Name") + .HasName("discord_webhooks_pkey"); + + b.ToTable("discord_webhooks", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.EmailOutboxMessage", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AttemptCount") + .HasColumnType("integer") + .HasColumnName("attempt_count"); + + b.Property("AttemptStartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("attempt_started_at"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("LastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("last_error"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("next_attempt_at"); + + b.Property>("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("Recipient") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("recipient"); + + b.Property("RecipientName") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("recipient_name"); + + b.Property("SentAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("sent_at"); + + b.Property("Status") + .HasColumnType("email_status") + .HasColumnName("status"); + + b.Property("Type") + .HasColumnType("email_type") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("email_outbox_pkey"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("Recipient"); + + b.HasIndex("Status", "NextAttemptAt"); + + b.ToTable("email_outbox", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.EmailProviderBlacklist", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Domain") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("domain") + .UseCollation("ndcoll"); + + b.HasKey("Id") + .HasName("email_provider_blacklist_pkey"); + + b.HasIndex("Domain") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(b.HasIndex("Domain"), new[] { "ndcoll" }); + + b.ToTable("email_provider_blacklist", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.HasKey("Id") + .HasName("public_shares_pkey"); + + b.HasIndex("OwnerId"); + + b.ToTable("public_shares", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShareShocker", b => + { + b.Property("PublicShareId") + .HasColumnType("uuid") + .HasColumnName("public_share_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("Cooldown") + .HasColumnType("integer") + .HasColumnName("cooldown"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("PublicShareId", "ShockerId") + .HasName("public_share_shockers_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("public_share_shockers", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeviceId") + .HasColumnType("uuid") + .HasColumnName("device_id"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("Model") + .HasColumnType("shocker_model_type") + .HasColumnName("model"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("name"); + + b.Property("RfId") + .HasColumnType("integer") + .HasColumnName("rf_id"); + + b.HasKey("Id") + .HasName("shockers_pkey"); + + b.HasIndex("DeviceId"); + + b.ToTable("shockers", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerControlLog", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ControlledByUserId") + .HasColumnType("uuid") + .HasColumnName("controlled_by_user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("CustomName") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("custom_name"); + + b.Property("Duration") + .HasColumnType("bigint") + .HasColumnName("duration"); + + b.Property("Intensity") + .HasColumnType("smallint") + .HasColumnName("intensity"); + + b.Property("LiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("live_control"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("Type") + .HasColumnType("control_type") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("shocker_control_logs_pkey"); + + b.HasIndex("ControlledByUserId"); + + b.HasIndex("ShockerId"); + + b.ToTable("shocker_control_logs", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.HasKey("Id") + .HasName("shocker_share_codes_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("shocker_share_codes", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.User", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ActivatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("activated_at"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("Email") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("name") + .UseCollation("ndcoll"); + + b.Property("PasswordHash") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("password_hash") + .UseCollation("C"); + + b.PrimitiveCollection>("Roles") + .IsRequired() + .HasColumnType("role_type[]") + .HasColumnName("roles"); + + b.Property("SecurityStamp") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("security_stamp") + .HasDefaultValueSql("gen_random_uuid()"); + + b.HasKey("Id") + .HasName("users_pkey"); + + b.HasIndex("Email") + .IsUnique(); + + b.HasIndex("Name") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(b.HasIndex("Name"), new[] { "ndcoll" }); + + b.ToTable("users", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserActivationRequest", b => + { + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("EmailSendAttempts") + .HasColumnType("integer") + .HasColumnName("email_send_attempts"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.HasKey("UserId") + .HasName("user_activation_requests_pkey"); + + b.HasIndex("TokenHash") + .IsUnique(); + + b.ToTable("user_activation_requests", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserDeactivation", b => + { + b.Property("DeactivatedUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DeactivatedByUserId") + .HasColumnType("uuid") + .HasColumnName("deactivated_by_user_id"); + + b.Property("DeleteLater") + .HasColumnType("boolean") + .HasColumnName("delete_later"); + + b.Property("UserModerationId") + .HasColumnType("uuid") + .HasColumnName("user_moderation_id"); + + b.HasKey("DeactivatedUserId") + .HasName("user_deactivations_pkey"); + + b.HasIndex("DeactivatedByUserId"); + + b.ToTable("user_deactivations", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserEmailChange", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("NewEmail") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email_new"); + + b.Property("OldEmail") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("email_old"); + + b.Property("SecurityStampAtCreate") + .HasColumnType("uuid") + .HasColumnName("security_stamp_at_create"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("used_at"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_email_changes_pkey"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("UsedAt"); + + b.HasIndex("UserId"); + + b.ToTable("user_email_changes", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameBlacklist", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("MatchType") + .HasColumnType("match_type_enum") + .HasColumnName("match_type"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("value") + .UseCollation("ndcoll"); + + b.HasKey("Id") + .HasName("user_name_blacklist_pkey"); + + b.HasIndex("Value") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.UseCollation(b.HasIndex("Value"), new[] { "ndcoll" }); + + b.ToTable("user_name_blacklist", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityAlwaysColumn(b.Property("Id")); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("OldName") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("old_name"); + + b.HasKey("Id", "UserId") + .HasName("user_name_changes_pkey"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("OldName"); + + b.HasIndex("UserId"); + + b.ToTable("user_name_changes", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserOAuthConnection", b => + { + b.Property("ProviderKey") + .HasColumnType("text") + .HasColumnName("provider_key") + .UseCollation("C"); + + b.Property("ExternalId") + .HasColumnType("text") + .HasColumnName("external_id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("DisplayName") + .HasColumnType("text") + .HasColumnName("display_name"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("ProviderKey", "ExternalId") + .HasName("user_oauth_connections_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("user_oauth_connections", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserPasswordReset", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("SecurityStampAtCreate") + .HasColumnType("uuid") + .HasColumnName("security_stamp_at_create"); + + b.Property("TokenHash") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("token_hash") + .UseCollation("C"); + + b.Property("UsedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("used_at"); + + b.Property("UserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_password_resets_pkey"); + + b.HasIndex("UserId"); + + b.ToTable("user_password_resets", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShare", b => + { + b.Property("SharedWithUserId") + .HasColumnType("uuid") + .HasColumnName("shared_with_user_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("SharedWithUserId", "ShockerId") + .HasName("user_shares_pkey"); + + b.HasIndex("SharedWithUserId"); + + b.HasIndex("ShockerId"); + + b.ToTable("user_shares", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("OwnerId") + .HasColumnType("uuid") + .HasColumnName("owner_id"); + + b.Property("RecipientUserId") + .HasColumnType("uuid") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("user_share_invites_pkey"); + + b.HasIndex("OwnerId"); + + b.HasIndex("RecipientUserId"); + + b.ToTable("user_share_invites", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInviteShocker", b => + { + b.Property("InviteId") + .HasColumnType("uuid") + .HasColumnName("invite_id"); + + b.Property("ShockerId") + .HasColumnType("uuid") + .HasColumnName("shocker_id"); + + b.Property("AllowLiveControl") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_livecontrol"); + + b.Property("AllowShock") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_shock"); + + b.Property("AllowSound") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_sound"); + + b.Property("AllowVibrate") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("allow_vibrate"); + + b.Property("IsPaused") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(false) + .HasColumnName("is_paused"); + + b.Property("MaxDuration") + .HasColumnType("integer") + .HasColumnName("max_duration"); + + b.Property("MaxIntensity") + .HasColumnType("smallint") + .HasColumnName("max_intensity"); + + b.HasKey("InviteId", "ShockerId") + .HasName("user_share_invite_shockers_pkey"); + + b.HasIndex("ShockerId"); + + b.ToTable("user_share_invite_shockers", (string)null); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiToken", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("ApiTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_api_tokens_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ApiTokenReport", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "ReportedByUser") + .WithMany("ReportedApiTokens") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_api_token_reports_reported_by_user_id"); + + b.Navigation("ReportedByUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("Devices") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_devices_owner_id"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.DeviceOtaUpdate", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Device", "Device") + .WithMany("OtaUpdates") + .HasForeignKey("DeviceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_device_ota_updates_device_id"); + + b.Navigation("Device"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("OwnedPublicShares") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_shares_owner_id"); + + b.Navigation("Owner"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShareShocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.PublicShare", "PublicShare") + .WithMany("ShockerMappings") + .HasForeignKey("PublicShareId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_share_shockers_public_share_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("PublicShareMappings") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_public_share_shockers_shocker_id"); + + b.Navigation("PublicShare"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Device", "Device") + .WithMany("Shockers") + .HasForeignKey("DeviceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shockers_device_id"); + + b.Navigation("Device"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerControlLog", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "ControlledByUser") + .WithMany("ShockerControlLogs") + .HasForeignKey("ControlledByUserId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_shocker_control_logs_controlled_by_user_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("ShockerControlLogs") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shocker_control_logs_shocker_id"); + + b.Navigation("ControlledByUser"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.ShockerShareCode", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("ShockerShareCodes") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_shocker_share_codes_shocker_id"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserActivationRequest", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithOne("UserActivationRequest") + .HasForeignKey("OpenShock.Common.OpenShockDb.UserActivationRequest", "UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_activation_requests_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserDeactivation", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "DeactivatedByUser") + .WithMany() + .HasForeignKey("DeactivatedByUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_deactivations_deactivated_by_user_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.User", "DeactivatedUser") + .WithOne("UserDeactivation") + .HasForeignKey("OpenShock.Common.OpenShockDb.UserDeactivation", "DeactivatedUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_deactivations_deactivated_user_id"); + + b.Navigation("DeactivatedByUser"); + + b.Navigation("DeactivatedUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserEmailChange", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("EmailChanges") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_email_changes_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserNameChange", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("NameChanges") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_name_changes_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserOAuthConnection", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("OAuthConnections") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_oauth_connections_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserPasswordReset", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "User") + .WithMany("PasswordResets") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_password_resets_user_id"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShare", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "SharedWithUser") + .WithMany("IncomingUserShares") + .HasForeignKey("SharedWithUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_shares_shared_with_user_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("UserShares") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_shares_shocker_id"); + + b.Navigation("SharedWithUser"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.User", "Owner") + .WithMany("OutgoingUserShareInvites") + .HasForeignKey("OwnerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invites_owner_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.User", "RecipientUser") + .WithMany("IncomingUserShareInvites") + .HasForeignKey("RecipientUserId") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_user_share_invites_recipient_user_id"); + + b.Navigation("Owner"); + + b.Navigation("RecipientUser"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInviteShocker", b => + { + b.HasOne("OpenShock.Common.OpenShockDb.UserShareInvite", "Invite") + .WithMany("ShockerMappings") + .HasForeignKey("InviteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invite_shockers_invite_id"); + + b.HasOne("OpenShock.Common.OpenShockDb.Shocker", "Shocker") + .WithMany("UserShareInviteShockerMappings") + .HasForeignKey("ShockerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_user_share_invite_shockers_shocker_id"); + + b.Navigation("Invite"); + + b.Navigation("Shocker"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Device", b => + { + b.Navigation("OtaUpdates"); + + b.Navigation("Shockers"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.PublicShare", b => + { + b.Navigation("ShockerMappings"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.Shocker", b => + { + b.Navigation("PublicShareMappings"); + + b.Navigation("ShockerControlLogs"); + + b.Navigation("ShockerShareCodes"); + + b.Navigation("UserShareInviteShockerMappings"); + + b.Navigation("UserShares"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.User", b => + { + b.Navigation("ApiTokens"); + + b.Navigation("Devices"); + + b.Navigation("EmailChanges"); + + b.Navigation("IncomingUserShareInvites"); + + b.Navigation("IncomingUserShares"); + + b.Navigation("NameChanges"); + + b.Navigation("OAuthConnections"); + + b.Navigation("OutgoingUserShareInvites"); + + b.Navigation("OwnedPublicShares"); + + b.Navigation("PasswordResets"); + + b.Navigation("ReportedApiTokens"); + + b.Navigation("ShockerControlLogs"); + + b.Navigation("UserActivationRequest"); + + b.Navigation("UserDeactivation"); + }); + + modelBuilder.Entity("OpenShock.Common.OpenShockDb.UserShareInvite", b => + { + b.Navigation("ShockerMappings"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.cs b/Common/Migrations/20260630103404_AddEmailOutbox.cs new file mode 100644 index 00000000..c92d6498 --- /dev/null +++ b/Common/Migrations/20260630103404_AddEmailOutbox.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.Migrations; +using OpenShock.Common.Models; + +#nullable disable + +namespace OpenShock.Common.Migrations +{ + /// + public partial class AddEmailOutbox : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterDatabase() + .Annotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .Annotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") + .Annotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") + .Annotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") + .Annotation("Npgsql:Enum:email_status", "sending,sent,failed") + .Annotation("Npgsql:Enum:email_type", "accountActivation,passwordReset,emailVerification,emailChangeNotice") + .Annotation("Npgsql:Enum:match_type_enum", "exact,contains") + .Annotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") + .Annotation("Npgsql:Enum:password_encryption_type", "pbkdf2,bcrypt_enhanced") + .Annotation("Npgsql:Enum:permission_type", "shockers.use,shockers.edit,shockers.pause,devices.edit,devices.auth") + .Annotation("Npgsql:Enum:role_type", "support,staff,admin,system") + .Annotation("Npgsql:Enum:shocker_model_type", "caiXianlin,petTrainer,petrainer998DR,wellturnT330") + .OldAnnotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .OldAnnotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") + .OldAnnotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") + .OldAnnotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") + .OldAnnotation("Npgsql:Enum:match_type_enum", "exact,contains") + .OldAnnotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") + .OldAnnotation("Npgsql:Enum:password_encryption_type", "pbkdf2,bcrypt_enhanced") + .OldAnnotation("Npgsql:Enum:permission_type", "shockers.use,shockers.edit,shockers.pause,devices.edit,devices.auth") + .OldAnnotation("Npgsql:Enum:role_type", "support,staff,admin,system") + .OldAnnotation("Npgsql:Enum:shocker_model_type", "caiXianlin,petTrainer,petrainer998DR,wellturnT330"); + + migrationBuilder.CreateTable( + name: "email_outbox", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + type = table.Column(type: "email_type", nullable: false), + recipient = table.Column(type: "character varying(320)", maxLength: 320, nullable: false), + recipient_name = table.Column(type: "character varying(32)", maxLength: 32, nullable: true), + payload = table.Column>(type: "jsonb", nullable: false), + status = table.Column(type: "email_status", nullable: false), + attempt_count = table.Column(type: "integer", nullable: false), + next_attempt_at = table.Column(type: "timestamp with time zone", nullable: true), + attempt_started_at = table.Column(type: "timestamp with time zone", nullable: true), + last_error = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: true), + created_at = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "CURRENT_TIMESTAMP"), + sent_at = table.Column(type: "timestamp with time zone", nullable: true), + failed_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("email_outbox_pkey", x => x.id); + }); + + migrationBuilder.CreateIndex( + name: "IX_email_outbox_created_at", + table: "email_outbox", + column: "created_at"); + + migrationBuilder.CreateIndex( + name: "IX_email_outbox_recipient", + table: "email_outbox", + column: "recipient"); + + migrationBuilder.CreateIndex( + name: "IX_email_outbox_status_next_attempt_at", + table: "email_outbox", + columns: new[] { "status", "next_attempt_at" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "email_outbox"); + + migrationBuilder.AlterDatabase() + .Annotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .Annotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") + .Annotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") + .Annotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") + .Annotation("Npgsql:Enum:match_type_enum", "exact,contains") + .Annotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") + .Annotation("Npgsql:Enum:password_encryption_type", "pbkdf2,bcrypt_enhanced") + .Annotation("Npgsql:Enum:permission_type", "shockers.use,shockers.edit,shockers.pause,devices.edit,devices.auth") + .Annotation("Npgsql:Enum:role_type", "support,staff,admin,system") + .Annotation("Npgsql:Enum:shocker_model_type", "caiXianlin,petTrainer,petrainer998DR,wellturnT330") + .OldAnnotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") + .OldAnnotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") + .OldAnnotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") + .OldAnnotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") + .OldAnnotation("Npgsql:Enum:email_status", "sending,sent,failed") + .OldAnnotation("Npgsql:Enum:email_type", "accountActivation,passwordReset,emailVerification,emailChangeNotice") + .OldAnnotation("Npgsql:Enum:match_type_enum", "exact,contains") + .OldAnnotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") + .OldAnnotation("Npgsql:Enum:password_encryption_type", "pbkdf2,bcrypt_enhanced") + .OldAnnotation("Npgsql:Enum:permission_type", "shockers.use,shockers.edit,shockers.pause,devices.edit,devices.auth") + .OldAnnotation("Npgsql:Enum:role_type", "support,staff,admin,system") + .OldAnnotation("Npgsql:Enum:shocker_model_type", "caiXianlin,petTrainer,petrainer998DR,wellturnT330"); + } + } +} diff --git a/Common/Migrations/OpenShockContextModelSnapshot.cs b/Common/Migrations/OpenShockContextModelSnapshot.cs index 81f0c618..cf496852 100644 --- a/Common/Migrations/OpenShockContextModelSnapshot.cs +++ b/Common/Migrations/OpenShockContextModelSnapshot.cs @@ -21,12 +21,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) #pragma warning disable 612, 618 modelBuilder .HasAnnotation("Npgsql:CollationDefinition:public.ndcoll", "und-u-ks-level2,und-u-ks-level2,icu,False") - .HasAnnotation("ProductVersion", "10.0.8") + .HasAnnotation("ProductVersion", "10.0.9") .HasAnnotation("Relational:MaxIdentifierLength", 63); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "configuration_value_type", new[] { "string", "bool", "int", "float", "json" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "sound", "vibrate", "shock", "stop" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "sending", "sent", "failed" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "accountActivation", "passwordReset", "emailVerification", "emailChangeNotice" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "password_encryption_type", new[] { "pbkdf2", "bcrypt_enhanced" }); @@ -426,6 +428,79 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("discord_webhooks", (string)null); }); + modelBuilder.Entity("OpenShock.Common.OpenShockDb.EmailOutboxMessage", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AttemptCount") + .HasColumnType("integer") + .HasColumnName("attempt_count"); + + b.Property("AttemptStartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("attempt_started_at"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("LastError") + .HasMaxLength(1024) + .HasColumnType("character varying(1024)") + .HasColumnName("last_error"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("next_attempt_at"); + + b.Property>("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("Recipient") + .IsRequired() + .HasMaxLength(320) + .HasColumnType("character varying(320)") + .HasColumnName("recipient"); + + b.Property("RecipientName") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("recipient_name"); + + b.Property("SentAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("sent_at"); + + b.Property("Status") + .HasColumnType("email_status") + .HasColumnName("status"); + + b.Property("Type") + .HasColumnType("email_type") + .HasColumnName("type"); + + b.HasKey("Id") + .HasName("email_outbox_pkey"); + + b.HasIndex("CreatedAt"); + + b.HasIndex("Recipient"); + + b.HasIndex("Status", "NextAttemptAt"); + + b.ToTable("email_outbox", (string)null); + }); + modelBuilder.Entity("OpenShock.Common.OpenShockDb.EmailProviderBlacklist", b => { b.Property("Id") diff --git a/Common/Models/EmailStatus.cs b/Common/Models/EmailStatus.cs new file mode 100644 index 00000000..b4fa4a8f --- /dev/null +++ b/Common/Models/EmailStatus.cs @@ -0,0 +1,32 @@ +namespace OpenShock.Common.Models; + +/// +/// Delivery state of an . +/// +/// +/// There are deliberately only three states. A separate "pending" state would carry no information +/// here: a message that has been enqueued but not yet delivered is simply one the consumer has not +/// finished sending, which is exactly what means. A row waiting for its next +/// retry is also (with a future NextAttemptAt), it is still in flight, +/// just not this instant. Collapsing "queued", "in progress" and "awaiting retry" into one state +/// keeps the claim query and the state machine trivial. +/// +public enum EmailStatus +{ + /// + /// The message still needs to be delivered. Covers freshly enqueued rows, rows currently being + /// attempted (see AttemptStartedAt), and rows waiting for a retry (see NextAttemptAt). + /// This is the only state the consumer claims from. + /// + Sending, + + /// The message was handed to the email provider successfully. Terminal. + Sent, + + /// + /// The message will not be delivered: it exhausted its retry budget, hit a permanent provider + /// error, or the underlying request it was for no longer exists. Terminal, but the row is kept + /// (never auto-deleted) so it stays inspectable and can be requeued by an operator. + /// + Failed +} diff --git a/Common/Models/EmailType.cs b/Common/Models/EmailType.cs new file mode 100644 index 00000000..bb9dc693 --- /dev/null +++ b/Common/Models/EmailType.cs @@ -0,0 +1,22 @@ +namespace OpenShock.Common.Models; + +/// +/// The kind of email an represents. The outbox stores +/// only the intent to send one of these, never a rendered body or a usable secret, the +/// consumer maps the type to the matching template and (for token-bearing types) mints a fresh +/// secret at send time. +/// +public enum EmailType +{ + /// Account activation / email confirmation for a newly created account. + AccountActivation, + + /// Password reset link. + PasswordReset, + + /// Verification of a newly requested email address (sent to the new address). + EmailVerification, + + /// Informational notice sent to the previous address when an email change is requested. Carries no secret. + EmailChangeNotice +} diff --git a/Common/OpenShockDb/EmailOutboxMessage.cs b/Common/OpenShockDb/EmailOutboxMessage.cs new file mode 100644 index 00000000..8d1413ba --- /dev/null +++ b/Common/OpenShockDb/EmailOutboxMessage.cs @@ -0,0 +1,144 @@ +using OpenShock.Common.Models; + +namespace OpenShock.Common.OpenShockDb; + +/// +/// A durable, application-owned record of an email the system intends to deliver. This table is the +/// source of truth for "we owe this user this email": a row is written in the same database +/// transaction as the business change that caused it, and a background consumer delivers it, +/// retrying until it succeeds or is exhausted. Rows are never auto-deleted, so failed sends remain +/// visible and can be requeued. +/// +/// +/// +/// Flow. The request handler creates this row (and any related request row, e.g. a +/// ) and commits. It does not send. A single background +/// consumer claims the row, renders the body, mints a fresh secret for token-bearing types, hands +/// the message to the email provider, and records the outcome. Delivery is therefore decoupled from +/// the HTTP request: a provider outage, a process restart, or a crash mid-send cannot lose the +/// email — the row simply stays and is retried. +/// +/// +/// +/// Why this lives in our database (rather than a job framework such as Hangfire). +/// This is intentionally an outbox, not a generic background-job queue, and the two solve +/// different problems: +/// +/// +/// +/// Transactional integrity. The email intent is committed in the same transaction +/// as the change that caused it (account created, reset requested, email changed). There is no +/// window where the business row commits but the email is lost, or where an email is scheduled +/// for a change that then rolls back. A separate job store cannot give this guarantee without, +/// in effect, reinventing an outbox alongside it. +/// +/// +/// Single source of truth. A job framework's record is "a method was scheduled and ran"; +/// this row is the domain fact "this user is owed this email, here is its delivery state". When +/// a user reports a missing email, this table answers it directly (who, which type, how many +/// attempts, last error, sent/failed) — no second system to correlate against. +/// +/// +/// No new infrastructure or operational surface. Reliability rides on PostgreSQL, which +/// is already the system's durable store. There is no extra schema owned by a third-party +/// scheduler, no dashboard to secure, and no job-serialization/versioning concerns. +/// +/// +/// Latency. Delivery is push-triggered (a Redis notification on enqueue) with a periodic +/// poll only as a safety net, so first-send is sub-second. A minute-granularity scheduler would +/// not meet that. +/// +/// +/// +/// This is not a claim that job frameworks are bad — for recurring/scheduled maintenance work they +/// are the right tool, and the codebase uses one for cron jobs. It is the narrower point that +/// transactional, must-not-be-lost, resendable email is an outbox-shaped problem, and modelling +/// it as a first-class table is the simplest design that satisfies those requirements. +/// +/// +/// +/// Why the related auth flows mint their token lazily. Because a working reset/verification +/// link cannot be reconstructed from the stored hash, "store the intent, mint on send" is the only +/// model that keeps the queue free of usable secrets while still allowing a resend. The consumer +/// generates the secret at send time and writes only its hash to the request row (see +/// etc., now nullable until first send). A resend produces +/// a brand-new token — which is exactly the desired behaviour for security links anyway. +/// +/// +/// +/// Delivery guarantee. This is at-least-once. The unavoidable edge case — the provider accepts +/// the message but the process dies before the row is marked — results +/// in one duplicate send on retry. For these email types that is harmless (a second activation / +/// reset / notice), and it is the correct trade against the alternative of silently losing mail. +/// +/// +public sealed class EmailOutboxMessage +{ + /// Primary key (UUIDv7, time-ordered so the queue drains roughly FIFO). + public required Guid Id { get; set; } + + /// Which email this is; selects the template and the send logic. + public required EmailType Type { get; set; } + + /// Destination email address. + public required string Recipient { get; set; } + + /// Optional display name for the recipient. + public string? RecipientName { get; set; } + + /// + /// Dynamic, non-secret parameters needed to render and send, stored as Postgres jsonb + /// (via a value converter). An open key/value bag keyed by the well-known names in + /// , so new email types need no schema change. Never contains + /// a rendered body or a usable secret. + /// + public required Dictionary Payload { get; set; } + + /// Delivery state. See . + public EmailStatus Status { get; set; } = EmailStatus.Sending; + + /// Number of send attempts started so far. Incremented when the row is claimed. + public int AttemptCount { get; set; } + + /// + /// Earliest time the next attempt may run. Null means "as soon as possible". Set into the future + /// after a transient failure to implement back-off. + /// + public DateTime? NextAttemptAt { get; set; } + + /// + /// When the current attempt was claimed, acting as a lease. A row whose lease is older than the + /// lease timeout is considered abandoned (the worker crashed mid-send) and may be re-claimed. + /// Cleared whenever the row reaches a resting state (sent, failed, or waiting for retry). + /// + public DateTime? AttemptStartedAt { get; set; } + + /// Last error recorded for a failed/retried attempt. Null while healthy. + public string? LastError { get; set; } + + /// When the row was created (i.e. when the email was enqueued). + public DateTime CreatedAt { get; set; } + + /// When the message was successfully handed to the provider. Null until then. + public DateTime? SentAt { get; set; } + + /// When the message reached the terminal state. + public DateTime? FailedAt { get; set; } + + /// + /// Builds a new enqueued message in the state. The caller adds + /// it to the context and commits it together with the related business change. + /// + public static EmailOutboxMessage Create(EmailType type, string recipient, string? recipientName, Dictionary payload) + { + return new EmailOutboxMessage + { + Id = Guid.CreateVersion7(), + Type = type, + Recipient = recipient, + RecipientName = recipientName, + Payload = payload, + Status = EmailStatus.Sending + }; + } +} diff --git a/Common/OpenShockDb/EmailOutboxPayloadKeys.cs b/Common/OpenShockDb/EmailOutboxPayloadKeys.cs new file mode 100644 index 00000000..121eed8b --- /dev/null +++ b/Common/OpenShockDb/EmailOutboxPayloadKeys.cs @@ -0,0 +1,23 @@ +namespace OpenShock.Common.OpenShockDb; + +/// +/// Well-known keys for the dynamic dictionary. The payload +/// is intentionally an open key/value bag (stored as Postgres jsonb) so new email types can +/// carry whatever references they need without a schema change; these constants just keep the +/// well-known keys typo-safe. Values are always non-secret references (ids / addresses) — never a +/// rendered body and never a usable token. +/// +public static class EmailOutboxPayloadKeys +{ + /// Target user id. Used by . + public const string UserId = "userId"; + + /// id. Used by . + public const string PasswordResetId = "passwordResetId"; + + /// id. Used by . + public const string EmailChangeId = "emailChangeId"; + + /// The newly requested address. Used by . + public const string NewEmail = "newEmail"; +} diff --git a/Common/OpenShockDb/OpenShockContext.cs b/Common/OpenShockDb/OpenShockContext.cs index 66de70f0..9c752d82 100644 --- a/Common/OpenShockDb/OpenShockContext.cs +++ b/Common/OpenShockDb/OpenShockContext.cs @@ -71,6 +71,8 @@ public static void ConfigureOptionsBuilder(DbContextOptionsBuilder optionsBuilde npgsqlBuilder.MapEnum(); npgsqlBuilder.MapEnum(); npgsqlBuilder.MapEnum(); + npgsqlBuilder.MapEnum(); + npgsqlBuilder.MapEnum(); }); if (debug) @@ -127,6 +129,8 @@ public static void ConfigureOptionsBuilder(DbContextOptionsBuilder optionsBuilde public DbSet UserNameBlacklists { get; set; } public DbSet EmailProviderBlacklists { get; set; } + + public DbSet EmailOutbox { get; set; } public DbSet DataProtectionKeys { get; set; } @@ -151,6 +155,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasPostgresEnum("shocker_model_type", ["caiXianlin", "petTrainer", "petrainer998DR", "wellturnT330"]) .HasPostgresEnum("match_type_enum", ["exact", "contains"]) .HasPostgresEnum("configuration_value_type", ["string", "bool", "int", "float", "json"]) + .HasPostgresEnum("email_type", ["accountActivation", "passwordReset", "emailVerification", "emailChangeNotice"]) + .HasPostgresEnum("email_status", ["sending", "sent", "failed"]) .HasCollation("public", "ndcoll", "und-u-ks-level2", "icu", false); // Add case-insensitive, accent-sensitive comparison collation modelBuilder.Entity(entity => @@ -872,6 +878,51 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasColumnName("created_at"); }); + modelBuilder.Entity(entity => + { + entity.HasKey(e => e.Id).HasName("email_outbox_pkey"); + + entity.ToTable("email_outbox"); + + // The consumer claims rows by (status, due time); created_at gives a stable FIFO order. + entity.HasIndex(e => new { e.Status, e.NextAttemptAt }); + entity.HasIndex(e => e.CreatedAt); + entity.HasIndex(e => e.Recipient); + + entity.Property(e => e.Id) + .ValueGeneratedNever() + .HasColumnName("id"); + entity.Property(e => e.Type) + .HasColumnName("type"); + entity.Property(e => e.Recipient) + .VarCharWithLength(HardLimits.EmailAddressMaxLength) + .HasColumnName("recipient"); + entity.Property(e => e.RecipientName) + .VarCharWithLength(HardLimits.UsernameMaxLength) + .HasColumnName("recipient_name"); + entity.Property(e => e.Payload) + .HasColumnType("jsonb") + .HasColumnName("payload"); + entity.Property(e => e.Status) + .HasColumnName("status"); + entity.Property(e => e.AttemptCount) + .HasColumnName("attempt_count"); + entity.Property(e => e.NextAttemptAt) + .HasColumnName("next_attempt_at"); + entity.Property(e => e.AttemptStartedAt) + .HasColumnName("attempt_started_at"); + entity.Property(e => e.LastError) + .VarCharWithLength(HardLimits.EmailOutboxLastErrorMaxLength) + .HasColumnName("last_error"); + entity.Property(e => e.CreatedAt) + .HasDefaultValueSql("CURRENT_TIMESTAMP") + .HasColumnName("created_at"); + entity.Property(e => e.SentAt) + .HasColumnName("sent_at"); + entity.Property(e => e.FailedAt) + .HasColumnName("failed_at"); + }); + modelBuilder.Entity(entity => { entity diff --git a/Common/OpenShockDb/UserActivationRequest.cs b/Common/OpenShockDb/UserActivationRequest.cs index c5c5e137..e08e236b 100644 --- a/Common/OpenShockDb/UserActivationRequest.cs +++ b/Common/OpenShockDb/UserActivationRequest.cs @@ -4,6 +4,11 @@ public sealed class UserActivationRequest { public required Guid UserId { get; set; } + /// + /// Hash of the secret activation token. The plaintext is never stored. Seeded when the request is + /// created and re-minted by the email outbox consumer on every (re)send, so the queue never holds + /// a usable activation link. See . + /// public required string TokenHash { get; set; } public int EmailSendAttempts { get; set; } diff --git a/Common/OpenShockDb/UserEmailChange.cs b/Common/OpenShockDb/UserEmailChange.cs index 9b1ecdb5..826a34a5 100644 --- a/Common/OpenShockDb/UserEmailChange.cs +++ b/Common/OpenShockDb/UserEmailChange.cs @@ -33,8 +33,10 @@ public sealed class UserEmailChange public required string NewEmail { get; set; } /// - /// Hash of the secret token sent to . The plaintext token is never - /// stored — it only exists in the verification email and in the request the user submits. + /// Hash of the secret token sent to . The plaintext token is never stored — + /// it only exists in the verification email and in the request the user submits. Seeded when the + /// request is created and re-minted by the email outbox consumer on every (re)send, so the queue + /// never holds a usable verification link. See . /// public required string TokenHash { get; set; } diff --git a/Common/OpenShockDb/UserPasswordReset.cs b/Common/OpenShockDb/UserPasswordReset.cs index 7aa444cb..d80f12b3 100644 --- a/Common/OpenShockDb/UserPasswordReset.cs +++ b/Common/OpenShockDb/UserPasswordReset.cs @@ -20,8 +20,10 @@ public sealed class UserPasswordReset public required Guid UserId { get; set; } /// - /// Hash of the secret token that was sent in the reset email. The plaintext token is never - /// stored — it only exists in the email link and in the request the user submits. + /// Hash of the secret token for the reset link. The plaintext token is never stored — it only + /// exists in the email link and in the request the user submits. Seeded when the request is + /// created and re-minted by the email outbox consumer on every (re)send, so the queue never + /// holds a usable reset link and a resend always supersedes earlier links. See . /// public required string TokenHash { get; set; } diff --git a/Common/Services/RedisPubSub/IRedisPubService.cs b/Common/Services/RedisPubSub/IRedisPubService.cs index 90dcdb36..935181d1 100644 --- a/Common/Services/RedisPubSub/IRedisPubService.cs +++ b/Common/Services/RedisPubSub/IRedisPubService.cs @@ -20,6 +20,14 @@ public interface IRedisPubService /// Task SendApiTokenUpdate(Guid tokenId); + /// + /// Notifies the email outbox consumer that one or more messages were just enqueued, so it drains + /// them without waiting for its next safety-net poll. Best-effort: a lost notification only delays + /// delivery until the next poll, it cannot lose the email (the row is already durably committed). + /// + /// + Task SendEmailOutboxPending(); + /// /// Used when a device comes online or changes its connection details like, gateway, firmware version, etc. /// diff --git a/Common/Services/RedisPubSub/RedisChannels.cs b/Common/Services/RedisPubSub/RedisChannels.cs index 60427a40..bcad13ff 100644 --- a/Common/Services/RedisPubSub/RedisChannels.cs +++ b/Common/Services/RedisPubSub/RedisChannels.cs @@ -11,4 +11,10 @@ public static class RedisChannels public static readonly RedisChannel DeviceStatus = new("device-status", RedisChannel.PatternMode.Literal); public static RedisChannel ApiTokenUpdate(Guid tokenId) => new($"api-token-update:{tokenId}", RedisChannel.PatternMode.Literal); + + /// + /// Fired when a row is added to the email outbox, so the email consumer drains it immediately + /// instead of waiting for its next poll. Payload-less: it is purely a "wake up and check" nudge. + /// + public static readonly RedisChannel EmailOutboxPending = new("email-outbox-pending", RedisChannel.PatternMode.Literal); } \ No newline at end of file diff --git a/Common/Services/RedisPubSub/RedisPubService.cs b/Common/Services/RedisPubSub/RedisPubService.cs index b8996bb2..a305558d 100644 --- a/Common/Services/RedisPubSub/RedisPubService.cs +++ b/Common/Services/RedisPubSub/RedisPubService.cs @@ -28,6 +28,12 @@ public Task SendApiTokenUpdate(Guid tokenId) return _subscriber.PublishAsync(RedisChannels.ApiTokenUpdate(tokenId), RedisValue.EmptyString); } + public Task SendEmailOutboxPending() + { + // Payload-less wake-up nudge; the consumer reads the actual rows from the database. + return _subscriber.PublishAsync(RedisChannels.EmailOutboxPending, RedisValue.EmptyString); + } + public Task SendDeviceOnlineStatus(Guid deviceId, bool isOnline) { return Publish(RedisChannels.DeviceStatus, DeviceStatus.Create(deviceId, DeviceBoolStateType.Online, isOnline)); From 4a4b4183662772620aafb3c8e2d5d199e6d4489d Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 13:18:02 +0200 Subject: [PATCH 02/15] fix(email): correct outbox enum labels + jsonb mapping; harden + docs CI caught two runtime failures (build passed, inserts 500'd): - Postgres enum labels for email_type must be snake_case to match Npgsql's default member translator (e.g. password_reset), not camelCase. - Dictionary -> jsonb needs Npgsql dynamic JSON; enable it via ConfigureDataSource(EnableDynamicJson) rather than a hand-rolled converter. Both verified against a real Postgres and the full MailTests suite (16/16). Hardening from a self-review pass: - Redis enqueue notification is now best-effort: a publish failure can no longer fail a request whose outbox row already committed (poll delivers it). - Deduplicate the password-reset / email-change validity checks in the dispatcher into a shared helper. - Fix back-off overflow for large attempt counts (cap in seconds before converting to TimeSpan). - Correct stale doc comments (jsonb via dynamic JSON; token hash seeded then re-minted, not nullable). Migration keeps its original id (only the enum labels changed). --- API/Services/Account/AccountService.cs | 30 +++++++++++++++---- API/Services/Email/IEmailService.cs | 2 +- .../Email/Mailjet/MailjetEmailService.cs | 2 +- .../Email/Outbox/EmailDispatchResult.cs | 2 +- .../Email/Outbox/EmailOutboxDispatcher.cs | 27 ++++++++++------- .../Email/Outbox/EmailOutboxRetryPolicy.cs | 4 +-- .../Email/Outbox/EmailOutboxWorker.cs | 4 +-- .../Email/Outbox/IEmailOutboxDispatcher.cs | 4 +-- .../20260630103404_AddEmailOutbox.Designer.cs | 2 +- .../20260630103404_AddEmailOutbox.cs | 4 +-- .../OpenShockContextModelSnapshot.cs | 2 +- Common/OpenShockDb/EmailOutboxMessage.cs | 18 ++++++----- Common/OpenShockDb/EmailOutboxPayloadKeys.cs | 2 +- Common/OpenShockDb/OpenShockContext.cs | 6 +++- Common/OpenShockDb/UserEmailChange.cs | 2 +- Common/OpenShockDb/UserPasswordReset.cs | 2 +- 16 files changed, 72 insertions(+), 41 deletions(-) diff --git a/API/Services/Account/AccountService.cs b/API/Services/Account/AccountService.cs index d4675100..f791e5fb 100644 --- a/API/Services/Account/AccountService.cs +++ b/API/Services/Account/AccountService.cs @@ -41,12 +41,30 @@ public AccountService(OpenShockContext db, IRedisPubService redisPubService, /// /// Seeds a random token hash for a freshly created request row. The plaintext is discarded - /// immediately — the email outbox consumer mints the real token (and overwrites this hash) when it + /// immediately: the email outbox consumer mints the real token (and overwrites this hash) when it /// sends, so this value is never the one delivered. It exists only to keep the column populated. /// private static string SeedTokenHash() => HashingUtils.HashToken(CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength)); + /// + /// Best-effort nudge to the email outbox consumer that a message was enqueued. Deliberately + /// swallows failures: the outbox row is already committed, so a dropped notification only delays + /// delivery to the consumer's next poll, and a transient Redis hiccup must not fail a request + /// whose work already succeeded. + /// + private async Task NotifyEmailOutboxAsync() + { + try + { + await _redisPubService.SendEmailOutboxPending(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to notify the email outbox consumer; delivery will fall back to polling"); + } + } + private async Task IsUserNameBlacklisted(string username) { await foreach (var entry in _db.UserNameBlacklists.AsNoTracking().AsAsyncEnumerable()) @@ -116,7 +134,7 @@ public async Task, AccountWithEmailOrUsernameExists>> Create new Dictionary { [EmailOutboxPayloadKeys.UserId] = user.Id.ToString() })); await _db.SaveChangesAsync(); - await _redisPubService.SendEmailOutboxPending(); + await NotifyEmailOutboxAsync(); return new Success(user); } @@ -199,7 +217,7 @@ public async Task, AccountWithEmailOrUsernameExists>> Create // Notify the outbox consumer only after a successful commit. if (!isEmailTrusted) { - await _redisPubService.SendEmailOutboxPending(); + await NotifyEmailOutboxAsync(); } return new Success(user); @@ -428,7 +446,7 @@ public async Task { [EmailOutboxPayloadKeys.PasswordResetId] = passwordReset.Id.ToString() })); await _db.SaveChangesAsync(); - await _redisPubService.SendEmailOutboxPending(); + await NotifyEmailOutboxAsync(); return new Success(); } @@ -595,14 +613,14 @@ public async Task { [EmailOutboxPayloadKeys.EmailChangeId] = emailChange.Id.ToString() })); _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.EmailChangeNotice, data.User.Email, data.User.Name, new Dictionary { [EmailOutboxPayloadKeys.NewEmail] = lowerCaseEmail })); await _db.SaveChangesAsync(); - await _redisPubService.SendEmailOutboxPending(); + await NotifyEmailOutboxAsync(); return new Success(); } diff --git a/API/Services/Email/IEmailService.cs b/API/Services/Email/IEmailService.cs index 4f8b1234..c096fa86 100644 --- a/API/Services/Email/IEmailService.cs +++ b/API/Services/Email/IEmailService.cs @@ -6,7 +6,7 @@ namespace OpenShock.API.Services.Email; /// Low-level email provider abstraction: renders a template and hands the message to the provider. /// Implementations return an instead of throwing, so the email outbox /// consumer (the only caller) can classify failures and decide whether to retry. Callers must not -/// invoke this directly to send transactional mail — enqueue an +/// invoke this directly to send transactional mail - enqueue an /// instead so the send is durable. /// public interface IEmailService diff --git a/API/Services/Email/Mailjet/MailjetEmailService.cs b/API/Services/Email/Mailjet/MailjetEmailService.cs index 4a49d69a..9fdd783e 100644 --- a/API/Services/Email/Mailjet/MailjetEmailService.cs +++ b/API/Services/Email/Mailjet/MailjetEmailService.cs @@ -75,7 +75,7 @@ private async Task SendMails(DirectMail[] mails, CancellationTo } catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException && !cancellationToken.IsCancellationRequested) { - // Network failure or request timeout — worth retrying. + // Network failure or request timeout - worth retrying. _logger.LogWarning(ex, "Transient failure sending mails {@Mails}", mails); return EmailSendResult.TransientFailure; } diff --git a/API/Services/Email/Outbox/EmailDispatchResult.cs b/API/Services/Email/Outbox/EmailDispatchResult.cs index 969957d8..f9c5def1 100644 --- a/API/Services/Email/Outbox/EmailDispatchResult.cs +++ b/API/Services/Email/Outbox/EmailDispatchResult.cs @@ -14,7 +14,7 @@ public enum EmailDispatchOutcome /// /// Nothing was sent because the underlying request no longer needs the email (expired, already - /// used, superseded, or gone). Terminal — there is nothing to retry. + /// used, superseded, or gone). Terminal - there is nothing to retry. /// Skipped } diff --git a/API/Services/Email/Outbox/EmailOutboxDispatcher.cs b/API/Services/Email/Outbox/EmailOutboxDispatcher.cs index d2e813f1..de820505 100644 --- a/API/Services/Email/Outbox/EmailOutboxDispatcher.cs +++ b/API/Services/Email/Outbox/EmailOutboxDispatcher.cs @@ -75,11 +75,8 @@ private async Task SendPasswordReset(EmailOutboxMessage mes .Include(r => r.User) .FirstOrDefaultAsync(r => r.Id == resetId, ct); if (reset is null) return EmailDispatchResult.Skip("Password reset no longer exists"); - if (reset.UsedAt is not null) return EmailDispatchResult.Skip("Password reset already used"); - if (reset.CreatedAt < DateTime.UtcNow - Duration.PasswordResetRequestLifetime) - return EmailDispatchResult.Skip("Password reset expired"); - if (reset.SecurityStampAtCreate != reset.User.SecurityStamp) - return EmailDispatchResult.Skip("Password reset superseded by a newer credential change"); + if (ValidatePending("Password reset", reset.UsedAt, reset.CreatedAt, Duration.PasswordResetRequestLifetime, + reset.SecurityStampAtCreate, reset.User.SecurityStamp) is { } resetSkip) return resetSkip; var token = await MintTokenAsync(db, h => reset.TokenHash = h, ct); var link = new Uri(_frontendOptions.BaseUrl, $"/#/account/password/recover/{reset.Id}/{token}"); @@ -95,11 +92,8 @@ private async Task SendEmailVerification(EmailOutboxMessage .Include(c => c.User) .FirstOrDefaultAsync(c => c.Id == changeId, ct); if (change is null) return EmailDispatchResult.Skip("Email change no longer exists"); - if (change.UsedAt is not null) return EmailDispatchResult.Skip("Email change already used"); - if (change.CreatedAt < DateTime.UtcNow - Duration.EmailChangeRequestLifetime) - return EmailDispatchResult.Skip("Email change expired"); - if (change.SecurityStampAtCreate != change.User.SecurityStamp) - return EmailDispatchResult.Skip("Email change superseded by a newer credential change"); + if (ValidatePending("Email change", change.UsedAt, change.CreatedAt, Duration.EmailChangeRequestLifetime, + change.SecurityStampAtCreate, change.User.SecurityStamp) is { } changeSkip) return changeSkip; var token = await MintTokenAsync(db, h => change.TokenHash = h, ct); var link = new Uri(_frontendOptions.BaseUrl, $"/verify-email?token={token}"); @@ -129,6 +123,19 @@ private static async Task MintTokenAsync(OpenShockContext db, Action + /// Shared validity check for a token-bearing request (password reset / email change): returns a + /// Skip result if it has been used, has expired, or has been superseded by a newer credential + /// change (security stamp rotated), otherwise null. + /// + private static EmailDispatchResult? ValidatePending(string what, DateTime? usedAt, DateTime createdAt, TimeSpan lifetime, Guid stampAtCreate, Guid currentStamp) + { + if (usedAt is not null) return EmailDispatchResult.Skip($"{what} already used"); + if (createdAt < DateTime.UtcNow - lifetime) return EmailDispatchResult.Skip($"{what} expired"); + if (stampAtCreate != currentStamp) return EmailDispatchResult.Skip($"{what} superseded by a newer credential change"); + return null; + } + private static EmailDispatchResult FromSend(EmailSendResult result) => result switch { EmailSendResult.Sent => EmailDispatchResult.Sent, diff --git a/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs b/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs index d1a3ef1d..33c4743d 100644 --- a/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs +++ b/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs @@ -23,7 +23,7 @@ public static class EmailOutboxRetryPolicy public static TimeSpan GetBaseDelay(int attemptCount) { // Compute in seconds (double) and cap before converting, so a large attempt count can't - // overflow TimeSpan — Math.Pow can reach infinity, which simply compares above the cap. + // overflow TimeSpan - Math.Pow can reach infinity, which simply compares above the cap. var maxDelay = Duration.EmailOutboxRetryMaxDelay; var seconds = Duration.EmailOutboxRetryBaseDelay.TotalSeconds * Math.Pow(2, Math.Max(0, attemptCount - 1)); return seconds >= maxDelay.TotalSeconds ? maxDelay : TimeSpan.FromSeconds(seconds); @@ -35,7 +35,7 @@ public static TimeSpan GetRetryDelay(int attemptCount) /// /// Applies a dispatch outcome to a message, advancing it to its next state: delivered, failed - /// (permanent / skipped / exhausted), or scheduled for another retry. Pure — only mutates + /// (permanent / skipped / exhausted), or scheduled for another retry. Pure - only mutates /// . /// public static void Apply(EmailOutboxMessage message, EmailDispatchResult result, DateTime nowUtc) diff --git a/API/Services/Email/Outbox/EmailOutboxWorker.cs b/API/Services/Email/Outbox/EmailOutboxWorker.cs index 1479054f..ce55042a 100644 --- a/API/Services/Email/Outbox/EmailOutboxWorker.cs +++ b/API/Services/Email/Outbox/EmailOutboxWorker.cs @@ -19,7 +19,7 @@ namespace OpenShock.API.Services.Email.Outbox; /// () still runs as a fallback: it picks up due retries /// (whose NextAttemptAt is in the future), re-claims leases abandoned by a crashed instance, /// and covers the rare case of a dropped notification. The notification can therefore be best-effort -/// — losing one only delays delivery to the next tick, it can never lose the email. +/// - losing one only delays delivery to the next tick, it can never lose the email. /// /// /// Runs in every API instance, safely. Rows are claimed with FOR UPDATE SKIP LOCKED, so @@ -110,7 +110,7 @@ private void OnPendingNotification(RedisChannel channel, RedisValue value) } catch (SemaphoreFullException) { - // Already pending a wake-up — nothing to do. + // Already pending a wake-up - nothing to do. } } diff --git a/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs b/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs index db25ece4..a2b143d6 100644 --- a/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs +++ b/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs @@ -6,7 +6,7 @@ namespace OpenShock.API.Services.Email.Outbox; /// Renders and sends a single . This is the one place email is /// actually sent: it maps the message's type and dynamic payload to a template, lazily mints (and /// persists the hash of) any secret token the email needs, hands the message to -/// , and reports the outcome. It does not own the message's lifecycle — +/// , and reports the outcome. It does not own the message's lifecycle - /// the consumer that calls it records the resulting state transition. /// public interface IEmailOutboxDispatcher @@ -15,7 +15,7 @@ public interface IEmailOutboxDispatcher /// Attempts to deliver . Any token row touched (e.g. a /// ) is updated via and saved before the /// send, so the link in the email always matches a stored hash. Never throws for ordinary send - /// failures — they are returned as . + /// failures - they are returned as . /// /// The message to send. Its status is not modified here. /// The context used to load/update related request rows. diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs index bdff376f..ab6bb127 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs +++ b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs @@ -31,7 +31,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "sound", "vibrate", "shock", "stop" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "sending", "sent", "failed" }); - NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "accountActivation", "passwordReset", "emailVerification", "emailChangeNotice" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "account_activation", "password_reset", "email_verification", "email_change_notice" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "password_encryption_type", new[] { "pbkdf2", "bcrypt_enhanced" }); diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.cs b/Common/Migrations/20260630103404_AddEmailOutbox.cs index c92d6498..3e88326f 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.cs +++ b/Common/Migrations/20260630103404_AddEmailOutbox.cs @@ -19,7 +19,7 @@ protected override void Up(MigrationBuilder migrationBuilder) .Annotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") .Annotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") .Annotation("Npgsql:Enum:email_status", "sending,sent,failed") - .Annotation("Npgsql:Enum:email_type", "accountActivation,passwordReset,emailVerification,emailChangeNotice") + .Annotation("Npgsql:Enum:email_type", "account_activation,password_reset,email_verification,email_change_notice") .Annotation("Npgsql:Enum:match_type_enum", "exact,contains") .Annotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") .Annotation("Npgsql:Enum:password_encryption_type", "pbkdf2,bcrypt_enhanced") @@ -98,7 +98,7 @@ protected override void Down(MigrationBuilder migrationBuilder) .OldAnnotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") .OldAnnotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") .OldAnnotation("Npgsql:Enum:email_status", "sending,sent,failed") - .OldAnnotation("Npgsql:Enum:email_type", "accountActivation,passwordReset,emailVerification,emailChangeNotice") + .OldAnnotation("Npgsql:Enum:email_type", "account_activation,password_reset,email_verification,email_change_notice") .OldAnnotation("Npgsql:Enum:match_type_enum", "exact,contains") .OldAnnotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") .OldAnnotation("Npgsql:Enum:password_encryption_type", "pbkdf2,bcrypt_enhanced") diff --git a/Common/Migrations/OpenShockContextModelSnapshot.cs b/Common/Migrations/OpenShockContextModelSnapshot.cs index cf496852..a74ca0f7 100644 --- a/Common/Migrations/OpenShockContextModelSnapshot.cs +++ b/Common/Migrations/OpenShockContextModelSnapshot.cs @@ -28,7 +28,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "sound", "vibrate", "shock", "stop" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "sending", "sent", "failed" }); - NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "accountActivation", "passwordReset", "emailVerification", "emailChangeNotice" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "account_activation", "password_reset", "email_verification", "email_change_notice" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "password_encryption_type", new[] { "pbkdf2", "bcrypt_enhanced" }); diff --git a/Common/OpenShockDb/EmailOutboxMessage.cs b/Common/OpenShockDb/EmailOutboxMessage.cs index 8d1413ba..0fae6f64 100644 --- a/Common/OpenShockDb/EmailOutboxMessage.cs +++ b/Common/OpenShockDb/EmailOutboxMessage.cs @@ -16,7 +16,7 @@ namespace OpenShock.Common.OpenShockDb; /// consumer claims the row, renders the body, mints a fresh secret for token-bearing types, hands /// the message to the email provider, and records the outcome. Delivery is therefore decoupled from /// the HTTP request: a provider outage, a process restart, or a crash mid-send cannot lose the -/// email — the row simply stays and is retried. +/// email - the row simply stays and is retried. /// /// /// @@ -36,7 +36,7 @@ namespace OpenShock.Common.OpenShockDb; /// Single source of truth. A job framework's record is "a method was scheduled and ran"; /// this row is the domain fact "this user is owed this email, here is its delivery state". When /// a user reports a missing email, this table answers it directly (who, which type, how many -/// attempts, last error, sent/failed) — no second system to correlate against. +/// attempts, last error, sent/failed) - no second system to correlate against. /// /// /// No new infrastructure or operational surface. Reliability rides on PostgreSQL, which @@ -50,7 +50,7 @@ namespace OpenShock.Common.OpenShockDb; /// /// /// -/// This is not a claim that job frameworks are bad — for recurring/scheduled maintenance work they +/// This is not a claim that job frameworks are bad - for recurring/scheduled maintenance work they /// are the right tool, and the codebase uses one for cron jobs. It is the narrower point that /// transactional, must-not-be-lost, resendable email is an outbox-shaped problem, and modelling /// it as a first-class table is the simplest design that satisfies those requirements. @@ -61,13 +61,14 @@ namespace OpenShock.Common.OpenShockDb; /// link cannot be reconstructed from the stored hash, "store the intent, mint on send" is the only /// model that keeps the queue free of usable secrets while still allowing a resend. The consumer /// generates the secret at send time and writes only its hash to the request row (see -/// etc., now nullable until first send). A resend produces -/// a brand-new token — which is exactly the desired behaviour for security links anyway. +/// etc.; the row is created with a seeded throwaway hash +/// that the consumer overwrites on first send). A resend produces a brand-new token, which is +/// exactly the desired behaviour for security links anyway. /// /// /// -/// Delivery guarantee. This is at-least-once. The unavoidable edge case — the provider accepts -/// the message but the process dies before the row is marked — results +/// Delivery guarantee. This is at-least-once. The unavoidable edge case - the provider accepts +/// the message but the process dies before the row is marked - results /// in one duplicate send on retry. For these email types that is harmless (a second activation / /// reset / notice), and it is the correct trade against the alternative of silently losing mail. /// @@ -88,7 +89,8 @@ public sealed class EmailOutboxMessage /// /// Dynamic, non-secret parameters needed to render and send, stored as Postgres jsonb - /// (via a value converter). An open key/value bag keyed by the well-known names in + /// (Npgsql maps the dictionary to jsonb via dynamic JSON; see EnableDynamicJson in the + /// context configuration). An open key/value bag keyed by the well-known names in /// , so new email types need no schema change. Never contains /// a rendered body or a usable secret. /// diff --git a/Common/OpenShockDb/EmailOutboxPayloadKeys.cs b/Common/OpenShockDb/EmailOutboxPayloadKeys.cs index 121eed8b..3e189664 100644 --- a/Common/OpenShockDb/EmailOutboxPayloadKeys.cs +++ b/Common/OpenShockDb/EmailOutboxPayloadKeys.cs @@ -4,7 +4,7 @@ namespace OpenShock.Common.OpenShockDb; /// Well-known keys for the dynamic dictionary. The payload /// is intentionally an open key/value bag (stored as Postgres jsonb) so new email types can /// carry whatever references they need without a schema change; these constants just keep the -/// well-known keys typo-safe. Values are always non-secret references (ids / addresses) — never a +/// well-known keys typo-safe. Values are always non-secret references (ids / addresses) - never a /// rendered body and never a usable token. /// public static class EmailOutboxPayloadKeys diff --git a/Common/OpenShockDb/OpenShockContext.cs b/Common/OpenShockDb/OpenShockContext.cs index 9c752d82..2638ffdc 100644 --- a/Common/OpenShockDb/OpenShockContext.cs +++ b/Common/OpenShockDb/OpenShockContext.cs @@ -63,6 +63,10 @@ public static void ConfigureOptionsBuilder(DbContextOptionsBuilder optionsBuilde { optionsBuilder.UseNpgsql(connectionString, npgsqlBuilder => { + // Required so Npgsql can serialize the email outbox's Dictionary payload + // to its jsonb column. Without dynamic JSON, Npgsql maps Dictionary to + // hstore by default and writing it to a jsonb column fails at runtime. + npgsqlBuilder.ConfigureDataSource(dataSourceBuilder => dataSourceBuilder.EnableDynamicJson()); npgsqlBuilder.MapEnum(); npgsqlBuilder.MapEnum(); npgsqlBuilder.MapEnum(); @@ -155,7 +159,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasPostgresEnum("shocker_model_type", ["caiXianlin", "petTrainer", "petrainer998DR", "wellturnT330"]) .HasPostgresEnum("match_type_enum", ["exact", "contains"]) .HasPostgresEnum("configuration_value_type", ["string", "bool", "int", "float", "json"]) - .HasPostgresEnum("email_type", ["accountActivation", "passwordReset", "emailVerification", "emailChangeNotice"]) + .HasPostgresEnum("email_type", ["account_activation", "password_reset", "email_verification", "email_change_notice"]) .HasPostgresEnum("email_status", ["sending", "sent", "failed"]) .HasCollation("public", "ndcoll", "und-u-ks-level2", "icu", false); // Add case-insensitive, accent-sensitive comparison collation diff --git a/Common/OpenShockDb/UserEmailChange.cs b/Common/OpenShockDb/UserEmailChange.cs index 826a34a5..57817ff7 100644 --- a/Common/OpenShockDb/UserEmailChange.cs +++ b/Common/OpenShockDb/UserEmailChange.cs @@ -33,7 +33,7 @@ public sealed class UserEmailChange public required string NewEmail { get; set; } /// - /// Hash of the secret token sent to . The plaintext token is never stored — + /// Hash of the secret token sent to . The plaintext token is never stored - /// it only exists in the verification email and in the request the user submits. Seeded when the /// request is created and re-minted by the email outbox consumer on every (re)send, so the queue /// never holds a usable verification link. See . diff --git a/Common/OpenShockDb/UserPasswordReset.cs b/Common/OpenShockDb/UserPasswordReset.cs index d80f12b3..bedcebb9 100644 --- a/Common/OpenShockDb/UserPasswordReset.cs +++ b/Common/OpenShockDb/UserPasswordReset.cs @@ -20,7 +20,7 @@ public sealed class UserPasswordReset public required Guid UserId { get; set; } /// - /// Hash of the secret token for the reset link. The plaintext token is never stored — it only + /// Hash of the secret token for the reset link. The plaintext token is never stored - it only /// exists in the email link and in the request the user submits. Seeded when the request is /// created and re-minted by the email outbox consumer on every (re)send, so the queue never /// holds a usable reset link and a resend always supersedes earlier links. See . From 01a35afdc57fee7f94871721b17e1a07c15b21e1 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 14:12:11 +0200 Subject: [PATCH 03/15] refactor(email): drive outbox delivery with Hangfire instead of a custom worker Move email sending out of the API host and into the Cron host, and hand the delivery machinery (retry back-off, scheduling, crash recovery, dashboard) to Hangfire rather than the bespoke worker. - Delete EmailOutboxWorker (polling loop, FOR UPDATE SKIP LOCKED claim, lease) and EmailOutboxRetryPolicy (+ test); sending is now a per-email Hangfire job EmailOutboxJob.SendAsync with [AutomaticRetry]. Transient failure throws so Hangfire retries; the job marks the row Failed on its last attempt (detected via PerformContext.RetryCount) so an exhausted row never stays stuck Queued. - Keep the transactional outbox: the API writes the Pending row in the business transaction and publishes the Redis nudge, nothing more. - Move the whole email stack (providers, templates, dispatcher, options, .liquid files) API -> Cron so no ASP.NET/DI logic or MailKit/Fluid lands in Common. - New coalescing EmailOutboxConsumer in Cron: Redis push + 1-minute safety sweep share one wake signal, claim Pending rows, and enqueue Hangfire jobs. - EmailStatus -> Pending, Queued, Sent, Failed; drop next_attempt_at, attempt_started_at, attempt_count (Hangfire owns retry state). Migration updated in place; anti-Hangfire docs rewritten for the hybrid design. --- .../Tests/EmailOutboxRetryPolicyTests.cs | 129 ------------ API/API.csproj | 11 - API/Program.cs | 2 - .../Email/Outbox/EmailOutboxRetryPolicy.cs | 90 --------- .../Email/Outbox/EmailOutboxWorker.cs | 189 ------------------ .../OpenShockDb/EmailOutboxMessageTests.cs | 5 +- Common/Constants/Constants.cs | 17 +- .../20260630103404_AddEmailOutbox.Designer.cs | 18 +- .../20260630103404_AddEmailOutbox.cs | 16 +- .../OpenShockContextModelSnapshot.cs | 18 +- Common/Models/EmailStatus.cs | 24 ++- Common/OpenShockDb/EmailOutboxMessage.cs | 75 ++----- Common/OpenShockDb/OpenShockContext.cs | 14 +- Cron/Cron.csproj | 11 + {API => Cron}/Options/MailJetOptions.cs | 2 +- {API => Cron}/Options/MailOptions.cs | 4 +- {API => Cron}/Options/SmtpOptions.cs | 2 +- Cron/Program.cs | 6 + .../Services/Email/EmailSendResult.cs | 2 +- .../Services/Email/EmailServiceExtension.cs | 15 +- .../Services/Email/EmailServiceTemplates.cs | 2 +- .../Services/Email/EmailServiceUtils.cs | 4 +- {API => Cron}/Services/Email/EmailTemplate.cs | 4 +- {API => Cron}/Services/Email/IEmailService.cs | 4 +- .../Services/Email/Mailjet/Mail/Contact.cs | 2 +- .../Services/Email/Mailjet/Mail/DirectMail.cs | 2 +- .../Services/Email/Mailjet/Mail/MailsWrap.cs | 2 +- .../Email/Mailjet/MailjetEmailService.cs | 6 +- .../Mailjet/MailjetEmailServiceExtension.cs | 4 +- .../Services/Email/NoneEmailService.cs | 4 +- .../Email/Outbox/EmailDispatchResult.cs | 2 +- .../Email/Outbox/EmailOutboxConsumer.cs | 159 +++++++++++++++ .../Email/Outbox/EmailOutboxDispatcher.cs | 4 +- Cron/Services/Email/Outbox/EmailOutboxJob.cs | 120 +++++++++++ .../Email/Outbox/IEmailOutboxDispatcher.cs | 2 +- .../Services/Email/Smtp/SmtpEmailService.cs | 6 +- .../Email/Smtp/SmtpEmailServiceExtension.cs | 4 +- .../SmtpTemplates/AccountActivation.liquid | 0 .../SmtpTemplates/EmailChangeNotice.liquid | 0 .../SmtpTemplates/EmailVerification.liquid | 0 .../SmtpTemplates/PasswordReset.liquid | 0 41 files changed, 389 insertions(+), 592 deletions(-) delete mode 100644 API.IntegrationTests/Tests/EmailOutboxRetryPolicyTests.cs delete mode 100644 API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs delete mode 100644 API/Services/Email/Outbox/EmailOutboxWorker.cs rename {API => Cron}/Options/MailJetOptions.cs (93%) rename {API => Cron}/Options/MailOptions.cs (83%) rename {API => Cron}/Options/SmtpOptions.cs (95%) rename {API => Cron}/Services/Email/EmailSendResult.cs (94%) rename {API => Cron}/Services/Email/EmailServiceExtension.cs (85%) rename {API => Cron}/Services/Email/EmailServiceTemplates.cs (88%) rename {API => Cron}/Services/Email/EmailServiceUtils.cs (78%) rename {API => Cron}/Services/Email/EmailTemplate.cs (95%) rename {API => Cron}/Services/Email/IEmailService.cs (96%) rename {API => Cron}/Services/Email/Mailjet/Mail/Contact.cs (91%) rename {API => Cron}/Services/Email/Mailjet/Mail/DirectMail.cs (79%) rename {API => Cron}/Services/Email/Mailjet/Mail/MailsWrap.cs (61%) rename {API => Cron}/Services/Email/Mailjet/MailjetEmailService.cs (96%) rename {API => Cron}/Services/Email/Mailjet/MailjetEmailServiceExtension.cs (93%) rename {API => Cron}/Services/Email/NoneEmailService.cs (95%) rename {API => Cron}/Services/Email/Outbox/EmailDispatchResult.cs (97%) create mode 100644 Cron/Services/Email/Outbox/EmailOutboxConsumer.cs rename {API => Cron}/Services/Email/Outbox/EmailOutboxDispatcher.cs (98%) create mode 100644 Cron/Services/Email/Outbox/EmailOutboxJob.cs rename {API => Cron}/Services/Email/Outbox/IEmailOutboxDispatcher.cs (96%) rename {API => Cron}/Services/Email/Smtp/SmtpEmailService.cs (97%) rename {API => Cron}/Services/Email/Smtp/SmtpEmailServiceExtension.cs (89%) rename {API => Cron}/SmtpTemplates/AccountActivation.liquid (100%) rename {API => Cron}/SmtpTemplates/EmailChangeNotice.liquid (100%) rename {API => Cron}/SmtpTemplates/EmailVerification.liquid (100%) rename {API => Cron}/SmtpTemplates/PasswordReset.liquid (100%) diff --git a/API.IntegrationTests/Tests/EmailOutboxRetryPolicyTests.cs b/API.IntegrationTests/Tests/EmailOutboxRetryPolicyTests.cs deleted file mode 100644 index 5b5633c7..00000000 --- a/API.IntegrationTests/Tests/EmailOutboxRetryPolicyTests.cs +++ /dev/null @@ -1,129 +0,0 @@ -using OpenShock.API.Services.Email.Outbox; -using OpenShock.Common.Constants; -using OpenShock.Common.Models; -using OpenShock.Common.OpenShockDb; - -namespace OpenShock.API.IntegrationTests.Tests; - -/// -/// Pure unit tests for the email outbox retry/state machine. These don't touch the database or the -/// test host (no WebApplicationFactory data source), so they run without Docker. -/// -public class EmailOutboxRetryPolicyTests -{ - private static EmailOutboxMessage NewMessage(int attemptCount) - { - var message = EmailOutboxMessage.Create(EmailType.PasswordReset, "user@example.com", "User", - new Dictionary()); - message.AttemptCount = attemptCount; - return message; - } - - [Test] - public async Task GetBaseDelay_GrowsExponentiallyFromBaseDelay() - { - await Assert.That(EmailOutboxRetryPolicy.GetBaseDelay(1)).IsEqualTo(Duration.EmailOutboxRetryBaseDelay); - await Assert.That(EmailOutboxRetryPolicy.GetBaseDelay(2)).IsEqualTo(Duration.EmailOutboxRetryBaseDelay * 2); - await Assert.That(EmailOutboxRetryPolicy.GetBaseDelay(3)).IsEqualTo(Duration.EmailOutboxRetryBaseDelay * 4); - } - - [Test] - public async Task GetBaseDelay_IsCappedAtMaxDelay() - { - await Assert.That(EmailOutboxRetryPolicy.GetBaseDelay(1000)).IsEqualTo(Duration.EmailOutboxRetryMaxDelay); - } - - [Test] - public async Task GetRetryDelay_StaysWithinJitterBounds() - { - var baseDelay = EmailOutboxRetryPolicy.GetBaseDelay(2); - var upper = baseDelay + TimeSpan.FromSeconds(EmailOutboxRetryPolicy.MaxJitterSeconds); - - for (var i = 0; i < 200; i++) - { - var delay = EmailOutboxRetryPolicy.GetRetryDelay(2); - await Assert.That(delay).IsGreaterThanOrEqualTo(baseDelay); - await Assert.That(delay).IsLessThanOrEqualTo(upper); - } - } - - [Test] - public async Task Apply_Sent_MarksSentAndClearsState() - { - var message = NewMessage(1); - var now = DateTime.UtcNow; - - EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Sent, now); - - await Assert.That(message.Status).IsEqualTo(EmailStatus.Sent); - await Assert.That(message.SentAt).IsEqualTo(now); - await Assert.That(message.AttemptStartedAt).IsNull(); - await Assert.That(message.LastError).IsNull(); - } - - [Test] - public async Task Apply_Skipped_MarksFailedWithReason() - { - var message = NewMessage(1); - - EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Skip("password reset expired"), DateTime.UtcNow); - - await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); - await Assert.That(message.FailedAt).IsNotNull(); - await Assert.That(message.LastError!).Contains("password reset expired"); - } - - [Test] - public async Task Apply_PermanentFailure_MarksFailed() - { - var message = NewMessage(1); - - EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Permanent("rejected recipient"), DateTime.UtcNow); - - await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); - await Assert.That(message.FailedAt).IsNotNull(); - await Assert.That(message.LastError).IsEqualTo("rejected recipient"); - } - - [Test] - public async Task Apply_TransientFailure_BelowMaxAttempts_SchedulesRetry() - { - var message = NewMessage(1); - var now = DateTime.UtcNow; - - EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Transient("smtp timeout"), now); - - await Assert.That(message.Status).IsEqualTo(EmailStatus.Sending); - await Assert.That(message.NextAttemptAt).IsNotNull(); - await Assert.That(message.NextAttemptAt!.Value).IsGreaterThan(now); - await Assert.That(message.AttemptStartedAt).IsNull(); - await Assert.That(message.LastError).IsEqualTo("smtp timeout"); - } - - [Test] - public async Task Apply_TransientFailure_AtMaxAttempts_MarksFailed() - { - var message = NewMessage(EmailOutboxRetryPolicy.MaxAttempts); - - EmailOutboxRetryPolicy.Apply(message, EmailDispatchResult.Transient("smtp timeout"), DateTime.UtcNow); - - await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); - await Assert.That(message.FailedAt).IsNotNull(); - } - - [Test] - public async Task Truncate_LimitsToMaxLength() - { - var longText = new string('x', HardLimits.EmailOutboxLastErrorMaxLength + 50); - - var result = EmailOutboxRetryPolicy.Truncate(longText); - - await Assert.That(result!.Length).IsEqualTo(HardLimits.EmailOutboxLastErrorMaxLength); - } - - [Test] - public async Task Truncate_Null_ReturnsNull() - { - await Assert.That(EmailOutboxRetryPolicy.Truncate(null)).IsNull(); - } -} diff --git a/API/API.csproj b/API/API.csproj index c61e25fb..6b49e1ac 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -12,8 +12,6 @@ - - @@ -24,15 +22,6 @@ - - - - PreserveNewest - PreserveNewest - - - diff --git a/API/Program.cs b/API/Program.cs index 8a92776f..83719d63 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -4,7 +4,6 @@ using OpenShock.API.Realtime; using OpenShock.API.Services.Account; using OpenShock.API.Services.DeviceUpdate; -using OpenShock.API.Services.Email; using OpenShock.API.Services.LCGNodeProvisioner; using OpenShock.API.Services.OAuthConnection; using OpenShock.API.Services.Token; @@ -109,7 +108,6 @@ static void DefaultOptions(RemoteAuthenticationOptions options, string provider) builder.AddSwaggerExt(); builder.AddCloudflareTurnstileService(); -await builder.AddEmailService(); //services.AddHealthChecks().AddCheck("database"); diff --git a/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs b/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs deleted file mode 100644 index 33c4743d..00000000 --- a/API/Services/Email/Outbox/EmailOutboxRetryPolicy.cs +++ /dev/null @@ -1,90 +0,0 @@ -using OpenShock.Common.Constants; -using OpenShock.Common.Models; -using OpenShock.Common.OpenShockDb; - -namespace OpenShock.API.Services.Email.Outbox; - -/// -/// Pure retry/state-machine logic for the email outbox, factored out of the consumer so it can be -/// unit-tested without a database or a running host. -/// -public static class EmailOutboxRetryPolicy -{ - /// Maximum number of attempts before a transiently-failing message is given up as failed. - public const int MaxAttempts = 10; - - /// Upper bound (seconds) of the random jitter added to each retry delay. - public const int MaxJitterSeconds = 30; - - /// - /// Deterministic part of the back-off: exponential in the attempt count, capped at - /// . Exposed (without jitter) for testing. - /// - public static TimeSpan GetBaseDelay(int attemptCount) - { - // Compute in seconds (double) and cap before converting, so a large attempt count can't - // overflow TimeSpan - Math.Pow can reach infinity, which simply compares above the cap. - var maxDelay = Duration.EmailOutboxRetryMaxDelay; - var seconds = Duration.EmailOutboxRetryBaseDelay.TotalSeconds * Math.Pow(2, Math.Max(0, attemptCount - 1)); - return seconds >= maxDelay.TotalSeconds ? maxDelay : TimeSpan.FromSeconds(seconds); - } - - /// Back-off delay for the next retry: plus a random jitter. - public static TimeSpan GetRetryDelay(int attemptCount) - => GetBaseDelay(attemptCount) + TimeSpan.FromSeconds(Random.Shared.Next(0, MaxJitterSeconds + 1)); - - /// - /// Applies a dispatch outcome to a message, advancing it to its next state: delivered, failed - /// (permanent / skipped / exhausted), or scheduled for another retry. Pure - only mutates - /// . - /// - public static void Apply(EmailOutboxMessage message, EmailDispatchResult result, DateTime nowUtc) - { - switch (result.Outcome) - { - case EmailDispatchOutcome.Sent: - message.Status = EmailStatus.Sent; - message.SentAt = nowUtc; - message.AttemptStartedAt = null; - message.LastError = null; - break; - - case EmailDispatchOutcome.Skipped: - message.Status = EmailStatus.Failed; - message.FailedAt = nowUtc; - message.AttemptStartedAt = null; - message.LastError = Truncate($"Skipped: {result.Detail}"); - break; - - case EmailDispatchOutcome.PermanentFailure: - message.Status = EmailStatus.Failed; - message.FailedAt = nowUtc; - message.AttemptStartedAt = null; - message.LastError = Truncate(result.Detail); - break; - - case EmailDispatchOutcome.TransientFailure: - message.LastError = Truncate(result.Detail); - message.AttemptStartedAt = null; - if (message.AttemptCount >= MaxAttempts) - { - message.Status = EmailStatus.Failed; - message.FailedAt = nowUtc; - } - else - { - message.NextAttemptAt = nowUtc + GetRetryDelay(message.AttemptCount); - } - break; - } - } - - /// Clamps an error string to the column's maximum length. - public static string? Truncate(string? value) - { - if (value is null) return null; - return value.Length <= HardLimits.EmailOutboxLastErrorMaxLength - ? value - : value[..HardLimits.EmailOutboxLastErrorMaxLength]; - } -} diff --git a/API/Services/Email/Outbox/EmailOutboxWorker.cs b/API/Services/Email/Outbox/EmailOutboxWorker.cs deleted file mode 100644 index ce55042a..00000000 --- a/API/Services/Email/Outbox/EmailOutboxWorker.cs +++ /dev/null @@ -1,189 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using OpenShock.Common.Constants; -using OpenShock.Common.Models; -using OpenShock.Common.OpenShockDb; -using OpenShock.Common.Services.RedisPubSub; -using StackExchange.Redis; - -namespace OpenShock.API.Services.Email.Outbox; - -/// -/// The email outbox consumer: the single component that delivers enqueued -/// rows, retrying until they succeed or are exhausted. -/// -/// -/// -/// Push first, poll as a safety net. Delivery is triggered by a Redis notification published -/// the instant a message is enqueued (), so first-send -/// latency is sub-second rather than tied to a polling interval. A periodic tick -/// () still runs as a fallback: it picks up due retries -/// (whose NextAttemptAt is in the future), re-claims leases abandoned by a crashed instance, -/// and covers the rare case of a dropped notification. The notification can therefore be best-effort -/// - losing one only delays delivery to the next tick, it can never lose the email. -/// -/// -/// Runs in every API instance, safely. Rows are claimed with FOR UPDATE SKIP LOCKED, so -/// when several instances react to the same notification each grabs a disjoint set of rows and they -/// drain in parallel without ever sending the same message twice in a batch. Claiming also stamps a -/// lease (AttemptStartedAt); a row whose lease is older than -/// is considered abandoned and re-claimed, which is how -/// a send interrupted by a crash recovers. -/// -/// -/// This lives in the API host rather than the cron/Hangfire host on purpose: it needs -/// (which the API host owns) and sub-second latency, which a -/// minute-granularity scheduler cannot provide. Periodic maintenance of this table (pruning -/// old delivered rows) is the appropriate cron job, and is separate from sending. -/// -/// -public sealed class EmailOutboxWorker : BackgroundService -{ - private const int BatchSize = 20; - - private readonly IDbContextFactory _dbContextFactory; - private readonly IEmailOutboxDispatcher _dispatcher; - private readonly ISubscriber _subscriber; - private readonly ILogger _logger; - - // Coalescing wake signal: capacity 1, so any number of notifications between drains collapse into - // a single "there is work" wake-up. - private readonly SemaphoreSlim _wakeSignal = new(0, 1); - - /// DI constructor. - public EmailOutboxWorker( - IDbContextFactory dbContextFactory, - IEmailOutboxDispatcher dispatcher, - IConnectionMultiplexer connectionMultiplexer, - ILogger logger) - { - _dbContextFactory = dbContextFactory; - _dispatcher = dispatcher; - _subscriber = connectionMultiplexer.GetSubscriber(); - _logger = logger; - } - - /// - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - await _subscriber.SubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); - - try - { - while (!stoppingToken.IsCancellationRequested) - { - try - { - await DrainAsync(stoppingToken); - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) - { - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Email outbox drain failed"); - } - - // Wait for a push notification or the fallback poll interval, whichever comes first. - try - { - await _wakeSignal.WaitAsync(Duration.EmailOutboxPollInterval, stoppingToken); - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) - { - break; - } - } - } - finally - { - await _subscriber.UnsubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); - } - } - - private void OnPendingNotification(RedisChannel channel, RedisValue value) - { - // Release the coalescing signal; ignore if already signalled. - try - { - _wakeSignal.Release(); - } - catch (SemaphoreFullException) - { - // Already pending a wake-up - nothing to do. - } - } - - private async Task DrainAsync(CancellationToken cancellationToken) - { - int processed; - do - { - processed = await ProcessBatchAsync(cancellationToken); - } - while (processed >= BatchSize && !cancellationToken.IsCancellationRequested); - } - - private async Task ProcessBatchAsync(CancellationToken cancellationToken) - { - await using var db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - - var nowUtc = DateTime.UtcNow; - var leaseCutoff = nowUtc - Duration.EmailOutboxLeaseTimeout; - - // Atomically claim a batch of due rows. FOR UPDATE SKIP LOCKED lets concurrent instances take - // disjoint sets; stamping the lease + bumping the attempt count is what reserves them. - List claimed; - await using (var transaction = await db.Database.BeginTransactionAsync(cancellationToken)) - { - claimed = await db.EmailOutbox.FromSql( - $""" - SELECT * FROM email_outbox - WHERE status = {EmailStatus.Sending} - AND (next_attempt_at IS NULL OR next_attempt_at <= {nowUtc}) - AND (attempt_started_at IS NULL OR attempt_started_at < {leaseCutoff}) - ORDER BY created_at - LIMIT {BatchSize} - FOR UPDATE SKIP LOCKED - """).ToListAsync(cancellationToken); - - if (claimed.Count == 0) - { - await transaction.CommitAsync(cancellationToken); - return 0; - } - - foreach (var message in claimed) - { - message.AttemptStartedAt = nowUtc; - message.AttemptCount++; - } - - await db.SaveChangesAsync(cancellationToken); - await transaction.CommitAsync(cancellationToken); - } - - // Send outside the claim transaction so a slow provider never holds row locks. Each message is - // marked durably right after its attempt, so a crash mid-batch only re-sends the ones whose - // lease has not yet been resolved. - foreach (var message in claimed) - { - var result = await _dispatcher.SendAsync(message, db, cancellationToken); - EmailOutboxRetryPolicy.Apply(message, result, DateTime.UtcNow); - await db.SaveChangesAsync(cancellationToken); - - if (message.Status == EmailStatus.Failed) - _logger.LogWarning("Email outbox message {MessageId} ({Type}) failed after {Attempts} attempt(s): {Detail}", - message.Id, message.Type, message.AttemptCount, result.Detail); - } - - return claimed.Count; - } - - /// - public override void Dispose() - { - _wakeSignal.Dispose(); - base.Dispose(); - } -} diff --git a/Common.Tests/OpenShockDb/EmailOutboxMessageTests.cs b/Common.Tests/OpenShockDb/EmailOutboxMessageTests.cs index e164fefc..0a2a6b9c 100644 --- a/Common.Tests/OpenShockDb/EmailOutboxMessageTests.cs +++ b/Common.Tests/OpenShockDb/EmailOutboxMessageTests.cs @@ -6,7 +6,7 @@ namespace OpenShock.Common.Tests.OpenShockDb; public class EmailOutboxMessageTests { [Test] - public async Task Create_InitializesAsSendingWithGivenFields() + public async Task Create_InitializesAsPendingWithGivenFields() { var payload = new Dictionary { @@ -15,11 +15,10 @@ public async Task Create_InitializesAsSendingWithGivenFields() var message = EmailOutboxMessage.Create(EmailType.PasswordReset, "user@example.com", "User", payload); - await Assert.That(message.Status).IsEqualTo(EmailStatus.Sending); + await Assert.That(message.Status).IsEqualTo(EmailStatus.Pending); await Assert.That(message.Type).IsEqualTo(EmailType.PasswordReset); await Assert.That(message.Recipient).IsEqualTo("user@example.com"); await Assert.That(message.RecipientName).IsEqualTo("User"); - await Assert.That(message.AttemptCount).IsEqualTo(0); await Assert.That(message.SentAt).IsNull(); await Assert.That(message.FailedAt).IsNull(); await Assert.That(message.Id).IsNotEqualTo(Guid.Empty); diff --git a/Common/Constants/Constants.cs b/Common/Constants/Constants.cs index bbafc408..3773eb1c 100644 --- a/Common/Constants/Constants.cs +++ b/Common/Constants/Constants.cs @@ -21,17 +21,8 @@ public static class Duration public static readonly TimeSpan DeactivatedAccountRetentionTime = TimeSpan.FromDays(14); - /// How long a claimed email-outbox row may be in flight before it is treated as - /// abandoned (worker crashed mid-send) and eligible to be re-claimed. - public static readonly TimeSpan EmailOutboxLeaseTimeout = TimeSpan.FromMinutes(5); - - /// Safety-net poll interval for the email-outbox consumer. Normal delivery is push-driven - /// (Redis notification on enqueue); this only catches due retries and any missed notification. - public static readonly TimeSpan EmailOutboxPollInterval = TimeSpan.FromSeconds(15); - - /// Base delay for the first email-outbox retry; doubles each subsequent attempt. - public static readonly TimeSpan EmailOutboxRetryBaseDelay = TimeSpan.FromSeconds(30); - - /// Upper bound on the exponential email-outbox retry back-off. - public static readonly TimeSpan EmailOutboxRetryMaxDelay = TimeSpan.FromHours(2); + /// Safety-net sweep interval for the email-outbox consumer. Normal hand-off is push-driven + /// (Redis notification on enqueue); this only catches a missed notification. Hangfire owns retry + /// timing for messages already handed off, so this need not be tight. + public static readonly TimeSpan EmailOutboxPollInterval = TimeSpan.FromMinutes(1); } diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs index ab6bb127..1ec182e7 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs +++ b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs @@ -30,7 +30,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "configuration_value_type", new[] { "string", "bool", "int", "float", "json" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "sound", "vibrate", "shock", "stop" }); - NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "sending", "sent", "failed" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "pending", "queued", "sent", "failed" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "account_activation", "password_reset", "email_verification", "email_change_notice" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); @@ -437,14 +437,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("uuid") .HasColumnName("id"); - b.Property("AttemptCount") - .HasColumnType("integer") - .HasColumnName("attempt_count"); - - b.Property("AttemptStartedAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("attempt_started_at"); - b.Property("CreatedAt") .ValueGeneratedOnAdd() .HasColumnType("timestamp with time zone") @@ -460,10 +452,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("character varying(1024)") .HasColumnName("last_error"); - b.Property("NextAttemptAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("next_attempt_at"); - b.Property>("Payload") .IsRequired() .HasColumnType("jsonb") @@ -495,11 +483,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasKey("Id") .HasName("email_outbox_pkey"); - b.HasIndex("CreatedAt"); - b.HasIndex("Recipient"); - b.HasIndex("Status", "NextAttemptAt"); + b.HasIndex("Status", "CreatedAt"); b.ToTable("email_outbox", (string)null); }); diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.cs b/Common/Migrations/20260630103404_AddEmailOutbox.cs index 3e88326f..b2a67ff8 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.cs +++ b/Common/Migrations/20260630103404_AddEmailOutbox.cs @@ -18,7 +18,7 @@ protected override void Up(MigrationBuilder migrationBuilder) .Annotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") .Annotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") .Annotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") - .Annotation("Npgsql:Enum:email_status", "sending,sent,failed") + .Annotation("Npgsql:Enum:email_status", "pending,queued,sent,failed") .Annotation("Npgsql:Enum:email_type", "account_activation,password_reset,email_verification,email_change_notice") .Annotation("Npgsql:Enum:match_type_enum", "exact,contains") .Annotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") @@ -47,9 +47,6 @@ protected override void Up(MigrationBuilder migrationBuilder) recipient_name = table.Column(type: "character varying(32)", maxLength: 32, nullable: true), payload = table.Column>(type: "jsonb", nullable: false), status = table.Column(type: "email_status", nullable: false), - attempt_count = table.Column(type: "integer", nullable: false), - next_attempt_at = table.Column(type: "timestamp with time zone", nullable: true), - attempt_started_at = table.Column(type: "timestamp with time zone", nullable: true), last_error = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: true), created_at = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "CURRENT_TIMESTAMP"), sent_at = table.Column(type: "timestamp with time zone", nullable: true), @@ -60,20 +57,15 @@ protected override void Up(MigrationBuilder migrationBuilder) table.PrimaryKey("email_outbox_pkey", x => x.id); }); - migrationBuilder.CreateIndex( - name: "IX_email_outbox_created_at", - table: "email_outbox", - column: "created_at"); - migrationBuilder.CreateIndex( name: "IX_email_outbox_recipient", table: "email_outbox", column: "recipient"); migrationBuilder.CreateIndex( - name: "IX_email_outbox_status_next_attempt_at", + name: "IX_email_outbox_status_created_at", table: "email_outbox", - columns: new[] { "status", "next_attempt_at" }); + columns: new[] { "status", "created_at" }); } /// @@ -97,7 +89,7 @@ protected override void Down(MigrationBuilder migrationBuilder) .OldAnnotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") .OldAnnotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") .OldAnnotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") - .OldAnnotation("Npgsql:Enum:email_status", "sending,sent,failed") + .OldAnnotation("Npgsql:Enum:email_status", "pending,queued,sent,failed") .OldAnnotation("Npgsql:Enum:email_type", "account_activation,password_reset,email_verification,email_change_notice") .OldAnnotation("Npgsql:Enum:match_type_enum", "exact,contains") .OldAnnotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") diff --git a/Common/Migrations/OpenShockContextModelSnapshot.cs b/Common/Migrations/OpenShockContextModelSnapshot.cs index a74ca0f7..c9b15626 100644 --- a/Common/Migrations/OpenShockContextModelSnapshot.cs +++ b/Common/Migrations/OpenShockContextModelSnapshot.cs @@ -27,7 +27,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "configuration_value_type", new[] { "string", "bool", "int", "float", "json" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "sound", "vibrate", "shock", "stop" }); - NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "sending", "sent", "failed" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "pending", "queued", "sent", "failed" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "account_activation", "password_reset", "email_verification", "email_change_notice" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); @@ -434,14 +434,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("uuid") .HasColumnName("id"); - b.Property("AttemptCount") - .HasColumnType("integer") - .HasColumnName("attempt_count"); - - b.Property("AttemptStartedAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("attempt_started_at"); - b.Property("CreatedAt") .ValueGeneratedOnAdd() .HasColumnType("timestamp with time zone") @@ -457,10 +449,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("character varying(1024)") .HasColumnName("last_error"); - b.Property("NextAttemptAt") - .HasColumnType("timestamp with time zone") - .HasColumnName("next_attempt_at"); - b.Property>("Payload") .IsRequired() .HasColumnType("jsonb") @@ -492,11 +480,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id") .HasName("email_outbox_pkey"); - b.HasIndex("CreatedAt"); - b.HasIndex("Recipient"); - b.HasIndex("Status", "NextAttemptAt"); + b.HasIndex("Status", "CreatedAt"); b.ToTable("email_outbox", (string)null); }); diff --git a/Common/Models/EmailStatus.cs b/Common/Models/EmailStatus.cs index b4fa4a8f..fa599d87 100644 --- a/Common/Models/EmailStatus.cs +++ b/Common/Models/EmailStatus.cs @@ -4,21 +4,25 @@ namespace OpenShock.Common.Models; /// Delivery state of an . /// /// -/// There are deliberately only three states. A separate "pending" state would carry no information -/// here: a message that has been enqueued but not yet delivered is simply one the consumer has not -/// finished sending, which is exactly what means. A row waiting for its next -/// retry is also (with a future NextAttemptAt), it is still in flight, -/// just not this instant. Collapsing "queued", "in progress" and "awaiting retry" into one state -/// keeps the claim query and the state machine trivial. +/// The row is written by the API as in the same transaction as the business +/// change. The Cron consumer claims pending rows and hands each to Hangfire for delivery, flipping it +/// to ; from there Hangfire owns execution, retry scheduling, and crash recovery +/// until the message reaches a terminal or state. /// public enum EmailStatus { /// - /// The message still needs to be delivered. Covers freshly enqueued rows, rows currently being - /// attempted (see AttemptStartedAt), and rows waiting for a retry (see NextAttemptAt). - /// This is the only state the consumer claims from. + /// Freshly enqueued by the API and not yet handed to the sender. This is the only state the Cron + /// consumer claims from. /// - Sending, + Pending, + + /// + /// Handed off to Hangfire for delivery: a send job is enqueued, running, or scheduled for a + /// retry. The consumer never re-claims a row in this state - Hangfire drives it to a terminal + /// state. + /// + Queued, /// The message was handed to the email provider successfully. Terminal. Sent, diff --git a/Common/OpenShockDb/EmailOutboxMessage.cs b/Common/OpenShockDb/EmailOutboxMessage.cs index 0fae6f64..79b7bda2 100644 --- a/Common/OpenShockDb/EmailOutboxMessage.cs +++ b/Common/OpenShockDb/EmailOutboxMessage.cs @@ -11,50 +11,34 @@ namespace OpenShock.Common.OpenShockDb; /// /// /// -/// Flow. The request handler creates this row (and any related request row, e.g. a -/// ) and commits. It does not send. A single background -/// consumer claims the row, renders the body, mints a fresh secret for token-bearing types, hands +/// Flow. The API request handler creates this row (and any related request row, e.g. a +/// ) and commits it as . It does +/// not send. The Cron host's outbox consumer claims pending rows and hands each to Hangfire, +/// which runs the send job: it renders the body, mints a fresh secret for token-bearing types, hands /// the message to the email provider, and records the outcome. Delivery is therefore decoupled from -/// the HTTP request: a provider outage, a process restart, or a crash mid-send cannot lose the -/// email - the row simply stays and is retried. +/// the HTTP request: a provider outage, a process restart, or a crash mid-send cannot lose the email. /// /// /// -/// Why this lives in our database (rather than a job framework such as Hangfire). -/// This is intentionally an outbox, not a generic background-job queue, and the two solve -/// different problems: +/// Why this row exists alongside Hangfire (it is an outbox, not a duplicate job queue). +/// The two layers own different things and are deliberately combined: /// /// /// -/// Transactional integrity. The email intent is committed in the same transaction -/// as the change that caused it (account created, reset requested, email changed). There is no -/// window where the business row commits but the email is lost, or where an email is scheduled -/// for a change that then rolls back. A separate job store cannot give this guarantee without, -/// in effect, reinventing an outbox alongside it. +/// This row owns transactional integrity + the audit fact. The email intent is committed +/// in the same transaction as the change that caused it (account created, reset +/// requested, email changed), so there is no window where the business row commits but the email +/// is lost, nor an email scheduled for a change that then rolls back. The row is also the domain +/// answer to "is this user owed this email, and what is its delivery state" (who, which type, +/// how many attempts, last error, sent/failed) - a single place to look, kept forever. /// /// -/// Single source of truth. A job framework's record is "a method was scheduled and ran"; -/// this row is the domain fact "this user is owed this email, here is its delivery state". When -/// a user reports a missing email, this table answers it directly (who, which type, how many -/// attempts, last error, sent/failed) - no second system to correlate against. -/// -/// -/// No new infrastructure or operational surface. Reliability rides on PostgreSQL, which -/// is already the system's durable store. There is no extra schema owned by a third-party -/// scheduler, no dashboard to secure, and no job-serialization/versioning concerns. -/// -/// -/// Latency. Delivery is push-triggered (a Redis notification on enqueue) with a periodic -/// poll only as a safety net, so first-send is sub-second. A minute-granularity scheduler would -/// not meet that. +/// Hangfire owns the delivery machinery. Once the consumer hands a row off, Hangfire owns +/// durable execution, retry scheduling, crash requeue, and the operator dashboard - the parts a +/// bespoke worker would otherwise have to reinvent. The back-off curve itself is supplied by +/// EmailOutboxRetryPolicy, which the send job feeds into Hangfire's scheduler. /// /// -/// -/// This is not a claim that job frameworks are bad - for recurring/scheduled maintenance work they -/// are the right tool, and the codebase uses one for cron jobs. It is the narrower point that -/// transactional, must-not-be-lost, resendable email is an outbox-shaped problem, and modelling -/// it as a first-class table is the simplest design that satisfies those requirements. -/// /// /// /// Why the related auth flows mint their token lazily. Because a working reset/verification @@ -96,24 +80,9 @@ public sealed class EmailOutboxMessage /// public required Dictionary Payload { get; set; } - /// Delivery state. See . - public EmailStatus Status { get; set; } = EmailStatus.Sending; - - /// Number of send attempts started so far. Incremented when the row is claimed. - public int AttemptCount { get; set; } - - /// - /// Earliest time the next attempt may run. Null means "as soon as possible". Set into the future - /// after a transient failure to implement back-off. - /// - public DateTime? NextAttemptAt { get; set; } - - /// - /// When the current attempt was claimed, acting as a lease. A row whose lease is older than the - /// lease timeout is considered abandoned (the worker crashed mid-send) and may be re-claimed. - /// Cleared whenever the row reaches a resting state (sent, failed, or waiting for retry). - /// - public DateTime? AttemptStartedAt { get; set; } + /// Delivery state. See . Retry count and scheduling are owned by + /// Hangfire once the row is handed off, so they are not tracked here. + public EmailStatus Status { get; set; } = EmailStatus.Pending; /// Last error recorded for a failed/retried attempt. Null while healthy. public string? LastError { get; set; } @@ -128,7 +97,7 @@ public sealed class EmailOutboxMessage public DateTime? FailedAt { get; set; } /// - /// Builds a new enqueued message in the state. The caller adds + /// Builds a new enqueued message in the state. The caller adds /// it to the context and commits it together with the related business change. /// public static EmailOutboxMessage Create(EmailType type, string recipient, string? recipientName, Dictionary payload) @@ -140,7 +109,7 @@ public static EmailOutboxMessage Create(EmailType type, string recipient, string Recipient = recipient, RecipientName = recipientName, Payload = payload, - Status = EmailStatus.Sending + Status = EmailStatus.Pending }; } } diff --git a/Common/OpenShockDb/OpenShockContext.cs b/Common/OpenShockDb/OpenShockContext.cs index 2638ffdc..eba5a91d 100644 --- a/Common/OpenShockDb/OpenShockContext.cs +++ b/Common/OpenShockDb/OpenShockContext.cs @@ -160,7 +160,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasPostgresEnum("match_type_enum", ["exact", "contains"]) .HasPostgresEnum("configuration_value_type", ["string", "bool", "int", "float", "json"]) .HasPostgresEnum("email_type", ["account_activation", "password_reset", "email_verification", "email_change_notice"]) - .HasPostgresEnum("email_status", ["sending", "sent", "failed"]) + .HasPostgresEnum("email_status", ["pending", "queued", "sent", "failed"]) .HasCollation("public", "ndcoll", "und-u-ks-level2", "icu", false); // Add case-insensitive, accent-sensitive comparison collation modelBuilder.Entity(entity => @@ -888,9 +888,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.ToTable("email_outbox"); - // The consumer claims rows by (status, due time); created_at gives a stable FIFO order. - entity.HasIndex(e => new { e.Status, e.NextAttemptAt }); - entity.HasIndex(e => e.CreatedAt); + // The consumer claims fresh (Pending) rows in FIFO order; (status, created_at) serves both + // the status filter and the ordering. Retry scheduling after hand-off is owned by Hangfire. + entity.HasIndex(e => new { e.Status, e.CreatedAt }); entity.HasIndex(e => e.Recipient); entity.Property(e => e.Id) @@ -909,12 +909,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasColumnName("payload"); entity.Property(e => e.Status) .HasColumnName("status"); - entity.Property(e => e.AttemptCount) - .HasColumnName("attempt_count"); - entity.Property(e => e.NextAttemptAt) - .HasColumnName("next_attempt_at"); - entity.Property(e => e.AttemptStartedAt) - .HasColumnName("attempt_started_at"); entity.Property(e => e.LastError) .VarCharWithLength(HardLimits.EmailOutboxLastErrorMaxLength) .HasColumnName("last_error"); diff --git a/Cron/Cron.csproj b/Cron/Cron.csproj index 3334a884..054f49c6 100644 --- a/Cron/Cron.csproj +++ b/Cron/Cron.csproj @@ -5,6 +5,8 @@ + + @@ -16,6 +18,15 @@ + + + + PreserveNewest + PreserveNewest + + + diff --git a/API/Options/MailJetOptions.cs b/Cron/Options/MailJetOptions.cs similarity index 93% rename from API/Options/MailJetOptions.cs rename to Cron/Options/MailJetOptions.cs index 44691434..af098e1c 100644 --- a/API/Options/MailJetOptions.cs +++ b/Cron/Options/MailJetOptions.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Options; using System.ComponentModel.DataAnnotations; -namespace OpenShock.API.Options; +namespace OpenShock.Cron.Options; public sealed class MailJetOptions { diff --git a/API/Options/MailOptions.cs b/Cron/Options/MailOptions.cs similarity index 83% rename from API/Options/MailOptions.cs rename to Cron/Options/MailOptions.cs index b4fd21a5..493b2065 100644 --- a/API/Options/MailOptions.cs +++ b/Cron/Options/MailOptions.cs @@ -1,7 +1,7 @@ using System.ComponentModel.DataAnnotations; -using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Cron.Services.Email.Mailjet.Mail; -namespace OpenShock.API.Options; +namespace OpenShock.Cron.Options; public sealed class MailOptions { diff --git a/API/Options/SmtpOptions.cs b/Cron/Options/SmtpOptions.cs similarity index 95% rename from API/Options/SmtpOptions.cs rename to Cron/Options/SmtpOptions.cs index 73ae1f6f..5c8e3bc3 100644 --- a/API/Options/SmtpOptions.cs +++ b/Cron/Options/SmtpOptions.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Options; using System.ComponentModel.DataAnnotations; -namespace OpenShock.API.Options; +namespace OpenShock.Cron.Options; public sealed class SmtpOptions { diff --git a/Cron/Program.cs b/Cron/Program.cs index 43b5cbb2..2d7d7d98 100644 --- a/Cron/Program.cs +++ b/Cron/Program.cs @@ -3,6 +3,7 @@ using OpenShock.Common; using OpenShock.Common.Extensions; using OpenShock.Cron; +using OpenShock.Cron.Services.Email; using OpenShock.Cron.Utils; using OpenShock.Common.Swagger; @@ -21,6 +22,11 @@ c.UseNpgsqlConnection(databaseOptions.Conn))); builder.Services.AddHangfireServer(); +// Registers the email providers, the outbox dispatcher, the per-email Hangfire send job, and the +// outbox consumer that hands pending rows off to Hangfire. The API host only writes outbox rows; +// all sending happens here. +await builder.AddEmailService(); + builder.AddSwaggerExt(); var app = builder.Build(); diff --git a/API/Services/Email/EmailSendResult.cs b/Cron/Services/Email/EmailSendResult.cs similarity index 94% rename from API/Services/Email/EmailSendResult.cs rename to Cron/Services/Email/EmailSendResult.cs index 53d2e11f..ede4061e 100644 --- a/API/Services/Email/EmailSendResult.cs +++ b/Cron/Services/Email/EmailSendResult.cs @@ -1,4 +1,4 @@ -namespace OpenShock.API.Services.Email; +namespace OpenShock.Cron.Services.Email; /// /// Outcome of a single attempt to hand an email to the provider. Drives the outbox consumer's diff --git a/API/Services/Email/EmailServiceExtension.cs b/Cron/Services/Email/EmailServiceExtension.cs similarity index 85% rename from API/Services/Email/EmailServiceExtension.cs rename to Cron/Services/Email/EmailServiceExtension.cs index bcc92d9e..356e366b 100644 --- a/API/Services/Email/EmailServiceExtension.cs +++ b/Cron/Services/Email/EmailServiceExtension.cs @@ -1,9 +1,9 @@ -using OpenShock.API.Options; -using OpenShock.API.Services.Email.Mailjet; -using OpenShock.API.Services.Email.Outbox; -using OpenShock.API.Services.Email.Smtp; +using OpenShock.Cron.Options; +using OpenShock.Cron.Services.Email.Mailjet; +using OpenShock.Cron.Services.Email.Outbox; +using OpenShock.Cron.Services.Email.Smtp; -namespace OpenShock.API.Services.Email; +namespace OpenShock.Cron.Services.Email; public static class EmailServiceExtension { @@ -12,9 +12,10 @@ public static async Task AddEmailService(this WebApplicat var mailOptions = builder.Configuration.GetRequiredSection(MailOptions.SectionName).Get() ?? throw new NullReferenceException(); // The outbox dispatcher + consumer drive all transactional email regardless of provider; even - // with mail disabled the consumer runs and marks messages terminal (the no-op provider). + // with mail disabled the consumer runs and the send job marks messages terminal (no-op provider). builder.Services.AddSingleton(); - builder.Services.AddHostedService(); + builder.Services.AddScoped(); + builder.Services.AddHostedService(); if (mailOptions.Type == MailOptions.MailType.None) { diff --git a/API/Services/Email/EmailServiceTemplates.cs b/Cron/Services/Email/EmailServiceTemplates.cs similarity index 88% rename from API/Services/Email/EmailServiceTemplates.cs rename to Cron/Services/Email/EmailServiceTemplates.cs index 80b2287b..b84d56ee 100644 --- a/API/Services/Email/EmailServiceTemplates.cs +++ b/Cron/Services/Email/EmailServiceTemplates.cs @@ -1,4 +1,4 @@ -namespace OpenShock.API.Services.Email; +namespace OpenShock.Cron.Services.Email; public sealed class EmailServiceTemplates { diff --git a/API/Services/Email/EmailServiceUtils.cs b/Cron/Services/Email/EmailServiceUtils.cs similarity index 78% rename from API/Services/Email/EmailServiceUtils.cs rename to Cron/Services/Email/EmailServiceUtils.cs index b596e19b..e209583f 100644 --- a/API/Services/Email/EmailServiceUtils.cs +++ b/Cron/Services/Email/EmailServiceUtils.cs @@ -1,8 +1,8 @@ using System.Net.Mail; using MimeKit; -using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Cron.Services.Email.Mailjet.Mail; -namespace OpenShock.API.Services.Email; +namespace OpenShock.Cron.Services.Email; public static class EmailServiceUtils { diff --git a/API/Services/Email/EmailTemplate.cs b/Cron/Services/Email/EmailTemplate.cs similarity index 95% rename from API/Services/Email/EmailTemplate.cs rename to Cron/Services/Email/EmailTemplate.cs index 37ab285c..aa01b548 100644 --- a/API/Services/Email/EmailTemplate.cs +++ b/Cron/Services/Email/EmailTemplate.cs @@ -1,8 +1,8 @@ using System.Text.Encodings.Web; using Fluid; -using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Cron.Services.Email.Mailjet.Mail; -namespace OpenShock.API.Services.Email; +namespace OpenShock.Cron.Services.Email; public sealed class EmailTemplate { diff --git a/API/Services/Email/IEmailService.cs b/Cron/Services/Email/IEmailService.cs similarity index 96% rename from API/Services/Email/IEmailService.cs rename to Cron/Services/Email/IEmailService.cs index c096fa86..f520e6e4 100644 --- a/API/Services/Email/IEmailService.cs +++ b/Cron/Services/Email/IEmailService.cs @@ -1,6 +1,6 @@ -using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Cron.Services.Email.Mailjet.Mail; -namespace OpenShock.API.Services.Email; +namespace OpenShock.Cron.Services.Email; /// /// Low-level email provider abstraction: renders a template and hands the message to the provider. diff --git a/API/Services/Email/Mailjet/Mail/Contact.cs b/Cron/Services/Email/Mailjet/Mail/Contact.cs similarity index 91% rename from API/Services/Email/Mailjet/Mail/Contact.cs rename to Cron/Services/Email/Mailjet/Mail/Contact.cs index afc664db..677bbde7 100644 --- a/API/Services/Email/Mailjet/Mail/Contact.cs +++ b/Cron/Services/Email/Mailjet/Mail/Contact.cs @@ -1,7 +1,7 @@ using System.ComponentModel.DataAnnotations; using System.Diagnostics.CodeAnalysis; -namespace OpenShock.API.Services.Email.Mailjet.Mail; +namespace OpenShock.Cron.Services.Email.Mailjet.Mail; public class Contact { diff --git a/API/Services/Email/Mailjet/Mail/DirectMail.cs b/Cron/Services/Email/Mailjet/Mail/DirectMail.cs similarity index 79% rename from API/Services/Email/Mailjet/Mail/DirectMail.cs rename to Cron/Services/Email/Mailjet/Mail/DirectMail.cs index 8d185c81..abad0105 100644 --- a/API/Services/Email/Mailjet/Mail/DirectMail.cs +++ b/Cron/Services/Email/Mailjet/Mail/DirectMail.cs @@ -1,4 +1,4 @@ -namespace OpenShock.API.Services.Email.Mailjet.Mail; +namespace OpenShock.Cron.Services.Email.Mailjet.Mail; public sealed class DirectMail { diff --git a/API/Services/Email/Mailjet/Mail/MailsWrap.cs b/Cron/Services/Email/Mailjet/Mail/MailsWrap.cs similarity index 61% rename from API/Services/Email/Mailjet/Mail/MailsWrap.cs rename to Cron/Services/Email/Mailjet/Mail/MailsWrap.cs index 3d6296e2..6652dc13 100644 --- a/API/Services/Email/Mailjet/Mail/MailsWrap.cs +++ b/Cron/Services/Email/Mailjet/Mail/MailsWrap.cs @@ -1,4 +1,4 @@ -namespace OpenShock.API.Services.Email.Mailjet.Mail; +namespace OpenShock.Cron.Services.Email.Mailjet.Mail; public sealed class MailsWrap { diff --git a/API/Services/Email/Mailjet/MailjetEmailService.cs b/Cron/Services/Email/Mailjet/MailjetEmailService.cs similarity index 96% rename from API/Services/Email/Mailjet/MailjetEmailService.cs rename to Cron/Services/Email/Mailjet/MailjetEmailService.cs index 9fdd783e..f31f127a 100644 --- a/API/Services/Email/Mailjet/MailjetEmailService.cs +++ b/Cron/Services/Email/Mailjet/MailjetEmailService.cs @@ -1,11 +1,11 @@ -using OpenShock.API.Options; -using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Cron.Options; +using OpenShock.Cron.Services.Email.Mailjet.Mail; using System.Net.Mime; using System.Text; using System.Text.Json; using OpenShock.Common.JsonSerialization; -namespace OpenShock.API.Services.Email.Mailjet; +namespace OpenShock.Cron.Services.Email.Mailjet; public sealed class MailjetEmailService : IEmailService, IDisposable { diff --git a/API/Services/Email/Mailjet/MailjetEmailServiceExtension.cs b/Cron/Services/Email/Mailjet/MailjetEmailServiceExtension.cs similarity index 93% rename from API/Services/Email/Mailjet/MailjetEmailServiceExtension.cs rename to Cron/Services/Email/Mailjet/MailjetEmailServiceExtension.cs index 31fe0758..b37f6552 100644 --- a/API/Services/Email/Mailjet/MailjetEmailServiceExtension.cs +++ b/Cron/Services/Email/Mailjet/MailjetEmailServiceExtension.cs @@ -1,9 +1,9 @@ -using OpenShock.API.Options; +using OpenShock.Cron.Options; using System.Net.Http.Headers; using System.Text; using Microsoft.Extensions.Options; -namespace OpenShock.API.Services.Email.Mailjet; +namespace OpenShock.Cron.Services.Email.Mailjet; public static class MailjetEmailServiceExtension { diff --git a/API/Services/Email/NoneEmailService.cs b/Cron/Services/Email/NoneEmailService.cs similarity index 95% rename from API/Services/Email/NoneEmailService.cs rename to Cron/Services/Email/NoneEmailService.cs index e7656d7c..1ab9fada 100644 --- a/API/Services/Email/NoneEmailService.cs +++ b/Cron/Services/Email/NoneEmailService.cs @@ -1,6 +1,6 @@ -using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Cron.Services.Email.Mailjet.Mail; -namespace OpenShock.API.Services.Email; +namespace OpenShock.Cron.Services.Email; /// /// This is a noop implementation of the email service. It does nothing. diff --git a/API/Services/Email/Outbox/EmailDispatchResult.cs b/Cron/Services/Email/Outbox/EmailDispatchResult.cs similarity index 97% rename from API/Services/Email/Outbox/EmailDispatchResult.cs rename to Cron/Services/Email/Outbox/EmailDispatchResult.cs index f9c5def1..b21194f3 100644 --- a/API/Services/Email/Outbox/EmailDispatchResult.cs +++ b/Cron/Services/Email/Outbox/EmailDispatchResult.cs @@ -1,4 +1,4 @@ -namespace OpenShock.API.Services.Email.Outbox; +namespace OpenShock.Cron.Services.Email.Outbox; /// What happened when the consumer tried to dispatch an outbox message. public enum EmailDispatchOutcome diff --git a/Cron/Services/Email/Outbox/EmailOutboxConsumer.cs b/Cron/Services/Email/Outbox/EmailOutboxConsumer.cs new file mode 100644 index 00000000..9b3e8d0b --- /dev/null +++ b/Cron/Services/Email/Outbox/EmailOutboxConsumer.cs @@ -0,0 +1,159 @@ +using Hangfire; +using Microsoft.EntityFrameworkCore; +using OpenShock.Common.Constants; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; +using OpenShock.Common.Services.RedisPubSub; +using StackExchange.Redis; + +namespace OpenShock.Cron.Services.Email.Outbox; + +/// +/// The email-outbox hand-off: claims freshly enqueued () rows and +/// hands each to Hangfire as an . It does not send or retry - +/// once a row is enqueued, Hangfire owns its delivery, retry scheduling, and crash recovery. +/// +/// +/// +/// Push first, sweep as a safety net. Hand-off is triggered by a Redis notification published +/// the instant a message is enqueued (), so the job is +/// queued within sub-seconds. A periodic sweep () also +/// runs as a fallback to cover a dropped notification. Because both feed a single coalescing wake +/// signal, the push and the sweep can never fire "on top of each other": any number of signals +/// between drains collapse into one drain, so a notification that lands just before a sweep simply +/// shares the same pass. +/// +/// +/// Claims are safe across instances. Rows are claimed with FOR UPDATE SKIP LOCKED, so if +/// the Cron host is scaled out each instance grabs a disjoint set. Hangfire jobs are enqueued before +/// the claim transaction commits: if the process dies in between, the transaction rolls back and the +/// rows stay to be re-handed-off next pass - at worst a duplicate +/// job, which the idempotent absorbs. +/// +/// +public sealed class EmailOutboxConsumer : BackgroundService +{ + private const int BatchSize = 50; + + private readonly IDbContextFactory _dbContextFactory; + private readonly IBackgroundJobClient _jobClient; + private readonly ISubscriber _subscriber; + private readonly ILogger _logger; + + // Coalescing wake signal: capacity 1, so any number of notifications between drains collapse into + // a single "there is work" wake-up. + private readonly SemaphoreSlim _wakeSignal = new(0, 1); + + /// DI constructor. + public EmailOutboxConsumer( + IDbContextFactory dbContextFactory, + IBackgroundJobClient jobClient, + IConnectionMultiplexer connectionMultiplexer, + ILogger logger) + { + _dbContextFactory = dbContextFactory; + _jobClient = jobClient; + _subscriber = connectionMultiplexer.GetSubscriber(); + _logger = logger; + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await _subscriber.SubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); + + try + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + await DrainAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Email outbox hand-off failed"); + } + + try + { + await _wakeSignal.WaitAsync(Duration.EmailOutboxPollInterval, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + } + finally + { + await _subscriber.UnsubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); + } + } + + private void OnPendingNotification(RedisChannel channel, RedisValue value) + { + try + { + _wakeSignal.Release(); + } + catch (SemaphoreFullException) + { + // Already pending a wake-up - nothing to do. + } + } + + private async Task DrainAsync(CancellationToken cancellationToken) + { + int handed; + do + { + handed = await HandOffBatchAsync(cancellationToken); + } + while (handed >= BatchSize && !cancellationToken.IsCancellationRequested); + } + + private async Task HandOffBatchAsync(CancellationToken cancellationToken) + { + await using var db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken); + + var claimed = await db.EmailOutbox.FromSql( + $""" + SELECT * FROM email_outbox + WHERE status = {EmailStatus.Pending} + ORDER BY created_at + LIMIT {BatchSize} + FOR UPDATE SKIP LOCKED + """).ToListAsync(cancellationToken); + + if (claimed.Count == 0) + { + await transaction.CommitAsync(cancellationToken); + return 0; + } + + foreach (var message in claimed) + { + // Enqueue first, flip to Queued second: if we crash here the rollback leaves the row Pending + // (re-handed-off next pass), never Queued-with-no-job. + _jobClient.Enqueue(job => job.SendAsync(message.Id, null, CancellationToken.None)); + message.Status = EmailStatus.Queued; + } + + await db.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + return claimed.Count; + } + + /// + public override void Dispose() + { + _wakeSignal.Dispose(); + base.Dispose(); + } +} diff --git a/API/Services/Email/Outbox/EmailOutboxDispatcher.cs b/Cron/Services/Email/Outbox/EmailOutboxDispatcher.cs similarity index 98% rename from API/Services/Email/Outbox/EmailOutboxDispatcher.cs rename to Cron/Services/Email/Outbox/EmailOutboxDispatcher.cs index de820505..51a6ca0a 100644 --- a/API/Services/Email/Outbox/EmailOutboxDispatcher.cs +++ b/Cron/Services/Email/Outbox/EmailOutboxDispatcher.cs @@ -1,12 +1,12 @@ using Microsoft.EntityFrameworkCore; -using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Cron.Services.Email.Mailjet.Mail; using OpenShock.Common.Constants; using OpenShock.Common.Models; using OpenShock.Common.OpenShockDb; using OpenShock.Common.Options; using OpenShock.Common.Utils; -namespace OpenShock.API.Services.Email.Outbox; +namespace OpenShock.Cron.Services.Email.Outbox; /// public sealed class EmailOutboxDispatcher : IEmailOutboxDispatcher diff --git a/Cron/Services/Email/Outbox/EmailOutboxJob.cs b/Cron/Services/Email/Outbox/EmailOutboxJob.cs new file mode 100644 index 00000000..1b088ee0 --- /dev/null +++ b/Cron/Services/Email/Outbox/EmailOutboxJob.cs @@ -0,0 +1,120 @@ +using Hangfire; +using Hangfire.Server; +using Microsoft.EntityFrameworkCore; +using OpenShock.Common.Constants; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.Cron.Services.Email.Outbox; + +/// +/// The Hangfire job that delivers a single . One job instance per +/// email, so each message has an independent retry timeline. +/// +/// +/// Hangfire owns the delivery machinery: supplies the back-off +/// curve, the persistent retry schedule, and crash requeue; the job only maps a dispatch outcome onto +/// the row's terminal state. A transient failure is surfaced by throwing, which is what makes +/// Hangfire schedule the next retry. On the final attempt (detected via the RetryCount job +/// parameter) the row is marked before the throw, so an exhausted +/// message never stays stuck looking like it is still in flight. +/// +public sealed class EmailOutboxJob +{ + /// Number of automatic retries before a transiently-failing message is given up as failed. + public const int MaxRetries = 10; + + private readonly IDbContextFactory _dbContextFactory; + private readonly IEmailOutboxDispatcher _dispatcher; + private readonly ILogger _logger; + + /// DI constructor. + public EmailOutboxJob(IDbContextFactory dbContextFactory, IEmailOutboxDispatcher dispatcher, ILogger logger) + { + _dbContextFactory = dbContextFactory; + _dispatcher = dispatcher; + _logger = logger; + } + + /// Delivers the message with the given id. See the type remarks for the retry contract. + [AutomaticRetry(Attempts = MaxRetries, OnAttemptsExceeded = AttemptsExceededAction.Fail)] + public async Task SendAsync(Guid messageId, PerformContext? context, CancellationToken cancellationToken) + { + await using var db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + + var message = await db.EmailOutbox.FirstOrDefaultAsync(m => m.Id == messageId, cancellationToken); + if (message is null) + { + _logger.LogWarning("Email outbox message {MessageId} no longer exists; nothing to send", messageId); + return; + } + + // Idempotency: a duplicate enqueue or a crash-requeue may re-run a message already resolved. + if (message.Status is EmailStatus.Sent or EmailStatus.Failed) return; + + var result = await _dispatcher.SendAsync(message, db, cancellationToken); + var nowUtc = DateTime.UtcNow; + + switch (result.Outcome) + { + case EmailDispatchOutcome.Sent: + message.Status = EmailStatus.Sent; + message.SentAt = nowUtc; + message.LastError = null; + await db.SaveChangesAsync(cancellationToken); + return; + + case EmailDispatchOutcome.Skipped: + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + message.LastError = Truncate($"Skipped: {result.Detail}"); + await db.SaveChangesAsync(cancellationToken); + return; + + case EmailDispatchOutcome.PermanentFailure: + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + message.LastError = Truncate(result.Detail); + await db.SaveChangesAsync(cancellationToken); + return; + + case EmailDispatchOutcome.TransientFailure: + message.LastError = Truncate(result.Detail); + + // RetryCount is the number of retries already performed (0 on the first run). When it has + // reached the cap this is the last attempt, so record the terminal failure on the row; + // the throw below then lets Hangfire move the job to its Failed state too. + var retryCount = context?.GetJobParameter("RetryCount") ?? 0; + if (retryCount >= MaxRetries) + { + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + await db.SaveChangesAsync(cancellationToken); + _logger.LogWarning("Email outbox message {MessageId} ({Type}) failed after {Attempts} attempt(s): {Detail}", + message.Id, message.Type, retryCount + 1, result.Detail); + } + else + { + await db.SaveChangesAsync(cancellationToken); + } + + throw new EmailOutboxTransientException(result.Detail ?? "Transient email send failure"); + + default: + throw new InvalidOperationException($"Unhandled dispatch outcome {result.Outcome}"); + } + } + + /// Clamps an error string to the column's maximum length. + private static string? Truncate(string? value) + => value is null || value.Length <= HardLimits.EmailOutboxLastErrorMaxLength + ? value + : value[..HardLimits.EmailOutboxLastErrorMaxLength]; +} + +/// Thrown to signal a transient send failure so Hangfire schedules a retry on its back-off curve. +public sealed class EmailOutboxTransientException : Exception +{ + /// Creates the exception with the underlying transient failure detail. + public EmailOutboxTransientException(string message) : base(message) { } +} diff --git a/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs b/Cron/Services/Email/Outbox/IEmailOutboxDispatcher.cs similarity index 96% rename from API/Services/Email/Outbox/IEmailOutboxDispatcher.cs rename to Cron/Services/Email/Outbox/IEmailOutboxDispatcher.cs index a2b143d6..6d6c98e7 100644 --- a/API/Services/Email/Outbox/IEmailOutboxDispatcher.cs +++ b/Cron/Services/Email/Outbox/IEmailOutboxDispatcher.cs @@ -1,6 +1,6 @@ using OpenShock.Common.OpenShockDb; -namespace OpenShock.API.Services.Email.Outbox; +namespace OpenShock.Cron.Services.Email.Outbox; /// /// Renders and sends a single . This is the one place email is diff --git a/API/Services/Email/Smtp/SmtpEmailService.cs b/Cron/Services/Email/Smtp/SmtpEmailService.cs similarity index 97% rename from API/Services/Email/Smtp/SmtpEmailService.cs rename to Cron/Services/Email/Smtp/SmtpEmailService.cs index 86101987..c783f85b 100644 --- a/API/Services/Email/Smtp/SmtpEmailService.cs +++ b/Cron/Services/Email/Smtp/SmtpEmailService.cs @@ -1,10 +1,10 @@ using MailKit.Net.Smtp; using MimeKit; using MimeKit.Text; -using OpenShock.API.Options; -using OpenShock.API.Services.Email.Mailjet.Mail; +using OpenShock.Cron.Options; +using OpenShock.Cron.Services.Email.Mailjet.Mail; -namespace OpenShock.API.Services.Email.Smtp; +namespace OpenShock.Cron.Services.Email.Smtp; public sealed class SmtpEmailService : IEmailService { diff --git a/API/Services/Email/Smtp/SmtpEmailServiceExtension.cs b/Cron/Services/Email/Smtp/SmtpEmailServiceExtension.cs similarity index 89% rename from API/Services/Email/Smtp/SmtpEmailServiceExtension.cs rename to Cron/Services/Email/Smtp/SmtpEmailServiceExtension.cs index a4be55f4..6636011f 100644 --- a/API/Services/Email/Smtp/SmtpEmailServiceExtension.cs +++ b/Cron/Services/Email/Smtp/SmtpEmailServiceExtension.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Options; -using OpenShock.API.Options; +using OpenShock.Cron.Options; -namespace OpenShock.API.Services.Email.Smtp; +namespace OpenShock.Cron.Services.Email.Smtp; public static class SmtpEmailServiceExtension { diff --git a/API/SmtpTemplates/AccountActivation.liquid b/Cron/SmtpTemplates/AccountActivation.liquid similarity index 100% rename from API/SmtpTemplates/AccountActivation.liquid rename to Cron/SmtpTemplates/AccountActivation.liquid diff --git a/API/SmtpTemplates/EmailChangeNotice.liquid b/Cron/SmtpTemplates/EmailChangeNotice.liquid similarity index 100% rename from API/SmtpTemplates/EmailChangeNotice.liquid rename to Cron/SmtpTemplates/EmailChangeNotice.liquid diff --git a/API/SmtpTemplates/EmailVerification.liquid b/Cron/SmtpTemplates/EmailVerification.liquid similarity index 100% rename from API/SmtpTemplates/EmailVerification.liquid rename to Cron/SmtpTemplates/EmailVerification.liquid diff --git a/API/SmtpTemplates/PasswordReset.liquid b/Cron/SmtpTemplates/PasswordReset.liquid similarity index 100% rename from API/SmtpTemplates/PasswordReset.liquid rename to Cron/SmtpTemplates/PasswordReset.liquid From a54bb3252dd0a4298aed9d6e36a858bfe40da628 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 14:25:00 +0200 Subject: [PATCH 04/15] fix(email): stop logging recipient address in SMTP error logs (CodeQL) --- Cron/Services/Email/Smtp/SmtpEmailService.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Cron/Services/Email/Smtp/SmtpEmailService.cs b/Cron/Services/Email/Smtp/SmtpEmailService.cs index c783f85b..683bd590 100644 --- a/Cron/Services/Email/Smtp/SmtpEmailService.cs +++ b/Cron/Services/Email/Smtp/SmtpEmailService.cs @@ -83,14 +83,16 @@ private async Task SendMail(Contact to, EmailTemplate templa // A 5xx reply (e.g. mailbox unavailable, message rejected) won't be fixed by retrying; // 4xx replies are temporary. var permanent = (int)ex.StatusCode is >= 500 and <= 599; - _logger.LogError(ex, "SMTP command failed with status {StatusCode} sending to {Recipient}", ex.StatusCode, to.Email); + // Don't log the recipient address - the outbox row (keyed by message id) already records it. + _logger.LogError(ex, "SMTP command failed with status {StatusCode}", ex.StatusCode); return permanent ? EmailSendResult.PermanentFailure : EmailSendResult.TransientFailure; } catch (Exception ex) { // Connection, TLS, auth, protocol and timeout failures are all treated as temporary; the // retry budget bounds how long a genuinely broken configuration keeps being attempted. - _logger.LogError(ex, "Transient SMTP failure sending to {Recipient}", to.Email); + // Don't log the recipient address - the outbox row (keyed by message id) already records it. + _logger.LogError(ex, "Transient SMTP failure while sending email"); return EmailSendResult.TransientFailure; } } From 5e4d1e2eff6b25822ce12e2db71ab3b44abe4d4c Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 14:56:37 +0200 Subject: [PATCH 05/15] test(email): run Cron delivery in-process for full integration + queue tests The API integration MailTests asserted real SMTP delivery, but delivery moved to the Cron host, so the API-only test host no longer sent anything and they failed. Boot the Cron pipeline in-process alongside the API (shared Postgres/Redis/Mailpit) so the outbox is actually drained and delivered. - WebApplicationFactory boots cronhost::Program (aliased to avoid the Program clash); its consumer + Hangfire deliver the rows the API enqueues. - Register FrontendOptions in Cron/Program (the dispatcher builds links with it). - Make Hangfire's QueuePollInterval configurable: production keeps the 15s default (no extra DB load); the test sets 0.5s so jobs land in the wait window. - Copy Cron's Liquid templates next to the test binaries (relative-path load). - Add CronOutboxQueueTests: drive the outbox job's state machine through each case (Sent / Skipped / Permanent / Transient-retry / already-terminal no-op). --- .../API.IntegrationTests.csproj | 11 ++ .../Tests/CronOutboxQueueTests.cs | 162 ++++++++++++++++++ API.IntegrationTests/WebApplicationFactory.cs | 39 ++++- Cron/Program.cs | 19 +- 4 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 API.IntegrationTests/Tests/CronOutboxQueueTests.cs diff --git a/API.IntegrationTests/API.IntegrationTests.csproj b/API.IntegrationTests/API.IntegrationTests.csproj index 06b883f5..33729f71 100644 --- a/API.IntegrationTests/API.IntegrationTests.csproj +++ b/API.IntegrationTests/API.IntegrationTests.csproj @@ -1,6 +1,9 @@ + + @@ -18,6 +21,14 @@ + + + + PreserveNewest + + + diff --git a/API.IntegrationTests/Tests/CronOutboxQueueTests.cs b/API.IntegrationTests/Tests/CronOutboxQueueTests.cs new file mode 100644 index 00000000..f2a431c9 --- /dev/null +++ b/API.IntegrationTests/Tests/CronOutboxQueueTests.cs @@ -0,0 +1,162 @@ +extern alias cronhost; + +using cronhost::OpenShock.Cron.Services.Email.Outbox; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using OpenShock.API.IntegrationTests.Helpers; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.API.IntegrationTests.Tests; + +/// +/// Tests how the Cron email-outbox handles each kind of queue case - +/// the delivery state machine - independently of the provider, Hangfire, and the consumer. Each test +/// seeds a row in the state (so the live consumer never claims it), +/// invokes the job directly with a stub dispatcher returning a chosen outcome, and asserts the row's +/// resulting terminal state. +/// +public sealed class CronOutboxQueueTests +{ + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required WebApplicationFactory WebApplicationFactory { get; init; } + + [Test] + public async Task Sent_MarksRowSent_AndClearsError() + { + var id = await SeedQueuedMessageAsync(lastError: "previous transient blip"); + var job = CreateJob(new StubDispatcher(EmailDispatchResult.Sent)); + + await job.SendAsync(id, null, CancellationToken.None); + + var row = await GetAsync(id); + await Assert.That(row.Status).IsEqualTo(EmailStatus.Sent); + await Assert.That(row.SentAt).IsNotNull(); + await Assert.That(row.FailedAt).IsNull(); + await Assert.That(row.LastError).IsNull(); + } + + [Test] + public async Task Skipped_MarksRowFailed_WithSkippedReason() + { + var id = await SeedQueuedMessageAsync(); + var job = CreateJob(new StubDispatcher(EmailDispatchResult.Skip("reset already used"))); + + await job.SendAsync(id, null, CancellationToken.None); + + var row = await GetAsync(id); + await Assert.That(row.Status).IsEqualTo(EmailStatus.Failed); + await Assert.That(row.FailedAt).IsNotNull(); + await Assert.That(row.LastError).IsNotNull(); + await Assert.That(row.LastError!).StartsWith("Skipped:"); + } + + [Test] + public async Task Permanent_MarksRowFailed_WithDetail() + { + var id = await SeedQueuedMessageAsync(); + var job = CreateJob(new StubDispatcher(EmailDispatchResult.Permanent("Provider rejected the message"))); + + await job.SendAsync(id, null, CancellationToken.None); + + var row = await GetAsync(id); + await Assert.That(row.Status).IsEqualTo(EmailStatus.Failed); + await Assert.That(row.FailedAt).IsNotNull(); + await Assert.That(row.LastError).IsEqualTo("Provider rejected the message"); + } + + [Test] + public async Task Transient_Throws_AndLeavesRowQueuedForRetry() + { + var id = await SeedQueuedMessageAsync(); + var job = CreateJob(new StubDispatcher(EmailDispatchResult.Transient("smtp timeout"))); + + // A transient failure surfaces as a throw - that is what makes Hangfire schedule the retry. + EmailOutboxTransientException? thrown = null; + try + { + await job.SendAsync(id, null, CancellationToken.None); + } + catch (EmailOutboxTransientException ex) + { + thrown = ex; + } + + await Assert.That(thrown).IsNotNull(); + + var row = await GetAsync(id); + await Assert.That(row.Status).IsEqualTo(EmailStatus.Queued); // still in flight, not terminal + await Assert.That(row.SentAt).IsNull(); + await Assert.That(row.FailedAt).IsNull(); + await Assert.That(row.LastError).IsEqualTo("smtp timeout"); + } + + [Test] + public async Task AlreadyTerminal_IsNoOp_AndDoesNotInvokeDispatcher() + { + var id = await SeedQueuedMessageAsync(status: EmailStatus.Sent); + var stub = new StubDispatcher(EmailDispatchResult.Sent, throwIfCalled: true); + var job = CreateJob(stub); + + // A duplicate enqueue or crash-requeue must not re-send an already-resolved message. + await job.SendAsync(id, null, CancellationToken.None); + + await Assert.That(stub.WasCalled).IsFalse(); + var row = await GetAsync(id); + await Assert.That(row.Status).IsEqualTo(EmailStatus.Sent); + } + + // --- Helpers --- + + private EmailOutboxJob CreateJob(IEmailOutboxDispatcher dispatcher) => new( + WebApplicationFactory.Services.GetRequiredService>(), + dispatcher, + NullLogger.Instance); + + private async Task SeedQueuedMessageAsync(EmailStatus status = EmailStatus.Queued, string? lastError = null) + { + var factory = WebApplicationFactory.Services.GetRequiredService>(); + await using var db = await factory.CreateDbContextAsync(); + + var message = EmailOutboxMessage.Create( + EmailType.PasswordReset, + TestHelper.UniqueEmail("cron-queue"), + "Queue Test", + new Dictionary { ["dummy"] = "value" }); + message.Status = status; + message.LastError = lastError; + + db.EmailOutbox.Add(message); + await db.SaveChangesAsync(); + return message.Id; + } + + private async Task GetAsync(Guid id) + { + var factory = WebApplicationFactory.Services.GetRequiredService>(); + await using var db = await factory.CreateDbContextAsync(); + return await db.EmailOutbox.AsNoTracking().FirstAsync(m => m.Id == id); + } + + private sealed class StubDispatcher : IEmailOutboxDispatcher + { + private readonly EmailDispatchResult _result; + private readonly bool _throwIfCalled; + + public bool WasCalled { get; private set; } + + public StubDispatcher(EmailDispatchResult result, bool throwIfCalled = false) + { + _result = result; + _throwIfCalled = throwIfCalled; + } + + public Task SendAsync(EmailOutboxMessage message, OpenShockContext db, CancellationToken cancellationToken = default) + { + WasCalled = true; + if (_throwIfCalled) throw new InvalidOperationException("Dispatcher must not be invoked for an already-terminal row"); + return Task.FromResult(_result); + } + } +} diff --git a/API.IntegrationTests/WebApplicationFactory.cs b/API.IntegrationTests/WebApplicationFactory.cs index 536cb6dd..401a657f 100644 --- a/API.IntegrationTests/WebApplicationFactory.cs +++ b/API.IntegrationTests/WebApplicationFactory.cs @@ -1,4 +1,6 @@ -using System.Threading.RateLimiting; +extern alias cronhost; + +using System.Threading.RateLimiting; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.RateLimiting; @@ -28,12 +30,41 @@ public class WebApplicationFactory : WebApplicationFactory, IAsyncIniti public MailpitHelper CreateMailpitHelper() => new(Mailpit.ApiBaseUrl); + // The Cron host, booted in-process so the email outbox is actually drained and delivered. It reads + // the same OPENSHOCK__* environment variables this factory sets (shared DB / Redis / Mailpit), so + // it sees the same outbox rows the API writes. + private CronHost? _cronHost; + + /// Service provider of the in-process Cron host (email dispatcher, outbox job, etc.). + public IServiceProvider CronServices => _cronHost?.Services + ?? throw new InvalidOperationException("Cron host not initialized"); + public Task InitializeAsync() { - _ = Server; + _ = Server; // Boots the API host, which sets the shared OPENSHOCK__* environment variables. + _cronHost = new CronHost(); + _ = _cronHost.Services; // Boots the Cron host: outbox consumer + Hangfire delivery now run. return Task.CompletedTask; } + protected override void Dispose(bool disposing) + { + if (disposing) _cronHost?.Dispose(); + base.Dispose(disposing); + } + + /// Minimal factory that boots the Cron host (cronhost::Program) for delivery. + private sealed class CronHost : WebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.ConfigureServices(services => + { + services.AddSerilog(configuration => configuration.WriteTo.Console(LogEventLevel.Warning)); + }); + } + } + protected override void ConfigureClient(HttpClient client) { base.ConfigureClient(client); @@ -65,6 +96,10 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) { "OPENSHOCK__REDIS__CONN", Redis.Container.GetConnectionString() }, + // Tests only: make the Cron host's Hangfire workers pick up enqueued email jobs fast so + // delivery lands inside the Mailpit wait window. Production keeps Hangfire's 15s default. + { "OPENSHOCK__HANGFIRE__QUEUEPOLLINTERVAL", "00:00:00.5" }, + { "OPENSHOCK__FRONTEND__BASEURL", "https://openshock.app" }, { "OPENSHOCK__FRONTEND__SHORTURL", "https://openshock.app" }, { "OPENSHOCK__FRONTEND__COOKIEDOMAIN", "openshock.app,localhost" }, diff --git a/Cron/Program.cs b/Cron/Program.cs index 2d7d7d98..56f2e491 100644 --- a/Cron/Program.cs +++ b/Cron/Program.cs @@ -12,14 +12,24 @@ var redisOptions = builder.RegisterRedisOptions(); var databaseOptions = builder.RegisterDatabaseOptions(); builder.RegisterMetricsOptions(); +// The outbox dispatcher builds activation/reset links, so it needs the frontend base URL. +builder.RegisterFrontendOptions(); builder.Services.AddOpenShockMemDB(redisOptions); builder.Services.AddOpenShockDB(databaseOptions); builder.Services.AddOpenShockServices(); +// Hangfire workers fetch enqueued jobs by polling the queue table. The default interval (15s) is +// left as-is in production - email delivery within that window is fine and it adds no DB load. Only +// the integration test overrides it (via OpenShock:Hangfire:QueuePollInterval) to run fast. +var hangfireStorageOptions = new PostgreSqlStorageOptions(); +if (builder.Configuration.GetValue("OpenShock:Hangfire:QueuePollInterval") is { } queuePollInterval) + hangfireStorageOptions.QueuePollInterval = queuePollInterval; + builder.Services.AddHangfire(hangfire => - hangfire.UsePostgreSqlStorage(c => - c.UseNpgsqlConnection(databaseOptions.Conn))); + hangfire.UsePostgreSqlStorage( + c => c.UseNpgsqlConnection(databaseOptions.Conn), + hangfireStorageOptions)); builder.Services.AddHangfireServer(); // Registers the email providers, the outbox dispatcher, the per-email Hangfire send job, and the @@ -48,4 +58,7 @@ jobManager.AddOrUpdate(cronJob.Name, cronJob.Job, cronJob.Schedule); } -await app.RunAsync(); \ No newline at end of file +await app.RunAsync(); + +// Expose Program for integration tests (so the test host can boot the Cron pipeline in-process). +public partial class Program; \ No newline at end of file From 5d13931dbd94c7f98a8bc46559a1ef7d253564be Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 16:22:25 +0200 Subject: [PATCH 06/15] refactor(email): own the outbox queue end-to-end; Hangfire-scheduled batch delivery Make email_outbox the single source of truth for delivery state (status, attempt_count, next_attempt_at, last_error) instead of handing retry and scheduling to Hangfire. The delivery job claims due rows with FOR UPDATE SKIP LOCKED, leases them (Sending), and records the outcome on the row. - Replace the per-email Hangfire job + [AutomaticRetry] and the custom BackgroundService poll loop with EmailOutboxDeliveryJob ([CronJob] every minute) + EmailOutboxNotificationListener (enqueues it on the Redis pending nudge for instant delivery). No background polling loop. - Add Sending/Skipped states; Skipped is a terminal no-op kept distinct from Failed so operators never requeue it. - Self-owned retry: EmailOutboxRetryPolicy (capped-exponential backoff, 11 attempts = parity with the old AutomaticRetry) plus a delivery lease so a crashed send is reclaimed. - xmin optimistic-concurrency token so a lapsed-lease reclaim can't clobber attempt count / terminal state; per-message reload + isolation so one failure can't abort the batch; drain loop for backlogs. - Pure EmailOutboxStateMachine unit-tested in new Cron.Tests project; DB round-trip / enum-mapping coverage in EmailOutboxPersistenceTests. --- .../Tests/CronOutboxQueueTests.cs | 162 ------------------ .../Tests/EmailOutboxPersistenceTests.cs | 98 +++++++++++ API.IntegrationTests/WebApplicationFactory.cs | 7 +- API/Services/Account/AccountService.cs | 22 +-- Common/Constants/Constants.cs | 5 - .../20260630103404_AddEmailOutbox.Designer.cs | 22 ++- .../20260630103404_AddEmailOutbox.cs | 12 +- .../OpenShockContextModelSnapshot.cs | 22 ++- Common/Models/EmailStatus.cs | 41 +++-- Common/OpenShockDb/EmailOutboxMessage.cs | 86 +++++----- Common/OpenShockDb/OpenShockContext.cs | 25 ++- Cron.Tests/Cron.Tests.csproj | 11 ++ .../Outbox/EmailOutboxStateMachineTests.cs | 143 ++++++++++++++++ Cron/Jobs/EmailOutboxDeliveryJob.cs | 157 +++++++++++++++++ Cron/Program.cs | 7 +- Cron/Services/Email/EmailServiceExtension.cs | 9 +- .../Email/Outbox/EmailOutboxConsumer.cs | 159 ----------------- Cron/Services/Email/Outbox/EmailOutboxJob.cs | 120 ------------- .../Outbox/EmailOutboxNotificationListener.cs | 67 ++++++++ .../Email/Outbox/EmailOutboxRetryPolicy.cs | 45 +++++ .../Email/Outbox/EmailOutboxStateMachine.cs | 88 ++++++++++ OpenShockBackend.slnx | 1 + dotnet-tools.json | 13 ++ 23 files changed, 790 insertions(+), 532 deletions(-) delete mode 100644 API.IntegrationTests/Tests/CronOutboxQueueTests.cs create mode 100644 API.IntegrationTests/Tests/EmailOutboxPersistenceTests.cs create mode 100644 Cron.Tests/Cron.Tests.csproj create mode 100644 Cron.Tests/Services/Email/Outbox/EmailOutboxStateMachineTests.cs create mode 100644 Cron/Jobs/EmailOutboxDeliveryJob.cs delete mode 100644 Cron/Services/Email/Outbox/EmailOutboxConsumer.cs delete mode 100644 Cron/Services/Email/Outbox/EmailOutboxJob.cs create mode 100644 Cron/Services/Email/Outbox/EmailOutboxNotificationListener.cs create mode 100644 Cron/Services/Email/Outbox/EmailOutboxRetryPolicy.cs create mode 100644 Cron/Services/Email/Outbox/EmailOutboxStateMachine.cs create mode 100644 dotnet-tools.json diff --git a/API.IntegrationTests/Tests/CronOutboxQueueTests.cs b/API.IntegrationTests/Tests/CronOutboxQueueTests.cs deleted file mode 100644 index f2a431c9..00000000 --- a/API.IntegrationTests/Tests/CronOutboxQueueTests.cs +++ /dev/null @@ -1,162 +0,0 @@ -extern alias cronhost; - -using cronhost::OpenShock.Cron.Services.Email.Outbox; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; -using OpenShock.API.IntegrationTests.Helpers; -using OpenShock.Common.Models; -using OpenShock.Common.OpenShockDb; - -namespace OpenShock.API.IntegrationTests.Tests; - -/// -/// Tests how the Cron email-outbox handles each kind of queue case - -/// the delivery state machine - independently of the provider, Hangfire, and the consumer. Each test -/// seeds a row in the state (so the live consumer never claims it), -/// invokes the job directly with a stub dispatcher returning a chosen outcome, and asserts the row's -/// resulting terminal state. -/// -public sealed class CronOutboxQueueTests -{ - [ClassDataSource(Shared = SharedType.PerTestSession)] - public required WebApplicationFactory WebApplicationFactory { get; init; } - - [Test] - public async Task Sent_MarksRowSent_AndClearsError() - { - var id = await SeedQueuedMessageAsync(lastError: "previous transient blip"); - var job = CreateJob(new StubDispatcher(EmailDispatchResult.Sent)); - - await job.SendAsync(id, null, CancellationToken.None); - - var row = await GetAsync(id); - await Assert.That(row.Status).IsEqualTo(EmailStatus.Sent); - await Assert.That(row.SentAt).IsNotNull(); - await Assert.That(row.FailedAt).IsNull(); - await Assert.That(row.LastError).IsNull(); - } - - [Test] - public async Task Skipped_MarksRowFailed_WithSkippedReason() - { - var id = await SeedQueuedMessageAsync(); - var job = CreateJob(new StubDispatcher(EmailDispatchResult.Skip("reset already used"))); - - await job.SendAsync(id, null, CancellationToken.None); - - var row = await GetAsync(id); - await Assert.That(row.Status).IsEqualTo(EmailStatus.Failed); - await Assert.That(row.FailedAt).IsNotNull(); - await Assert.That(row.LastError).IsNotNull(); - await Assert.That(row.LastError!).StartsWith("Skipped:"); - } - - [Test] - public async Task Permanent_MarksRowFailed_WithDetail() - { - var id = await SeedQueuedMessageAsync(); - var job = CreateJob(new StubDispatcher(EmailDispatchResult.Permanent("Provider rejected the message"))); - - await job.SendAsync(id, null, CancellationToken.None); - - var row = await GetAsync(id); - await Assert.That(row.Status).IsEqualTo(EmailStatus.Failed); - await Assert.That(row.FailedAt).IsNotNull(); - await Assert.That(row.LastError).IsEqualTo("Provider rejected the message"); - } - - [Test] - public async Task Transient_Throws_AndLeavesRowQueuedForRetry() - { - var id = await SeedQueuedMessageAsync(); - var job = CreateJob(new StubDispatcher(EmailDispatchResult.Transient("smtp timeout"))); - - // A transient failure surfaces as a throw - that is what makes Hangfire schedule the retry. - EmailOutboxTransientException? thrown = null; - try - { - await job.SendAsync(id, null, CancellationToken.None); - } - catch (EmailOutboxTransientException ex) - { - thrown = ex; - } - - await Assert.That(thrown).IsNotNull(); - - var row = await GetAsync(id); - await Assert.That(row.Status).IsEqualTo(EmailStatus.Queued); // still in flight, not terminal - await Assert.That(row.SentAt).IsNull(); - await Assert.That(row.FailedAt).IsNull(); - await Assert.That(row.LastError).IsEqualTo("smtp timeout"); - } - - [Test] - public async Task AlreadyTerminal_IsNoOp_AndDoesNotInvokeDispatcher() - { - var id = await SeedQueuedMessageAsync(status: EmailStatus.Sent); - var stub = new StubDispatcher(EmailDispatchResult.Sent, throwIfCalled: true); - var job = CreateJob(stub); - - // A duplicate enqueue or crash-requeue must not re-send an already-resolved message. - await job.SendAsync(id, null, CancellationToken.None); - - await Assert.That(stub.WasCalled).IsFalse(); - var row = await GetAsync(id); - await Assert.That(row.Status).IsEqualTo(EmailStatus.Sent); - } - - // --- Helpers --- - - private EmailOutboxJob CreateJob(IEmailOutboxDispatcher dispatcher) => new( - WebApplicationFactory.Services.GetRequiredService>(), - dispatcher, - NullLogger.Instance); - - private async Task SeedQueuedMessageAsync(EmailStatus status = EmailStatus.Queued, string? lastError = null) - { - var factory = WebApplicationFactory.Services.GetRequiredService>(); - await using var db = await factory.CreateDbContextAsync(); - - var message = EmailOutboxMessage.Create( - EmailType.PasswordReset, - TestHelper.UniqueEmail("cron-queue"), - "Queue Test", - new Dictionary { ["dummy"] = "value" }); - message.Status = status; - message.LastError = lastError; - - db.EmailOutbox.Add(message); - await db.SaveChangesAsync(); - return message.Id; - } - - private async Task GetAsync(Guid id) - { - var factory = WebApplicationFactory.Services.GetRequiredService>(); - await using var db = await factory.CreateDbContextAsync(); - return await db.EmailOutbox.AsNoTracking().FirstAsync(m => m.Id == id); - } - - private sealed class StubDispatcher : IEmailOutboxDispatcher - { - private readonly EmailDispatchResult _result; - private readonly bool _throwIfCalled; - - public bool WasCalled { get; private set; } - - public StubDispatcher(EmailDispatchResult result, bool throwIfCalled = false) - { - _result = result; - _throwIfCalled = throwIfCalled; - } - - public Task SendAsync(EmailOutboxMessage message, OpenShockContext db, CancellationToken cancellationToken = default) - { - WasCalled = true; - if (_throwIfCalled) throw new InvalidOperationException("Dispatcher must not be invoked for an already-terminal row"); - return Task.FromResult(_result); - } - } -} diff --git a/API.IntegrationTests/Tests/EmailOutboxPersistenceTests.cs b/API.IntegrationTests/Tests/EmailOutboxPersistenceTests.cs new file mode 100644 index 00000000..ccb267cd --- /dev/null +++ b/API.IntegrationTests/Tests/EmailOutboxPersistenceTests.cs @@ -0,0 +1,98 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using OpenShock.API.IntegrationTests.Helpers; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; + +namespace OpenShock.API.IntegrationTests.Tests; + +/// +/// Database round-trip coverage for the email_outbox row against real Postgres: that every +/// (including the new sending/skipped enum values) and the new +/// attempt/next-attempt columns persist and read back, and that the delivery job's claim predicate +/// (the enum filter used by the FOR UPDATE SKIP LOCKED query) selects the right rows. The pure +/// transition logic is covered by the Cron unit tests; this guards the persistence mapping the unit +/// tests can't. +/// +public sealed class EmailOutboxPersistenceTests +{ + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required WebApplicationFactory WebApplicationFactory { get; init; } + + [Test] + public async Task EveryStatus_AndAttemptColumns_RoundTripThroughPostgres() + { + var factory = WebApplicationFactory.Services.GetRequiredService>(); + + foreach (var status in Enum.GetValues()) + { + Guid id; + await using (var db = await factory.CreateDbContextAsync()) + { + var message = EmailOutboxMessage.Create( + EmailType.PasswordReset, TestHelper.UniqueEmail("outbox-persist"), "Persist", + new Dictionary { ["k"] = "v" }); + message.Status = status; + message.AttemptCount = 3; + message.LastError = "boom"; + db.EmailOutbox.Add(message); + await db.SaveChangesAsync(); + id = message.Id; + } + + await using var verifyDb = await factory.CreateDbContextAsync(); + var loaded = await verifyDb.EmailOutbox.AsNoTracking().FirstAsync(m => m.Id == id); + + await Assert.That(loaded.Status).IsEqualTo(status); + await Assert.That(loaded.AttemptCount).IsEqualTo(3); + await Assert.That(loaded.LastError).IsEqualTo("boom"); + } + } + + [Test] + public async Task ClaimPredicate_SelectsDuePendingAndLapsedSending_NotTerminalOrFuture() + { + var factory = WebApplicationFactory.Services.GetRequiredService>(); + var recipient = TestHelper.UniqueEmail("outbox-claim"); + var past = DateTime.UtcNow - TimeSpan.FromMinutes(5); + var future = DateTime.UtcNow + TimeSpan.FromHours(1); + + var duePending = await SeedAsync(factory, recipient, EmailStatus.Pending, past); + var lapsedSending = await SeedAsync(factory, recipient, EmailStatus.Sending, past); + await SeedAsync(factory, recipient, EmailStatus.Pending, future); // scheduled retry, not yet due + await SeedAsync(factory, recipient, EmailStatus.Sent, past); // terminal + + await using var db = await factory.CreateDbContextAsync(); + + // The delivery job's claim predicate (enum filter + dueness), scoped to this test's recipient so + // it never touches rows from other tests and needs no FOR UPDATE. + var claimed = await db.EmailOutbox + .FromSql( + $""" + SELECT * FROM email_outbox + WHERE recipient = {recipient} + AND next_attempt_at <= now() + AND (status = {EmailStatus.Pending} OR status = {EmailStatus.Sending}) + """) + .AsNoTracking() + .Select(m => m.Id) + .ToListAsync(); + + await Assert.That(claimed).Contains(duePending); + await Assert.That(claimed).Contains(lapsedSending); + await Assert.That(claimed.Count).IsEqualTo(2); + } + + private static async Task SeedAsync(IDbContextFactory factory, string recipient, EmailStatus status, DateTime nextAttemptAt) + { + await using var db = await factory.CreateDbContextAsync(); + var message = EmailOutboxMessage.Create( + EmailType.PasswordReset, recipient, "Claim", + new Dictionary { ["k"] = "v" }); + message.Status = status; + message.NextAttemptAt = nextAttemptAt; + db.EmailOutbox.Add(message); + await db.SaveChangesAsync(); + return message.Id; + } +} diff --git a/API.IntegrationTests/WebApplicationFactory.cs b/API.IntegrationTests/WebApplicationFactory.cs index 401a657f..0c25c2f4 100644 --- a/API.IntegrationTests/WebApplicationFactory.cs +++ b/API.IntegrationTests/WebApplicationFactory.cs @@ -43,7 +43,7 @@ public Task InitializeAsync() { _ = Server; // Boots the API host, which sets the shared OPENSHOCK__* environment variables. _cronHost = new CronHost(); - _ = _cronHost.Services; // Boots the Cron host: outbox consumer + Hangfire delivery now run. + _ = _cronHost.Services; // Boots the Cron host: the outbox delivery job + notification listener run. return Task.CompletedTask; } @@ -96,8 +96,9 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) { "OPENSHOCK__REDIS__CONN", Redis.Container.GetConnectionString() }, - // Tests only: make the Cron host's Hangfire workers pick up enqueued email jobs fast so - // delivery lands inside the Mailpit wait window. Production keeps Hangfire's 15s default. + // Tests only: make the Cron host's Hangfire workers pick up enqueued jobs fast. The Redis + // pending notification enqueues the email delivery job the instant a row is written, so this + // is what lands mail inside the Mailpit wait window. Production keeps Hangfire's 15s default. { "OPENSHOCK__HANGFIRE__QUEUEPOLLINTERVAL", "00:00:00.5" }, { "OPENSHOCK__FRONTEND__BASEURL", "https://openshock.app" }, diff --git a/API/Services/Account/AccountService.cs b/API/Services/Account/AccountService.cs index f791e5fb..4aa51351 100644 --- a/API/Services/Account/AccountService.cs +++ b/API/Services/Account/AccountService.cs @@ -27,7 +27,7 @@ public sealed class AccountService : IAccountService /// DI Constructor /// /// - /// Used to notify the email outbox consumer that mail was enqueued. + /// Used to notify the email outbox delivery job that mail was enqueued. /// /// public AccountService(OpenShockContext db, IRedisPubService redisPubService, @@ -41,16 +41,16 @@ public AccountService(OpenShockContext db, IRedisPubService redisPubService, /// /// Seeds a random token hash for a freshly created request row. The plaintext is discarded - /// immediately: the email outbox consumer mints the real token (and overwrites this hash) when it + /// immediately: the email outbox delivery job mints the real token (and overwrites this hash) when it /// sends, so this value is never the one delivered. It exists only to keep the column populated. /// private static string SeedTokenHash() => HashingUtils.HashToken(CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength)); /// - /// Best-effort nudge to the email outbox consumer that a message was enqueued. Deliberately + /// Best-effort nudge to the email outbox delivery job that a message was enqueued. Deliberately /// swallows failures: the outbox row is already committed, so a dropped notification only delays - /// delivery to the consumer's next poll, and a transient Redis hiccup must not fail a request + /// delivery to the next scheduled delivery sweep, and a transient Redis hiccup must not fail a request /// whose work already succeeded. /// private async Task NotifyEmailOutboxAsync() @@ -61,7 +61,7 @@ private async Task NotifyEmailOutboxAsync() } catch (Exception ex) { - _logger.LogWarning(ex, "Failed to notify the email outbox consumer; delivery will fall back to polling"); + _logger.LogWarning(ex, "Failed to notify the email outbox delivery job; delivery will fall back to the scheduled sweep"); } } @@ -122,7 +122,7 @@ public async Task, AccountWithEmailOrUsernameExists>> Create var user = accountCreate.AsT0.Value; - // The real activation token is minted by the outbox consumer at send time; here we record the + // The real activation token is minted by the outbox delivery job at send time; here we record the // request (with a seeded hash) and durably enqueue the email. user.UserActivationRequest = new UserActivationRequest { @@ -184,7 +184,7 @@ public async Task, AccountWithEmailOrUsernameExists>> Create await _db.SaveChangesAsync(); // If email isn't trusted, create an activation request and durably enqueue the activation - // email. The token itself is minted by the outbox consumer at send time. + // email. The token itself is minted by the outbox delivery job at send time. if (!isEmailTrusted) { user.UserActivationRequest = new UserActivationRequest @@ -214,7 +214,7 @@ public async Task, AccountWithEmailOrUsernameExists>> Create await tx.CommitAsync(); - // Notify the outbox consumer only after a successful commit. + // Notify the outbox delivery job only after a successful commit. if (!isEmailTrusted) { await NotifyEmailOutboxAsync(); @@ -432,7 +432,7 @@ public async Task= 3) return new TooManyPasswordResets(); - // The reset token is minted lazily by the outbox consumer at send time (and re-minted on any + // The reset token is minted lazily by the outbox delivery job at send time (and re-minted on any // resend), so no usable reset link is ever stored. Here we only record the request and // durably enqueue the email. var passwordReset = new UserPasswordReset @@ -596,7 +596,7 @@ public async Task x.Email == lowerCaseEmail)) return new EmailAlreadyInUse(); - // The verification token is minted lazily by the outbox consumer at send time, so no usable + // The verification token is minted lazily by the outbox delivery job at send time, so no usable // verification link is ever stored. var emailChange = new UserEmailChange { @@ -613,7 +613,7 @@ public async Task { [EmailOutboxPayloadKeys.EmailChangeId] = emailChange.Id.ToString() })); _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.EmailChangeNotice, data.User.Email, data.User.Name, diff --git a/Common/Constants/Constants.cs b/Common/Constants/Constants.cs index 3773eb1c..e8ffbbcd 100644 --- a/Common/Constants/Constants.cs +++ b/Common/Constants/Constants.cs @@ -20,9 +20,4 @@ public static class Duration public static readonly TimeSpan DeviceKeepAliveTimeout = TimeSpan.FromSeconds(35); public static readonly TimeSpan DeactivatedAccountRetentionTime = TimeSpan.FromDays(14); - - /// Safety-net sweep interval for the email-outbox consumer. Normal hand-off is push-driven - /// (Redis notification on enqueue); this only catches a missed notification. Hangfire owns retry - /// timing for messages already handed off, so this need not be tight. - public static readonly TimeSpan EmailOutboxPollInterval = TimeSpan.FromMinutes(1); } diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs index 1ec182e7..5b22a735 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs +++ b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs @@ -30,7 +30,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "configuration_value_type", new[] { "string", "bool", "int", "float", "json" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "sound", "vibrate", "shock", "stop" }); - NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "pending", "queued", "sent", "failed" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "pending", "sending", "sent", "failed", "skipped" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "account_activation", "password_reset", "email_verification", "email_change_notice" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); @@ -437,6 +437,12 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("uuid") .HasColumnName("id"); + b.Property("AttemptCount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0) + .HasColumnName("attempt_count"); + b.Property("CreatedAt") .ValueGeneratedOnAdd() .HasColumnType("timestamp with time zone") @@ -452,6 +458,12 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("character varying(1024)") .HasColumnName("last_error"); + b.Property("NextAttemptAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("next_attempt_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + b.Property>("Payload") .IsRequired() .HasColumnType("jsonb") @@ -480,12 +492,18 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("email_type") .HasColumnName("type"); + b.Property("xmin") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + b.HasKey("Id") .HasName("email_outbox_pkey"); b.HasIndex("Recipient"); - b.HasIndex("Status", "CreatedAt"); + b.HasIndex("Status", "NextAttemptAt"); b.ToTable("email_outbox", (string)null); }); diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.cs b/Common/Migrations/20260630103404_AddEmailOutbox.cs index b2a67ff8..02406e23 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.cs +++ b/Common/Migrations/20260630103404_AddEmailOutbox.cs @@ -18,7 +18,7 @@ protected override void Up(MigrationBuilder migrationBuilder) .Annotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") .Annotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") .Annotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") - .Annotation("Npgsql:Enum:email_status", "pending,queued,sent,failed") + .Annotation("Npgsql:Enum:email_status", "pending,sending,sent,failed,skipped") .Annotation("Npgsql:Enum:email_type", "account_activation,password_reset,email_verification,email_change_notice") .Annotation("Npgsql:Enum:match_type_enum", "exact,contains") .Annotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") @@ -47,10 +47,14 @@ protected override void Up(MigrationBuilder migrationBuilder) recipient_name = table.Column(type: "character varying(32)", maxLength: 32, nullable: true), payload = table.Column>(type: "jsonb", nullable: false), status = table.Column(type: "email_status", nullable: false), + attempt_count = table.Column(type: "integer", nullable: false, defaultValue: 0), + next_attempt_at = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "CURRENT_TIMESTAMP"), last_error = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: true), created_at = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "CURRENT_TIMESTAMP"), sent_at = table.Column(type: "timestamp with time zone", nullable: true), failed_at = table.Column(type: "timestamp with time zone", nullable: true) + // xmin is the Postgres system column (mapped as the concurrency token in the model); + // it is not a real user column, so it must not be created here. }, constraints: table => { @@ -63,9 +67,9 @@ protected override void Up(MigrationBuilder migrationBuilder) column: "recipient"); migrationBuilder.CreateIndex( - name: "IX_email_outbox_status_created_at", + name: "IX_email_outbox_status_next_attempt_at", table: "email_outbox", - columns: new[] { "status", "created_at" }); + columns: new[] { "status", "next_attempt_at" }); } /// @@ -89,7 +93,7 @@ protected override void Down(MigrationBuilder migrationBuilder) .OldAnnotation("Npgsql:Enum:configuration_value_type", "string,bool,int,float,json") .OldAnnotation("Npgsql:Enum:control_limit_mode", "clamp,lerp") .OldAnnotation("Npgsql:Enum:control_type", "sound,vibrate,shock,stop") - .OldAnnotation("Npgsql:Enum:email_status", "pending,queued,sent,failed") + .OldAnnotation("Npgsql:Enum:email_status", "pending,sending,sent,failed,skipped") .OldAnnotation("Npgsql:Enum:email_type", "account_activation,password_reset,email_verification,email_change_notice") .OldAnnotation("Npgsql:Enum:match_type_enum", "exact,contains") .OldAnnotation("Npgsql:Enum:ota_update_status", "started,running,finished,error,timeout") diff --git a/Common/Migrations/OpenShockContextModelSnapshot.cs b/Common/Migrations/OpenShockContextModelSnapshot.cs index c9b15626..2db87ee7 100644 --- a/Common/Migrations/OpenShockContextModelSnapshot.cs +++ b/Common/Migrations/OpenShockContextModelSnapshot.cs @@ -27,7 +27,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "configuration_value_type", new[] { "string", "bool", "int", "float", "json" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_limit_mode", new[] { "clamp", "lerp" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "control_type", new[] { "sound", "vibrate", "shock", "stop" }); - NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "pending", "queued", "sent", "failed" }); + NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_status", new[] { "pending", "sending", "sent", "failed", "skipped" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "email_type", new[] { "account_activation", "password_reset", "email_verification", "email_change_notice" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "match_type_enum", new[] { "exact", "contains" }); NpgsqlModelBuilderExtensions.HasPostgresEnum(modelBuilder, "ota_update_status", new[] { "started", "running", "finished", "error", "timeout" }); @@ -434,6 +434,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("uuid") .HasColumnName("id"); + b.Property("AttemptCount") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasDefaultValue(0) + .HasColumnName("attempt_count"); + b.Property("CreatedAt") .ValueGeneratedOnAdd() .HasColumnType("timestamp with time zone") @@ -449,6 +455,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("character varying(1024)") .HasColumnName("last_error"); + b.Property("NextAttemptAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasColumnName("next_attempt_at") + .HasDefaultValueSql("CURRENT_TIMESTAMP"); + b.Property>("Payload") .IsRequired() .HasColumnType("jsonb") @@ -477,12 +489,18 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("email_type") .HasColumnName("type"); + b.Property("xmin") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + b.HasKey("Id") .HasName("email_outbox_pkey"); b.HasIndex("Recipient"); - b.HasIndex("Status", "CreatedAt"); + b.HasIndex("Status", "NextAttemptAt"); b.ToTable("email_outbox", (string)null); }); diff --git a/Common/Models/EmailStatus.cs b/Common/Models/EmailStatus.cs index fa599d87..f3c5a8d5 100644 --- a/Common/Models/EmailStatus.cs +++ b/Common/Models/EmailStatus.cs @@ -5,32 +5,47 @@ namespace OpenShock.Common.Models; /// /// /// The row is written by the API as in the same transaction as the business -/// change. The Cron consumer claims pending rows and hands each to Hangfire for delivery, flipping it -/// to ; from there Hangfire owns execution, retry scheduling, and crash recovery -/// until the message reaches a terminal or state. +/// change. The Cron delivery job is the sole executor: it claims a due row (flipping it to +/// under a lease), hands it to the email provider, and records the outcome +/// directly on the row - a terminal , , or , +/// or back to with a future retry time. Retry count and scheduling live in the +/// row itself ( / +/// ), so the table is the complete, queryable +/// source of truth for every email's delivery state. /// public enum EmailStatus { /// - /// Freshly enqueued by the API and not yet handed to the sender. This is the only state the Cron - /// consumer claims from. + /// Due for delivery: either freshly enqueued, or a transient failure scheduled for a later retry + /// (distinguished by being in the + /// future and > 0). A row is eligible + /// to be claimed once its next-attempt time has passed. /// Pending, /// - /// Handed off to Hangfire for delivery: a send job is enqueued, running, or scheduled for a - /// retry. The consumer never re-claims a row in this state - Hangfire drives it to a terminal - /// state. + /// Claimed by an executor and currently being delivered. The claim carries a lease + /// ( set to the lease expiry); if the + /// process dies mid-send, the lease lapses and the row is reclaimed - so a crash cannot strand a + /// message in this state. /// - Queued, + Sending, /// The message was handed to the email provider successfully. Terminal. Sent, /// - /// The message will not be delivered: it exhausted its retry budget, hit a permanent provider - /// error, or the underlying request it was for no longer exists. Terminal, but the row is kept - /// (never auto-deleted) so it stays inspectable and can be requeued by an operator. + /// Delivery was abandoned: it exhausted its retry budget or hit a permanent provider error. The + /// row is kept (never auto-deleted) so it stays inspectable and can be requeued by an operator. + /// Terminal. /// - Failed + Failed, + + /// + /// The email was intentionally not sent because the underlying request no longer needs it (the + /// request was used, expired, superseded by a newer credential change, or no longer exists). This + /// is a successful no-op, kept distinct from so operators never mistake it for + /// a delivery problem or requeue it. Terminal. + /// + Skipped } diff --git a/Common/OpenShockDb/EmailOutboxMessage.cs b/Common/OpenShockDb/EmailOutboxMessage.cs index 79b7bda2..77882db8 100644 --- a/Common/OpenShockDb/EmailOutboxMessage.cs +++ b/Common/OpenShockDb/EmailOutboxMessage.cs @@ -4,57 +4,50 @@ namespace OpenShock.Common.OpenShockDb; /// /// A durable, application-owned record of an email the system intends to deliver. This table is the -/// source of truth for "we owe this user this email": a row is written in the same database -/// transaction as the business change that caused it, and a background consumer delivers it, -/// retrying until it succeeds or is exhausted. Rows are never auto-deleted, so failed sends remain -/// visible and can be requeued. +/// source of truth for "we owe this user this email" and for its delivery progress: +/// a row is written in the same database transaction as the business change that caused it, and the +/// Cron delivery job delivers it, retrying on its own schedule until it succeeds or is exhausted. Rows are +/// never auto-deleted, so failed sends remain visible and can be requeued. /// /// /// /// Flow. The API request handler creates this row (and any related request row, e.g. a /// ) and commits it as . It does -/// not send. The Cron host's outbox consumer claims pending rows and hands each to Hangfire, -/// which runs the send job: it renders the body, mints a fresh secret for token-bearing types, hands -/// the message to the email provider, and records the outcome. Delivery is therefore decoupled from -/// the HTTP request: a provider outage, a process restart, or a crash mid-send cannot lose the email. +/// not send. The Cron host's outbox delivery job claims due rows, marks each +/// under a lease, renders the body, mints a fresh secret for +/// token-bearing types, hands the message to the email provider, and records the outcome on this row. +/// Delivery is therefore decoupled from the HTTP request: a provider outage, a process restart, or a +/// crash mid-send cannot lose the email. /// /// /// -/// Why this row exists alongside Hangfire (it is an outbox, not a duplicate job queue). -/// The two layers own different things and are deliberately combined: +/// This row owns the whole lifecycle - there is no second queue. Unlike the typical +/// outbox-in-front-of-a-job-queue pattern, delivery state is not handed off to an external scheduler. +/// Attempt count and the next-retry time live here ( / +/// ), the back-off curve is supplied by EmailOutboxRetryPolicy, and +/// the delivery job claims work straight from this table with FOR UPDATE SKIP LOCKED. So a single +/// SQL query answers "is this user owed this email, what is its delivery state, how many attempts, when +/// is the next one, and what was the last error" - all in a table we own and can query, with nothing +/// mirrored into a third-party store. /// -/// -/// -/// This row owns transactional integrity + the audit fact. The email intent is committed -/// in the same transaction as the change that caused it (account created, reset -/// requested, email changed), so there is no window where the business row commits but the email -/// is lost, nor an email scheduled for a change that then rolls back. The row is also the domain -/// answer to "is this user owed this email, and what is its delivery state" (who, which type, -/// how many attempts, last error, sent/failed) - a single place to look, kept forever. -/// -/// -/// Hangfire owns the delivery machinery. Once the consumer hands a row off, Hangfire owns -/// durable execution, retry scheduling, crash requeue, and the operator dashboard - the parts a -/// bespoke worker would otherwise have to reinvent. The back-off curve itself is supplied by -/// EmailOutboxRetryPolicy, which the send job feeds into Hangfire's scheduler. -/// -/// /// /// -/// Why the related auth flows mint their token lazily. Because a working reset/verification -/// link cannot be reconstructed from the stored hash, "store the intent, mint on send" is the only -/// model that keeps the queue free of usable secrets while still allowing a resend. The consumer -/// generates the secret at send time and writes only its hash to the request row (see -/// etc.; the row is created with a seeded throwaway hash -/// that the consumer overwrites on first send). A resend produces a brand-new token, which is -/// exactly the desired behaviour for security links anyway. +/// Why the related auth flows mint their token lazily, and resends invalidate older links. +/// A working reset/verification link cannot be reconstructed from the stored hash, so "store the +/// intent, mint on send" is the only model that keeps the queue free of usable secrets while still +/// allowing a resend. The delivery job generates the secret at send time and writes only its hash to the +/// request row (see etc.; the row is created with a seeded +/// throwaway hash that the delivery job overwrites on first send). Because each send overwrites that hash, +/// every newly sent link invalidates all previously sent ones - only the most recent email's +/// link is ever valid, which is the desired behaviour for security links. /// /// /// /// Delivery guarantee. This is at-least-once. The unavoidable edge case - the provider accepts /// the message but the process dies before the row is marked - results -/// in one duplicate send on retry. For these email types that is harmless (a second activation / -/// reset / notice), and it is the correct trade against the alternative of silently losing mail. +/// in one duplicate send on retry (carrying a fresh link that supersedes the first, per above). For +/// these email types that is harmless, and it is the correct trade against the alternative of silently +/// losing mail. /// /// public sealed class EmailOutboxMessage @@ -80,11 +73,24 @@ public sealed class EmailOutboxMessage /// public required Dictionary Payload { get; set; } - /// Delivery state. See . Retry count and scheduling are owned by - /// Hangfire once the row is handed off, so they are not tracked here. + /// Delivery state. See . public EmailStatus Status { get; set; } = EmailStatus.Pending; - /// Last error recorded for a failed/retried attempt. Null while healthy. + /// + /// How many delivery attempts have been started for this message. Incremented when the delivery job + /// claims the row, so a crash that strands the row in still + /// counts as an attempt and the message cannot be retried forever. + /// + public int AttemptCount { get; set; } + + /// + /// When the row is next eligible for delivery. Set to the creation time on enqueue (so it is due + /// immediately), pushed into the future by the retry back-off after a transient failure, and used + /// as the lease expiry while so a crashed attempt is reclaimed. + /// + public DateTime NextAttemptAt { get; set; } + + /// Last error recorded for a failed/retried/skipped attempt. Null while healthy. public string? LastError { get; set; } /// When the row was created (i.e. when the email was enqueued). @@ -97,8 +103,8 @@ public sealed class EmailOutboxMessage public DateTime? FailedAt { get; set; } /// - /// Builds a new enqueued message in the state. The caller adds - /// it to the context and commits it together with the related business change. + /// Builds a new enqueued message in the state, due immediately. + /// The caller adds it to the context and commits it together with the related business change. /// public static EmailOutboxMessage Create(EmailType type, string recipient, string? recipientName, Dictionary payload) { diff --git a/Common/OpenShockDb/OpenShockContext.cs b/Common/OpenShockDb/OpenShockContext.cs index eba5a91d..d5223771 100644 --- a/Common/OpenShockDb/OpenShockContext.cs +++ b/Common/OpenShockDb/OpenShockContext.cs @@ -160,7 +160,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasPostgresEnum("match_type_enum", ["exact", "contains"]) .HasPostgresEnum("configuration_value_type", ["string", "bool", "int", "float", "json"]) .HasPostgresEnum("email_type", ["account_activation", "password_reset", "email_verification", "email_change_notice"]) - .HasPostgresEnum("email_status", ["pending", "queued", "sent", "failed"]) + .HasPostgresEnum("email_status", ["pending", "sending", "sent", "failed", "skipped"]) .HasCollation("public", "ndcoll", "und-u-ks-level2", "icu", false); // Add case-insensitive, accent-sensitive comparison collation modelBuilder.Entity(entity => @@ -888,9 +888,20 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.ToTable("email_outbox"); - // The consumer claims fresh (Pending) rows in FIFO order; (status, created_at) serves both - // the status filter and the ordering. Retry scheduling after hand-off is owned by Hangfire. - entity.HasIndex(e => new { e.Status, e.CreatedAt }); + // Optimistic concurrency for the deferred delivery write: if a lapsed lease lets another run + // reclaim a Sending row mid-batch, the loser's UPDATE matches no row (xmin advanced) and + // throws DbUpdateConcurrencyException instead of clobbering attempt count / terminal state. + // Maps the Postgres xmin system column as a store-generated concurrency token (the manual + // form of the removed UseXminAsConcurrencyToken helper; no real column is created). + entity.Property("xmin") + .HasColumnName("xmin") + .HasColumnType("xid") + .ValueGeneratedOnAddOrUpdate() + .IsConcurrencyToken(); + + // The consumer claims due rows ordered by next_attempt_at; (status, next_attempt_at) serves + // both the status filter and the ordering for the FOR UPDATE SKIP LOCKED claim query. + entity.HasIndex(e => new { e.Status, e.NextAttemptAt }); entity.HasIndex(e => e.Recipient); entity.Property(e => e.Id) @@ -909,6 +920,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasColumnName("payload"); entity.Property(e => e.Status) .HasColumnName("status"); + entity.Property(e => e.AttemptCount) + .HasDefaultValue(0) + .HasColumnName("attempt_count"); + entity.Property(e => e.NextAttemptAt) + .HasDefaultValueSql("CURRENT_TIMESTAMP") + .HasColumnName("next_attempt_at"); entity.Property(e => e.LastError) .VarCharWithLength(HardLimits.EmailOutboxLastErrorMaxLength) .HasColumnName("last_error"); diff --git a/Cron.Tests/Cron.Tests.csproj b/Cron.Tests/Cron.Tests.csproj new file mode 100644 index 00000000..0234ae28 --- /dev/null +++ b/Cron.Tests/Cron.Tests.csproj @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/Cron.Tests/Services/Email/Outbox/EmailOutboxStateMachineTests.cs b/Cron.Tests/Services/Email/Outbox/EmailOutboxStateMachineTests.cs new file mode 100644 index 00000000..6bb8d1ea --- /dev/null +++ b/Cron.Tests/Services/Email/Outbox/EmailOutboxStateMachineTests.cs @@ -0,0 +1,143 @@ +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; +using OpenShock.Cron.Services.Email.Outbox; + +namespace OpenShock.Cron.Tests.Services.Email.Outbox; + +/// +/// Unit tests for the email-outbox delivery state machine - the pure transitions the delivery job +/// applies when it claims a due row and when it records a send outcome. No database, provider, or +/// background host: these pin the delivery contract (terminal states, the Skipped no-op, retry +/// scheduling, the lease, and the exhaustion cut-off) in isolation. The end-to-end DB + delivery-job +/// path is covered by the integration MailTests and EmailOutboxPersistenceTests. +/// +public sealed class EmailOutboxStateMachineTests +{ + private static EmailOutboxMessage NewMessage() => EmailOutboxMessage.Create( + EmailType.PasswordReset, "user@example.com", "User", + new Dictionary { ["dummy"] = "value" }); + + private static readonly DateTime Now = new(2026, 6, 30, 12, 0, 0, DateTimeKind.Utc); + + // --- TryClaim --- + + [Test] + public async Task TryClaim_FreshRow_MarksSendingUnderLease_AndCountsAttempt() + { + var message = NewMessage(); + + var claimed = EmailOutboxStateMachine.TryClaim(message, Now); + + await Assert.That(claimed).IsTrue(); + await Assert.That(message.Status).IsEqualTo(EmailStatus.Sending); + await Assert.That(message.AttemptCount).IsEqualTo(1); + // Lease pushes the next-attempt time out so no other pass reclaims it while it is in flight. + await Assert.That(message.NextAttemptAt).IsEqualTo(Now + EmailOutboxRetryPolicy.DeliveryLease); + } + + [Test] + public async Task TryClaim_WhenAttemptsExhausted_FailsInPlace_AndDoesNotClaim() + { + var message = NewMessage(); + message.AttemptCount = EmailOutboxRetryPolicy.MaxAttempts; // next claim would exceed the budget + + var claimed = EmailOutboxStateMachine.TryClaim(message, Now); + + await Assert.That(claimed).IsFalse(); + await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); + await Assert.That(message.FailedAt).IsEqualTo(Now); + await Assert.That(message.LastError).IsNotNull(); + } + + // --- ApplyResult: terminal outcomes --- + + [Test] + public async Task ApplyResult_Sent_MarksSent_AndClearsError() + { + var message = NewMessage(); + message.LastError = "previous transient blip"; + EmailOutboxStateMachine.TryClaim(message, Now); + + EmailOutboxStateMachine.ApplyResult(message, EmailDispatchResult.Sent, Now); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Sent); + await Assert.That(message.SentAt).IsEqualTo(Now); + await Assert.That(message.FailedAt).IsNull(); + await Assert.That(message.LastError).IsNull(); + } + + [Test] + public async Task ApplyResult_Skipped_MarksSkipped_NotFailed() + { + var message = NewMessage(); + EmailOutboxStateMachine.TryClaim(message, Now); + + EmailOutboxStateMachine.ApplyResult(message, EmailDispatchResult.Skip("reset already used"), Now); + + // A skip is a successful no-op: it must be its own terminal state, never Failed (which would + // invite an operator to requeue it). + await Assert.That(message.Status).IsEqualTo(EmailStatus.Skipped); + await Assert.That(message.FailedAt).IsNull(); + await Assert.That(message.LastError).IsEqualTo("reset already used"); + } + + [Test] + public async Task ApplyResult_Permanent_MarksFailed_WithDetail() + { + var message = NewMessage(); + EmailOutboxStateMachine.TryClaim(message, Now); + + EmailOutboxStateMachine.ApplyResult(message, EmailDispatchResult.Permanent("Provider rejected the message"), Now); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); + await Assert.That(message.FailedAt).IsEqualTo(Now); + await Assert.That(message.LastError).IsEqualTo("Provider rejected the message"); + } + + // --- ApplyResult: transient retry / exhaustion --- + + [Test] + public async Task ApplyResult_Transient_WithAttemptsLeft_RequeuesWithBackoff() + { + var message = NewMessage(); + EmailOutboxStateMachine.TryClaim(message, Now); // AttemptCount = 1 + + EmailOutboxStateMachine.ApplyResult(message, EmailDispatchResult.Transient("smtp timeout"), Now); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Pending); // due again later, not terminal + await Assert.That(message.SentAt).IsNull(); + await Assert.That(message.FailedAt).IsNull(); + await Assert.That(message.LastError).IsEqualTo("smtp timeout"); + // Rescheduled into the future on our own back-off curve. + await Assert.That(message.NextAttemptAt).IsEqualTo(Now + EmailOutboxRetryPolicy.BackoffFor(1)); + await Assert.That(message.NextAttemptAt).IsGreaterThan(Now); + } + + [Test] + public async Task ApplyResult_Transient_OnFinalAttempt_MarksFailed() + { + var message = NewMessage(); + message.AttemptCount = EmailOutboxRetryPolicy.MaxAttempts; // this attempt is the last allowed + + EmailOutboxStateMachine.ApplyResult(message, EmailDispatchResult.Transient("smtp timeout"), Now); + + await Assert.That(message.Status).IsEqualTo(EmailStatus.Failed); + await Assert.That(message.FailedAt).IsEqualTo(Now); + await Assert.That(message.LastError).IsEqualTo("smtp timeout"); + } + + // --- Back-off curve --- + + [Test] + public async Task BackoffFor_IsMonotonic_AndCapped() + { + var previous = TimeSpan.Zero; + for (var attempt = 1; attempt <= EmailOutboxRetryPolicy.MaxAttempts; attempt++) + { + var delay = EmailOutboxRetryPolicy.BackoffFor(attempt); + await Assert.That(delay).IsGreaterThanOrEqualTo(previous); // never decreases + await Assert.That(delay).IsLessThanOrEqualTo(TimeSpan.FromHours(1)); // capped + previous = delay; + } + } +} diff --git a/Cron/Jobs/EmailOutboxDeliveryJob.cs b/Cron/Jobs/EmailOutboxDeliveryJob.cs new file mode 100644 index 00000000..c0510a42 --- /dev/null +++ b/Cron/Jobs/EmailOutboxDeliveryJob.cs @@ -0,0 +1,157 @@ +using Microsoft.EntityFrameworkCore; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; +using OpenShock.Cron.Attributes; +using OpenShock.Cron.Services.Email.Outbox; + +namespace OpenShock.Cron.Jobs; + +/// +/// Claims due rows from the email_outbox table and delivers them, recording attempt count, retry time, +/// and terminal state back on each row. The table is the only place delivery state lives - Hangfire +/// just schedules this job. Runs on the recurring schedule below (retry sweep + safety net) and is also +/// enqueued on demand by the instant a row is written, so +/// fresh mail goes out without waiting for the next minute. +/// +/// +/// Concurrent runs are safe: a row is claimed with FOR UPDATE SKIP LOCKED and leased (Sending), and the +/// terminal write uses an optimistic-concurrency token (xmin), so if a lapsed lease lets another run +/// reclaim a row mid-batch the loser's write fails with +/// instead of clobbering it. Each message is delivered in isolation (its own reload + change-tracker +/// reset and its own try/catch), so one failure never aborts the rest of the batch, and the whole +/// backlog is drained batch by batch rather than 50 per run. +/// +[CronJob("* * * * *")] // Every minute (https://crontab.guru/) +public sealed class EmailOutboxDeliveryJob +{ + private const int BatchSize = 50; + + private readonly OpenShockContext _db; + private readonly IEmailOutboxDispatcher _dispatcher; + private readonly ILogger _logger; + + /// + /// DI constructor + /// + /// + /// + /// + public EmailOutboxDeliveryJob(OpenShockContext db, IEmailOutboxDispatcher dispatcher, ILogger logger) + { + _db = db; + _dispatcher = dispatcher; + _logger = logger; + } + + public async Task Execute() + { + int claimed; + do + { + var ids = await ClaimDueBatchAsync(); + claimed = ids.Count; + + foreach (var id in ids) + { + try + { + await DeliverAsync(id); + } + catch (DbUpdateConcurrencyException) + { + // A lapsed lease let another run reclaim this row mid-batch; it now owns delivery. + _logger.LogDebug("Email outbox message {MessageId} was reclaimed concurrently; skipping", id); + } + catch (Exception ex) + { + // Isolate the failure to this message: the row stays Sending and is retried when its + // lease lapses, while the rest of the batch still goes out. + _logger.LogError(ex, "Unexpected error delivering email outbox message {MessageId}", id); + } + } + } + while (claimed >= BatchSize); + } + + /// + /// Atomically claims up to due rows - rows + /// past their next-attempt time, or rows whose lease has lapsed (a + /// crashed prior run) - flipping each to with a fresh lease and an + /// incremented attempt count. A row that has run out of attempts is failed in place instead. The lock + /// is held only for this short transaction; the sends happen afterwards, outside it. Returns the ids + /// now owned by this run. + /// + private async Task> ClaimDueBatchAsync() + { + _db.ChangeTracker.Clear(); + + await using var transaction = await _db.Database.BeginTransactionAsync(); + + var due = await _db.EmailOutbox.FromSql( + $""" + SELECT * FROM email_outbox + WHERE next_attempt_at <= now() + AND (status = {EmailStatus.Pending} OR status = {EmailStatus.Sending}) + ORDER BY next_attempt_at + LIMIT {BatchSize} + FOR UPDATE SKIP LOCKED + """).ToListAsync(); + + if (due.Count == 0) + { + await transaction.CommitAsync(); + return []; + } + + var nowUtc = DateTime.UtcNow; + var claimed = new List(due.Count); + + foreach (var message in due) + { + if (EmailOutboxStateMachine.TryClaim(message, nowUtc)) + { + claimed.Add(message.Id); + } + else + { + _logger.LogWarning("Email outbox message {MessageId} ({Type}) exhausted via lease reclaim after {Attempts} attempt(s)", + message.Id, message.Type, message.AttemptCount - 1); + } + } + + await _db.SaveChangesAsync(); + await transaction.CommitAsync(); + return claimed; + } + + /// + /// Delivers a single claimed message and records the outcome on its row. Reloads the row fresh (its + /// own change-tracker reset) so a failure on the previous message can't carry over and so the write + /// carries the current concurrency token; if the row is no longer + /// another run already resolved it and this one bows out. + /// + private async Task DeliverAsync(Guid messageId) + { + _db.ChangeTracker.Clear(); + + var message = await _db.EmailOutbox.FirstOrDefaultAsync(m => m.Id == messageId); + if (message is null) + { + _logger.LogWarning("Email outbox message {MessageId} no longer exists; nothing to send", messageId); + return; + } + + if (message.Status != EmailStatus.Sending) return; + + var result = await _dispatcher.SendAsync(message, _db); + + if (result.Outcome == EmailDispatchOutcome.TransientFailure && message.AttemptCount >= EmailOutboxRetryPolicy.MaxAttempts) + { + _logger.LogWarning("Email outbox message {MessageId} ({Type}) failed after {Attempts} attempt(s): {Detail}", + message.Id, message.Type, message.AttemptCount, result.Detail); + } + + EmailOutboxStateMachine.ApplyResult(message, result, DateTime.UtcNow); + await _db.SaveChangesAsync(); + } +} diff --git a/Cron/Program.cs b/Cron/Program.cs index 56f2e491..4846e81e 100644 --- a/Cron/Program.cs +++ b/Cron/Program.cs @@ -32,9 +32,10 @@ hangfireStorageOptions)); builder.Services.AddHangfireServer(); -// Registers the email providers, the outbox dispatcher, the per-email Hangfire send job, and the -// outbox consumer that hands pending rows off to Hangfire. The API host only writes outbox rows; -// all sending happens here. +// Registers the email providers, the outbox dispatcher, and the Redis notification listener. Delivery +// is the EmailOutboxDeliveryJob, driven through Hangfire (recurring every-minute sweep auto-registered +// via [CronJob], plus on-demand enqueue from the listener); all retry/lease/state lives on the +// email_outbox row, not in Hangfire. The API host only writes outbox rows; all sending happens here. await builder.AddEmailService(); builder.AddSwaggerExt(); diff --git a/Cron/Services/Email/EmailServiceExtension.cs b/Cron/Services/Email/EmailServiceExtension.cs index 356e366b..f841bed8 100644 --- a/Cron/Services/Email/EmailServiceExtension.cs +++ b/Cron/Services/Email/EmailServiceExtension.cs @@ -11,11 +11,12 @@ public static async Task AddEmailService(this WebApplicat { var mailOptions = builder.Configuration.GetRequiredSection(MailOptions.SectionName).Get() ?? throw new NullReferenceException(); - // The outbox dispatcher + consumer drive all transactional email regardless of provider; even - // with mail disabled the consumer runs and the send job marks messages terminal (no-op provider). + // The outbox dispatcher delivers all transactional email regardless of provider; even with mail + // disabled the delivery job still runs and marks messages terminal (no-op provider). Delivery is + // driven through Hangfire (a recurring sweep job, auto-registered via [CronJob]; plus the + // notification listener that enqueues it on demand) - there is no background polling loop. builder.Services.AddSingleton(); - builder.Services.AddScoped(); - builder.Services.AddHostedService(); + builder.Services.AddHostedService(); if (mailOptions.Type == MailOptions.MailType.None) { diff --git a/Cron/Services/Email/Outbox/EmailOutboxConsumer.cs b/Cron/Services/Email/Outbox/EmailOutboxConsumer.cs deleted file mode 100644 index 9b3e8d0b..00000000 --- a/Cron/Services/Email/Outbox/EmailOutboxConsumer.cs +++ /dev/null @@ -1,159 +0,0 @@ -using Hangfire; -using Microsoft.EntityFrameworkCore; -using OpenShock.Common.Constants; -using OpenShock.Common.Models; -using OpenShock.Common.OpenShockDb; -using OpenShock.Common.Services.RedisPubSub; -using StackExchange.Redis; - -namespace OpenShock.Cron.Services.Email.Outbox; - -/// -/// The email-outbox hand-off: claims freshly enqueued () rows and -/// hands each to Hangfire as an . It does not send or retry - -/// once a row is enqueued, Hangfire owns its delivery, retry scheduling, and crash recovery. -/// -/// -/// -/// Push first, sweep as a safety net. Hand-off is triggered by a Redis notification published -/// the instant a message is enqueued (), so the job is -/// queued within sub-seconds. A periodic sweep () also -/// runs as a fallback to cover a dropped notification. Because both feed a single coalescing wake -/// signal, the push and the sweep can never fire "on top of each other": any number of signals -/// between drains collapse into one drain, so a notification that lands just before a sweep simply -/// shares the same pass. -/// -/// -/// Claims are safe across instances. Rows are claimed with FOR UPDATE SKIP LOCKED, so if -/// the Cron host is scaled out each instance grabs a disjoint set. Hangfire jobs are enqueued before -/// the claim transaction commits: if the process dies in between, the transaction rolls back and the -/// rows stay to be re-handed-off next pass - at worst a duplicate -/// job, which the idempotent absorbs. -/// -/// -public sealed class EmailOutboxConsumer : BackgroundService -{ - private const int BatchSize = 50; - - private readonly IDbContextFactory _dbContextFactory; - private readonly IBackgroundJobClient _jobClient; - private readonly ISubscriber _subscriber; - private readonly ILogger _logger; - - // Coalescing wake signal: capacity 1, so any number of notifications between drains collapse into - // a single "there is work" wake-up. - private readonly SemaphoreSlim _wakeSignal = new(0, 1); - - /// DI constructor. - public EmailOutboxConsumer( - IDbContextFactory dbContextFactory, - IBackgroundJobClient jobClient, - IConnectionMultiplexer connectionMultiplexer, - ILogger logger) - { - _dbContextFactory = dbContextFactory; - _jobClient = jobClient; - _subscriber = connectionMultiplexer.GetSubscriber(); - _logger = logger; - } - - /// - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - await _subscriber.SubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); - - try - { - while (!stoppingToken.IsCancellationRequested) - { - try - { - await DrainAsync(stoppingToken); - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) - { - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Email outbox hand-off failed"); - } - - try - { - await _wakeSignal.WaitAsync(Duration.EmailOutboxPollInterval, stoppingToken); - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) - { - break; - } - } - } - finally - { - await _subscriber.UnsubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); - } - } - - private void OnPendingNotification(RedisChannel channel, RedisValue value) - { - try - { - _wakeSignal.Release(); - } - catch (SemaphoreFullException) - { - // Already pending a wake-up - nothing to do. - } - } - - private async Task DrainAsync(CancellationToken cancellationToken) - { - int handed; - do - { - handed = await HandOffBatchAsync(cancellationToken); - } - while (handed >= BatchSize && !cancellationToken.IsCancellationRequested); - } - - private async Task HandOffBatchAsync(CancellationToken cancellationToken) - { - await using var db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken); - - var claimed = await db.EmailOutbox.FromSql( - $""" - SELECT * FROM email_outbox - WHERE status = {EmailStatus.Pending} - ORDER BY created_at - LIMIT {BatchSize} - FOR UPDATE SKIP LOCKED - """).ToListAsync(cancellationToken); - - if (claimed.Count == 0) - { - await transaction.CommitAsync(cancellationToken); - return 0; - } - - foreach (var message in claimed) - { - // Enqueue first, flip to Queued second: if we crash here the rollback leaves the row Pending - // (re-handed-off next pass), never Queued-with-no-job. - _jobClient.Enqueue(job => job.SendAsync(message.Id, null, CancellationToken.None)); - message.Status = EmailStatus.Queued; - } - - await db.SaveChangesAsync(cancellationToken); - await transaction.CommitAsync(cancellationToken); - return claimed.Count; - } - - /// - public override void Dispose() - { - _wakeSignal.Dispose(); - base.Dispose(); - } -} diff --git a/Cron/Services/Email/Outbox/EmailOutboxJob.cs b/Cron/Services/Email/Outbox/EmailOutboxJob.cs deleted file mode 100644 index 1b088ee0..00000000 --- a/Cron/Services/Email/Outbox/EmailOutboxJob.cs +++ /dev/null @@ -1,120 +0,0 @@ -using Hangfire; -using Hangfire.Server; -using Microsoft.EntityFrameworkCore; -using OpenShock.Common.Constants; -using OpenShock.Common.Models; -using OpenShock.Common.OpenShockDb; - -namespace OpenShock.Cron.Services.Email.Outbox; - -/// -/// The Hangfire job that delivers a single . One job instance per -/// email, so each message has an independent retry timeline. -/// -/// -/// Hangfire owns the delivery machinery: supplies the back-off -/// curve, the persistent retry schedule, and crash requeue; the job only maps a dispatch outcome onto -/// the row's terminal state. A transient failure is surfaced by throwing, which is what makes -/// Hangfire schedule the next retry. On the final attempt (detected via the RetryCount job -/// parameter) the row is marked before the throw, so an exhausted -/// message never stays stuck looking like it is still in flight. -/// -public sealed class EmailOutboxJob -{ - /// Number of automatic retries before a transiently-failing message is given up as failed. - public const int MaxRetries = 10; - - private readonly IDbContextFactory _dbContextFactory; - private readonly IEmailOutboxDispatcher _dispatcher; - private readonly ILogger _logger; - - /// DI constructor. - public EmailOutboxJob(IDbContextFactory dbContextFactory, IEmailOutboxDispatcher dispatcher, ILogger logger) - { - _dbContextFactory = dbContextFactory; - _dispatcher = dispatcher; - _logger = logger; - } - - /// Delivers the message with the given id. See the type remarks for the retry contract. - [AutomaticRetry(Attempts = MaxRetries, OnAttemptsExceeded = AttemptsExceededAction.Fail)] - public async Task SendAsync(Guid messageId, PerformContext? context, CancellationToken cancellationToken) - { - await using var db = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - - var message = await db.EmailOutbox.FirstOrDefaultAsync(m => m.Id == messageId, cancellationToken); - if (message is null) - { - _logger.LogWarning("Email outbox message {MessageId} no longer exists; nothing to send", messageId); - return; - } - - // Idempotency: a duplicate enqueue or a crash-requeue may re-run a message already resolved. - if (message.Status is EmailStatus.Sent or EmailStatus.Failed) return; - - var result = await _dispatcher.SendAsync(message, db, cancellationToken); - var nowUtc = DateTime.UtcNow; - - switch (result.Outcome) - { - case EmailDispatchOutcome.Sent: - message.Status = EmailStatus.Sent; - message.SentAt = nowUtc; - message.LastError = null; - await db.SaveChangesAsync(cancellationToken); - return; - - case EmailDispatchOutcome.Skipped: - message.Status = EmailStatus.Failed; - message.FailedAt = nowUtc; - message.LastError = Truncate($"Skipped: {result.Detail}"); - await db.SaveChangesAsync(cancellationToken); - return; - - case EmailDispatchOutcome.PermanentFailure: - message.Status = EmailStatus.Failed; - message.FailedAt = nowUtc; - message.LastError = Truncate(result.Detail); - await db.SaveChangesAsync(cancellationToken); - return; - - case EmailDispatchOutcome.TransientFailure: - message.LastError = Truncate(result.Detail); - - // RetryCount is the number of retries already performed (0 on the first run). When it has - // reached the cap this is the last attempt, so record the terminal failure on the row; - // the throw below then lets Hangfire move the job to its Failed state too. - var retryCount = context?.GetJobParameter("RetryCount") ?? 0; - if (retryCount >= MaxRetries) - { - message.Status = EmailStatus.Failed; - message.FailedAt = nowUtc; - await db.SaveChangesAsync(cancellationToken); - _logger.LogWarning("Email outbox message {MessageId} ({Type}) failed after {Attempts} attempt(s): {Detail}", - message.Id, message.Type, retryCount + 1, result.Detail); - } - else - { - await db.SaveChangesAsync(cancellationToken); - } - - throw new EmailOutboxTransientException(result.Detail ?? "Transient email send failure"); - - default: - throw new InvalidOperationException($"Unhandled dispatch outcome {result.Outcome}"); - } - } - - /// Clamps an error string to the column's maximum length. - private static string? Truncate(string? value) - => value is null || value.Length <= HardLimits.EmailOutboxLastErrorMaxLength - ? value - : value[..HardLimits.EmailOutboxLastErrorMaxLength]; -} - -/// Thrown to signal a transient send failure so Hangfire schedules a retry on its back-off curve. -public sealed class EmailOutboxTransientException : Exception -{ - /// Creates the exception with the underlying transient failure detail. - public EmailOutboxTransientException(string message) : base(message) { } -} diff --git a/Cron/Services/Email/Outbox/EmailOutboxNotificationListener.cs b/Cron/Services/Email/Outbox/EmailOutboxNotificationListener.cs new file mode 100644 index 00000000..b884c632 --- /dev/null +++ b/Cron/Services/Email/Outbox/EmailOutboxNotificationListener.cs @@ -0,0 +1,67 @@ +using Hangfire; +using OpenShock.Common.Services.RedisPubSub; +using OpenShock.Common.Utils; +using OpenShock.Cron.Jobs; +using StackExchange.Redis; + +namespace OpenShock.Cron.Services.Email.Outbox; + +/// +/// Bridges the Redis "email enqueued" notification to Hangfire. When the API publishes +/// after writing an outbox row, this enqueues an +/// immediate so fresh mail is delivered within Hangfire's pickup +/// latency instead of waiting for the next scheduled (every-minute) sweep. +/// +/// +/// It is event-driven - a Redis subscription, not a polling loop - and holds no delivery state: the +/// notification is a payload-less nudge, and the job reads the actual due rows from the table. A +/// dropped notification or a failed enqueue only delays delivery to the next scheduled sweep, so the +/// callback never throws. +/// +public sealed class EmailOutboxNotificationListener : IHostedService +{ + private readonly ISubscriber _subscriber; + private readonly IBackgroundJobClient _jobClient; + private readonly ILogger _logger; + + /// DI constructor. + public EmailOutboxNotificationListener( + IConnectionMultiplexer connectionMultiplexer, + IBackgroundJobClient jobClient, + ILogger logger) + { + _subscriber = connectionMultiplexer.GetSubscriber(); + _jobClient = jobClient; + _logger = logger; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + => _subscriber.SubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); + + private void OnPendingNotification(RedisChannel channel, RedisValue value) + { + // Offload off the Redis subscription's message-processing thread: enqueueing is a synchronous + // Hangfire/Postgres write, and blocking it here would serialize every other notification behind + // this DB round-trip (see RedisSubscriberService for the same pattern). + OsTask.Run(EnqueueDeliveryAsync); + } + + private Task EnqueueDeliveryAsync() + { + try + { + _jobClient.Enqueue(job => job.Execute()); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to enqueue email outbox delivery from pending notification; delivery falls back to the scheduled sweep"); + } + + return Task.CompletedTask; + } + + /// + public Task StopAsync(CancellationToken cancellationToken) + => _subscriber.UnsubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); +} diff --git a/Cron/Services/Email/Outbox/EmailOutboxRetryPolicy.cs b/Cron/Services/Email/Outbox/EmailOutboxRetryPolicy.cs new file mode 100644 index 00000000..b462c33e --- /dev/null +++ b/Cron/Services/Email/Outbox/EmailOutboxRetryPolicy.cs @@ -0,0 +1,45 @@ +namespace OpenShock.Cron.Services.Email.Outbox; + +/// +/// The outbox's self-owned delivery policy: how many times a transiently-failing email is retried, +/// how long to wait before each retry, and how long a claimed ("Sending") row is leased before another +/// executor may reclaim it. All retry state lives on the row itself, so this is the only place the +/// curve is defined - there is no external job scheduler to keep in sync. +/// +public static class EmailOutboxRetryPolicy +{ + /// + /// Maximum number of delivery attempts before a transiently-failing message is given up as + /// . An attempt is counted when the row is + /// claimed, so this also bounds crash/lease-expiry reclaim loops. Matches the prior Hangfire policy + /// (one initial run plus ten retries). + /// + public const int MaxAttempts = 11; + + /// Delay before the first retry; each subsequent retry doubles it up to . + private static readonly TimeSpan BaseBackoff = TimeSpan.FromSeconds(30); + + /// Upper bound on the retry back-off, so a long outage doesn't push retries hours apart. + private static readonly TimeSpan MaxBackoff = TimeSpan.FromHours(1); + + /// + /// How long a claimed row stays leased to its executor. If the executor doesn't reach a terminal + /// state (or schedule the next retry) within this window - e.g. it crashed mid-send - the row + /// becomes due again and another pass reclaims it. Comfortably longer than a healthy send. + /// + public static readonly TimeSpan DeliveryLease = TimeSpan.FromMinutes(5); + + /// + /// The back-off before retrying after failed attempts (1-based): + /// capped exponential growth (30s, 1m, 2m, 4m, ... up to 1h). + /// + public static TimeSpan BackoffFor(int attemptNumber) + { + if (attemptNumber < 1) attemptNumber = 1; + + // Cap the shift so 2^exponent can never overflow before the Min clamp. + var exponent = Math.Min(attemptNumber - 1, 16); + var scaled = BaseBackoff * Math.Pow(2, exponent); + return scaled < MaxBackoff ? scaled : MaxBackoff; + } +} diff --git a/Cron/Services/Email/Outbox/EmailOutboxStateMachine.cs b/Cron/Services/Email/Outbox/EmailOutboxStateMachine.cs new file mode 100644 index 00000000..f90ca38a --- /dev/null +++ b/Cron/Services/Email/Outbox/EmailOutboxStateMachine.cs @@ -0,0 +1,88 @@ +using OpenShock.Common.Constants; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; +using OpenShock.Common.Utils; + +namespace OpenShock.Cron.Services.Email.Outbox; + +/// +/// The pure state transitions of an : claiming a due row for delivery +/// and recording a delivery outcome. Kept free of I/O so the delivery contract (including the retry +/// back-off and the exhaustion cut-off) is unit-testable in isolation; the delivery job is the only +/// caller and owns the surrounding database transaction. +/// +public static class EmailOutboxStateMachine +{ + /// + /// Claims for a delivery attempt: counts the attempt and, unless the + /// attempt budget is now exhausted, marks it with a fresh lease. + /// Returns true if the caller should now deliver it, or false if it was failed in + /// place because it ran out of attempts (only reachable via repeated lease-expiry reclaims of a + /// send that keeps crashing). + /// + public static bool TryClaim(EmailOutboxMessage message, DateTime nowUtc) + { + message.AttemptCount++; + + if (message.AttemptCount > EmailOutboxRetryPolicy.MaxAttempts) + { + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + message.LastError = $"Exhausted after {message.AttemptCount - 1} attempt(s) (delivery did not complete)".Truncate(HardLimits.EmailOutboxLastErrorMaxLength); + return false; + } + + message.Status = EmailStatus.Sending; + message.NextAttemptAt = nowUtc + EmailOutboxRetryPolicy.DeliveryLease; + return true; + } + + /// + /// Records the result of a delivery attempt on : a terminal + /// //, + /// or - for a transient failure with attempts to spare - back to + /// with the next-attempt time pushed out on the back-off curve. Assumes has + /// already counted this attempt. + /// + public static void ApplyResult(EmailOutboxMessage message, EmailDispatchResult result, DateTime nowUtc) + { + switch (result.Outcome) + { + case EmailDispatchOutcome.Sent: + message.Status = EmailStatus.Sent; + message.SentAt = nowUtc; + message.LastError = null; + break; + + case EmailDispatchOutcome.Skipped: + // A successful no-op (request used/expired/superseded/gone). Distinct from Failed so it + // is never mistaken for a delivery problem or requeued. + message.Status = EmailStatus.Skipped; + message.LastError = result.Detail?.Truncate(HardLimits.EmailOutboxLastErrorMaxLength); + break; + + case EmailDispatchOutcome.PermanentFailure: + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + message.LastError = result.Detail?.Truncate(HardLimits.EmailOutboxLastErrorMaxLength); + break; + + case EmailDispatchOutcome.TransientFailure: + message.LastError = result.Detail?.Truncate(HardLimits.EmailOutboxLastErrorMaxLength); + if (message.AttemptCount >= EmailOutboxRetryPolicy.MaxAttempts) + { + message.Status = EmailStatus.Failed; + message.FailedAt = nowUtc; + } + else + { + message.Status = EmailStatus.Pending; + message.NextAttemptAt = nowUtc + EmailOutboxRetryPolicy.BackoffFor(message.AttemptCount); + } + break; + + default: + throw new InvalidOperationException($"Unhandled dispatch outcome {result.Outcome}"); + } + } +} diff --git a/OpenShockBackend.slnx b/OpenShockBackend.slnx index 7143254a..156ce6b7 100644 --- a/OpenShockBackend.slnx +++ b/OpenShockBackend.slnx @@ -5,6 +5,7 @@ + \ No newline at end of file diff --git a/dotnet-tools.json b/dotnet-tools.json new file mode 100644 index 00000000..807729e4 --- /dev/null +++ b/dotnet-tools.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "dotnet-ef": { + "version": "10.0.9", + "commands": [ + "dotnet-ef" + ], + "rollForward": false + } + } +} \ No newline at end of file From d17b4f4979e3b46e87b063c0d732f240a4ecd51e Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 17:00:43 +0200 Subject: [PATCH 07/15] Delete dotnet-tools.json --- dotnet-tools.json | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 dotnet-tools.json diff --git a/dotnet-tools.json b/dotnet-tools.json deleted file mode 100644 index 807729e4..00000000 --- a/dotnet-tools.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": 1, - "isRoot": true, - "tools": { - "dotnet-ef": { - "version": "10.0.9", - "commands": [ - "dotnet-ef" - ], - "rollForward": false - } - } -} \ No newline at end of file From cd44d4b3ef2d076b571473e3c007c443b3bd3cc3 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 17:15:05 +0200 Subject: [PATCH 08/15] Update EmailOutboxNotificationListener.cs --- .../Outbox/EmailOutboxNotificationListener.cs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/Cron/Services/Email/Outbox/EmailOutboxNotificationListener.cs b/Cron/Services/Email/Outbox/EmailOutboxNotificationListener.cs index b884c632..0c8b666d 100644 --- a/Cron/Services/Email/Outbox/EmailOutboxNotificationListener.cs +++ b/Cron/Services/Email/Outbox/EmailOutboxNotificationListener.cs @@ -1,6 +1,5 @@ using Hangfire; using OpenShock.Common.Services.RedisPubSub; -using OpenShock.Common.Utils; using OpenShock.Cron.Jobs; using StackExchange.Redis; @@ -36,18 +35,10 @@ public EmailOutboxNotificationListener( } /// - public Task StartAsync(CancellationToken cancellationToken) - => _subscriber.SubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); + public Task StartAsync(CancellationToken cancellationToken) => + _subscriber.SubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); private void OnPendingNotification(RedisChannel channel, RedisValue value) - { - // Offload off the Redis subscription's message-processing thread: enqueueing is a synchronous - // Hangfire/Postgres write, and blocking it here would serialize every other notification behind - // this DB round-trip (see RedisSubscriberService for the same pattern). - OsTask.Run(EnqueueDeliveryAsync); - } - - private Task EnqueueDeliveryAsync() { try { @@ -57,11 +48,9 @@ private Task EnqueueDeliveryAsync() { _logger.LogWarning(ex, "Failed to enqueue email outbox delivery from pending notification; delivery falls back to the scheduled sweep"); } - - return Task.CompletedTask; } /// - public Task StopAsync(CancellationToken cancellationToken) - => _subscriber.UnsubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); + public Task StopAsync(CancellationToken cancellationToken) => + _subscriber.UnsubscribeAsync(RedisChannels.EmailOutboxPending, OnPendingNotification); } From c94ba4fa4e89f602b727437691fa66ea1e0600af Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 20:22:35 +0200 Subject: [PATCH 09/15] test(email): drive outbox delivery directly so integration tests are deterministic WebApplicationFactory stops the Cron host at WebApplication.Build(), so the recurring [CronJob] email-outbox sweep registered after Build() in Cron/Program.cs never runs under tests. That left email-flow integration tests depending solely on the one-shot Redis "pending" nudge with no periodic fallback, so mail was never drained inside the Mailpit wait window and 8 flow tests failed. Run the same EmailOutboxDeliveryJob.Execute() the production sweep runs, on a fast 250ms loop in the test host. It is concurrency-safe (FOR UPDATE SKIP LOCKED + xmin), so it composes with the still-live notification listener. --- API.IntegrationTests/WebApplicationFactory.cs | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/API.IntegrationTests/WebApplicationFactory.cs b/API.IntegrationTests/WebApplicationFactory.cs index 0c25c2f4..340e3a7e 100644 --- a/API.IntegrationTests/WebApplicationFactory.cs +++ b/API.IntegrationTests/WebApplicationFactory.cs @@ -14,6 +14,7 @@ using Serilog; using Serilog.Events; using TUnit.Core.Interfaces; +using EmailOutboxDeliveryJob = cronhost::OpenShock.Cron.Jobs.EmailOutboxDeliveryJob; namespace OpenShock.API.IntegrationTests; @@ -35,6 +36,16 @@ public class WebApplicationFactory : WebApplicationFactory, IAsyncIniti // it sees the same outbox rows the API writes. private CronHost? _cronHost; + // Drives the email-outbox delivery job in the test host. In production the job runs on a recurring + // every-minute Hangfire sweep (registered in Cron/Program.cs *after* WebApplication.Build()) plus an + // on-demand enqueue from the Redis "pending" nudge. WebApplicationFactory stops the Cron host at + // Build(), so that post-build recurring registration never runs here and a per-minute sweep would be + // far too coarse for the Mailpit wait window anyway. To keep delivery deterministic we run the very + // same job directly on a fast loop; it is built to run concurrently (FOR UPDATE SKIP LOCKED + xmin), + // so it composes safely with the still-live notification listener. + private CancellationTokenSource? _deliveryPumpCts; + private Task? _deliveryPump; + /// Service provider of the in-process Cron host (email dispatcher, outbox job, etc.). public IServiceProvider CronServices => _cronHost?.Services ?? throw new InvalidOperationException("Cron host not initialized"); @@ -44,12 +55,59 @@ public Task InitializeAsync() _ = Server; // Boots the API host, which sets the shared OPENSHOCK__* environment variables. _cronHost = new CronHost(); _ = _cronHost.Services; // Boots the Cron host: the outbox delivery job + notification listener run. + StartOutboxDeliveryPump(_cronHost.Services); return Task.CompletedTask; } + private void StartOutboxDeliveryPump(IServiceProvider cronServices) + { + _deliveryPumpCts = new CancellationTokenSource(); + var cancellationToken = _deliveryPumpCts.Token; + + _deliveryPump = Task.Run(async () => + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await using var scope = cronServices.CreateAsyncScope(); + var job = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + await job.Execute(); + } + catch + { + // Best effort: a transient hiccup (e.g. a row reclaimed by the live notification + // listener mid-batch) is simply picked up again on the next tick. + } + + try + { + await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken); + } + catch (OperationCanceledException) + { + return; + } + } + }, cancellationToken); + } + protected override void Dispose(bool disposing) { - if (disposing) _cronHost?.Dispose(); + if (disposing) + { + _deliveryPumpCts?.Cancel(); + try + { + _deliveryPump?.GetAwaiter().GetResult(); + } + catch (OperationCanceledException) + { + // Expected when the pump is cancelled on shutdown. + } + _deliveryPumpCts?.Dispose(); + _cronHost?.Dispose(); + } base.Dispose(disposing); } From a6fc2a7aba501b13f54a891aad94a25f25cd074b Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 21:02:38 +0200 Subject: [PATCH 10/15] Update EmailOutboxDispatcher.cs --- Cron/Services/Email/Outbox/EmailOutboxDispatcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cron/Services/Email/Outbox/EmailOutboxDispatcher.cs b/Cron/Services/Email/Outbox/EmailOutboxDispatcher.cs index 51a6ca0a..dc3d2587 100644 --- a/Cron/Services/Email/Outbox/EmailOutboxDispatcher.cs +++ b/Cron/Services/Email/Outbox/EmailOutboxDispatcher.cs @@ -147,7 +147,7 @@ private static async Task MintTokenAsync(OpenShockContext db, Action Date: Tue, 30 Jun 2026 22:46:27 +0200 Subject: [PATCH 11/15] feat(email): newest-wins coalescing for transactional emails Add an opaque per-row coalesce key so only the newest pending request of a kind is delivered (account activation, password reset, email-change verification), while the email-change notice keeps delivering every occurrence. - EmailOutboxMessage.CoalesceKey (nullable) + coalesce_key column/index; the domain stamps the key via EmailOutboxCoalesceKeys (activation/pwreset/emailchange:{userId}). - Delivery job skips a row as Skipped("Superseded by a newer request") when a strictly newer sibling shares its key; null key never coalesces. Fixes stale redrive double-sends and out-of-order delivery without requiring delivery ordering. - Per-type EmailOutboxMessage factories (ForAccountActivation/ForPasswordReset/ ForEmailVerification/ForEmailChangeNotice) own the (type, payload key, coalesce key) triple so call sites can't drift; AccountService call sites collapsed to one line each. - Rewrite the two sibling-request mail tests to assert newest-wins delivery (older row Skipped) and preserve completion-time stamp-rotation invalidation via atomic DB seeding. - Document the outbox design in docs/email-outbox-design.md. --- .../Tests/EmailOutboxPersistenceTests.cs | 2 + API.IntegrationTests/Tests/MailTests.cs | 238 ++++++++++++------ API/Services/Account/AccountService.cs | 15 +- Common/Constants/HardLimits.cs | 1 + .../20260630103404_AddEmailOutbox.Designer.cs | 7 + .../20260630103404_AddEmailOutbox.cs | 6 + .../OpenShockContextModelSnapshot.cs | 7 + Common/OpenShockDb/EmailOutboxCoalesceKeys.cs | 27 ++ Common/OpenShockDb/EmailOutboxMessage.cs | 44 +++- Common/OpenShockDb/OpenShockContext.cs | 6 + Cron/Jobs/EmailOutboxDeliveryJob.cs | 21 ++ 11 files changed, 284 insertions(+), 90 deletions(-) create mode 100644 Common/OpenShockDb/EmailOutboxCoalesceKeys.cs diff --git a/API.IntegrationTests/Tests/EmailOutboxPersistenceTests.cs b/API.IntegrationTests/Tests/EmailOutboxPersistenceTests.cs index ccb267cd..5755e190 100644 --- a/API.IntegrationTests/Tests/EmailOutboxPersistenceTests.cs +++ b/API.IntegrationTests/Tests/EmailOutboxPersistenceTests.cs @@ -35,6 +35,7 @@ public async Task EveryStatus_AndAttemptColumns_RoundTripThroughPostgres() message.Status = status; message.AttemptCount = 3; message.LastError = "boom"; + message.CoalesceKey = "pwreset:round-trip"; db.EmailOutbox.Add(message); await db.SaveChangesAsync(); id = message.Id; @@ -46,6 +47,7 @@ public async Task EveryStatus_AndAttemptColumns_RoundTripThroughPostgres() await Assert.That(loaded.Status).IsEqualTo(status); await Assert.That(loaded.AttemptCount).IsEqualTo(3); await Assert.That(loaded.LastError).IsEqualTo("boom"); + await Assert.That(loaded.CoalesceKey).IsEqualTo("pwreset:round-trip"); } } diff --git a/API.IntegrationTests/Tests/MailTests.cs b/API.IntegrationTests/Tests/MailTests.cs index 8f8cd793..03126974 100644 --- a/API.IntegrationTests/Tests/MailTests.cs +++ b/API.IntegrationTests/Tests/MailTests.cs @@ -3,7 +3,10 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using OpenShock.API.IntegrationTests.Helpers; +using OpenShock.Common.Constants; +using OpenShock.Common.Models; using OpenShock.Common.OpenShockDb; +using OpenShock.Common.Utils; namespace OpenShock.API.IntegrationTests.Tests; @@ -363,7 +366,7 @@ public async Task PasswordResetCheck_InvalidToken_Returns404() } [Test] - public async Task ChangeEmailFlow_SecondPendingRequest_InvalidatedAfterFirstCompletes() + public async Task ChangeEmailFlow_NewerRequest_SupersedesOlder_OnlyNewestVerificationDelivered() { var oldEmail = TestHelper.UniqueEmail("mail-chgemail-sibling-old"); var firstNewEmail = TestHelper.UniqueEmail("mail-chgemail-sibling-first"); @@ -372,107 +375,124 @@ public async Task ChangeEmailFlow_SecondPendingRequest_InvalidatedAfterFirstComp const string password = "SecurePassword123#"; using var mailpit = WebApplicationFactory.CreateMailpitHelper(); - var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, username, oldEmail, password); - using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); + var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, username, oldEmail, password); - // Initiate two concurrent email change requests - var firstInit = await client.PostAsync("/1/account/email-change", TestHelper.JsonContent(new + // Seed two pending email-change requests for the same user, sharing one coalesce key, committed + // together so both are pending before the delivery job runs. Newest-wins coalescing must deliver + // only the newer verification (to secondNewEmail) and skip the older (to firstNewEmail). + Guid olderOutboxId, newerOutboxId; + await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) { - currentPassword = password, - email = firstNewEmail - })); - await Assert.That(firstInit.StatusCode).IsEqualTo(HttpStatusCode.OK); + var db = scope.ServiceProvider.GetRequiredService(); + var stamp = await db.Users.Where(u => u.Id == userId).Select(u => u.SecurityStamp).FirstAsync(); - var secondInit = await client.PostAsync("/1/account/email-change", TestHelper.JsonContent(new - { - currentPassword = password, - email = secondNewEmail - })); - await Assert.That(secondInit.StatusCode).IsEqualTo(HttpStatusCode.OK); + var older = NewPendingEmailChange(userId, oldEmail, firstNewEmail, username, stamp); + var newer = NewPendingEmailChange(userId, oldEmail, secondNewEmail, username, stamp); - var firstMessage = await mailpit.WaitForMessageAsync(firstNewEmail); - var secondMessage = await mailpit.WaitForMessageAsync(secondNewEmail); - await Assert.That(firstMessage).IsNotNull(); - await Assert.That(secondMessage).IsNotNull(); + db.UserEmailChanges.AddRange(older.Change, newer.Change); + db.EmailOutbox.AddRange(older.Outbox, newer.Outbox); + await db.SaveChangesAsync(); - var firstFull = await mailpit.GetMessageAsync(firstMessage!.Id); - var secondFull = await mailpit.GetMessageAsync(secondMessage!.Id); - var firstToken = ExtractQueryParam(firstFull!.Html, "token"); - var secondToken = ExtractQueryParam(secondFull!.Html, "token"); - await Assert.That(firstToken).IsNotNull().And.IsNotEmpty(); - await Assert.That(secondToken).IsNotNull().And.IsNotEmpty(); + olderOutboxId = older.Outbox.Id; + newerOutboxId = newer.Outbox.Id; + } - using var anonClient = WebApplicationFactory.CreateClient(); + // Only the newest request's verification email is delivered; the older is skipped as superseded. + var message = await mailpit.WaitForMessageAsync(secondNewEmail); + await Assert.That(message).IsNotNull(); - // Complete the first request — email becomes firstNewEmail - var firstVerify = await anonClient.PostAsync($"/1/account/email-change/verify?token={firstToken}", null); - await Assert.That(firstVerify.StatusCode).IsEqualTo(HttpStatusCode.OK); + var newerOutbox = await WaitForOutboxStatusAsync(newerOutboxId, EmailStatus.Sent); + var olderOutbox = await WaitForOutboxStatusAsync(olderOutboxId, EmailStatus.Skipped); + await Assert.That(newerOutbox.Status).IsEqualTo(EmailStatus.Sent); + await Assert.That(olderOutbox.Status).IsEqualTo(EmailStatus.Skipped); - // Second pending request is now invalid: its SecurityStampAtCreate snapshot no longer matches User.SecurityStamp. - var secondVerify = await anonClient.PostAsync($"/1/account/email-change/verify?token={secondToken}", null); - await Assert.That(secondVerify.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + // The superseded request's address never receives anything. + var firstInbox = await mailpit.SearchByRecipientAsync(firstNewEmail); + await Assert.That(firstInbox.Count).IsEqualTo(0); - await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); - var db = scope.ServiceProvider.GetRequiredService(); - var afterUser = await db.Users.AsNoTracking().FirstAsync(u => u.Id == user.Id); - await Assert.That(afterUser.Email).IsEqualTo(firstNewEmail); + // The delivered (newer) verification completes, switching the email to secondNewEmail. + var full = await mailpit.GetMessageAsync(message!.Id); + var token = ExtractQueryParam(full!.Html, "token"); + await Assert.That(token).IsNotNull().And.IsNotEmpty(); + + using var anonClient = WebApplicationFactory.CreateClient(); + var verify = await anonClient.PostAsync($"/1/account/email-change/verify?token={token}", null); + await Assert.That(verify.StatusCode).IsEqualTo(HttpStatusCode.OK); + + await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var afterUser = await db.Users.AsNoTracking().FirstAsync(u => u.Id == userId); + await Assert.That(afterUser.Email).IsEqualTo(secondNewEmail); + } } [Test] - public async Task PasswordResetFlow_SecondPendingResetInvalidatedAfterFirstCompletes() + public async Task PasswordResetFlow_NewerRequest_SupersedesOlder_OnlyNewestDelivered() { var email = TestHelper.UniqueEmail("mail-pwreset-sibling"); var username = TestHelper.UniqueUsername("mailpwresetsibling"); - const string firstNewPassword = "FirstNewPassword123#"; - const string secondNewPassword = "SecondNewPassword456#"; + const string newPassword = "FreshPassword123#"; using var mailpit = WebApplicationFactory.CreateMailpitHelper(); - await TestHelper.CreateUserInDb(WebApplicationFactory, username, email, "OldPassword123#"); + var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, username, email, "OldPassword123#"); + + // Seed two pending password-reset requests for the same user, sharing one coalesce key, committed + // together so both are pending before the delivery job runs. The older must be superseded. + Guid olderResetId, newerResetId, olderOutboxId, newerOutboxId; + await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var stamp = await db.Users.Where(u => u.Id == userId).Select(u => u.SecurityStamp).FirstAsync(); + + var older = NewPendingReset(userId, email, username, stamp); + var newer = NewPendingReset(userId, email, username, stamp); + + db.UserPasswordResets.AddRange(older.Reset, newer.Reset); + db.EmailOutbox.AddRange(older.Outbox, newer.Outbox); + await db.SaveChangesAsync(); + + olderResetId = older.Reset.Id; + newerResetId = newer.Reset.Id; + olderOutboxId = older.Outbox.Id; + newerOutboxId = newer.Outbox.Id; + } + + // Only the newest request's email is delivered; the older is skipped as superseded. + var message = await mailpit.WaitForMessageAsync(email); + await Assert.That(message).IsNotNull(); + + var newerOutbox = await WaitForOutboxStatusAsync(newerOutboxId, EmailStatus.Sent); + var olderOutbox = await WaitForOutboxStatusAsync(olderOutboxId, EmailStatus.Skipped); + await Assert.That(newerOutbox.Status).IsEqualTo(EmailStatus.Sent); + await Assert.That(olderOutbox.Status).IsEqualTo(EmailStatus.Skipped); + + var delivered = await mailpit.SearchByRecipientAsync(email); + await Assert.That(delivered.Count).IsEqualTo(1); + + // The delivered link belongs to the newer reset, and it completes. + var full = await mailpit.GetMessageAsync(message!.Id); + var (resetId, secret) = ExtractPasswordResetParams(full!.Html); + await Assert.That(resetId).IsEqualTo(newerResetId.ToString()); using var client = WebApplicationFactory.CreateClient(); + var complete = await client.PostAsync( + $"/1/account/password-reset/{resetId}/{secret}/complete", + TestHelper.JsonContent(new { password = newPassword })); + await Assert.That(complete.StatusCode).IsEqualTo(HttpStatusCode.OK); - // Fire two reset requests back-to-back, then wait for both emails to land. - var firstInit = await client.PostAsync("/2/account/password-reset", TestHelper.JsonContent(new { email, turnstileResponse = "valid-token" })); - await Assert.That(firstInit.StatusCode).IsEqualTo(HttpStatusCode.OK); - - var secondInit = await client.PostAsync("/2/account/password-reset", TestHelper.JsonContent(new { email, turnstileResponse = "valid-token" })); - await Assert.That(secondInit.StatusCode).IsEqualTo(HttpStatusCode.OK); - - var messages = await mailpit.WaitForMessagesAsync(email, minCount: 2); - await Assert.That(messages.Count).IsGreaterThanOrEqualTo(2); - - // We don't care which of the two emails came from which request — the scenario is just - // "two valid pending resets exist; completing either must invalidate the other". Pick the - // first two distinct reset-id/secret pairs we see and call them A and B. - var fullA = await mailpit.GetMessageAsync(messages[0].Id); - var fullB = await mailpit.GetMessageAsync(messages[1].Id); - var (resetIdA, secretA) = ExtractPasswordResetParams(fullA!.Html); - var (resetIdB, secretB) = ExtractPasswordResetParams(fullB!.Html); - await Assert.That(resetIdA).IsNotNull().And.IsNotEmpty(); - await Assert.That(resetIdB).IsNotNull().And.IsNotEmpty(); - await Assert.That(resetIdA).IsNotEqualTo(resetIdB); - - // Complete reset A - var completeA = await client.PostAsync( - $"/1/account/password-reset/{resetIdA}/{secretA}/complete", - TestHelper.JsonContent(new { password = firstNewPassword })); - await Assert.That(completeA.StatusCode).IsEqualTo(HttpStatusCode.OK); - - // Reset B (sibling) must no longer be usable - var checkB = await client.GetAsync( - $"/1/account/password-reset/{resetIdB}/{secretB}"); - await Assert.That(checkB.StatusCode).IsEqualTo(HttpStatusCode.NotFound); - - var completeB = await client.PostAsync( - $"/1/account/password-reset/{resetIdB}/{secretB}/complete", - TestHelper.JsonContent(new { password = secondNewPassword })); - await Assert.That(completeB.StatusCode).IsEqualTo(HttpStatusCode.NotFound); - - // Password from the winning reset works + // Completing the surviving reset rotated the security stamp, so the superseded sibling is dead + // at redemption too (its SecurityStampAtCreate no longer matches) - not just undelivered. + var completeOlder = await client.PostAsync( + $"/1/account/password-reset/{olderResetId}/irrelevant-secret/complete", + TestHelper.JsonContent(new { password = "AnotherPassword789#" })); + await Assert.That(completeOlder.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + + // The new password from the surviving reset works. var loginResponse = await client.PostAsync("/2/account/login", TestHelper.JsonContent(new { usernameOrEmail = email, - password = firstNewPassword, + password = newPassword, turnstileResponse = "valid-token" })); await Assert.That(loginResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); @@ -502,6 +522,66 @@ public async Task ChangeEmail_AlreadyInUse_Returns409() // --- Helpers --- + /// + /// Builds a pending and its matching outbox row (carrying + /// ), exactly as the API would, for direct DB seeding. Each call mints + /// fresh UUIDv7 ids, so calling it twice in order yields a strictly-older then strictly-newer pair. + /// + private static (UserPasswordReset Reset, EmailOutboxMessage Outbox) NewPendingReset( + Guid userId, string email, string? recipientName, Guid stamp) + { + var reset = new UserPasswordReset + { + Id = Guid.CreateVersion7(), + UserId = userId, + TokenHash = HashingUtils.HashToken(CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength)), + SecurityStampAtCreate = stamp, + CreatedAt = DateTime.UtcNow + }; + var outbox = EmailOutboxMessage.ForPasswordReset(reset.Id, userId, email, recipientName); + return (reset, outbox); + } + + /// + /// Builds a pending and its matching verification outbox row (carrying + /// ) for direct DB seeding. Ordering mirrors . + /// + private static (UserEmailChange Change, EmailOutboxMessage Outbox) NewPendingEmailChange( + Guid userId, string oldEmail, string newEmail, string? recipientName, Guid stamp) + { + var change = new UserEmailChange + { + Id = Guid.CreateVersion7(), + UserId = userId, + OldEmail = oldEmail, + NewEmail = newEmail, + TokenHash = HashingUtils.HashToken(CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength)), + SecurityStampAtCreate = stamp, + CreatedAt = DateTime.UtcNow + }; + var outbox = EmailOutboxMessage.ForEmailVerification(change.Id, userId, newEmail, recipientName); + return (change, outbox); + } + + /// + /// Polls the outbox row until it reaches or the timeout elapses, returning + /// the last-read row either way so the caller's assertion reports the actual state. + /// + private async Task WaitForOutboxStatusAsync(Guid outboxId, EmailStatus status, TimeSpan? timeout = null) + { + var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(15)); + EmailOutboxMessage? row = null; + while (DateTime.UtcNow < deadline) + { + await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + row = await db.EmailOutbox.AsNoTracking().FirstOrDefaultAsync(m => m.Id == outboxId); + if (row is not null && row.Status == status) return row; + await Task.Delay(200); + } + return row ?? throw new InvalidOperationException($"Outbox row {outboxId} not found"); + } + /// /// Extracts a query parameter value from a URL embedded in HTML (first <a href> containing the param). /// diff --git a/API/Services/Account/AccountService.cs b/API/Services/Account/AccountService.cs index 4aa51351..1ab0968a 100644 --- a/API/Services/Account/AccountService.cs +++ b/API/Services/Account/AccountService.cs @@ -130,8 +130,7 @@ public async Task, AccountWithEmailOrUsernameExists>> Create TokenHash = SeedTokenHash() }; - _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.AccountActivation, email, username, - new Dictionary { [EmailOutboxPayloadKeys.UserId] = user.Id.ToString() })); + _db.EmailOutbox.Add(EmailOutboxMessage.ForAccountActivation(user.Id, email, username)); await _db.SaveChangesAsync(); await NotifyEmailOutboxAsync(); @@ -194,8 +193,7 @@ public async Task, AccountWithEmailOrUsernameExists>> Create CreatedAt = creationTime }; - _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.AccountActivation, email, username, - new Dictionary { [EmailOutboxPayloadKeys.UserId] = user.Id.ToString() })); + _db.EmailOutbox.Add(EmailOutboxMessage.ForAccountActivation(user.Id, email, username)); await _db.SaveChangesAsync(); } @@ -443,8 +441,7 @@ public async Task { [EmailOutboxPayloadKeys.PasswordResetId] = passwordReset.Id.ToString() })); + _db.EmailOutbox.Add(EmailOutboxMessage.ForPasswordReset(passwordReset.Id, user.User.Id, user.User.Email, user.User.Name)); await _db.SaveChangesAsync(); await NotifyEmailOutboxAsync(); @@ -614,10 +611,8 @@ public async Task { [EmailOutboxPayloadKeys.EmailChangeId] = emailChange.Id.ToString() })); - _db.EmailOutbox.Add(EmailOutboxMessage.Create(EmailType.EmailChangeNotice, data.User.Email, data.User.Name, - new Dictionary { [EmailOutboxPayloadKeys.NewEmail] = lowerCaseEmail })); + _db.EmailOutbox.Add(EmailOutboxMessage.ForEmailVerification(emailChange.Id, data.User.Id, lowerCaseEmail, data.User.Name)); + _db.EmailOutbox.Add(EmailOutboxMessage.ForEmailChangeNotice(lowerCaseEmail, data.User.Email, data.User.Name)); await _db.SaveChangesAsync(); await NotifyEmailOutboxAsync(); diff --git a/Common/Constants/HardLimits.cs b/Common/Constants/HardLimits.cs index 716727cd..3cbc0660 100644 --- a/Common/Constants/HardLimits.cs +++ b/Common/Constants/HardLimits.cs @@ -42,6 +42,7 @@ public static class HardLimits public const int PasswordHashMaxLength = 100; public const int EmailOutboxLastErrorMaxLength = 1024; + public const int EmailOutboxCoalesceKeyMaxLength = 128; public const int UserEmailChangeSecretMaxLength = 128; public const int UserActivationRequestSecretMaxLength = 128; diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs index 5b22a735..dcf525ba 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs +++ b/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs @@ -443,6 +443,11 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasDefaultValue(0) .HasColumnName("attempt_count"); + b.Property("CoalesceKey") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("coalesce_key"); + b.Property("CreatedAt") .ValueGeneratedOnAdd() .HasColumnType("timestamp with time zone") @@ -501,6 +506,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.HasKey("Id") .HasName("email_outbox_pkey"); + b.HasIndex("CoalesceKey"); + b.HasIndex("Recipient"); b.HasIndex("Status", "NextAttemptAt"); diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.cs b/Common/Migrations/20260630103404_AddEmailOutbox.cs index 02406e23..4ee2a542 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.cs +++ b/Common/Migrations/20260630103404_AddEmailOutbox.cs @@ -46,6 +46,7 @@ protected override void Up(MigrationBuilder migrationBuilder) recipient = table.Column(type: "character varying(320)", maxLength: 320, nullable: false), recipient_name = table.Column(type: "character varying(32)", maxLength: 32, nullable: true), payload = table.Column>(type: "jsonb", nullable: false), + coalesce_key = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), status = table.Column(type: "email_status", nullable: false), attempt_count = table.Column(type: "integer", nullable: false, defaultValue: 0), next_attempt_at = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "CURRENT_TIMESTAMP"), @@ -61,6 +62,11 @@ protected override void Up(MigrationBuilder migrationBuilder) table.PrimaryKey("email_outbox_pkey", x => x.id); }); + migrationBuilder.CreateIndex( + name: "IX_email_outbox_coalesce_key", + table: "email_outbox", + column: "coalesce_key"); + migrationBuilder.CreateIndex( name: "IX_email_outbox_recipient", table: "email_outbox", diff --git a/Common/Migrations/OpenShockContextModelSnapshot.cs b/Common/Migrations/OpenShockContextModelSnapshot.cs index 2db87ee7..8b3a88ae 100644 --- a/Common/Migrations/OpenShockContextModelSnapshot.cs +++ b/Common/Migrations/OpenShockContextModelSnapshot.cs @@ -440,6 +440,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasDefaultValue(0) .HasColumnName("attempt_count"); + b.Property("CoalesceKey") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("coalesce_key"); + b.Property("CreatedAt") .ValueGeneratedOnAdd() .HasColumnType("timestamp with time zone") @@ -498,6 +503,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Id") .HasName("email_outbox_pkey"); + b.HasIndex("CoalesceKey"); + b.HasIndex("Recipient"); b.HasIndex("Status", "NextAttemptAt"); diff --git a/Common/OpenShockDb/EmailOutboxCoalesceKeys.cs b/Common/OpenShockDb/EmailOutboxCoalesceKeys.cs new file mode 100644 index 00000000..c7e5e31e --- /dev/null +++ b/Common/OpenShockDb/EmailOutboxCoalesceKeys.cs @@ -0,0 +1,27 @@ +namespace OpenShock.Common.OpenShockDb; + +/// +/// Builders for the opaque . The domain that enqueues a +/// message decides whether older requests of the same kind should be superseded by a newer one, and +/// expresses that purely by the key it stamps (or by leaving it null). The delivery queue never +/// parses these strings - it only compares them - so the "always deliver vs newest-wins" policy lives +/// here on the domain side, never in the queue. +/// +/// +/// The kind prefix is baked into the key so different email kinds for the same user never coalesce +/// into each other (a pending reset must not suppress a pending activation). Scope is per user: only +/// the newest activation / reset / verification for a given user is delivered. +/// intentionally has no builder - it is an +/// always-deliver type, so it carries a null key. +/// +public static class EmailOutboxCoalesceKeys +{ + /// Newest account-activation email per user wins. + public static string AccountActivation(Guid userId) => $"activation:{userId}"; + + /// Newest password-reset email per user wins. + public static string PasswordReset(Guid userId) => $"pwreset:{userId}"; + + /// Newest email-change verification per user wins. + public static string EmailVerification(Guid userId) => $"emailchange:{userId}"; +} diff --git a/Common/OpenShockDb/EmailOutboxMessage.cs b/Common/OpenShockDb/EmailOutboxMessage.cs index 77882db8..186baa3f 100644 --- a/Common/OpenShockDb/EmailOutboxMessage.cs +++ b/Common/OpenShockDb/EmailOutboxMessage.cs @@ -73,6 +73,15 @@ public sealed class EmailOutboxMessage /// public required Dictionary Payload { get; set; } + /// + /// Optional opaque coalescing key. When set, only the newest outbox row sharing this key is + /// actually delivered; older siblings are skipped as superseded (see the delivery job). The key is + /// stamped by the domain that enqueues the message (e.g. pwreset:{userId}) and is treated as + /// an opaque string by the queue - the delivery path never parses it. A null key means + /// "always deliver, never coalesce" (every row stands on its own). + /// + public string? CoalesceKey { get; set; } + /// Delivery state. See . public EmailStatus Status { get; set; } = EmailStatus.Pending; @@ -106,7 +115,12 @@ public sealed class EmailOutboxMessage /// Builds a new enqueued message in the state, due immediately. /// The caller adds it to the context and commits it together with the related business change. /// - public static EmailOutboxMessage Create(EmailType type, string recipient, string? recipientName, Dictionary payload) + /// + /// Optional opaque coalescing key (see ). Pass a stable per-intent key + /// (e.g. pwreset:{userId}) for "only the newest such request is delivered"; leave + /// null for "always deliver" types. + /// + public static EmailOutboxMessage Create(EmailType type, string recipient, string? recipientName, Dictionary payload, string? coalesceKey = null) { return new EmailOutboxMessage { @@ -115,7 +129,35 @@ public static EmailOutboxMessage Create(EmailType type, string recipient, string Recipient = recipient, RecipientName = recipientName, Payload = payload, + CoalesceKey = coalesceKey, Status = EmailStatus.Pending }; } + + // Per-type builders: each one owns the (type, payload key, coalesce key) triple for its email so + // callers never repeat it - and so the three can never drift out of sync. Newest-wins types key on + // the user; the change notice is always-delivered and carries no key. + + /// Account-activation email for . Newest activation per user wins. + public static EmailOutboxMessage ForAccountActivation(Guid userId, string recipient, string? recipientName) => + Create(EmailType.AccountActivation, recipient, recipientName, + new Dictionary { [EmailOutboxPayloadKeys.UserId] = userId.ToString() }, + EmailOutboxCoalesceKeys.AccountActivation(userId)); + + /// Password-reset email for reset . Newest reset per user wins. + public static EmailOutboxMessage ForPasswordReset(Guid passwordResetId, Guid userId, string recipient, string? recipientName) => + Create(EmailType.PasswordReset, recipient, recipientName, + new Dictionary { [EmailOutboxPayloadKeys.PasswordResetId] = passwordResetId.ToString() }, + EmailOutboxCoalesceKeys.PasswordReset(userId)); + + /// Email-change verification for change . Newest change per user wins. + public static EmailOutboxMessage ForEmailVerification(Guid emailChangeId, Guid userId, string recipient, string? recipientName) => + Create(EmailType.EmailVerification, recipient, recipientName, + new Dictionary { [EmailOutboxPayloadKeys.EmailChangeId] = emailChangeId.ToString() }, + EmailOutboxCoalesceKeys.EmailVerification(userId)); + + /// Informational notice to a previous address that the email is being changed. Always delivered (no coalescing). + public static EmailOutboxMessage ForEmailChangeNotice(string newEmail, string recipient, string? recipientName) => + Create(EmailType.EmailChangeNotice, recipient, recipientName, + new Dictionary { [EmailOutboxPayloadKeys.NewEmail] = newEmail }); } diff --git a/Common/OpenShockDb/OpenShockContext.cs b/Common/OpenShockDb/OpenShockContext.cs index d5223771..75a685a7 100644 --- a/Common/OpenShockDb/OpenShockContext.cs +++ b/Common/OpenShockDb/OpenShockContext.cs @@ -903,6 +903,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) // both the status filter and the ordering for the FOR UPDATE SKIP LOCKED claim query. entity.HasIndex(e => new { e.Status, e.NextAttemptAt }); entity.HasIndex(e => e.Recipient); + // The delivery job resolves newest-wins coalescing by looking up siblings that share a + // coalesce key, so index it. + entity.HasIndex(e => e.CoalesceKey); entity.Property(e => e.Id) .ValueGeneratedNever() @@ -918,6 +921,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.Payload) .HasColumnType("jsonb") .HasColumnName("payload"); + entity.Property(e => e.CoalesceKey) + .VarCharWithLength(HardLimits.EmailOutboxCoalesceKeyMaxLength) + .HasColumnName("coalesce_key"); entity.Property(e => e.Status) .HasColumnName("status"); entity.Property(e => e.AttemptCount) diff --git a/Cron/Jobs/EmailOutboxDeliveryJob.cs b/Cron/Jobs/EmailOutboxDeliveryJob.cs index c0510a42..4b933e80 100644 --- a/Cron/Jobs/EmailOutboxDeliveryJob.cs +++ b/Cron/Jobs/EmailOutboxDeliveryJob.cs @@ -143,6 +143,27 @@ private async Task DeliverAsync(Guid messageId) if (message.Status != EmailStatus.Sending) return; + // Newest-wins coalescing: if a strictly-newer row shares this row's coalesce key, this row is a + // stale duplicate (a superseded request, or a redrive of an old dead row after a newer one was + // already sent) - skip it instead of sending. The queue only compares the opaque key; what it + // means (and whether to set one at all) is the enqueueing domain's choice. A null key never + // coalesces. Rows are time-ordered by their UUIDv7 id, so "newest" is just the max id in the group. + if (message.CoalesceKey is not null) + { + var newestId = await _db.EmailOutbox + .Where(m => m.CoalesceKey == message.CoalesceKey) + .OrderByDescending(m => m.CreatedAt).ThenByDescending(m => m.Id) + .Select(m => m.Id) + .FirstAsync(); + + if (newestId != message.Id) + { + EmailOutboxStateMachine.ApplyResult(message, EmailDispatchResult.Skip("Superseded by a newer request"), DateTime.UtcNow); + await _db.SaveChangesAsync(); + return; + } + } + var result = await _dispatcher.SendAsync(message, _db); if (result.Outcome == EmailDispatchOutcome.TransientFailure && message.AttemptCount >= EmailOutboxRetryPolicy.MaxAttempts) From 28e4f415b6f141d126fdcc9c65c803a485931f07 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 23:07:00 +0200 Subject: [PATCH 12/15] refactor(email): use attempt_count as the delivery-lease fencing token instead of xmin Guard the deferred terminal write with attempt_count (already incremented on every claim) instead of the row's xmin: mark it IsConcurrencyToken so the write only conflicts on an actual lease reclaim, with no false-conflict surface from unrelated updates to the row. The AddEmailOutbox migration is regenerated with the EF tool (MigrationOpenShockContext) rather than hand-edited, so it now reflects the current model (coalesce_key column + index, attempt_count fencing token, no xmin shadow property) as tool-generated output. --- API.IntegrationTests/WebApplicationFactory.cs | 3 ++- ...> 20260630210423_AddEmailOutbox.Designer.cs} | 9 ++------- ...tbox.cs => 20260630210423_AddEmailOutbox.cs} | 2 -- .../Migrations/OpenShockContextModelSnapshot.cs | 7 +------ Common/OpenShockDb/OpenShockContext.cs | 17 ++++++----------- Cron/Jobs/EmailOutboxDeliveryJob.cs | 7 ++++--- 6 files changed, 15 insertions(+), 30 deletions(-) rename Common/Migrations/{20260630103404_AddEmailOutbox.Designer.cs => 20260630210423_AddEmailOutbox.Designer.cs} (99%) rename Common/Migrations/{20260630103404_AddEmailOutbox.cs => 20260630210423_AddEmailOutbox.cs} (97%) diff --git a/API.IntegrationTests/WebApplicationFactory.cs b/API.IntegrationTests/WebApplicationFactory.cs index 340e3a7e..fedc449d 100644 --- a/API.IntegrationTests/WebApplicationFactory.cs +++ b/API.IntegrationTests/WebApplicationFactory.cs @@ -41,7 +41,8 @@ public class WebApplicationFactory : WebApplicationFactory, IAsyncIniti // on-demand enqueue from the Redis "pending" nudge. WebApplicationFactory stops the Cron host at // Build(), so that post-build recurring registration never runs here and a per-minute sweep would be // far too coarse for the Mailpit wait window anyway. To keep delivery deterministic we run the very - // same job directly on a fast loop; it is built to run concurrently (FOR UPDATE SKIP LOCKED + xmin), + // same job directly on a fast loop; it is built to run concurrently (FOR UPDATE SKIP LOCKED + an + // attempt_count fencing token), // so it composes safely with the still-live notification listener. private CancellationTokenSource? _deliveryPumpCts; private Task? _deliveryPump; diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs b/Common/Migrations/20260630210423_AddEmailOutbox.Designer.cs similarity index 99% rename from Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs rename to Common/Migrations/20260630210423_AddEmailOutbox.Designer.cs index dcf525ba..32ebce5b 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.Designer.cs +++ b/Common/Migrations/20260630210423_AddEmailOutbox.Designer.cs @@ -15,7 +15,7 @@ namespace OpenShock.Common.Migrations { [DbContext(typeof(MigrationOpenShockContext))] - [Migration("20260630103404_AddEmailOutbox")] + [Migration("20260630210423_AddEmailOutbox")] partial class AddEmailOutbox { /// @@ -438,6 +438,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnName("id"); b.Property("AttemptCount") + .IsConcurrencyToken() .ValueGeneratedOnAdd() .HasColumnType("integer") .HasDefaultValue(0) @@ -497,12 +498,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("email_type") .HasColumnName("type"); - b.Property("xmin") - .IsConcurrencyToken() - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("xid") - .HasColumnName("xmin"); - b.HasKey("Id") .HasName("email_outbox_pkey"); diff --git a/Common/Migrations/20260630103404_AddEmailOutbox.cs b/Common/Migrations/20260630210423_AddEmailOutbox.cs similarity index 97% rename from Common/Migrations/20260630103404_AddEmailOutbox.cs rename to Common/Migrations/20260630210423_AddEmailOutbox.cs index 4ee2a542..8f0ca811 100644 --- a/Common/Migrations/20260630103404_AddEmailOutbox.cs +++ b/Common/Migrations/20260630210423_AddEmailOutbox.cs @@ -54,8 +54,6 @@ protected override void Up(MigrationBuilder migrationBuilder) created_at = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "CURRENT_TIMESTAMP"), sent_at = table.Column(type: "timestamp with time zone", nullable: true), failed_at = table.Column(type: "timestamp with time zone", nullable: true) - // xmin is the Postgres system column (mapped as the concurrency token in the model); - // it is not a real user column, so it must not be created here. }, constraints: table => { diff --git a/Common/Migrations/OpenShockContextModelSnapshot.cs b/Common/Migrations/OpenShockContextModelSnapshot.cs index 8b3a88ae..ee0dea15 100644 --- a/Common/Migrations/OpenShockContextModelSnapshot.cs +++ b/Common/Migrations/OpenShockContextModelSnapshot.cs @@ -435,6 +435,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnName("id"); b.Property("AttemptCount") + .IsConcurrencyToken() .ValueGeneratedOnAdd() .HasColumnType("integer") .HasDefaultValue(0) @@ -494,12 +495,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("email_type") .HasColumnName("type"); - b.Property("xmin") - .IsConcurrencyToken() - .ValueGeneratedOnAddOrUpdate() - .HasColumnType("xid") - .HasColumnName("xmin"); - b.HasKey("Id") .HasName("email_outbox_pkey"); diff --git a/Common/OpenShockDb/OpenShockContext.cs b/Common/OpenShockDb/OpenShockContext.cs index 75a685a7..d0475343 100644 --- a/Common/OpenShockDb/OpenShockContext.cs +++ b/Common/OpenShockDb/OpenShockContext.cs @@ -888,17 +888,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.ToTable("email_outbox"); - // Optimistic concurrency for the deferred delivery write: if a lapsed lease lets another run - // reclaim a Sending row mid-batch, the loser's UPDATE matches no row (xmin advanced) and - // throws DbUpdateConcurrencyException instead of clobbering attempt count / terminal state. - // Maps the Postgres xmin system column as a store-generated concurrency token (the manual - // form of the removed UseXminAsConcurrencyToken helper; no real column is created). - entity.Property("xmin") - .HasColumnName("xmin") - .HasColumnType("xid") - .ValueGeneratedOnAddOrUpdate() - .IsConcurrencyToken(); - // The consumer claims due rows ordered by next_attempt_at; (status, next_attempt_at) serves // both the status filter and the ordering for the FOR UPDATE SKIP LOCKED claim query. entity.HasIndex(e => new { e.Status, e.NextAttemptAt }); @@ -926,8 +915,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasColumnName("coalesce_key"); entity.Property(e => e.Status) .HasColumnName("status"); + // attempt_count doubles as the delivery lease's fencing token: every claim increments it, and + // the deferred terminal write is guarded by it (IsConcurrencyToken). If a lapsed lease lets + // another run reclaim a Sending row mid-batch (bumping attempt_count), the original run's + // UPDATE matches no row and throws DbUpdateConcurrencyException instead of clobbering the + // reclaimer's claim / terminal state. entity.Property(e => e.AttemptCount) .HasDefaultValue(0) + .IsConcurrencyToken() .HasColumnName("attempt_count"); entity.Property(e => e.NextAttemptAt) .HasDefaultValueSql("CURRENT_TIMESTAMP") diff --git a/Cron/Jobs/EmailOutboxDeliveryJob.cs b/Cron/Jobs/EmailOutboxDeliveryJob.cs index 4b933e80..59f45bef 100644 --- a/Cron/Jobs/EmailOutboxDeliveryJob.cs +++ b/Cron/Jobs/EmailOutboxDeliveryJob.cs @@ -14,9 +14,10 @@ namespace OpenShock.Cron.Jobs; /// fresh mail goes out without waiting for the next minute. /// /// -/// Concurrent runs are safe: a row is claimed with FOR UPDATE SKIP LOCKED and leased (Sending), and the -/// terminal write uses an optimistic-concurrency token (xmin), so if a lapsed lease lets another run -/// reclaim a row mid-batch the loser's write fails with +/// Concurrent runs are safe: a row is claimed with FOR UPDATE SKIP LOCKED and leased (Sending) with an +/// incremented attempt_count, which acts as the lease's fencing token. The terminal write is guarded by +/// it (mapped as a concurrency token), so if a lapsed lease lets another run reclaim a row mid-batch +/// (bumping attempt_count) the loser's write fails with /// instead of clobbering it. Each message is delivered in isolation (its own reload + change-tracker /// reset and its own try/catch), so one failure never aborts the rest of the batch, and the whole /// backlog is drained batch by batch rather than 50 per run. From d4676a09aa480ef22a73afcae1bbaa0f4f243466 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 23:44:57 +0200 Subject: [PATCH 13/15] test(email): scope API integration tests to outbox enqueue, drop in-process Cron The API's job for transactional email is to atomically write the EmailOutboxMessage row; it never sends. Booting the Cron host + a delivery pump inside the API tests was wrong and caused races (a background drain loop mutating rows that schema/predicate tests seed, and two executors racing the coalescing). - WebApplicationFactory no longer boots the Cron host, SMTP server, or a delivery loop - just the API + Postgres + Redis. - MailTests now assert the correct outbox row is enqueued (type, recipient, coalesce key, payload) or that nothing is enqueued for rejected requests; the emailed-link end-to-end flows and newest-wins coalescing move to Cron.IntegrationTests. - Drop the Cron project reference, SmtpTemplates copy, and the now-unused Mailpit fixtures. --- .../API.IntegrationTests.csproj | 11 - API.IntegrationTests/Docker/TestMailServer.cs | 36 -- API.IntegrationTests/Helpers/MailpitHelper.cs | 163 ------ API.IntegrationTests/Tests/MailTests.cs | 535 +++--------------- API.IntegrationTests/WebApplicationFactory.cs | 124 +--- 5 files changed, 86 insertions(+), 783 deletions(-) delete mode 100644 API.IntegrationTests/Docker/TestMailServer.cs delete mode 100644 API.IntegrationTests/Helpers/MailpitHelper.cs diff --git a/API.IntegrationTests/API.IntegrationTests.csproj b/API.IntegrationTests/API.IntegrationTests.csproj index 33729f71..06b883f5 100644 --- a/API.IntegrationTests/API.IntegrationTests.csproj +++ b/API.IntegrationTests/API.IntegrationTests.csproj @@ -1,9 +1,6 @@ - - @@ -21,14 +18,6 @@ - - - - PreserveNewest - - - diff --git a/API.IntegrationTests/Docker/TestMailServer.cs b/API.IntegrationTests/Docker/TestMailServer.cs deleted file mode 100644 index 53ec2366..00000000 --- a/API.IntegrationTests/Docker/TestMailServer.cs +++ /dev/null @@ -1,36 +0,0 @@ -using DotNet.Testcontainers.Builders; -using DotNet.Testcontainers.Containers; -using TUnit.Core.Interfaces; - -namespace OpenShock.API.IntegrationTests.Docker; - -public sealed class TestMailServer : IAsyncInitializer, IAsyncDisposable -{ - [ClassDataSource(Shared = SharedType.PerTestSession)] - public required DockerNetwork DockerNetwork { get; init; } - - private IContainer? _container; - public IContainer Container - { - get - { - _container ??= new ContainerBuilder("axllent/mailpit:latest") - .WithNetwork(DockerNetwork.Instance) - .WithName($"tunit-mailpit-{Guid.CreateVersion7()}") - .WithPortBinding(1025, true) - .WithPortBinding(8025, true) - .WithWaitStrategy(Wait.ForUnixContainer() - .UntilHttpRequestIsSucceeded(r => r.ForPort(8025).ForPath("/api/v1/info"))) - .Build(); - - return _container; - } - } - - public string SmtpHost => Container.Hostname; - public int SmtpPort => Container.GetMappedPublicPort(1025); - public string ApiBaseUrl => $"http://{Container.Hostname}:{Container.GetMappedPublicPort(8025)}"; - - public Task InitializeAsync() => Container.StartAsync(); - public ValueTask DisposeAsync() => Container.DisposeAsync(); -} diff --git a/API.IntegrationTests/Helpers/MailpitHelper.cs b/API.IntegrationTests/Helpers/MailpitHelper.cs deleted file mode 100644 index ad2ee0b7..00000000 --- a/API.IntegrationTests/Helpers/MailpitHelper.cs +++ /dev/null @@ -1,163 +0,0 @@ -using System.Net.Http.Json; -using System.Text.Json.Serialization; - -namespace OpenShock.API.IntegrationTests.Helpers; - -/// -/// Helper for querying the Mailpit HTTP API in integration tests. -/// -public sealed class MailpitHelper : IDisposable -{ - private readonly HttpClient _client; - - public MailpitHelper(string apiBaseUrl) - { - _client = new HttpClient { BaseAddress = new Uri(apiBaseUrl) }; - } - - /// - /// Polls until at least one email arrives for the given recipient address, or the timeout elapses. - /// Returns null if no message arrived within the timeout. - /// - /// - /// Uses Mailpit's server-side search so the lookup is unaffected by how many unrelated messages - /// have accumulated in the inbox. Listing endpoints page at 50 by default which silently hid - /// matches once enough emails piled up across a test session. - /// - public async Task WaitForMessageAsync( - string toAddress, - TimeSpan? timeout = null, - CancellationToken cancellationToken = default) - { - var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(15)); - while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) - { - var match = (await SearchByRecipientAsync(toAddress, limit: 1, cancellationToken)).FirstOrDefault(); - if (match is not null) return match; - await Task.Delay(300, cancellationToken); - } - return null; - } - - /// - /// Polls until at least emails are present for the recipient, or - /// the timeout elapses. Useful when a test expects multiple emails to arrive (e.g. two reset - /// requests in a row) and needs to disambiguate them. - /// - public async Task> WaitForMessagesAsync( - string toAddress, - int minCount, - TimeSpan? timeout = null, - CancellationToken cancellationToken = default) - { - var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(15)); - while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) - { - var matches = await SearchByRecipientAsync(toAddress, limit: Math.Max(minCount, 10), cancellationToken); - if (matches.Count >= minCount) return matches; - await Task.Delay(300, cancellationToken); - } - return await SearchByRecipientAsync(toAddress, limit: Math.Max(minCount, 10), cancellationToken); - } - - /// - /// Returns all messages currently in Mailpit addressed to the given recipient. - /// Server-side filtered via the Mailpit search API. - /// - public async Task> SearchByRecipientAsync( - string toAddress, - int limit = 10, - CancellationToken cancellationToken = default) - { - var query = Uri.EscapeDataString($"to:{toAddress}"); - var response = await _client.GetFromJsonAsync( - $"/api/v1/search?query={query}&limit={limit}", cancellationToken); - return response?.Messages ?? []; - } - - /// - /// Returns all messages in Mailpit (no filtering). - /// - public async Task> GetAllMessagesAsync( - int limit = 50, - CancellationToken cancellationToken = default) - { - var response = await _client.GetFromJsonAsync( - $"/api/v1/messages?limit={limit}", cancellationToken); - return response?.Messages ?? []; - } - - /// - /// Fetches the full HTML body of a message by its ID. - /// - public async Task GetMessageAsync(string messageId, CancellationToken cancellationToken = default) - { - return await _client.GetFromJsonAsync( - $"/api/v1/message/{messageId}", - cancellationToken); - } - - /// - /// Deletes all messages from Mailpit (useful for test isolation between test classes). - /// - public Task DeleteAllMessagesAsync(CancellationToken cancellationToken = default) - => _client.DeleteAsync("/api/v1/messages", cancellationToken); - - public void Dispose() => _client.Dispose(); - - // --- DTOs --- - - public sealed class MailpitSearchResponse - { - [JsonPropertyName("messages")] - public List Messages { get; init; } = []; - } - - public sealed class MailpitMessage - { - [JsonPropertyName("ID")] - public string Id { get; init; } = string.Empty; - - [JsonPropertyName("Subject")] - public string Subject { get; init; } = string.Empty; - - [JsonPropertyName("From")] - public MailpitContact? From { get; init; } - - [JsonPropertyName("To")] - public List? To { get; init; } - - [JsonPropertyName("Snippet")] - public string Snippet { get; init; } = string.Empty; - } - - public sealed class MailpitFullMessage - { - [JsonPropertyName("ID")] - public string Id { get; init; } = string.Empty; - - [JsonPropertyName("Subject")] - public string Subject { get; init; } = string.Empty; - - [JsonPropertyName("From")] - public MailpitContact? From { get; init; } - - [JsonPropertyName("To")] - public List? To { get; init; } - - [JsonPropertyName("HTML")] - public string Html { get; init; } = string.Empty; - - [JsonPropertyName("Text")] - public string Text { get; init; } = string.Empty; - } - - public sealed class MailpitContact - { - [JsonPropertyName("Name")] - public string Name { get; init; } = string.Empty; - - [JsonPropertyName("Address")] - public string Address { get; init; } = string.Empty; - } -} diff --git a/API.IntegrationTests/Tests/MailTests.cs b/API.IntegrationTests/Tests/MailTests.cs index 03126974..7aa6a2f4 100644 --- a/API.IntegrationTests/Tests/MailTests.cs +++ b/API.IntegrationTests/Tests/MailTests.cs @@ -1,21 +1,20 @@ using System.Net; -using System.Text.RegularExpressions; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using OpenShock.API.IntegrationTests.Helpers; -using OpenShock.Common.Constants; using OpenShock.Common.Models; using OpenShock.Common.OpenShockDb; -using OpenShock.Common.Utils; namespace OpenShock.API.IntegrationTests.Tests; /// -/// Tests that verify emails are actually delivered via SMTP to Mailpit. -/// Each test uses a unique recipient address via so Mailpit -/// lookups never collide with other tests in the session. +/// The API's responsibility for transactional email is to write the correct +/// row (and its business row) atomically - it never sends mail. These tests assert exactly that: the right +/// outbox row is enqueued (type, recipient, coalesce key, payload) or, for rejected requests, that nothing +/// is enqueued. Actual delivery, lazy token minting, the emailed-link flows, and newest-wins coalescing are +/// the Cron host's job and are covered by Cron.IntegrationTests. /// -public sealed partial class MailTests +public sealed class MailTests { [ClassDataSource(Shared = SharedType.PerTestSession)] public required WebApplicationFactory WebApplicationFactory { get; init; } @@ -23,11 +22,10 @@ public sealed partial class MailTests // --- Account Activation --- [Test] - public async Task V2Signup_SendsAccountActivationEmail() + public async Task V2Signup_EnqueuesActivationOutbox() { var email = TestHelper.UniqueEmail("mail-activation"); var username = TestHelper.UniqueUsername("mailactivation"); - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); using var client = WebApplicationFactory.CreateClient(); var response = await client.PostAsync("/2/account/signup", TestHelper.JsonContent(new @@ -37,53 +35,17 @@ public async Task V2Signup_SendsAccountActivationEmail() email, turnstileResponse = "valid-token" })); - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); - var message = await mailpit.WaitForMessageAsync(email); - await Assert.That(message).IsNotNull(); - await Assert.That(message!.To?.Select(c => c.Address)).Contains(email); - } - - [Test] - public async Task ActivationFlow_ViaEmailLink_ActivatesAccount() - { - var email = TestHelper.UniqueEmail("mail-activate-flow"); - var username = TestHelper.UniqueUsername("mailactivateflow"); - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); - using var client = WebApplicationFactory.CreateClient(); - - // Sign up — this triggers an activation email - var signupResponse = await client.PostAsync("/2/account/signup", TestHelper.JsonContent(new - { - username, - password = "SecurePassword123#", - email, - turnstileResponse = "valid-token" - })); - await Assert.That(signupResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); - - // Wait for and retrieve the activation email - var message = await mailpit.WaitForMessageAsync(email); - await Assert.That(message).IsNotNull(); - - var fullMessage = await mailpit.GetMessageAsync(message!.Id); - await Assert.That(fullMessage).IsNotNull(); - - // Extract the activation token from the link in the email HTML - var token = ExtractQueryParam(fullMessage!.Html, "token"); - await Assert.That(token).IsNotNull().And.IsNotEmpty(); - - // Use the token to activate the account - var activateResponse = await client.PostAsync($"/1/account/activate?token={token}", null); - await Assert.That(activateResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); - - // Confirm the user is now activated in the DB await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); var db = scope.ServiceProvider.GetRequiredService(); - var user = await db.Users.FirstOrDefaultAsync(u => u.Email == email); - await Assert.That(user).IsNotNull(); - await Assert.That(user!.ActivatedAt).IsNotNull(); + var user = await db.Users.AsNoTracking().FirstAsync(u => u.Email == email); + + var row = await db.EmailOutbox.AsNoTracking().SingleAsync(m => m.Recipient == email); + await Assert.That(row.Type).IsEqualTo(EmailType.AccountActivation); + await Assert.That(row.Status).IsEqualTo(EmailStatus.Pending); + await Assert.That(row.CoalesceKey).IsEqualTo(EmailOutboxCoalesceKeys.AccountActivation(user.Id)); + await Assert.That(row.Payload[EmailOutboxPayloadKeys.UserId]).IsEqualTo(user.Id.ToString()); } // --- Password Reset --- @@ -111,13 +73,11 @@ public async Task ResetPasswordAlias_Retired_Returns410Gone() } [Test] - public async Task V2PasswordReset_SendsPasswordResetEmail() + public async Task V2PasswordReset_EnqueuesResetOutbox() { var email = TestHelper.UniqueEmail("mail-pwreset-v2"); var username = TestHelper.UniqueUsername("mailpwresetv2"); - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); - - await TestHelper.CreateUserInDb(WebApplicationFactory, username, email, "OldPassword123#"); + var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, username, email, "OldPassword123#"); using var client = WebApplicationFactory.CreateClient(); var response = await client.PostAsync("/2/account/password-reset", TestHelper.JsonContent(new @@ -125,77 +85,71 @@ public async Task V2PasswordReset_SendsPasswordResetEmail() email, turnstileResponse = "valid-token" })); - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); - var message = await mailpit.WaitForMessageAsync(email); - await Assert.That(message).IsNotNull(); - await Assert.That(message!.To?.Select(c => c.Address)).Contains(email); + await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var reset = await db.UserPasswordResets.AsNoTracking().SingleAsync(r => r.UserId == userId); + + var row = await db.EmailOutbox.AsNoTracking().SingleAsync(m => m.Recipient == email); + await Assert.That(row.Type).IsEqualTo(EmailType.PasswordReset); + await Assert.That(row.Status).IsEqualTo(EmailStatus.Pending); + await Assert.That(row.CoalesceKey).IsEqualTo(EmailOutboxCoalesceKeys.PasswordReset(userId)); + await Assert.That(row.Payload[EmailOutboxPayloadKeys.PasswordResetId]).IsEqualTo(reset.Id.ToString()); } [Test] - public async Task PasswordResetFlow_ViaEmailLink_ChangesPassword() + public async Task PasswordResetComplete_LegacyRecoverRoute_Returns410Gone() { - var email = TestHelper.UniqueEmail("mail-pwreset-flow"); - var username = TestHelper.UniqueUsername("mailpwresetflow"); - const string newPassword = "NewSecurePassword456#"; - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); + using var client = WebApplicationFactory.CreateClient(); - await TestHelper.CreateUserInDb(WebApplicationFactory, username, email, "OldPassword123#"); + var response = await client.PostAsync( + $"/1/account/recover/{Guid.CreateVersion7()}/somesecret", + TestHelper.JsonContent(new { password = "LegacyNewPassword456#" })); - using var client = WebApplicationFactory.CreateClient(); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Gone); + } - // Initiate password reset - var resetResponse = await client.PostAsync("/2/account/password-reset", TestHelper.JsonContent(new { email, turnstileResponse = "valid-token" })); - await Assert.That(resetResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); + [Test] + public async Task PasswordResetCheck_LegacyHeadRecoverRoute_Returns410Gone() + { + using var client = WebApplicationFactory.CreateClient(); - // Wait for reset email and extract the link - var message = await mailpit.WaitForMessageAsync(email); - await Assert.That(message).IsNotNull(); + var response = await client.SendAsync(new HttpRequestMessage( + HttpMethod.Head, $"/1/account/recover/{Guid.CreateVersion7()}/somesecret")); - var fullMessage = await mailpit.GetMessageAsync(message!.Id); - await Assert.That(fullMessage).IsNotNull(); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Gone); + } - // Link format: /#/account/password/recover/{id}/{secret} - var (resetId, secret) = ExtractPasswordResetParams(fullMessage!.Html); - await Assert.That(resetId).IsNotNull().And.IsNotEmpty(); - await Assert.That(secret).IsNotNull().And.IsNotEmpty(); + [Test] + public async Task PasswordResetCheck_InvalidToken_Returns404() + { + var email = TestHelper.UniqueEmail("mail-pwreset-check-invalid"); + var username = TestHelper.UniqueUsername("mailpwresetcheckinvalid"); + await TestHelper.CreateUserInDb(WebApplicationFactory, username, email, "OldPassword123#"); - // Verify the reset token is valid - var checkResponse = await client.GetAsync($"/1/account/password-reset/{resetId}/{secret}"); - await Assert.That(checkResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); + using var client = WebApplicationFactory.CreateClient(); - // Complete the reset with a new password - var completeResponse = await client.PostAsync( - $"/1/account/password-reset/{resetId}/{secret}/complete", - TestHelper.JsonContent(new { password = newPassword })); - await Assert.That(completeResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); + var bogusId = Guid.CreateVersion7(); + const string bogusSecret = "thisisnotarealtokenatallzz"; - // Confirm we can log in with the new password - var loginResponse = await client.PostAsync("/2/account/login", TestHelper.JsonContent(new - { - usernameOrEmail = email, - password = newPassword, - turnstileResponse = "valid-token" - })); - await Assert.That(loginResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); + var response = await client.GetAsync($"/1/account/password-reset/{bogusId}/{bogusSecret}"); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.NotFound); } // --- Change Email --- [Test] - public async Task ChangeEmailFlow_ViaEmailLink_ChangesEmail() + public async Task ChangeEmail_EnqueuesVerificationAndNotice() { - var oldEmail = TestHelper.UniqueEmail("mail-chgemail-flow-old"); - var newEmail = TestHelper.UniqueEmail("mail-chgemail-flow-new"); - var username = TestHelper.UniqueUsername("mailchgemailflow"); + var oldEmail = TestHelper.UniqueEmail("mail-chgemail-notice-old"); + var newEmail = TestHelper.UniqueEmail("mail-chgemail-notice-new"); + var username = TestHelper.UniqueUsername("mailchgemailnotice"); const string password = "SecurePassword123#"; - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, username, oldEmail, password); using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); - // Initiate the email change var initiateResponse = await client.PostAsync("/1/account/email-change", TestHelper.JsonContent(new { currentPassword = password, @@ -203,49 +157,30 @@ public async Task ChangeEmailFlow_ViaEmailLink_ChangesEmail() })); await Assert.That(initiateResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); - // The verification email goes to the NEW address - var message = await mailpit.WaitForMessageAsync(newEmail); - await Assert.That(message).IsNotNull(); - - var fullMessage = await mailpit.GetMessageAsync(message!.Id); - await Assert.That(fullMessage).IsNotNull(); + await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var change = await db.UserEmailChanges.AsNoTracking().SingleAsync(c => c.UserId == user.Id); - var token = ExtractQueryParam(fullMessage!.Html, "token"); - await Assert.That(token).IsNotNull().And.IsNotEmpty(); + // Verification to the NEW address: coalesced per user, references the change row by id. + var verification = await db.EmailOutbox.AsNoTracking().SingleAsync(m => m.Recipient == newEmail); + await Assert.That(verification.Type).IsEqualTo(EmailType.EmailVerification); + await Assert.That(verification.Status).IsEqualTo(EmailStatus.Pending); + await Assert.That(verification.CoalesceKey).IsEqualTo(EmailOutboxCoalesceKeys.EmailVerification(user.Id)); + await Assert.That(verification.Payload[EmailOutboxPayloadKeys.EmailChangeId]).IsEqualTo(change.Id.ToString()); - // Email is not changed yet - await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - var beforeUser = await db.Users.AsNoTracking().FirstAsync(u => u.Id == user.Id); - await Assert.That(beforeUser.Email).IsEqualTo(oldEmail); - } - - // Use the token to complete the change - using var anonClient = WebApplicationFactory.CreateClient(); - var verifyResponse = await anonClient.PostAsync($"/1/account/email-change/verify?token={token}", null); - await Assert.That(verifyResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); - - // Email is now updated - await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - var afterUser = await db.Users.AsNoTracking().FirstAsync(u => u.Id == user.Id); - await Assert.That(afterUser.Email).IsEqualTo(newEmail); - } - - // Re-using the same token must now fail - var replayResponse = await anonClient.PostAsync($"/1/account/email-change/verify?token={token}", null); - await Assert.That(replayResponse.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + // Notice to the OLD address: always delivered (no coalesce key), carries the new address as data. + var notice = await db.EmailOutbox.AsNoTracking().SingleAsync(m => m.Recipient == oldEmail); + await Assert.That(notice.Type).IsEqualTo(EmailType.EmailChangeNotice); + await Assert.That(notice.CoalesceKey).IsNull(); + await Assert.That(notice.Payload[EmailOutboxPayloadKeys.NewEmail]).IsEqualTo(newEmail); } [Test] - public async Task ChangeEmail_WrongPassword_Returns403_AndSendsNoEmail() + public async Task ChangeEmail_WrongPassword_Returns403_AndEnqueuesNothing() { var oldEmail = TestHelper.UniqueEmail("mail-chgemail-badpwd-old"); var newEmail = TestHelper.UniqueEmail("mail-chgemail-badpwd-new"); var username = TestHelper.UniqueUsername("mailchgemailbadpwd"); - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, username, oldEmail, "CorrectPassword123#"); using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); @@ -255,60 +190,21 @@ public async Task ChangeEmail_WrongPassword_Returns403_AndSendsNoEmail() currentPassword = "WrongPassword!", email = newEmail })); - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Forbidden); - // Neither the verification (new address) nor the change notice (old address) should have been dispatched. - var verification = await mailpit.WaitForMessageAsync(newEmail, TimeSpan.FromSeconds(2)); - await Assert.That(verification).IsNull(); - var notice = await mailpit.WaitForMessageAsync(oldEmail, TimeSpan.FromSeconds(2)); - await Assert.That(notice).IsNull(); - } - - [Test] - public async Task ChangeEmailFlow_SendsNoticeToOldEmail() - { - var oldEmail = TestHelper.UniqueEmail("mail-chgemail-notice-old"); - var newEmail = TestHelper.UniqueEmail("mail-chgemail-notice-new"); - var username = TestHelper.UniqueUsername("mailchgemailnotice"); - const string password = "SecurePassword123#"; - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); - - var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, username, oldEmail, password); - using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); - - var initiateResponse = await client.PostAsync("/1/account/email-change", TestHelper.JsonContent(new - { - currentPassword = password, - email = newEmail - })); - await Assert.That(initiateResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); - - // Verification email lands at the new address... - var verification = await mailpit.WaitForMessageAsync(newEmail); - await Assert.That(verification).IsNotNull(); - await Assert.That(verification!.To?.Select(c => c.Address)).Contains(newEmail); - - // ...and a notice lands at the OLD address, mentioning the new address. - var notice = await mailpit.WaitForMessageAsync(oldEmail); - await Assert.That(notice).IsNotNull(); - await Assert.That(notice!.To?.Select(c => c.Address)).Contains(oldEmail); - - var noticeFull = await mailpit.GetMessageAsync(notice.Id); - await Assert.That(noticeFull).IsNotNull(); - await Assert.That(noticeFull!.Html).Contains(newEmail); - - // The notice must not contain a verification link — it's informational only. - await Assert.That(noticeFull.Html).DoesNotContain("token="); + await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var enqueued = await db.EmailOutbox.AsNoTracking() + .CountAsync(m => m.Recipient == newEmail || m.Recipient == oldEmail); + await Assert.That(enqueued).IsEqualTo(0); } [Test] - public async Task ChangeEmail_Unchanged_Returns400_AndSendsNoEmail() + public async Task ChangeEmail_Unchanged_Returns400_AndEnqueuesNothing() { var email = TestHelper.UniqueEmail("mail-chgemail-unchanged"); var username = TestHelper.UniqueUsername("mailchgemailunchanged"); const string password = "SecurePassword123#"; - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); var user = await TestHelper.CreateAndLoginUser(WebApplicationFactory, username, email, password); using var client = TestHelper.CreateAuthenticatedClient(WebApplicationFactory, user.SessionToken); @@ -318,184 +214,12 @@ public async Task ChangeEmail_Unchanged_Returns400_AndSendsNoEmail() currentPassword = password, email })); - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); - // No emails at all — neither verification nor notice. - var any = await mailpit.WaitForMessageAsync(email, TimeSpan.FromSeconds(2)); - await Assert.That(any).IsNull(); - } - - [Test] - public async Task PasswordResetComplete_LegacyRecoverRoute_Returns410Gone() - { - using var client = WebApplicationFactory.CreateClient(); - - var response = await client.PostAsync( - $"/1/account/recover/{Guid.CreateVersion7()}/somesecret", - TestHelper.JsonContent(new { password = "LegacyNewPassword456#" })); - - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Gone); - } - - [Test] - public async Task PasswordResetCheck_LegacyHeadRecoverRoute_Returns410Gone() - { - using var client = WebApplicationFactory.CreateClient(); - - var response = await client.SendAsync(new HttpRequestMessage( - HttpMethod.Head, $"/1/account/recover/{Guid.CreateVersion7()}/somesecret")); - - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Gone); - } - - [Test] - public async Task PasswordResetCheck_InvalidToken_Returns404() - { - var email = TestHelper.UniqueEmail("mail-pwreset-check-invalid"); - var username = TestHelper.UniqueUsername("mailpwresetcheckinvalid"); - await TestHelper.CreateUserInDb(WebApplicationFactory, username, email, "OldPassword123#"); - - using var client = WebApplicationFactory.CreateClient(); - - var bogusId = Guid.CreateVersion7(); - const string bogusSecret = "thisisnotarealtokenatallzz"; - - var response = await client.GetAsync($"/1/account/password-reset/{bogusId}/{bogusSecret}"); - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.NotFound); - } - - [Test] - public async Task ChangeEmailFlow_NewerRequest_SupersedesOlder_OnlyNewestVerificationDelivered() - { - var oldEmail = TestHelper.UniqueEmail("mail-chgemail-sibling-old"); - var firstNewEmail = TestHelper.UniqueEmail("mail-chgemail-sibling-first"); - var secondNewEmail = TestHelper.UniqueEmail("mail-chgemail-sibling-second"); - var username = TestHelper.UniqueUsername("mailchgemailsibling"); - const string password = "SecurePassword123#"; - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); - - var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, username, oldEmail, password); - - // Seed two pending email-change requests for the same user, sharing one coalesce key, committed - // together so both are pending before the delivery job runs. Newest-wins coalescing must deliver - // only the newer verification (to secondNewEmail) and skip the older (to firstNewEmail). - Guid olderOutboxId, newerOutboxId; - await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - var stamp = await db.Users.Where(u => u.Id == userId).Select(u => u.SecurityStamp).FirstAsync(); - - var older = NewPendingEmailChange(userId, oldEmail, firstNewEmail, username, stamp); - var newer = NewPendingEmailChange(userId, oldEmail, secondNewEmail, username, stamp); - - db.UserEmailChanges.AddRange(older.Change, newer.Change); - db.EmailOutbox.AddRange(older.Outbox, newer.Outbox); - await db.SaveChangesAsync(); - - olderOutboxId = older.Outbox.Id; - newerOutboxId = newer.Outbox.Id; - } - - // Only the newest request's verification email is delivered; the older is skipped as superseded. - var message = await mailpit.WaitForMessageAsync(secondNewEmail); - await Assert.That(message).IsNotNull(); - - var newerOutbox = await WaitForOutboxStatusAsync(newerOutboxId, EmailStatus.Sent); - var olderOutbox = await WaitForOutboxStatusAsync(olderOutboxId, EmailStatus.Skipped); - await Assert.That(newerOutbox.Status).IsEqualTo(EmailStatus.Sent); - await Assert.That(olderOutbox.Status).IsEqualTo(EmailStatus.Skipped); - - // The superseded request's address never receives anything. - var firstInbox = await mailpit.SearchByRecipientAsync(firstNewEmail); - await Assert.That(firstInbox.Count).IsEqualTo(0); - - // The delivered (newer) verification completes, switching the email to secondNewEmail. - var full = await mailpit.GetMessageAsync(message!.Id); - var token = ExtractQueryParam(full!.Html, "token"); - await Assert.That(token).IsNotNull().And.IsNotEmpty(); - - using var anonClient = WebApplicationFactory.CreateClient(); - var verify = await anonClient.PostAsync($"/1/account/email-change/verify?token={token}", null); - await Assert.That(verify.StatusCode).IsEqualTo(HttpStatusCode.OK); - - await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - var afterUser = await db.Users.AsNoTracking().FirstAsync(u => u.Id == userId); - await Assert.That(afterUser.Email).IsEqualTo(secondNewEmail); - } - } - - [Test] - public async Task PasswordResetFlow_NewerRequest_SupersedesOlder_OnlyNewestDelivered() - { - var email = TestHelper.UniqueEmail("mail-pwreset-sibling"); - var username = TestHelper.UniqueUsername("mailpwresetsibling"); - const string newPassword = "FreshPassword123#"; - using var mailpit = WebApplicationFactory.CreateMailpitHelper(); - - var userId = await TestHelper.CreateUserInDb(WebApplicationFactory, username, email, "OldPassword123#"); - - // Seed two pending password-reset requests for the same user, sharing one coalesce key, committed - // together so both are pending before the delivery job runs. The older must be superseded. - Guid olderResetId, newerResetId, olderOutboxId, newerOutboxId; - await using (var scope = WebApplicationFactory.Services.CreateAsyncScope()) - { - var db = scope.ServiceProvider.GetRequiredService(); - var stamp = await db.Users.Where(u => u.Id == userId).Select(u => u.SecurityStamp).FirstAsync(); - - var older = NewPendingReset(userId, email, username, stamp); - var newer = NewPendingReset(userId, email, username, stamp); - - db.UserPasswordResets.AddRange(older.Reset, newer.Reset); - db.EmailOutbox.AddRange(older.Outbox, newer.Outbox); - await db.SaveChangesAsync(); - - olderResetId = older.Reset.Id; - newerResetId = newer.Reset.Id; - olderOutboxId = older.Outbox.Id; - newerOutboxId = newer.Outbox.Id; - } - - // Only the newest request's email is delivered; the older is skipped as superseded. - var message = await mailpit.WaitForMessageAsync(email); - await Assert.That(message).IsNotNull(); - - var newerOutbox = await WaitForOutboxStatusAsync(newerOutboxId, EmailStatus.Sent); - var olderOutbox = await WaitForOutboxStatusAsync(olderOutboxId, EmailStatus.Skipped); - await Assert.That(newerOutbox.Status).IsEqualTo(EmailStatus.Sent); - await Assert.That(olderOutbox.Status).IsEqualTo(EmailStatus.Skipped); - - var delivered = await mailpit.SearchByRecipientAsync(email); - await Assert.That(delivered.Count).IsEqualTo(1); - - // The delivered link belongs to the newer reset, and it completes. - var full = await mailpit.GetMessageAsync(message!.Id); - var (resetId, secret) = ExtractPasswordResetParams(full!.Html); - await Assert.That(resetId).IsEqualTo(newerResetId.ToString()); - - using var client = WebApplicationFactory.CreateClient(); - var complete = await client.PostAsync( - $"/1/account/password-reset/{resetId}/{secret}/complete", - TestHelper.JsonContent(new { password = newPassword })); - await Assert.That(complete.StatusCode).IsEqualTo(HttpStatusCode.OK); - - // Completing the surviving reset rotated the security stamp, so the superseded sibling is dead - // at redemption too (its SecurityStampAtCreate no longer matches) - not just undelivered. - var completeOlder = await client.PostAsync( - $"/1/account/password-reset/{olderResetId}/irrelevant-secret/complete", - TestHelper.JsonContent(new { password = "AnotherPassword789#" })); - await Assert.That(completeOlder.StatusCode).IsEqualTo(HttpStatusCode.NotFound); - - // The new password from the surviving reset works. - var loginResponse = await client.PostAsync("/2/account/login", TestHelper.JsonContent(new - { - usernameOrEmail = email, - password = newPassword, - turnstileResponse = "valid-token" - })); - await Assert.That(loginResponse.StatusCode).IsEqualTo(HttpStatusCode.OK); + await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var enqueued = await db.EmailOutbox.AsNoTracking().CountAsync(m => m.Recipient == email); + await Assert.That(enqueued).IsEqualTo(0); } [Test] @@ -519,103 +243,4 @@ public async Task ChangeEmail_AlreadyInUse_Returns409() await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Conflict); } - - // --- Helpers --- - - /// - /// Builds a pending and its matching outbox row (carrying - /// ), exactly as the API would, for direct DB seeding. Each call mints - /// fresh UUIDv7 ids, so calling it twice in order yields a strictly-older then strictly-newer pair. - /// - private static (UserPasswordReset Reset, EmailOutboxMessage Outbox) NewPendingReset( - Guid userId, string email, string? recipientName, Guid stamp) - { - var reset = new UserPasswordReset - { - Id = Guid.CreateVersion7(), - UserId = userId, - TokenHash = HashingUtils.HashToken(CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength)), - SecurityStampAtCreate = stamp, - CreatedAt = DateTime.UtcNow - }; - var outbox = EmailOutboxMessage.ForPasswordReset(reset.Id, userId, email, recipientName); - return (reset, outbox); - } - - /// - /// Builds a pending and its matching verification outbox row (carrying - /// ) for direct DB seeding. Ordering mirrors . - /// - private static (UserEmailChange Change, EmailOutboxMessage Outbox) NewPendingEmailChange( - Guid userId, string oldEmail, string newEmail, string? recipientName, Guid stamp) - { - var change = new UserEmailChange - { - Id = Guid.CreateVersion7(), - UserId = userId, - OldEmail = oldEmail, - NewEmail = newEmail, - TokenHash = HashingUtils.HashToken(CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength)), - SecurityStampAtCreate = stamp, - CreatedAt = DateTime.UtcNow - }; - var outbox = EmailOutboxMessage.ForEmailVerification(change.Id, userId, newEmail, recipientName); - return (change, outbox); - } - - /// - /// Polls the outbox row until it reaches or the timeout elapses, returning - /// the last-read row either way so the caller's assertion reports the actual state. - /// - private async Task WaitForOutboxStatusAsync(Guid outboxId, EmailStatus status, TimeSpan? timeout = null) - { - var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(15)); - EmailOutboxMessage? row = null; - while (DateTime.UtcNow < deadline) - { - await using var scope = WebApplicationFactory.Services.CreateAsyncScope(); - var db = scope.ServiceProvider.GetRequiredService(); - row = await db.EmailOutbox.AsNoTracking().FirstOrDefaultAsync(m => m.Id == outboxId); - if (row is not null && row.Status == status) return row; - await Task.Delay(200); - } - return row ?? throw new InvalidOperationException($"Outbox row {outboxId} not found"); - } - - /// - /// Extracts a query parameter value from a URL embedded in HTML (first <a href> containing the param). - /// - private static string? ExtractQueryParam(string html, string paramName) - { - var hrefMatch = HrefRegex().Match(html); - while (hrefMatch.Success) - { - var href = hrefMatch.Groups[1].Value; - if (Uri.TryCreate(href, UriKind.Absolute, out var uri)) - { - var query = System.Web.HttpUtility.ParseQueryString(uri.Query); - var value = query[paramName]; - if (value is not null) return value; - } - hrefMatch = hrefMatch.NextMatch(); - } - return null; - } - - /// - /// Extracts the (passwordResetId, secret) pair from the password-reset URL embedded in email HTML. - /// URL pattern: /account/password/recover/{guid}/{secret} - /// - private static (string? ResetId, string? Secret) ExtractPasswordResetParams(string html) - { - var match = PasswordResetPathRegex().Match(html); - if (!match.Success) return (null, null); - return (match.Groups[1].Value, match.Groups[2].Value); - } - - [GeneratedRegex(@"href=""([^""]+)""", RegexOptions.IgnoreCase)] - private static partial Regex HrefRegex(); - - [GeneratedRegex(@"/account/password/recover/([0-9a-fA-F\-]+)/([A-Za-z0-9]+)", RegexOptions.IgnoreCase)] - private static partial Regex PasswordResetPathRegex(); } diff --git a/API.IntegrationTests/WebApplicationFactory.cs b/API.IntegrationTests/WebApplicationFactory.cs index fedc449d..289c548b 100644 --- a/API.IntegrationTests/WebApplicationFactory.cs +++ b/API.IntegrationTests/WebApplicationFactory.cs @@ -1,6 +1,4 @@ -extern alias cronhost; - -using System.Threading.RateLimiting; +using System.Threading.RateLimiting; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.RateLimiting; @@ -9,16 +7,17 @@ using Microsoft.Extensions.Http; using Microsoft.Extensions.Options; using OpenShock.API.IntegrationTests.Docker; -using OpenShock.API.IntegrationTests.Helpers; using OpenShock.API.IntegrationTests.HttpMessageHandlers; using Serilog; using Serilog.Events; -using TUnit.Core.Interfaces; -using EmailOutboxDeliveryJob = cronhost::OpenShock.Cron.Jobs.EmailOutboxDeliveryJob; namespace OpenShock.API.IntegrationTests; -public class WebApplicationFactory : WebApplicationFactory, IAsyncInitializer +// These tests exercise the API only. The API's job for transactional email is to write the +// EmailOutboxMessage row (and its business row) atomically - it never sends mail. Delivery, token +// minting, and newest-wins coalescing belong to the Cron host and are covered by Cron.IntegrationTests, +// so this factory deliberately boots no Cron host, no SMTP server, and no delivery loop. +public class WebApplicationFactory : WebApplicationFactory { [ClassDataSource(Shared = SharedType.PerTestSession)] public required InMemoryDatabase PostgreSql { get; init; } @@ -26,104 +25,6 @@ public class WebApplicationFactory : WebApplicationFactory, IAsyncIniti [ClassDataSource(Shared = SharedType.PerTestSession)] public required InMemoryRedis Redis { get; init; } - [ClassDataSource(Shared = SharedType.PerTestSession)] - public required TestMailServer Mailpit { get; init; } - - public MailpitHelper CreateMailpitHelper() => new(Mailpit.ApiBaseUrl); - - // The Cron host, booted in-process so the email outbox is actually drained and delivered. It reads - // the same OPENSHOCK__* environment variables this factory sets (shared DB / Redis / Mailpit), so - // it sees the same outbox rows the API writes. - private CronHost? _cronHost; - - // Drives the email-outbox delivery job in the test host. In production the job runs on a recurring - // every-minute Hangfire sweep (registered in Cron/Program.cs *after* WebApplication.Build()) plus an - // on-demand enqueue from the Redis "pending" nudge. WebApplicationFactory stops the Cron host at - // Build(), so that post-build recurring registration never runs here and a per-minute sweep would be - // far too coarse for the Mailpit wait window anyway. To keep delivery deterministic we run the very - // same job directly on a fast loop; it is built to run concurrently (FOR UPDATE SKIP LOCKED + an - // attempt_count fencing token), - // so it composes safely with the still-live notification listener. - private CancellationTokenSource? _deliveryPumpCts; - private Task? _deliveryPump; - - /// Service provider of the in-process Cron host (email dispatcher, outbox job, etc.). - public IServiceProvider CronServices => _cronHost?.Services - ?? throw new InvalidOperationException("Cron host not initialized"); - - public Task InitializeAsync() - { - _ = Server; // Boots the API host, which sets the shared OPENSHOCK__* environment variables. - _cronHost = new CronHost(); - _ = _cronHost.Services; // Boots the Cron host: the outbox delivery job + notification listener run. - StartOutboxDeliveryPump(_cronHost.Services); - return Task.CompletedTask; - } - - private void StartOutboxDeliveryPump(IServiceProvider cronServices) - { - _deliveryPumpCts = new CancellationTokenSource(); - var cancellationToken = _deliveryPumpCts.Token; - - _deliveryPump = Task.Run(async () => - { - while (!cancellationToken.IsCancellationRequested) - { - try - { - await using var scope = cronServices.CreateAsyncScope(); - var job = ActivatorUtilities.CreateInstance(scope.ServiceProvider); - await job.Execute(); - } - catch - { - // Best effort: a transient hiccup (e.g. a row reclaimed by the live notification - // listener mid-batch) is simply picked up again on the next tick. - } - - try - { - await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken); - } - catch (OperationCanceledException) - { - return; - } - } - }, cancellationToken); - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - _deliveryPumpCts?.Cancel(); - try - { - _deliveryPump?.GetAwaiter().GetResult(); - } - catch (OperationCanceledException) - { - // Expected when the pump is cancelled on shutdown. - } - _deliveryPumpCts?.Dispose(); - _cronHost?.Dispose(); - } - base.Dispose(disposing); - } - - /// Minimal factory that boots the Cron host (cronhost::Program) for delivery. - private sealed class CronHost : WebApplicationFactory - { - protected override void ConfigureWebHost(IWebHostBuilder builder) - { - builder.ConfigureServices(services => - { - services.AddSerilog(configuration => configuration.WriteTo.Console(LogEventLevel.Warning)); - }); - } - } - protected override void ConfigureClient(HttpClient client) { base.ConfigureClient(client); @@ -155,23 +56,10 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) { "OPENSHOCK__REDIS__CONN", Redis.Container.GetConnectionString() }, - // Tests only: make the Cron host's Hangfire workers pick up enqueued jobs fast. The Redis - // pending notification enqueues the email delivery job the instant a row is written, so this - // is what lands mail inside the Mailpit wait window. Production keeps Hangfire's 15s default. - { "OPENSHOCK__HANGFIRE__QUEUEPOLLINTERVAL", "00:00:00.5" }, - { "OPENSHOCK__FRONTEND__BASEURL", "https://openshock.app" }, { "OPENSHOCK__FRONTEND__SHORTURL", "https://openshock.app" }, { "OPENSHOCK__FRONTEND__COOKIEDOMAIN", "openshock.app,localhost" }, - { "OPENSHOCK__MAIL__TYPE", "SMTP" }, - { "OPENSHOCK__MAIL__SENDER__EMAIL", "system@openshock.org" }, - { "OPENSHOCK__MAIL__SENDER__NAME", "OpenShock" }, - { "OPENSHOCK__MAIL__SMTP__HOST", Mailpit.SmtpHost }, - { "OPENSHOCK__MAIL__SMTP__PORT", Mailpit.SmtpPort.ToString() }, - { "OPENSHOCK__MAIL__SMTP__ENABLESSL", "false" }, - { "OPENSHOCK__MAIL__SMTP__VERIFYCERTIFICATE", "false" }, - { "OPENSHOCK__TURNSTILE__ENABLED", "true" }, { "OPENSHOCK__TURNSTILE__SECRETKEY", "turnstile-secret-key" }, { "OPENSHOCK__TURNSTILE__SITEKEY", "turnstile-site-key" }, From 61739fe9055d35520384845e0a934d021ce44e0d Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 23:52:15 +0200 Subject: [PATCH 14/15] test(email): add Cron.IntegrationTests for delivery + coalescing The delivery behaviour removed from the API tests now lives in a Cron-side integration project that boots the real Cron host against throwaway Postgres / Redis / Mailpit containers. Delivery is driven deterministically - tests seed outbox rows then call RunDeliveryAsync() to run one pass of the real EmailOutboxDeliveryJob, so nothing races the assertions. Covers: password-reset delivery with lazy token minting (the emailed token verifies against the stored hash), the self-contained change notice, and newest-wins coalescing (older sibling Skipped, newest Sent, exactly one email). The Cron host doesn't migrate on boot, so the factory applies migrations to the fresh database first. --- .../Cron.IntegrationTests.csproj | 37 ++++ .../CronApplicationFactory.cs | 98 ++++++++++ Cron.IntegrationTests/Docker/DockerNetwork.cs | 15 ++ .../Docker/InMemoryDatabase.cs | 31 ++++ Cron.IntegrationTests/Docker/InMemoryRedis.cs | 27 +++ .../Docker/TestMailServer.cs | 36 ++++ .../Helpers/MailpitHelper.cs | 110 ++++++++++++ .../Tests/EmailOutboxDeliveryTests.cs | 169 ++++++++++++++++++ OpenShockBackend.slnx | 1 + 9 files changed, 524 insertions(+) create mode 100644 Cron.IntegrationTests/Cron.IntegrationTests.csproj create mode 100644 Cron.IntegrationTests/CronApplicationFactory.cs create mode 100644 Cron.IntegrationTests/Docker/DockerNetwork.cs create mode 100644 Cron.IntegrationTests/Docker/InMemoryDatabase.cs create mode 100644 Cron.IntegrationTests/Docker/InMemoryRedis.cs create mode 100644 Cron.IntegrationTests/Docker/TestMailServer.cs create mode 100644 Cron.IntegrationTests/Helpers/MailpitHelper.cs create mode 100644 Cron.IntegrationTests/Tests/EmailOutboxDeliveryTests.cs diff --git a/Cron.IntegrationTests/Cron.IntegrationTests.csproj b/Cron.IntegrationTests/Cron.IntegrationTests.csproj new file mode 100644 index 00000000..98a725c9 --- /dev/null +++ b/Cron.IntegrationTests/Cron.IntegrationTests.csproj @@ -0,0 +1,37 @@ + + + + + + + + + false + + + + + + + + + + + + + + + PreserveNewest + + + + + + + + <_Parameter1>$(SourceRevisionId) + + + + diff --git a/Cron.IntegrationTests/CronApplicationFactory.cs b/Cron.IntegrationTests/CronApplicationFactory.cs new file mode 100644 index 00000000..0fb22bc8 --- /dev/null +++ b/Cron.IntegrationTests/CronApplicationFactory.cs @@ -0,0 +1,98 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using OpenShock.Common.OpenShockDb; +using OpenShock.Cron.IntegrationTests.Docker; +using OpenShock.Cron.IntegrationTests.Helpers; +using OpenShock.Cron.Jobs; +using Serilog; +using Serilog.Events; +using TUnit.Core.Interfaces; + +namespace OpenShock.Cron.IntegrationTests; + +/// +/// Boots the real Cron host (its Program) against throwaway Postgres / Redis / Mailpit +/// containers. Delivery is driven deterministically: tests seed outbox rows and then call +/// to run one pass of the real - +/// there is no background loop, so nothing races the assertions. The Cron host doesn't migrate on +/// boot, so the factory applies migrations to the fresh database first. +/// +public class CronApplicationFactory : WebApplicationFactory, IAsyncInitializer +{ + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required InMemoryDatabase PostgreSql { get; init; } + + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required InMemoryRedis Redis { get; init; } + + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required TestMailServer Mailpit { get; init; } + + public MailpitHelper CreateMailpitHelper() => new(Mailpit.ApiBaseUrl); + + public IDbContextFactory DbContextFactory => + Services.GetRequiredService>(); + + public async Task InitializeAsync() + { + // The Cron host doesn't run migrations on boot, so apply them to the fresh database before + // anything touches it. Uses the same connection string the host will read from the env var. + await using (var migrationContext = new MigrationOpenShockContext( + PostgreSql.Container.GetConnectionString(), false, NullLoggerFactory.Instance)) + { + await migrationContext.Database.MigrateAsync(); + } + + _ = Services; // Boot the Cron host (email dispatcher, templates, db factory, etc.). + } + + /// + /// Runs one pass of the real delivery job in a fresh DI scope. Deterministic: it returns only + /// after the batch (claim → dispatch → record outcome) has completed. + /// + public async Task RunDeliveryAsync() + { + await using var scope = Services.CreateAsyncScope(); + var job = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + await job.Execute(); + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + var environmentVariables = new Dictionary + { + { "ASPNETCORE_UNDER_INTEGRATION_TEST", "1" }, + + { "OPENSHOCK__DB__CONN", PostgreSql.Container.GetConnectionString() }, + { "OPENSHOCK__DB__SKIPMIGRATION", "true" }, + { "OPENSHOCK__DB__DEBUG", "false" }, + + { "OPENSHOCK__REDIS__CONN", Redis.Container.GetConnectionString() }, + + { "OPENSHOCK__FRONTEND__BASEURL", "https://openshock.app" }, + { "OPENSHOCK__FRONTEND__SHORTURL", "https://openshock.app" }, + { "OPENSHOCK__FRONTEND__COOKIEDOMAIN", "openshock.app,localhost" }, + + { "OPENSHOCK__MAIL__TYPE", "SMTP" }, + { "OPENSHOCK__MAIL__SENDER__EMAIL", "system@openshock.org" }, + { "OPENSHOCK__MAIL__SENDER__NAME", "OpenShock" }, + { "OPENSHOCK__MAIL__SMTP__HOST", Mailpit.SmtpHost }, + { "OPENSHOCK__MAIL__SMTP__PORT", Mailpit.SmtpPort.ToString() }, + { "OPENSHOCK__MAIL__SMTP__ENABLESSL", "false" }, + { "OPENSHOCK__MAIL__SMTP__VERIFYCERTIFICATE", "false" }, + }; + + foreach (var envVar in environmentVariables) + { + Environment.SetEnvironmentVariable(envVar.Key, envVar.Value); + } + + builder.ConfigureServices(services => + { + services.AddSerilog(configuration => configuration.WriteTo.Console(LogEventLevel.Warning)); + }); + } +} diff --git a/Cron.IntegrationTests/Docker/DockerNetwork.cs b/Cron.IntegrationTests/Docker/DockerNetwork.cs new file mode 100644 index 00000000..7b4e9bb2 --- /dev/null +++ b/Cron.IntegrationTests/Docker/DockerNetwork.cs @@ -0,0 +1,15 @@ +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Networks; +using TUnit.Core.Interfaces; + +namespace OpenShock.Cron.IntegrationTests.Docker; + +public sealed class DockerNetwork : IAsyncInitializer, IAsyncDisposable +{ + public INetwork Instance { get; } = new NetworkBuilder() + .WithName($"tunit-cron-{Guid.CreateVersion7():N}") + .Build(); + + public Task InitializeAsync() => Instance.CreateAsync(); + public ValueTask DisposeAsync() => Instance.DisposeAsync(); +} diff --git a/Cron.IntegrationTests/Docker/InMemoryDatabase.cs b/Cron.IntegrationTests/Docker/InMemoryDatabase.cs new file mode 100644 index 00000000..9ce428e4 --- /dev/null +++ b/Cron.IntegrationTests/Docker/InMemoryDatabase.cs @@ -0,0 +1,31 @@ +using OpenShock.Common.Utils; +using Testcontainers.PostgreSql; +using TUnit.Core.Interfaces; + +namespace OpenShock.Cron.IntegrationTests.Docker; + +public sealed class InMemoryDatabase : IAsyncInitializer, IAsyncDisposable +{ + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required DockerNetwork DockerNetwork { get; init; } + + private PostgreSqlContainer? _container; + public PostgreSqlContainer Container + { + get + { + _container ??= new PostgreSqlBuilder(image: "postgres:latest") + .WithNetwork(DockerNetwork.Instance) + .WithName($"tunit-cron-postgresql-{Guid.CreateVersion7()}") + .WithDatabase("openshock") + .WithUsername("openshock") + .WithPassword(CryptoUtils.RandomAlphaNumericString(32)) + .Build(); + + return _container; + } + } + + public Task InitializeAsync() => Container.StartAsync(); + public ValueTask DisposeAsync() => Container.DisposeAsync(); +} diff --git a/Cron.IntegrationTests/Docker/InMemoryRedis.cs b/Cron.IntegrationTests/Docker/InMemoryRedis.cs new file mode 100644 index 00000000..65414a96 --- /dev/null +++ b/Cron.IntegrationTests/Docker/InMemoryRedis.cs @@ -0,0 +1,27 @@ +using Testcontainers.Redis; +using TUnit.Core.Interfaces; + +namespace OpenShock.Cron.IntegrationTests.Docker; + +public sealed class InMemoryRedis : IAsyncInitializer, IAsyncDisposable +{ + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required DockerNetwork DockerNetwork { get; init; } + + private RedisContainer? _container; + public RedisContainer Container + { + get + { + _container ??= new RedisBuilder(image: "redis/redis-stack-server:latest") + .WithNetwork(DockerNetwork.Instance) + .WithName($"tunit-cron-redis-{Guid.CreateVersion7()}") + .Build(); + + return _container; + } + } + + public Task InitializeAsync() => Container.StartAsync(); + public ValueTask DisposeAsync() => Container.DisposeAsync(); +} diff --git a/Cron.IntegrationTests/Docker/TestMailServer.cs b/Cron.IntegrationTests/Docker/TestMailServer.cs new file mode 100644 index 00000000..d122cd5e --- /dev/null +++ b/Cron.IntegrationTests/Docker/TestMailServer.cs @@ -0,0 +1,36 @@ +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using TUnit.Core.Interfaces; + +namespace OpenShock.Cron.IntegrationTests.Docker; + +public sealed class TestMailServer : IAsyncInitializer, IAsyncDisposable +{ + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required DockerNetwork DockerNetwork { get; init; } + + private IContainer? _container; + public IContainer Container + { + get + { + _container ??= new ContainerBuilder("axllent/mailpit:latest") + .WithNetwork(DockerNetwork.Instance) + .WithName($"tunit-cron-mailpit-{Guid.CreateVersion7()}") + .WithPortBinding(1025, true) + .WithPortBinding(8025, true) + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilHttpRequestIsSucceeded(r => r.ForPort(8025).ForPath("/api/v1/info"))) + .Build(); + + return _container; + } + } + + public string SmtpHost => Container.Hostname; + public int SmtpPort => Container.GetMappedPublicPort(1025); + public string ApiBaseUrl => $"http://{Container.Hostname}:{Container.GetMappedPublicPort(8025)}"; + + public Task InitializeAsync() => Container.StartAsync(); + public ValueTask DisposeAsync() => Container.DisposeAsync(); +} diff --git a/Cron.IntegrationTests/Helpers/MailpitHelper.cs b/Cron.IntegrationTests/Helpers/MailpitHelper.cs new file mode 100644 index 00000000..4f7b4122 --- /dev/null +++ b/Cron.IntegrationTests/Helpers/MailpitHelper.cs @@ -0,0 +1,110 @@ +using System.Net.Http.Json; +using System.Text.Json.Serialization; + +namespace OpenShock.Cron.IntegrationTests.Helpers; + +/// +/// Helper for querying the Mailpit HTTP API in integration tests. +/// +public sealed class MailpitHelper : IDisposable +{ + private readonly HttpClient _client; + + public MailpitHelper(string apiBaseUrl) + { + _client = new HttpClient { BaseAddress = new Uri(apiBaseUrl) }; + } + + /// + /// Polls until at least one email arrives for the given recipient address, or the timeout elapses. + /// Returns null if no message arrived within the timeout. + /// + public async Task WaitForMessageAsync( + string toAddress, + TimeSpan? timeout = null, + CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(15)); + while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) + { + var match = (await SearchByRecipientAsync(toAddress, limit: 1, cancellationToken)).FirstOrDefault(); + if (match is not null) return match; + await Task.Delay(300, cancellationToken); + } + return null; + } + + /// + /// Returns all messages currently in Mailpit addressed to the given recipient. + /// Server-side filtered via the Mailpit search API. + /// + public async Task> SearchByRecipientAsync( + string toAddress, + int limit = 10, + CancellationToken cancellationToken = default) + { + var query = Uri.EscapeDataString($"to:{toAddress}"); + var response = await _client.GetFromJsonAsync( + $"/api/v1/search?query={query}&limit={limit}", cancellationToken); + return response?.Messages ?? []; + } + + /// + /// Fetches the full HTML body of a message by its ID. + /// + public async Task GetMessageAsync(string messageId, CancellationToken cancellationToken = default) + { + return await _client.GetFromJsonAsync( + $"/api/v1/message/{messageId}", + cancellationToken); + } + + public void Dispose() => _client.Dispose(); + + // --- DTOs --- + + public sealed class MailpitSearchResponse + { + [JsonPropertyName("messages")] + public List Messages { get; init; } = []; + } + + public sealed class MailpitMessage + { + [JsonPropertyName("ID")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("Subject")] + public string Subject { get; init; } = string.Empty; + + [JsonPropertyName("To")] + public List? To { get; init; } + + [JsonPropertyName("Snippet")] + public string Snippet { get; init; } = string.Empty; + } + + public sealed class MailpitFullMessage + { + [JsonPropertyName("ID")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("HTML")] + public string Html { get; init; } = string.Empty; + + [JsonPropertyName("Text")] + public string Text { get; init; } = string.Empty; + + [JsonPropertyName("To")] + public List? To { get; init; } + } + + public sealed class MailpitContact + { + [JsonPropertyName("Name")] + public string Name { get; init; } = string.Empty; + + [JsonPropertyName("Address")] + public string Address { get; init; } = string.Empty; + } +} diff --git a/Cron.IntegrationTests/Tests/EmailOutboxDeliveryTests.cs b/Cron.IntegrationTests/Tests/EmailOutboxDeliveryTests.cs new file mode 100644 index 00000000..a2e2bcba --- /dev/null +++ b/Cron.IntegrationTests/Tests/EmailOutboxDeliveryTests.cs @@ -0,0 +1,169 @@ +using System.Text.RegularExpressions; +using Microsoft.EntityFrameworkCore; +using OpenShock.Common.Constants; +using OpenShock.Common.Models; +using OpenShock.Common.OpenShockDb; +using OpenShock.Common.Utils; + +namespace OpenShock.Cron.IntegrationTests.Tests; + +/// +/// Exercises the Cron host's actual email delivery: the outbox delivery job claims a row, the dispatcher +/// mints the token lazily and renders the template, and the SMTP provider hands it to Mailpit. These are +/// the behaviours that used to live (wrongly) in the API integration tests. +/// +public sealed partial class EmailOutboxDeliveryTests +{ + [ClassDataSource(Shared = SharedType.PerTestSession)] + public required CronApplicationFactory Factory { get; init; } + + [Test] + public async Task PasswordReset_IsDelivered_AndEmailedTokenMatchesStoredHash() + { + var email = UniqueEmail("cron-pwreset"); + using var mailpit = Factory.CreateMailpitHelper(); + + var (userId, stamp) = await AddUserAsync(email, "Reset User"); + var resetId = await AddPendingResetAsync(userId, stamp, email, "Reset User"); + + await Factory.RunDeliveryAsync(); + + // The email was delivered to the user. + var message = await mailpit.WaitForMessageAsync(email); + await Assert.That(message).IsNotNull(); + + // The outbox row is now terminal Sent. + await using (var db = await Factory.DbContextFactory.CreateDbContextAsync()) + { + var row = await db.EmailOutbox.AsNoTracking().SingleAsync(m => m.Recipient == email); + await Assert.That(row.Status).IsEqualTo(EmailStatus.Sent); + + // The token was minted lazily at send time: the link in the email must hash to the value now + // stored on the reset row (which is exactly what the API's redeem endpoint checks). + var full = await mailpit.GetMessageAsync(message!.Id); + var token = ExtractResetToken(full!.Html); + await Assert.That(token).IsNotNull().And.IsNotEmpty(); + + var reset = await db.UserPasswordResets.AsNoTracking().SingleAsync(r => r.Id == resetId); + await Assert.That(HashingUtils.VerifyToken(token!, reset.TokenHash).Verified).IsTrue(); + } + } + + [Test] + public async Task EmailChangeNotice_IsDelivered_ToOldAddress_WithoutAToken() + { + var oldEmail = UniqueEmail("cron-notice-old"); + var newEmail = UniqueEmail("cron-notice-new"); + using var mailpit = Factory.CreateMailpitHelper(); + + // The change notice is self-contained (no token, no row to load): just enqueue and deliver. + await using (var db = await Factory.DbContextFactory.CreateDbContextAsync()) + { + db.EmailOutbox.Add(EmailOutboxMessage.ForEmailChangeNotice(newEmail, oldEmail, "Notice User")); + await db.SaveChangesAsync(); + } + + await Factory.RunDeliveryAsync(); + + var message = await mailpit.WaitForMessageAsync(oldEmail); + await Assert.That(message).IsNotNull(); + + var full = await mailpit.GetMessageAsync(message!.Id); + await Assert.That(full!.Html).Contains(newEmail); + await Assert.That(full.Html).DoesNotContain("token="); + } + + [Test] + public async Task PasswordReset_NewerRequest_SupersedesOlder_OnlyNewestDelivered() + { + var email = UniqueEmail("cron-pwreset-coalesce"); + using var mailpit = Factory.CreateMailpitHelper(); + + var (userId, stamp) = await AddUserAsync(email, "Coalesce User"); + + // Two pending resets for the same user, sharing one coalesce key, committed together so both are + // pending before the delivery job runs. + Guid olderOutboxId, newerOutboxId; + await using (var db = await Factory.DbContextFactory.CreateDbContextAsync()) + { + var olderReset = NewResetRow(userId, stamp); + var olderOutbox = EmailOutboxMessage.ForPasswordReset(olderReset.Id, userId, email, "Coalesce User"); + var newerReset = NewResetRow(userId, stamp); + var newerOutbox = EmailOutboxMessage.ForPasswordReset(newerReset.Id, userId, email, "Coalesce User"); + + db.UserPasswordResets.AddRange(olderReset, newerReset); + db.EmailOutbox.AddRange(olderOutbox, newerOutbox); + await db.SaveChangesAsync(); + + olderOutboxId = olderOutbox.Id; + newerOutboxId = newerOutbox.Id; + } + + await Factory.RunDeliveryAsync(); + + // Exactly one email is delivered. + var delivered = await mailpit.SearchByRecipientAsync(email); + await Assert.That(delivered.Count).IsEqualTo(1); + + await using (var db = await Factory.DbContextFactory.CreateDbContextAsync()) + { + var older = await db.EmailOutbox.AsNoTracking().SingleAsync(m => m.Id == olderOutboxId); + var newer = await db.EmailOutbox.AsNoTracking().SingleAsync(m => m.Id == newerOutboxId); + await Assert.That(newer.Status).IsEqualTo(EmailStatus.Sent); + await Assert.That(older.Status).IsEqualTo(EmailStatus.Skipped); + } + } + + // --- Helpers --- + + private async Task<(Guid UserId, Guid Stamp)> AddUserAsync(string email, string name) + { + await using var db = await Factory.DbContextFactory.CreateDbContextAsync(); + var user = new User + { + Id = Guid.CreateVersion7(), + Name = name, + Email = email, + PasswordHash = HashingUtils.HashPassword("SeedPassword123#"), + SecurityStamp = Guid.CreateVersion7(), + CreatedAt = DateTime.UtcNow, + ActivatedAt = DateTime.UtcNow + }; + db.Users.Add(user); + await db.SaveChangesAsync(); + + // Read back the stamp in case the store generated it, so seeded SecurityStampAtCreate matches. + var stamp = await db.Users.AsNoTracking().Where(u => u.Id == user.Id).Select(u => u.SecurityStamp).FirstAsync(); + return (user.Id, stamp); + } + + private static UserPasswordReset NewResetRow(Guid userId, Guid stamp) => new() + { + Id = Guid.CreateVersion7(), + UserId = userId, + TokenHash = HashingUtils.HashToken(CryptoUtils.RandomAlphaNumericString(AuthConstants.GeneratedTokenLength)), + SecurityStampAtCreate = stamp, + CreatedAt = DateTime.UtcNow + }; + + private async Task AddPendingResetAsync(Guid userId, Guid stamp, string email, string name) + { + await using var db = await Factory.DbContextFactory.CreateDbContextAsync(); + var reset = NewResetRow(userId, stamp); + db.UserPasswordResets.Add(reset); + db.EmailOutbox.Add(EmailOutboxMessage.ForPasswordReset(reset.Id, userId, email, name)); + await db.SaveChangesAsync(); + return reset.Id; + } + + private static string UniqueEmail(string prefix) => $"{prefix}-{Guid.CreateVersion7().ToString("N")[..8]}@test.org"; + + private static string? ExtractResetToken(string html) + { + var match = ResetLinkRegex().Match(html); + return match.Success ? match.Groups[1].Value : null; + } + + [GeneratedRegex(@"/account/password/recover/[0-9a-fA-F\-]+/([A-Za-z0-9]+)", RegexOptions.IgnoreCase)] + private static partial Regex ResetLinkRegex(); +} diff --git a/OpenShockBackend.slnx b/OpenShockBackend.slnx index 156ce6b7..44e928f4 100644 --- a/OpenShockBackend.slnx +++ b/OpenShockBackend.slnx @@ -6,6 +6,7 @@ + \ No newline at end of file From 93de6d1e640a49d5128dbff83fb25009d8529b2b Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Tue, 30 Jun 2026 23:59:24 +0200 Subject: [PATCH 15/15] test(email): seed coalescing siblings in separate transactions; fix XML-doc warning The coalescing test seeded both reset rows in one transaction, so they shared a created_at (Postgres CURRENT_TIMESTAMP is transaction-start time). The tie fell through to a UUIDv7 id comparison, whose Postgres ordering doesn't match .NET creation order, so "newest" came out inverted and the newer row was wrongly skipped. Real requests are separate transactions with distinct timestamps, so production was never affected - seed each sibling in its own transaction to mirror that and make the test deterministic. Also drop the lone Create() tag that tripped CS1573 (fold it into the summary). --- Common/OpenShockDb/EmailOutboxMessage.cs | 8 ++--- .../Tests/EmailOutboxDeliveryTests.cs | 34 +++++++++---------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/Common/OpenShockDb/EmailOutboxMessage.cs b/Common/OpenShockDb/EmailOutboxMessage.cs index 186baa3f..3b0b3342 100644 --- a/Common/OpenShockDb/EmailOutboxMessage.cs +++ b/Common/OpenShockDb/EmailOutboxMessage.cs @@ -114,12 +114,10 @@ public sealed class EmailOutboxMessage /// /// Builds a new enqueued message in the state, due immediately. /// The caller adds it to the context and commits it together with the related business change. + /// is the optional opaque coalescing key (see ): + /// a stable per-intent key (e.g. pwreset:{userId}) makes only the newest such request deliver; + /// null means always deliver. Prefer the per-type factories ( etc.). /// - /// - /// Optional opaque coalescing key (see ). Pass a stable per-intent key - /// (e.g. pwreset:{userId}) for "only the newest such request is delivered"; leave - /// null for "always deliver" types. - /// public static EmailOutboxMessage Create(EmailType type, string recipient, string? recipientName, Dictionary payload, string? coalesceKey = null) { return new EmailOutboxMessage diff --git a/Cron.IntegrationTests/Tests/EmailOutboxDeliveryTests.cs b/Cron.IntegrationTests/Tests/EmailOutboxDeliveryTests.cs index a2e2bcba..48abae03 100644 --- a/Cron.IntegrationTests/Tests/EmailOutboxDeliveryTests.cs +++ b/Cron.IntegrationTests/Tests/EmailOutboxDeliveryTests.cs @@ -81,23 +81,11 @@ public async Task PasswordReset_NewerRequest_SupersedesOlder_OnlyNewestDelivered var (userId, stamp) = await AddUserAsync(email, "Coalesce User"); - // Two pending resets for the same user, sharing one coalesce key, committed together so both are - // pending before the delivery job runs. - Guid olderOutboxId, newerOutboxId; - await using (var db = await Factory.DbContextFactory.CreateDbContextAsync()) - { - var olderReset = NewResetRow(userId, stamp); - var olderOutbox = EmailOutboxMessage.ForPasswordReset(olderReset.Id, userId, email, "Coalesce User"); - var newerReset = NewResetRow(userId, stamp); - var newerOutbox = EmailOutboxMessage.ForPasswordReset(newerReset.Id, userId, email, "Coalesce User"); - - db.UserPasswordResets.AddRange(olderReset, newerReset); - db.EmailOutbox.AddRange(olderOutbox, newerOutbox); - await db.SaveChangesAsync(); - - olderOutboxId = olderOutbox.Id; - newerOutboxId = newerOutbox.Id; - } + // Two pending resets for the same user sharing one coalesce key, each committed in its own + // transaction so they get distinct created_at timestamps - exactly as two separate API requests + // would. Both are pending before delivery runs; the newest must win, the older is superseded. + var olderOutboxId = await AddPendingResetOutboxAsync(userId, stamp, email, "Coalesce User"); + var newerOutboxId = await AddPendingResetOutboxAsync(userId, stamp, email, "Coalesce User"); await Factory.RunDeliveryAsync(); @@ -156,6 +144,18 @@ private async Task AddPendingResetAsync(Guid userId, Guid stamp, string em return reset.Id; } + /// Seeds a reset + its outbox row in their own transaction and returns the outbox row id. + private async Task AddPendingResetOutboxAsync(Guid userId, Guid stamp, string email, string name) + { + await using var db = await Factory.DbContextFactory.CreateDbContextAsync(); + var reset = NewResetRow(userId, stamp); + var outbox = EmailOutboxMessage.ForPasswordReset(reset.Id, userId, email, name); + db.UserPasswordResets.Add(reset); + db.EmailOutbox.Add(outbox); + await db.SaveChangesAsync(); + return outbox.Id; + } + private static string UniqueEmail(string prefix) => $"{prefix}-{Guid.CreateVersion7().ToString("N")[..8]}@test.org"; private static string? ExtractResetToken(string html)