[codex] report background job failures and health

.env.example

@@ -46,6 +46,11 @@ JOBS_ENABLED=true
 INDEXER_INTERVAL=30
 TX_STATUS_INTERVAL=15
 REMINDER_CRON=0 9 * * *
+JOB_STALE_THRESHOLD_INDEXER_SECONDS=180
+JOB_STALE_THRESHOLD_TRANSACTION_STATUS_CHECKER_SECONDS=360
+JOB_STALE_THRESHOLD_LOAN_PAYMENT_REMINDER_SECONDS=93600
+JOB_STALE_THRESHOLD_NONCE_CLEANUP_SECONDS=7200
+JOB_STALE_THRESHOLD_SUPABASE_KEEPALIVE_SECONDS=345600
 
 # Sentry
 SENTRY_DSN=

context/progress-tracker.md

@@ -6,6 +6,15 @@ pure chore/docs commits). Direct pushes to main must also be logged here.
 
 ---
 
+## 2026-07-22
+
+- Added shared background-job monitoring for the indexer, transaction status
+  checker, loan payment reminders, nonce cleanup, and Supabase keep-alive.
+  Successful runs now publish Prometheus timestamps, failed runs track
+  consecutive failures and report to Sentry at power-of-two intervals to avoid
+  alert floods, and `/health` degrades when a job exceeds its independently
+  configurable staleness threshold.
+
 ## 2026-07-19
 
 - Wired `LiquidityContractClient` (restored under `src/blockchain/contracts/liquidity-contract.client.ts`) into `LiquidityService` constructor.

src/app.module.ts

@@ -29,10 +29,12 @@ import { CreditScoringModule } from './modules/credit-scoring/credit-scoring.mod
 import { AdminModule } from './modules/admin/admin.module';
 import { CorrelationIdMiddleware } from './common/logger/correlation-id.middleware';
 import { StateReconciliationModule } from './jobs/state-reconciliation/state-reconciliation.module';
+import { validateEnvironment } from './config/env';
+import { JobMonitorModule } from './jobs/monitoring/job-monitor.module';
 
 @Module({
   imports: [
-    ConfigModule.forRoot({ isGlobal: true }),
+    ConfigModule.forRoot({ isGlobal: true, validate: validateEnvironment }),
     SentryModule.forRoot(),
     ScheduleModule.forRoot(),
     LoggerModule,
@@ -45,6 +47,7 @@ import { StateReconciliationModule } from './jobs/state-reconciliation/state-rec
     AuthModule,
     HealthModule,
     MetricsModule,
+    JobMonitorModule,
     LoansModule,
     ReputationModule,
     UsersModule,

src/config/env.ts

@@ -0,0 +1,48 @@
+export const JOB_STALE_THRESHOLD_CONFIG = {
+  indexer: {
+    env: 'JOB_STALE_THRESHOLD_INDEXER_SECONDS',
+    defaultSeconds: 180,
+  },
+  transactionStatusChecker: {
+    env: 'JOB_STALE_THRESHOLD_TRANSACTION_STATUS_CHECKER_SECONDS',
+    defaultSeconds: 360,
+  },
+  loanPaymentReminder: {
+    env: 'JOB_STALE_THRESHOLD_LOAN_PAYMENT_REMINDER_SECONDS',
+    defaultSeconds: 93_600,
+  },
+  nonceCleanup: {
+    env: 'JOB_STALE_THRESHOLD_NONCE_CLEANUP_SECONDS',
+    defaultSeconds: 7_200,
+  },
+  supabaseKeepAlive: {
+    env: 'JOB_STALE_THRESHOLD_SUPABASE_KEEPALIVE_SECONDS',
+    defaultSeconds: 345_600,
+  },
+} as const;
+
+export type MonitoredJobName = keyof typeof JOB_STALE_THRESHOLD_CONFIG;
+
+export function validateEnvironment(
+  config: Record<string, unknown>,
+): Record<string, unknown> {
+  const validated = { ...config };
+
+  for (const { env, defaultSeconds } of Object.values(
+    JOB_STALE_THRESHOLD_CONFIG,
+  )) {
+    const rawValue = config[env];
+    const parsed =
+      rawValue === undefined || rawValue === ''
+        ? defaultSeconds
+        : Number(rawValue);
+
+    if (!Number.isInteger(parsed) || parsed <= 0) {
+      throw new Error(`${env} must be a positive integer number of seconds`);
+    }
+
+    validated[env] = parsed;
+  }
+
+  return validated;
+}

src/indexer/indexer.service.ts

@@ -5,6 +5,7 @@ import * as StellarSdk from 'stellar-sdk';
 import { SupabaseService } from '../database/supabase.client';
 import { SorobanService } from '../blockchain/soroban/soroban.service';
 import { EventParserService } from './event-parser.service';
+import { JobMonitorService } from '../jobs/monitoring/job-monitor.service';
 import {
   ParsedContractEvent,
   LoanEventType,
@@ -31,6 +32,7 @@ export class IndexerService {
     private readonly sorobanService: SorobanService,
     private readonly supabaseService: SupabaseService,
     private readonly eventParser: EventParserService,
+    private readonly jobMonitorService: JobMonitorService,
   ) {
     this.loanContractId =
       this.configService.get<string>('CREDIT_LINE_CONTRACT_ID') || '';
@@ -49,36 +51,48 @@ export class IndexerService {
     this.logger.log('Blockchain indexer cycle started');
 
     try {
-      await this.indexLoanContract();
-      await this.indexReputationContract();
+      const loanError = await this.indexLoanContract();
+      const reputationError = await this.indexReputationContract();
+      const runError = loanError ?? reputationError;
+
+      if (runError) {
+        this.jobMonitorService.recordFailure('indexer', runError);
+      } else {
+        this.jobMonitorService.recordSuccess('indexer');
+      }
     } catch (error) {
       this.logger.error({ error: error.message, stack: error.stack }, 'Indexer cycle failed');
+      this.jobMonitorService.recordFailure('indexer', error);
     } finally {
       this.isRunning = false;
     }
 
     this.logger.log('Blockchain indexer cycle completed');
   }
 
-  private async indexLoanContract(): Promise<void> {
+  private async indexLoanContract(): Promise<Error | null> {
     try {
       await this.indexContract(this.loanContractId, 'loan');
+      return null;
     } catch (error) {
       this.logger.error(
         { error: error.message, stack: error.stack },
         'Failed to index loan contract events — will retry next cycle',
       );
+      return error instanceof Error ? error : new Error(String(error));
     }
   }
 
-  private async indexReputationContract(): Promise<void> {
+  private async indexReputationContract(): Promise<Error | null> {
     try {
       await this.indexContract(this.reputationContractId, 'reputation');
+      return null;
     } catch (error) {
       this.logger.error(
         { error: error.message, stack: error.stack },
         'Failed to index reputation contract events — will retry next cycle',
       );
+      return error instanceof Error ? error : new Error(String(error));
     }
   }
 

src/jobs/loan-payment-reminder/loan-payment-reminder.service.ts

@@ -8,6 +8,7 @@ import {
   ReminderSummary,
   ReminderType,
 } from './interfaces/reminder.interfaces';
+import { JobMonitorService } from '../monitoring/job-monitor.service';
 
 const REMINDER_TITLES: Record<ReminderType, string> = {
   payment_reminder_3d: 'Payment Due in 3 Days',
@@ -37,12 +38,16 @@ export class LoanPaymentReminderService {
   private readonly logger = new Logger(LoanPaymentReminderService.name);
   private isRunning = false;
 
-  constructor(private readonly supabaseService: SupabaseService) {}
+  constructor(
+    private readonly supabaseService: SupabaseService,
+    private readonly jobMonitorService: JobMonitorService,
+  ) {}
 
   @Cron('0 9 * * *')
   async sendPaymentReminders(): Promise<void> {
     if (this.isRunning) return;
     this.isRunning = true;
+    let runFailure: unknown;
 
     const summary: ReminderSummary = {
       total: 0,
@@ -86,13 +91,23 @@ export class LoanPaymentReminderService {
           summary.created++;
           summary.breakdown[candidate.reminderType]++;
         } catch (error) {
+          runFailure ??= error;
           summary.failed++;
           this.logger.error(`Failed to process reminder candidate for loan ${candidate.loan.loan_id}: ${error.message}`);
         }
       }
     } catch (error) {
+      runFailure = error;
       this.logger.error(`Fatal error during reminder job: ${error.message}`);
     } finally {
+      if (runFailure) {
+        this.jobMonitorService.recordFailure(
+          'loanPaymentReminder',
+          runFailure,
+        );
+      } else {
+        this.jobMonitorService.recordSuccess('loanPaymentReminder');
+      }
       this.logger.log(`Reminder job complete — created: ${summary.created}, skipped: ${summary.skipped}, failed: ${summary.failed}`);
       this.isRunning = false;
     }

src/jobs/monitoring/job-monitor.module.ts

@@ -0,0 +1,9 @@
+import { Global, Module } from '@nestjs/common';
+import { JobMonitorService } from './job-monitor.service';
+
+@Global()
+@Module({
+  providers: [JobMonitorService],
+  exports: [JobMonitorService],
+})
+export class JobMonitorModule {}

src/jobs/monitoring/job-monitor.service.spec.ts

@@ -0,0 +1,90 @@
+import { ConfigService } from '@nestjs/config';
+import * as Sentry from '@sentry/nestjs';
+import {
+  JOB_STALE_THRESHOLD_CONFIG,
+  validateEnvironment,
+} from '../../config/env';
+import { MetricsService } from '../../modules/metrics/metrics.service';
+import { JobMonitorService } from './job-monitor.service';
+
+describe('JobMonitorService', () => {
+  const metricsService = {
+    setJobLastSuccessTimestamp: jest.fn(),
+    setJobConsecutiveFailures: jest.fn(),
+  };
+  const configService = {
+    get: jest.fn((_key: string, defaultValue: number) => defaultValue),
+  };
+
+  let service: JobMonitorService;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    service = new JobMonitorService(
+      configService as unknown as ConfigService,
+      metricsService as unknown as MetricsService,
+    );
+  });
+
+  it('records success timestamps and resets the failure gauge', () => {
+    service.recordFailure('indexer', new Error('temporary failure'));
+    service.recordSuccess('indexer');
+
+    expect(metricsService.setJobLastSuccessTimestamp).toHaveBeenCalledWith(
+      'indexer',
+      expect.any(Number),
+    );
+    expect(metricsService.setJobConsecutiveFailures).toHaveBeenLastCalledWith(
+      'indexer',
+      0,
+    );
+  });
+
+  it('captures the first failure and then powers of two to avoid alert noise', () => {
+    const captureException = jest
+      .spyOn(Sentry, 'captureException')
+      .mockReturnValue('event-id');
+
+    for (let failure = 1; failure <= 8; failure++) {
+      service.recordFailure('transactionStatusChecker', new Error('down'));
+    }
+
+    expect(captureException).toHaveBeenCalledTimes(4);
+    expect(captureException).toHaveBeenLastCalledWith(
+      expect.any(Error),
+      expect.objectContaining({
+        tags: { job: 'transactionStatusChecker' },
+        contexts: {
+          job: {
+            name: 'transactionStatusChecker',
+            consecutiveFailures: 8,
+          },
+        },
+      }),
+    );
+  });
+
+  it('reports a job as stale after its configured threshold', () => {
+    service.recordSuccess('indexer');
+    const statuses = service.getHealthStatuses(
+      new Date(Date.now() + 181_000),
+    );
+
+    expect(statuses.find((status) => status.job === 'indexer')).toEqual(
+      expect.objectContaining({ status: 'stale', thresholdSeconds: 180 }),
+    );
+  });
+
+  it('validates and applies job threshold defaults at startup', () => {
+    const validated = validateEnvironment({});
+    expect(
+      validated[JOB_STALE_THRESHOLD_CONFIG.indexer.env],
+    ).toBe(JOB_STALE_THRESHOLD_CONFIG.indexer.defaultSeconds);
+
+    expect(() =>
+      validateEnvironment({
+        [JOB_STALE_THRESHOLD_CONFIG.indexer.env]: '0',
+      }),
+    ).toThrow('must be a positive integer');
+  });
+});