From 1fd99c10cf219797b0a81d6c2d7b5530f1b9a42e Mon Sep 17 00:00:00 2001 From: samliok Date: Mon, 3 Aug 2026 17:36:34 -0400 Subject: [PATCH 1/3] Close the WAL when it is no longer in use Instance never closed the GarbageCollectedWAL it holds: createEpochConfig overwrote i.wal with a freshly created one on every epoch, and Stop left the last one open. Each dropped WAL keeps its underlying files open, so every epoch change leaked a set of file handles, and they were never released on shutdown. Close the previous WAL before replacing it and close the WAL in Stop. --- instance.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/instance.go b/instance.go index 1a0c0efe..4cda29f8 100644 --- a/instance.go +++ b/instance.go @@ -250,6 +250,18 @@ func (i *Instance) Stop() { i.stopValidator() i.stopNonValidator() + if err := i.closeWAL(); err != nil { + i.Config.Logger.Error("Error closing WAL on shutdown", zap.Error(err)) + } +} + +// closeWAL closes the WAL currently in use, if any. +// Must be called under the lock, and only once the epoch or non-validator using it has been stopped. +func (i *Instance) closeWAL() error { + if i.wal == nil { + return nil + } + return i.wal.Close() } func (i *Instance) stopNonValidator() { @@ -429,6 +441,14 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { return simplex.EpochConfig{}, err } + // The epoch that used the previous WAL has already been stopped by now, + // so close it before replacing it, otherwise we leak the files it holds open. + // Failing to close it doesn't prevent the new epoch from running, so as with + // garbage collecting the WAL on an epoch change, we only log the error. + if err := i.closeWAL(); err != nil { + i.Config.Logger.Error("Error closing the WAL of the previous epoch", zap.Error(err)) + } + wal, err := wal.NewGarbageCollectedWAL(i.Config.WALs, i.Config.WalCreator, &common.WALRetentionReader{}, i.Config.ParameterConfig.WALMaxEntryCount) if err != nil { return simplex.EpochConfig{}, fmt.Errorf("error creating garbage collected wal: %w", err) From 0c19eadc70afa38938886dee7c57f8f15151b28d Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 4 Aug 2026 17:15:05 -0400 Subject: [PATCH 2/3] update wal close to stopValidator --- instance.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/instance.go b/instance.go index 4cda29f8..8d481dc4 100644 --- a/instance.go +++ b/instance.go @@ -250,9 +250,6 @@ func (i *Instance) Stop() { i.stopValidator() i.stopNonValidator() - if err := i.closeWAL(); err != nil { - i.Config.Logger.Error("Error closing WAL on shutdown", zap.Error(err)) - } } // closeWAL closes the WAL currently in use, if any. @@ -261,7 +258,9 @@ func (i *Instance) closeWAL() error { if i.wal == nil { return nil } - return i.wal.Close() + err := i.wal.Close() + i.wal = nil + return err } func (i *Instance) stopNonValidator() { @@ -278,6 +277,10 @@ func (i *Instance) stopValidator() { i.e = nil i.epochOrNV = nil } + + if err := i.closeWAL(); err != nil { + i.Config.Logger.Error("Error closing the WAL of the previous epoch", zap.Error(err)) + } } func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error { @@ -441,14 +444,6 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { return simplex.EpochConfig{}, err } - // The epoch that used the previous WAL has already been stopped by now, - // so close it before replacing it, otherwise we leak the files it holds open. - // Failing to close it doesn't prevent the new epoch from running, so as with - // garbage collecting the WAL on an epoch change, we only log the error. - if err := i.closeWAL(); err != nil { - i.Config.Logger.Error("Error closing the WAL of the previous epoch", zap.Error(err)) - } - wal, err := wal.NewGarbageCollectedWAL(i.Config.WALs, i.Config.WalCreator, &common.WALRetentionReader{}, i.Config.ParameterConfig.WALMaxEntryCount) if err != nil { return simplex.EpochConfig{}, fmt.Errorf("error creating garbage collected wal: %w", err) @@ -598,6 +593,7 @@ func (i *Instance) transitionEpochValidator(epochChange epochChange) error { // Stop the epoch before doing anything else, so that we don't process any more messages while we are changing epochs. i.stopValidator() + // Wipe out the WALs from the config so we won't try to load them again i.Config.WALs = nil // On epoch change, garbage collect the WAL to remove all entries from previous epochs. From 2b8c627a0c25e0e659100d60fe24aefed6deb96c Mon Sep 17 00:00:00 2001 From: samliok Date: Tue, 4 Aug 2026 17:19:54 -0400 Subject: [PATCH 3/3] update wal close to stopValidator --- instance.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/instance.go b/instance.go index 8d481dc4..7410922f 100644 --- a/instance.go +++ b/instance.go @@ -591,15 +591,16 @@ func (i *Instance) transitionEpochValidator(epochChange epochChange) error { i.lock.Lock() defer i.lock.Unlock() + // On epoch change, garbage collect the WAL to remove all entries from previous epochs. + if err := i.wal.GarbageCollect(math.MaxUint64); err != nil { + i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) + } + // Stop the epoch before doing anything else, so that we don't process any more messages while we are changing epochs. i.stopValidator() // Wipe out the WALs from the config so we won't try to load them again i.Config.WALs = nil - // On epoch change, garbage collect the WAL to remove all entries from previous epochs. - if err := i.wal.GarbageCollect(math.MaxUint64); err != nil { - i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) - } return i.startAtEpoch(epochChange.validators, epochChange.epochNum) }