Gloas fix submit balances

rocketpool-cli/megapool/notify-final-balance.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"sort"
 	"strconv"
+	"time"
 
 	cliutils "github.com/rocket-pool/smartnode/rocketpool-cli/cli"
 	"github.com/rocket-pool/smartnode/rocketpool-cli/cli/color"
@@ -24,35 +25,145 @@ func getNotifiableValidator() (uint64, uint64, bool, error) {
 		return 0, 0, false, err
 	}
 	defer rp.Close()
-	// Get Megapool status
+	// Get Megapool status (finalized beacon state — required for final balance proofs)
+	fmt.Println("Loading megapool validators at the finalized beacon state...")
 	status, err := rp.MegapoolStatus(true)
 	if err != nil {
 		return 0, 0, false, err
 	}
 
-	exitingValidators := []api.MegapoolValidatorDetails{}
+	readyValidators := []api.MegapoolValidatorDetails{}
+	pendingValidators := []api.MegapoolValidatorDetails{}
+	// Beacon exit started, but notify-validator-exit has not been submitted yet.
+	needsExitNotify := []api.MegapoolValidatorDetails{}
 
 	for _, validator := range status.Megapool.Validators {
-		if validator.Exiting && validator.BeaconStatus.Status == beacon.ValidatorState_WithdrawalDone {
-			exitingValidators = append(exitingValidators, validator)
+		if validator.Exited {
+			continue
+		}
+		if validator.Exiting {
+			if validator.BeaconStatus.Status == beacon.ValidatorState_WithdrawalDone {
+				readyValidators = append(readyValidators, validator)
+			} else {
+				pendingValidators = append(pendingValidators, validator)
+			}
+			continue
+		}
+		// Exit is visible on the finalized beacon state but not yet recorded on the megapool.
+		if validator.Activated &&
+			validator.BeaconStatus.Exists &&
+			validator.BeaconStatus.ExitEpoch != 0 &&
+			validator.BeaconStatus.ExitEpoch != FarFutureEpoch {
+			needsExitNotify = append(needsExitNotify, validator)
 		}
 	}
-	if len(exitingValidators) > 0 {
-		sort.Sort(ByIndex(exitingValidators))
-		options := make([]string, len(exitingValidators))
-		for vi, v := range exitingValidators {
+	if len(readyValidators) > 0 {
+		sort.Sort(ByIndex(readyValidators))
+		options := make([]string, len(readyValidators))
+		for vi, v := range readyValidators {
 			options[vi] = fmt.Sprintf("ID: %d - Index: %d - Pubkey: 0x%s", v.ValidatorId, v.ValidatorIndex, v.PubKey.String())
 		}
 		selected, _ := prompt.Select("Please select a validator to notify the final balance:", options)
 
 		// Get validators
-		return uint64(exitingValidators[selected].ValidatorId), uint64(exitingValidators[selected].ValidatorIndex), true, nil
+		return uint64(readyValidators[selected].ValidatorId), uint64(readyValidators[selected].ValidatorIndex), true, nil
 
 	}
-	fmt.Println("No validators at the state where the full withdrawal can be proved")
+
+	fmt.Println("No validators at the state where the full withdrawal can be proved.")
+	printPendingFinalBalanceValidators(pendingValidators, needsExitNotify, status)
 	return 0, 0, false, nil
 }
 
+func printPendingFinalBalanceValidators(pending, needsExitNotify []api.MegapoolValidatorDetails, status api.MegapoolStatusResponse) {
+	currentEpoch := status.BeaconHead.FinalizedEpoch
+	if currentEpoch == 0 {
+		currentEpoch = status.BeaconHead.Epoch
+	}
+	secondsPerEpoch := status.SecondsPerEpoch
+	if secondsPerEpoch == 0 {
+		secondsPerEpoch = 384
+	}
+
+	if len(pending) == 0 && len(needsExitNotify) == 0 {
+		fmt.Println("There are also no megapool validators currently exiting on the finalized beacon state.")
+		fmt.Println("A final balance proof can only be submitted after:")
+		fmt.Println("  1. the validator has exited on the beacon chain")
+		fmt.Println("  2. notify-validator-exit has been run")
+		fmt.Println("  3. the validator reaches beacon status withdrawal_done (full balance withdrawn)")
+		return
+	}
+
+	if len(needsExitNotify) > 0 {
+		sort.Sort(ByIndex(needsExitNotify))
+		fmt.Printf("The following %d validator(s) have an exit visible on the finalized beacon state but still need notify-validator-exit first:\n", len(needsExitNotify))
+		fmt.Println()
+		for _, v := range needsExitNotify {
+			printPendingFinalBalanceValidator(v, currentEpoch, secondsPerEpoch)
+		}
+		fmt.Println("Run: rocketpool megapool notify-validator-exit")
+		fmt.Println()
+	}
+
+	if len(pending) > 0 {
+		sort.Sort(ByIndex(pending))
+		fmt.Printf("The following %d validator(s) are exiting on the megapool but not yet fully withdrawn (finalized epoch %d):\n", len(pending), currentEpoch)
+		fmt.Println()
+		for _, v := range pending {
+			printPendingFinalBalanceValidator(v, currentEpoch, secondsPerEpoch)
+		}
+	}
+
+	fmt.Println("A final balance proof can only be submitted once beacon status is withdrawal_done.")
+	fmt.Println("After withdrawable_epoch, the beacon chain still needs to process the full withdrawal (sweep); that can take additional time beyond the estimate above.")
+}
+
+func printPendingFinalBalanceValidator(v api.MegapoolValidatorDetails, currentEpoch, secondsPerEpoch uint64) {
+	bs := v.BeaconStatus
+	fmt.Printf("  ID %d - Index %d - status: %s\n", v.ValidatorId, v.ValidatorIndex, bs.Status)
+
+	exitEpoch := bs.ExitEpoch
+	withdrawableEpoch := bs.WithdrawableEpoch
+	if withdrawableEpoch == 0 || withdrawableEpoch == FarFutureEpoch {
+		// Prefer the megapool-recorded withdrawable epoch if beacon still reports FAR_FUTURE.
+		if v.WithdrawableEpoch != 0 && v.WithdrawableEpoch != FarFutureEpoch {
+			withdrawableEpoch = v.WithdrawableEpoch
+		}
+	}
+
+	if exitEpoch != 0 && exitEpoch != FarFutureEpoch {
+		fmt.Printf("    exit_epoch:          %d%s\n", exitEpoch, epochTimingSuffix(exitEpoch, currentEpoch, secondsPerEpoch))
+	} else {
+		fmt.Printf("    exit_epoch:          not yet set on finalized state\n")
+	}
+
+	if withdrawableEpoch != 0 && withdrawableEpoch != FarFutureEpoch {
+		fmt.Printf("    withdrawable_epoch:  %d%s\n", withdrawableEpoch, epochTimingSuffix(withdrawableEpoch, currentEpoch, secondsPerEpoch))
+		if currentEpoch >= withdrawableEpoch {
+			switch bs.Status {
+			case beacon.ValidatorState_WithdrawalPossible:
+				fmt.Printf("    note:                withdrawable; waiting for the beacon withdrawal sweep (full balance)\n")
+			case beacon.ValidatorState_ExitedUnslashed, beacon.ValidatorState_ExitedSlashed:
+				fmt.Printf("    note:                exited; waiting to become withdrawal_possible, then for the sweep\n")
+			default:
+				fmt.Printf("    note:                withdrawable epoch reached; waiting for full withdrawal (status %s)\n", bs.Status)
+			}
+		}
+	} else {
+		fmt.Printf("    withdrawable_epoch:  not yet set on finalized state\n")
+	}
+	fmt.Println()
+}
+
+func epochTimingSuffix(targetEpoch, currentEpoch, secondsPerEpoch uint64) string {
+	if targetEpoch <= currentEpoch {
+		return " (reached)"
+	}
+	remaining := targetEpoch - currentEpoch
+	wait := formatDaysHours(time.Duration(remaining*secondsPerEpoch) * time.Second)
+	return fmt.Sprintf(" (in %d epochs, ~%s)", remaining, wait)
+}
+
 func notifyFinalBalance(validatorId, validatorIndex, slot uint64, yes bool) error {
 
 	// Get RP client

rocketpool/api/megapool/status.go

@@ -64,6 +64,7 @@ func getStatus(c *cli.Command, finalizedState bool) (*api.MegapoolStatusResponse
 	if response.ShardCommitteePeriod == 0 {
 		response.ShardCommitteePeriod = 256
 	}
+	response.SecondsPerEpoch = eth2Config.SecondsPerEpoch
 
 	// Get latest delegate address
 	delegate, err := rp.GetContract("rocketMegapoolDelegate", nil)

rocketpool/watchtower/submit-network-balances-state_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rocket-pool/smartnode/shared/services/beacon"
 	"github.com/rocket-pool/smartnode/shared/services/config"
 	"github.com/rocket-pool/smartnode/shared/services/state"
+	"github.com/rocket-pool/smartnode/shared/types/eth2/generic"
 )
 
 const smallStateFixture = "../../shared/services/state/testdata/network_state.json.gz"
@@ -49,6 +50,28 @@ func (s *stubSmoothingPoolCalculator) GetSmoothingPoolShare(ns *state.NetworkSta
 	return ns.NetworkDetails.SmoothingPoolBalance, nil
 }
 
+// stubWithdrawalFinder satisfies withdrawalFinder for balance-calculation tests
+// that do not exercise the live beacon/EL withdrawal scan.
+type stubWithdrawalFinder struct {
+	withdrawal *generic.Withdrawal
+	err        error
+	calls      int
+}
+
+func (s *stubWithdrawalFinder) FindWithdrawal(slot uint64, validatorIndex uint64) (*generic.Withdrawal, error) {
+	s.calls++
+	if s.err != nil {
+		return nil, s.err
+	}
+	if s.withdrawal != nil {
+		return s.withdrawal, nil
+	}
+	return &generic.Withdrawal{
+		ValidatorIndex: validatorIndex,
+		Amount:         32000000000,
+	}, nil
+}
+
 func TestGetNetworkBalancesFromState(t *testing.T) {
 	provider, err := state.NewStaticNetworkStateProviderFromFile(smallStateFixture)
 	if err != nil {
@@ -64,6 +87,7 @@ func TestGetNetworkBalancesFromState(t *testing.T) {
 	cfg := config.NewRocketPoolConfig("", false)
 	rewardCalc := &stubRewardSplitCalculator{}
 	spCalc := &stubSmoothingPoolCalculator{}
+	wFinder := &stubWithdrawalFinder{}
 
 	elBlockHeader := &types.Header{
 		Number: new(big.Int).SetUint64(ns.ElBlockNumber),
@@ -75,7 +99,7 @@ func TestGetNetworkBalancesFromState(t *testing.T) {
 		cfg: cfg,
 	}
 
-	balances, err := task.getNetworkBalancesFromState(ns, elBlockHeader, slotTime, rewardCalc, spCalc)
+	balances, err := task.getNetworkBalancesFromState(ns, elBlockHeader, slotTime, rewardCalc, spCalc, wFinder)
 	if err != nil {
 		t.Fatalf("getNetworkBalancesFromState: %v", err)
 	}
@@ -243,7 +267,7 @@ func TestMegapoolBalanceWithDuplicatePubkey(t *testing.T) {
 	}
 
 	task := &submitNetworkBalances{}
-	details, err := task.getMegapoolBalanceDetails(megapoolAddrA, &restored, megapoolDetails, &stubRewardSplitCalculator{})
+	details, err := task.getMegapoolBalanceDetails(megapoolAddrA, &restored, megapoolDetails, &stubRewardSplitCalculator{}, &stubWithdrawalFinder{})
 	if err != nil {
 		t.Fatalf("getMegapoolBalanceDetails: %v", err)
 	}

rocketpool/watchtower/submit-network-balances.go

@@ -33,6 +33,7 @@ import (
 	rprewards "github.com/rocket-pool/smartnode/shared/services/rewards"
 	"github.com/rocket-pool/smartnode/shared/services/state"
 	"github.com/rocket-pool/smartnode/shared/services/wallet"
+	"github.com/rocket-pool/smartnode/shared/types/eth2/generic"
 )
 
 const (
@@ -52,6 +53,10 @@ type smoothingPoolShareCalculator interface {
 	GetSmoothingPoolShare(ns *state.NetworkState, elBlockHeader *types.Header, slotTime time.Time) (*big.Int, error)
 }
 
+type withdrawalFinder interface {
+	FindWithdrawal(slot uint64, validatorIndex uint64) (*generic.Withdrawal, error)
+}
+
 // Submit network balances task
 type submitNetworkBalances struct {
 	c         *cli.Command
@@ -447,8 +452,13 @@ func (t *submitNetworkBalances) getNetworkBalances(elBlockHeader *types.Header,
 
 	rewardCalc := &liveRewardSplitCalculator{rp: client}
 	spCalc := &liveSmoothingPoolCalculator{log: t.log, cfg: t.cfg, bc: t.bc, client: client}
+	elc, err := services.GetEthClient(t.c)
+	if err != nil {
+		return networkBalances{}, err
+	}
+	wFinder := services.NewWithdrawalFinder(t.bc, elc, ns.BeaconConfig)
 
-	return t.getNetworkBalancesFromState(ns, elBlockHeader, slotTime, rewardCalc, spCalc)
+	return t.getNetworkBalancesFromState(ns, elBlockHeader, slotTime, rewardCalc, spCalc, wFinder)
 }
 
 // getNetworkBalancesFromState computes the network balances from an already-loaded NetworkState.
@@ -458,6 +468,7 @@ func (t *submitNetworkBalances) getNetworkBalancesFromState(
 	slotTime time.Time,
 	rewardCalc rewardSplitCalculator,
 	spCalc smoothingPoolShareCalculator,
+	wFinder withdrawalFinder,
 ) (networkBalances, error) {
 
 	// Data
@@ -488,7 +499,7 @@ func (t *submitNetworkBalances) getNetworkBalancesFromState(
 		i := 0
 		for megapoolAddress, megapoolDetails := range state.MegapoolDetails {
 			var err error
-			megapoolBalanceDetails[i], err = t.getMegapoolBalanceDetails(megapoolAddress, state, megapoolDetails, rewardCalc)
+			megapoolBalanceDetails[i], err = t.getMegapoolBalanceDetails(megapoolAddress, state, megapoolDetails, rewardCalc, wFinder)
 			if err != nil {
 				return fmt.Errorf("error getting megapool balance details: %w", err)
 			}
@@ -564,7 +575,7 @@ func (t *submitNetworkBalances) getNetworkBalancesFromState(
 
 }
 
-func (t *submitNetworkBalances) getMegapoolBalanceDetails(megapoolAddress common.Address, state *state.NetworkState, megapoolDetails rpstate.NativeMegapoolDetails, rewardCalc rewardSplitCalculator) (megapoolBalanceDetail, error) {
+func (t *submitNetworkBalances) getMegapoolBalanceDetails(megapoolAddress common.Address, state *state.NetworkState, megapoolDetails rpstate.NativeMegapoolDetails, rewardCalc rewardSplitCalculator, wFinder withdrawalFinder) (megapoolBalanceDetail, error) {
 	megapoolBalanceDetails := megapoolBalanceDetail{}
 	megapoolValidators := state.MegapoolToPubkeysMap[megapoolAddress]
 	// iterate the megapoolValidators array
@@ -597,13 +608,13 @@ func (t *submitNetworkBalances) getMegapoolBalanceDetails(megapoolAddress common
 		if megapoolValidatorInfo.ValidatorInfo.Exiting {
 			if megapoolValidatorDetails.Balance == 0 {
 				// Find the withdrawn balance from the beacon chain
-				searchWithdrawSlot := megapoolValidatorDetails.WithdrawableEpoch * 32
+				searchWithdrawSlot := megapoolValidatorDetails.WithdrawableEpoch * state.BeaconConfig.SlotsPerEpoch
 				// Convert the validator index to a uint64
 				validatorIndex, err := strconv.ParseUint(megapoolValidatorDetails.Index, 10, 64)
 				if err != nil {
 					return megapoolBalanceDetails, fmt.Errorf("error converting validator index %s to uint64: %w", megapoolValidatorDetails.Index, err)
 				}
-				_, _, _, withdrawal, _, err := services.FindWithdrawalBlockAndArrayPosition(searchWithdrawSlot, validatorIndex, t.bc)
+				withdrawal, err := wFinder.FindWithdrawal(searchWithdrawSlot, validatorIndex)
 				if err != nil {
 					return megapoolBalanceDetails, fmt.Errorf("error finding withdrawal for validator %d: %w", validatorIndex, err)
 				}

shared/services/beacon/client/std-http-client.go

@@ -139,6 +139,11 @@ func (c *StandardHttpClient) GetEth2Config() (beacon.Eth2Config, error) {
 	if shardCommitteePeriod == 0 {
 		shardCommitteePeriod = 256
 	}
+	// A CL that predates Gloas omits GLOAS_FORK_EPOCH from the spec
+	gloasForkEpoch := beacon.FarFutureEpoch
+	if eth2Config.Data.GloasForkEpoch != nil {
+		gloasForkEpoch = uint64(*eth2Config.Data.GloasForkEpoch)
+	}
 	out := beacon.Eth2Config{
 		GenesisForkVersion:           genesis.Data.GenesisForkVersion,
 		GenesisValidatorsRoot:        genesis.Data.GenesisValidatorsRoot,
@@ -149,6 +154,7 @@ func (c *StandardHttpClient) GetEth2Config() (beacon.Eth2Config, error) {
 		SecondsPerEpoch:              uint64(eth2Config.Data.SecondsPerSlot * eth2Config.Data.SlotsPerEpoch),
 		EpochsPerSyncCommitteePeriod: uint64(eth2Config.Data.EpochsPerSyncCommitteePeriod),
 		ShardCommitteePeriod:         shardCommitteePeriod,
+		GloasForkEpoch:               gloasForkEpoch,
 	}
 	eth2ConfigCache.Store(&out)
 

shared/services/beacon/client/types.go

@@ -46,6 +46,7 @@ type Eth2ConfigResponse struct {
 		CapellaForkVersion           byteArray `json:"CAPELLA_FORK_VERSION"`
 		EpochsPerSyncCommitteePeriod uinteger  `json:"EPOCHS_PER_SYNC_COMMITTEE_PERIOD"`
 		ShardCommitteePeriod         uinteger  `json:"SHARD_COMMITTEE_PERIOD"`
+		GloasForkEpoch               *uinteger `json:"GLOAS_FORK_EPOCH"`
 	} `json:"data"`
 }
 type Eth2DepositContractResponse struct {

shared/services/beacon/config.go

@@ -3,11 +3,14 @@ package beacon
 import (
 	"encoding/json"
 	"fmt"
+	"math"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
+const FarFutureEpoch uint64 = 0xffffffffffffffff
+
 type Eth2Config struct {
 	GenesisForkVersion           []byte `json:"genesis_fork_version"`
 	GenesisValidatorsRoot        []byte `json:"genesis_validators_root"`
@@ -18,6 +21,7 @@ type Eth2Config struct {
 	SecondsPerEpoch              uint64 `json:"seconds_per_epoch"`
 	EpochsPerSyncCommitteePeriod uint64 `json:"epochs_per_sync_committee_period"`
 	ShardCommitteePeriod         uint64 `json:"shard_committee_period"`
+	GloasForkEpoch               uint64 `json:"gloas_fork_epoch"`
 }
 
 func (c *Eth2Config) MarshalJSON() ([]byte, error) {
@@ -61,6 +65,15 @@ func (c *Eth2Config) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
+// GloasActivationSlot returns the first slot of the Gloas fork, saturating at
+// MaxUint64 when the fork is not scheduled (FAR_FUTURE_EPOCH).
+func (c *Eth2Config) GloasActivationSlot() uint64 {
+	if c.SlotsPerEpoch == 0 || c.GloasForkEpoch > math.MaxUint64/c.SlotsPerEpoch {
+		return math.MaxUint64
+	}
+	return c.GloasForkEpoch * c.SlotsPerEpoch
+}
+
 // GetSlotTime returns the time of a given slot for the network described by Eth2Config.
 func (c *Eth2Config) GetSlotTime(slot uint64) time.Time {
 	// In the interest of keeping this pure, we'll just return genesis time for slots before genesis