feat: redundant rpcs

cmd/appstate.go

@@ -319,3 +319,15 @@ func (a *appState) useRpcAddr(chainName string, rpcAddr string) error {
 		return nil
 	})
 }
+
+func (a *appState) useBackupRpcAddrs(chainName string, rpcAddrs []string) error {
+	_, exists := a.config.Chains[chainName]
+	if !exists {
+		return fmt.Errorf("chain %s not found in config", chainName)
+	}
+
+	return a.performConfigLockingOperation(context.Background(), func() error {
+		a.config.Chains[chainName].ChainProvider.SetBackupRpcAddrs(rpcAddrs)
+		return nil
+	})
+}

cmd/chains.go

@@ -41,16 +41,17 @@ func chainsCmd(a *appState) *cobra.Command {
 		chainsAddDirCmd(a),
 		cmdChainsConfigure(a),
 		cmdChainsUseRpcAddr(a),
+		cmdChainsUseBackupRpcAddr(a),
 	)
 
 	return cmd
 }
 
 func cmdChainsUseRpcAddr(a *appState) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:     "set-rpc-addr chain_name  valid_rpc_url",
+		Use:     "set-rpc-addr chain_name valid_rpc_url",
 		Aliases: []string{"rpc"},
-		Short:   "Sets  chain's rpc address",
+		Short:   "Sets chain's rpc address",
 		Args:    withUsage(cobra.ExactArgs(2)),
 		Example: strings.TrimSpace(fmt.Sprintf(`
 $ %s chains set-rpc-addr ibc-0 https://abc.xyz.com:443
@@ -69,6 +70,37 @@ $ %s ch set-rpc-addr ibc-0 https://abc.xyz.com:443`, appName, appName)),
 	return cmd
 }
 
+func cmdChainsUseBackupRpcAddr(a *appState) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "set-backup-rpc-addrs chain_name comma_separated_valid_rpc_urls",
+		Aliases: []string{"set-backup-rpcs"},
+		Short:   "Sets chain's backup rpc addresses",
+		Args:    withUsage(cobra.ExactArgs(2)),
+		Example: strings.TrimSpace(fmt.Sprintf(`
+$ %s chains set-backup-rpc-addr ibc-0 https://abc.xyz.com:443,https://123.456.com:443
+$ %s ch set-backup-rpc-addr ibc-0 https://abc.xyz.com:443,https://123.456.com:443`, appName, appName)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			chainName := args[0]
+			rpc_addresses := args[1]
+
+			// split rpc_addresses by ','
+			rpc_addresses_list := strings.Split(rpc_addresses, ",")
+
+			// loop through and ensure valid
+			for _, rpc_address := range rpc_addresses_list {
+				rpc_address := rpc_address
+				if !isValidURL(rpc_address) {
+					return invalidRpcAddr(rpc_address)
+				}
+			}
+
+			return a.useBackupRpcAddrs(chainName, rpc_addresses_list)
+		},
+	}
+
+	return cmd
+}
+
 func chainsAddrCmd(a *appState) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "address chain_name",

cregistry/chain_info.go

@@ -209,6 +209,48 @@ func (c ChainInfo) GetRandomRPCEndpoint(ctx context.Context, forceAdd bool) (str
 	return endpoint, nil
 }
 
+// GetBackupRPCEndpoints returns a slice of strings to be used as fallback, backup RPC endpoints. forceAdd will
+// force the use of all available RPC endpoints, regardless of health.
+func (c ChainInfo) GetBackupRPCEndpoints(ctx context.Context, forceAdd bool, primaryRPC string, count uint64) ([]string, error) {
+	// if force add, get all rpcs, otherwise get only healthy ones
+	var rpcs []string
+	var err error
+	if forceAdd {
+		rpcs, err = c.GetAllRPCEndpoints()
+	} else {
+		rpcs, err = c.GetRPCEndpoints(ctx)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// if no rpcs, return error
+	if len(rpcs) == 0 {
+		if !forceAdd {
+			return nil, fmt.Errorf("no working RPCs found, consider using --force-add")
+		} else {
+			return nil, nil
+		}
+	}
+
+	// Select first two endpoints
+	backupRpcs := []string{}
+	for _, endpoint := range rpcs {
+		if len(backupRpcs) < 2 && primaryRPC != endpoint {
+			backupRpcs = append(backupRpcs, endpoint)
+		} else {
+			break
+		}
+	}
+
+	// Log endpoints
+	c.log.Info("Backup Endpoints selected",
+		zap.String("chain_name", c.ChainName),
+		zap.Strings("endpoints", backupRpcs),
+	)
+	return backupRpcs, nil
+}
+
 // GetAssetList returns the asset metadata from the cosmos chain registry for this particular chain.
 func (c ChainInfo) GetAssetList(ctx context.Context, testnet bool, name string) (AssetList, error) {
 	var chainRegURL string
@@ -265,10 +307,17 @@ func (c ChainInfo) GetChainConfig(ctx context.Context, forceAdd, testnet bool, n
 		return nil, err
 	}
 
+	// select 2 healthy endpoints as backup
+	backupRpcs, err := c.GetBackupRPCEndpoints(ctx, forceAdd, rpc, 2)
+	if err != nil {
+		return nil, err
+	}
+
 	return &cosmos.CosmosProviderConfig{
 		Key:              "default",
 		ChainID:          c.ChainID,
 		RPCAddr:          rpc,
+		BackupRPCAddrs:   backupRpcs,
 		AccountPrefix:    c.Bech32Prefix,
 		KeyringBackend:   "test",
 		GasAdjustment:    1.2,

interchaintest/backup_rpc_test.go

@@ -0,0 +1,219 @@
+package interchaintest_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	sdkmath "cosmossdk.io/math"
+	transfertypes "github.com/cosmos/ibc-go/v8/modules/apps/transfer/types"
+	chantypes "github.com/cosmos/ibc-go/v8/modules/core/04-channel/types"
+	relayertest "github.com/cosmos/relayer/v2/interchaintest"
+	"github.com/strangelove-ventures/interchaintest/v8"
+	"github.com/strangelove-ventures/interchaintest/v8/chain/cosmos"
+	"github.com/strangelove-ventures/interchaintest/v8/ibc"
+	icrelayer "github.com/strangelove-ventures/interchaintest/v8/relayer"
+	"github.com/strangelove-ventures/interchaintest/v8/relayer/rly"
+	"github.com/strangelove-ventures/interchaintest/v8/testreporter"
+	"github.com/strangelove-ventures/interchaintest/v8/testutil"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+)
+
+// TestBackupRpcs tests the functionality of falling back to secondary RPCs when the primary node goes offline or becomes unresponsive.
+func TestBackupRpcs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+
+	t.Parallel()
+
+	numVals := 5
+	numFullNodes := 0
+
+	cf := interchaintest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*interchaintest.ChainSpec{
+		{
+			Name:          "gaia",
+			Version:       "v7.0.0",
+			NumValidators: &numVals,
+			NumFullNodes:  &numFullNodes,
+
+			ChainConfig: ibc.ChainConfig{
+				GasPrices: "0.0uatom",
+			},
+		},
+		{
+			Name:          "osmosis",
+			Version:       "v11.0.0",
+			NumValidators: &numVals,
+			NumFullNodes:  &numFullNodes,
+			ChainConfig: ibc.ChainConfig{
+				GasPrices: "0.0uosmo",
+			},
+		},
+	})
+
+	chains, err := cf.Chains(t.Name())
+	require.NoError(t, err)
+	chainA, chainB := chains[0], chains[1]
+
+	ctx := context.Background()
+	client, network := interchaintest.DockerSetup(t)
+
+	image := relayertest.BuildRelayerImage(t)
+	rf := interchaintest.NewBuiltinRelayerFactory(
+		ibc.CosmosRly,
+		zaptest.NewLogger(t),
+		icrelayer.CustomDockerImage(image, "latest", "100:1000"),
+		icrelayer.ImagePull(false),
+		icrelayer.StartupFlags("--processor", "events", "--block-history", "100"),
+	)
+
+	r := rf.Build(t, client, network)
+
+	const pathName = "chainA-chainB"
+
+	ic := interchaintest.NewInterchain().
+		AddChain(chainA).
+		AddChain(chainB).
+		AddRelayer(r, "relayer").
+		AddLink(interchaintest.InterchainLink{
+			Chain1:  chainA,
+			Chain2:  chainB,
+			Relayer: r,
+			Path:    pathName,
+		})
+
+	rep := testreporter.NewNopReporter()
+	eRep := rep.RelayerExecReporter(t)
+
+	require.NoError(t, ic.Build(ctx, eRep, interchaintest.InterchainBuildOptions{
+		TestName:         t.Name(),
+		Client:           client,
+		NetworkID:        network,
+		SkipPathCreation: false,
+	}))
+
+	t.Cleanup(func() {
+		_ = ic.Close()
+	})
+
+	// Create and fund user accs & assert initial balances.
+	initBal := sdkmath.NewInt(1_000_000_000_000)
+	users := interchaintest.GetAndFundTestUsers(t, ctx, t.Name(), initBal, chainA, chainB)
+	require.NoError(t, testutil.WaitForBlocks(ctx, 2, chainA, chainB))
+
+	userA := users[0]
+	userB := users[1]
+
+	userABal, err := chainA.GetBalance(ctx, userA.FormattedAddress(), chainA.Config().Denom)
+	require.NoError(t, err)
+	require.True(t, initBal.Equal(userABal))
+
+	userBBal, err := chainB.GetBalance(ctx, userB.FormattedAddress(), chainB.Config().Denom)
+	require.NoError(t, err)
+	require.True(t, initBal.Equal(userBBal))
+
+	rly := r.(*rly.CosmosRelayer)
+
+	// Get all chains
+	for _, chain := range [](*cosmos.CosmosChain){chainA.(*cosmos.CosmosChain), chainB.(*cosmos.CosmosChain)} {
+
+		addrs := []string{}
+
+		// loop through nodes to collect rpc addrs
+		for _, node := range chain.Validators {
+			rpc := fmt.Sprintf("http://%s:26657", node.Name())
+			addrs = append(addrs, rpc)
+		}
+
+		cmd := []string{"rly", "chains", "set-rpc-addr", chain.Config().Name, addrs[numVals-1], "--home", rly.HomeDir()}
+		res := r.Exec(ctx, eRep, cmd, nil)
+		require.NoError(t, res.Err)
+
+		cmd = []string{"rly", "chains", "set-backup-rpc-addrs", chain.Config().Name, strings.Join(addrs[:numVals-1], ","), "--home", rly.HomeDir()}
+		res = r.Exec(ctx, eRep, cmd, nil)
+		require.NoError(t, res.Err)
+	}
+
+	err = r.StartRelayer(ctx, eRep, pathName)
+	require.NoError(t, err)
+
+	t.Cleanup(
+		func() {
+			err := r.StopRelayer(ctx, eRep)
+			if err != nil {
+				t.Logf("an error occurred while stopping the relayer: %s", err)
+			}
+		},
+	)
+
+	// Send transfers that should succeed & assert balances.
+	channels, err := r.GetChannels(ctx, eRep, chainA.Config().ChainID)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(channels))
+
+	channel := channels[0]
+
+	transferAmount := sdkmath.NewInt(1_000)
+
+	transferAB := ibc.WalletAmount{
+		Address: userB.FormattedAddress(),
+		Denom:   chainA.Config().Denom,
+		Amount:  transferAmount,
+	}
+
+	// wait 10 blocks
+	require.NoError(t, testutil.WaitForBlocks(ctx, 10, chainA, chainB))
+
+	// turn off last nodes on both chains
+	val := chainA.(*cosmos.CosmosChain).Validators[numVals-1]
+	err = val.StopContainer(ctx)
+	require.NoError(t, err)
+
+	val = chainB.(*cosmos.CosmosChain).Validators[numVals-1]
+	err = val.StopContainer(ctx)
+	require.NoError(t, err)
+
+	// wait 10 blocks
+	require.NoError(t, testutil.WaitForBlocks(ctx, 10, chainA, chainB))
+
+	// send ibc tx from chain a to b
+	tx, err := chainA.SendIBCTransfer(ctx, channel.ChannelID, userA.KeyName(), transferAB, ibc.TransferOptions{})
+	require.NoError(t, err)
+	require.NoError(t, tx.Validate())
+
+	// get chain b height
+	bHeight, err := chainB.Height(ctx)
+	require.NoError(t, err)
+
+	// Poll for MsgRecvPacket on b chain
+	_, err = cosmos.PollForMessage[*chantypes.MsgRecvPacket](ctx, chainB.(*cosmos.CosmosChain), cosmos.DefaultEncoding().InterfaceRegistry, bHeight, bHeight+20, nil)
+	require.NoError(t, err)
+
+	// get chain a height
+	aHeight, err := chainA.Height(ctx)
+	require.NoError(t, err)
+
+	// poll for acknowledge on 'a' chain
+	_, err = testutil.PollForAck(ctx, chainA.(*cosmos.CosmosChain), aHeight, aHeight+30, tx.Packet)
+	require.NoError(t, err)
+
+	// Compose the ibc denom for balance assertions on the counterparty and assert balances.
+	denom := transfertypes.GetPrefixedDenom(
+		channel.Counterparty.PortID,
+		channel.Counterparty.ChannelID,
+		chainA.Config().Denom,
+	)
+	trace := transfertypes.ParseDenomTrace(denom)
+
+	// validate user balances on both chains
+	userABal, err = chainA.GetBalance(ctx, userA.FormattedAddress(), chainA.Config().Denom)
+	require.NoError(t, err)
+	require.True(t, userABal.Equal(initBal.Sub(transferAmount)))
+
+	userBBal, err = chainB.GetBalance(ctx, userB.FormattedAddress(), trace.IBCDenom())
+	require.NoError(t, err)
+	require.True(t, userBBal.Equal(transferAmount))
+}

relayer/chain.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/avast/retry-go/v4"
-	"net/url"
 	"time"
 
+	"github.com/avast/retry-go/v4"
+
 	"github.com/cosmos/cosmos-sdk/crypto/hd"
 	clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types"
 	"github.com/cosmos/relayer/v2/relayer/provider"
@@ -31,7 +31,6 @@ type Chain struct {
 
 	ChainProvider provider.ChainProvider
 	Chainid       string `yaml:"chain-id" json:"chain-id"`
-	RPCAddr       string `yaml:"rpc-addr" json:"rpc-addr"`
 
 	PathEnd *PathEnd `yaml:"-" json:"-"`
 
@@ -137,12 +136,6 @@ func (c Chains) Gets(chainIDs ...string) (map[string]*Chain, error) {
 	return out, nil
 }
 
-// GetRPCPort returns the port configured for the chain
-func (c *Chain) GetRPCPort() string {
-	u, _ := url.Parse(c.RPCAddr)
-	return u.Port()
-}
-
 // CreateTestKey creates a key for test chain
 func (c *Chain) CreateTestKey() error {
 	if c.ChainProvider.KeyExists(c.ChainProvider.Key()) {