Skip to content

Commit

Permalink
fix: catch context errors from http provider (celestiaorg#1160)
Browse files Browse the repository at this point in the history
It seems from attempts to perform state sync that the light client is
not correctly detecting when the context gets cancelled or the deadline
is exceeded as the error is being wrapped

```
�[90m12:53PM�[0m �[32mINF�[0m error from light block request from primary, removing... �[36merror=�[0m�[31m"post failed: Post \"https://rpc-snapscale.lunaroasis.net:443\": context deadline exceeded"�[0m �[36mheight=�[0m354290 �[36mmodule=�[0mlight �[36mprimary=�[0m{}
```

This PR checks to see if `context deadline exceeded` is contained within
the error and then returns the actual context error
  • Loading branch information
cmwaters authored Jan 4, 2024
1 parent 39777b4 commit 52633f4
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 2 deletions.
16 changes: 16 additions & 0 deletions light/provider/http/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,14 @@ OUTER_LOOP:
time.Sleep(backoffTimeout(uint16(attempt)))
continue

// NOTE: it seems like the context errors are being wrapped in a way that
// makes them hard to detect. For now, we just check the error string.
case strings.Contains(err.Error(), context.DeadlineExceeded.Error()):
return nil, context.DeadlineExceeded

case ctx.Err() != nil:
return nil, ctx.Err()

// context canceled or connection refused we return the error
default:
return nil, err
Expand Down Expand Up @@ -202,6 +210,14 @@ func (p *http) signedHeader(ctx context.Context, height *int64) (*types.SignedHe
time.Sleep(backoffTimeout(uint16(attempt)))
continue

// NOTE: it seems like the context errors are being wrapped in a way that
// makes them hard to detect. For now, we just check the error string.
case strings.Contains(err.Error(), context.DeadlineExceeded.Error()):
return nil, context.DeadlineExceeded

case ctx.Err() != nil:
return nil, ctx.Err()

// either context was cancelled or connection refused.
default:
return nil, err
Expand Down
35 changes: 33 additions & 2 deletions light/provider/http/http_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
lighthttp "github.com/cometbft/cometbft/light/provider/http"
rpcclient "github.com/cometbft/cometbft/rpc/client"
rpchttp "github.com/cometbft/cometbft/rpc/client/http"
ctypes "github.com/cometbft/cometbft/rpc/core/types"
rpctest "github.com/cometbft/cometbft/rpc/test"
"github.com/cometbft/cometbft/types"
)
Expand Down Expand Up @@ -46,10 +47,9 @@ func TestProvider(t *testing.T) {
chainID := genDoc.ChainID

c, err := rpchttp.New(rpcAddr, "/websocket")
require.Nil(t, err)
require.NoError(t, err)

p := lighthttp.NewWithClient(chainID, c)
require.NoError(t, err)
require.NotNil(t, p)

// let it produce some blocks
Expand Down Expand Up @@ -82,6 +82,21 @@ func TestProvider(t *testing.T) {
require.Nil(t, lb)
assert.Equal(t, provider.ErrLightBlockNotFound, err)

// fetching with the context cancelled
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = p.LightBlock(ctx, lower+3)
require.Error(t, err)
require.Equal(t, context.Canceled, err)

// fetching with the deadline exceeded (a mock RPC client is used to simulate this)
c2, err := newMockHTTP(rpcAddr)
require.NoError(t, err)
p2 := lighthttp.NewWithClient(chainID, c2)
_, err = p2.LightBlock(context.Background(), 0)
require.Error(t, err)
require.Equal(t, context.DeadlineExceeded, err)

// stop the full node and check that a no response error is returned
rpctest.StopTendermint(node)
time.Sleep(10 * time.Second)
Expand All @@ -91,3 +106,19 @@ func TestProvider(t *testing.T) {
require.Contains(t, err.Error(), "connection refused")
require.Nil(t, lb)
}

type mockHTTP struct {
*rpchttp.HTTP
}

func newMockHTTP(remote string) (*mockHTTP, error) {
c, err := rpchttp.New(remote, "/websocket")
if err != nil {
return nil, err
}
return &mockHTTP{c}, nil
}

func (m *mockHTTP) Validators(ctx context.Context, height *int64, page, perPage *int) (*ctypes.ResultValidators, error) {
return nil, fmt.Errorf("post failed: %w", context.DeadlineExceeded)
}

0 comments on commit 52633f4

Please sign in to comment.