diff --git a/ais/backend/ais.go b/ais/backend/ais.go index b78c0378d78..e13c640fc54 100644 --- a/ais/backend/ais.go +++ b/ais/backend/ais.go @@ -45,6 +45,7 @@ type ( url string uuid string bp api.BaseParams + bpL api.BaseParams // long & list } AISbp struct { t core.TargetPut @@ -94,6 +95,9 @@ func extractErrCode(e error, uuid string) (int, error) { if e == nil { return http.StatusOK, nil } + if cos.IsClientTimeout(e) { + return http.StatusRequestTimeout, e + } herr := cmn.Err2HTTPErr(e) if herr == nil { return http.StatusInternalServerError, e @@ -257,7 +261,9 @@ func (r *remAis) init(alias string, confURLs []string, cfg *cmn.ClusterConfig) ( var ( url string remSmap, smap *meta.Smap - cliH, cliTLS = remaisClients(&cfg.Client) + + clientL http.Client + cliH, cliTLS = remaisClients(&cfg.Client) ) for _, u := range confURLs { client := cliH @@ -289,10 +295,17 @@ func (r *remAis) init(alias string, confURLs []string, cfg *cmn.ClusterConfig) ( r.smap, r.url = remSmap, url if cos.IsHTTPS(url) { r.bp = api.BaseParams{Client: cliTLS, URL: url, UA: ua} + clientL = *cliTLS } else { r.bp = api.BaseParams{Client: cliH, URL: url, UA: ua} + clientL = *cliH } + + r.bpL = r.bp + clientL.Timeout = cfg.Client.TimeoutLong.D() + r.bpL.Client = &clientL r.uuid = remSmap.UUID + return } @@ -446,7 +459,7 @@ func (m *AISbp) ListObjects(remoteBck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRe unsetUUID(&bck) var lstRes *cmn.LsoRes - if lstRes, err = api.ListObjectsPage(remAis.bp, bck, remoteMsg, api.ListArgs{}); err != nil { + if lstRes, err = api.ListObjectsPage(remAis.bpL, bck, remoteMsg, api.ListArgs{}); err != nil { ecode, err = extractErrCode(err, remAis.uuid) return } @@ -540,6 +553,7 @@ func (m *AISbp) HeadObj(_ context.Context, lom *core.LOM, _ *http.Request) (oa * return } +// TODO: retry func (m *AISbp) GetObj(_ context.Context, lom *core.LOM, owt cmn.OWT, _ *http.Request) (ecode int, err error) { var ( remAis *remAis @@ -551,7 +565,7 @@ func (m *AISbp) GetObj(_ context.Context, lom *core.LOM, owt cmn.OWT, _ *http.Re return } unsetUUID(&remoteBck) - if r, size, err = api.GetObjectReader(remAis.bp, remoteBck, lom.ObjName, nil /*api.GetArgs*/); err != nil { + if r, size, err = api.GetObjectReader(remAis.bpL, remoteBck, lom.ObjName, nil /*api.GetArgs*/); err != nil { return extractErrCode(err, remAis.uuid) } params := core.AllocPutParams() @@ -564,6 +578,9 @@ func (m *AISbp) GetObj(_ context.Context, lom *core.LOM, owt cmn.OWT, _ *http.Re } err = m.t.PutObject(lom, params) core.FreePutParams(params) + + // TODO: retry upon 'unreachable' or timeout + return extractErrCode(err, remAis.uuid) } @@ -599,11 +616,12 @@ func (m *AISbp) GetObjReader(_ context.Context, lom *core.LOM, offset, length in res.ExpCksum = oa.Cksum lom.SetCksum(nil) } - res.R, res.Size, res.Err = api.GetObjectReader(remAis.bp, remoteBck, lom.ObjName, args) + res.R, res.Size, res.Err = api.GetObjectReader(remAis.bpL, remoteBck, lom.ObjName, args) res.ErrCode, res.Err = extractErrCode(res.Err, remAis.uuid) return } +// TODO: retry upon 'unreachable' or timeout func (m *AISbp) PutObj(r io.ReadCloser, lom *core.LOM, _ *http.Request) (ecode int, err error) { var ( oah api.ObjAttrs @@ -617,7 +635,7 @@ func (m *AISbp) PutObj(r io.ReadCloser, lom *core.LOM, _ *http.Request) (ecode i unsetUUID(&remoteBck) size := lom.Lsize(true) // _special_ as it's still a workfile at this point args := api.PutArgs{ - BaseParams: remAis.bp, + BaseParams: remAis.bpL, Bck: remoteBck, ObjName: lom.ObjName, Cksum: lom.Checksum(), diff --git a/api/ls.go b/api/ls.go index ac79ea102b8..87c9f4eae03 100644 --- a/api/ls.go +++ b/api/ls.go @@ -5,8 +5,6 @@ package api import ( - "context" - "errors" "net/http" "net/url" "strconv" @@ -204,7 +202,7 @@ func lsoPage(reqParams *ReqParams) (_ *cmn.LsoRes, err error) { if _, err = reqParams.DoReqAny(page); err == nil { return page, nil } - if !errors.Is(err, context.DeadlineExceeded) { + if !cos.IsClientTimeout(err) { break } client := *reqParams.BaseParams.Client diff --git a/cmn/config.go b/cmn/config.go index 8176d445235..5ed6dc50954 100644 --- a/cmn/config.go +++ b/cmn/config.go @@ -880,15 +880,25 @@ func (c *LogConf) Validate() error { // ClientConf // //////////////// +const ( + minClientTimeout = time.Second + maxClientTimeout = 30 * time.Minute + + errExpectedRange = "(expected range [1s, 30m] or zero)" +) + func (c *ClientConf) Validate() error { - if j := c.Timeout.D(); j < time.Second || j > 2*time.Minute { - return fmt.Errorf("invalid client.client_timeout=%s (expected range [1s, 2m])", j) + if j := c.Timeout.D(); j != 0 && (j < minClientTimeout || j > maxClientTimeout) { + return fmt.Errorf("invalid client_timeout=%s %s", j, errExpectedRange) + } + if j := c.TimeoutLong.D(); j != 0 && (j < minClientTimeout || j > maxClientTimeout) { + return fmt.Errorf("invalid client_long_timeout=%s %s", j, errExpectedRange) } - if j := c.TimeoutLong.D(); j < 30*time.Second || j < c.Timeout.D() || j > 30*time.Minute { - return fmt.Errorf("invalid client.client_long_timeout=%s (expected range [30s, 30m])", j) + if j := c.TimeoutLong.D(); j != 0 && j < c.Timeout.D() { + return fmt.Errorf("client_long_timeout=%s cannot be less than client_timeout=%s", j, c.Timeout.D()) } - if j := c.ListObjTimeout.D(); j < 2*time.Second || j > 15*time.Minute { - return fmt.Errorf("invalid client.list_timeout=%s (expected range [2s, 15m])", j) + if j := c.ListObjTimeout.D(); j != 0 && (j < minClientTimeout || j > maxClientTimeout) { + return fmt.Errorf("invalid list_timeout=%s %s", j, errExpectedRange) } return nil } diff --git a/cmn/cos/err.go b/cmn/cos/err.go index a3439db8e1a..35826269f05 100644 --- a/cmn/cos/err.go +++ b/cmn/cos/err.go @@ -188,6 +188,10 @@ func IsErrDNSLookup(err error) bool { return errors.As(err, &wrapped) } +func IsClientTimeout(err error) bool { + return errors.Is(err, context.DeadlineExceeded) +} + func IsUnreachable(err error, status int) bool { return IsErrConnectionRefused(err) || IsErrDNSLookup(err) ||