Skip to content

Commit 917a81e

Browse files
author
Dean Karn
authored
Customized retrier (#50)
1 parent ce7aa8a commit 917a81e

File tree

13 files changed

+1033
-9
lines changed

13 files changed

+1033
-9
lines changed

CHANGELOG.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88

9+
## [5.29.0] - 2024-03-24
10+
### Added
11+
- `asciiext` package for ASCII related functions.
12+
- `errorsext.Retrier` configurable retry helper for any fallible operation.
13+
- `httpext.Retrier` configurable retry helper for HTTP requests and parsing of responses.
14+
- `httpext.DecodeResponseAny` non-generic helper for decoding HTTP responses.
15+
- `httpext.HasRetryAfter` helper for checking if a response has a `Retry-After` header and returning duration to wait.
16+
917
## [5.28.1] - 2024-02-14
1018
### Fixed
1119
- Additional supported types, cast to `sql.Valuer` supported types, they need to be returned to the driver for evaluation.
@@ -120,7 +128,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
120128
### Added
121129
- Added `timext.NanoTime` for fast low level monotonic time with nanosecond precision.
122130

123-
[Unreleased]: https://github.com/go-playground/pkg/compare/v5.28.1...HEAD
131+
[Unreleased]: https://github.com/go-playground/pkg/compare/v5.29.0...HEAD
132+
[5.29.0]: https://github.com/go-playground/pkg/compare/v5.28.1..v5.29.0
124133
[5.28.1]: https://github.com/go-playground/pkg/compare/v5.28.0..v5.28.1
125134
[5.28.0]: https://github.com/go-playground/pkg/compare/v5.27.0..v5.28.0
126135
[5.27.0]: https://github.com/go-playground/pkg/compare/v5.26.0..v5.27.0

README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# pkg
22

3-
![Project status](https://img.shields.io/badge/version-5.28.0-green.svg)
3+
![Project status](https://img.shields.io/badge/version-5.29.0-green.svg)
44
[![Lint & Test](https://github.com/go-playground/pkg/actions/workflows/go.yml/badge.svg)](https://github.com/go-playground/pkg/actions/workflows/go.yml)
55
[![Coverage Status](https://coveralls.io/repos/github/go-playground/pkg/badge.svg?branch=master)](https://coveralls.io/github/go-playground/pkg?branch=master)
66
[![GoDoc](https://godoc.org/github.com/go-playground/pkg?status.svg)](https://pkg.go.dev/mod/github.com/go-playground/pkg/v5)
@@ -23,7 +23,7 @@ This is a place to put common reusable code that is not quite a library but exte
2323
- Generic Mutex and RWMutex.
2424
- Bytes helper placeholders units eg. MB, MiB, GB, ...
2525
- Detachable context.
26-
- Error retryable helper functions.
26+
- Retrier for helping with any fallible operation.
2727
- Proper RFC3339Nano definition.
2828
- unsafe []byte->string & string->[]byte helper functions.
2929
- HTTP helper functions and constant placeholders.

_examples/net/http/retrier/main.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
"net/http/httptest"
8+
"time"
9+
10+
appext "github.com/go-playground/pkg/v5/app"
11+
errorsext "github.com/go-playground/pkg/v5/errors"
12+
httpext "github.com/go-playground/pkg/v5/net/http"
13+
. "github.com/go-playground/pkg/v5/values/result"
14+
)
15+
16+
// customize as desired to meet your needs including custom retryable status codes, errors etc.
17+
var retrier = httpext.NewRetryer()
18+
19+
func main() {
20+
ctx := appext.Context().Build()
21+
22+
type Test struct {
23+
Date time.Time
24+
}
25+
var count int
26+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
27+
if count < 2 {
28+
count++
29+
http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
30+
return
31+
}
32+
_ = httpext.JSON(w, http.StatusOK, Test{Date: time.Now().UTC()})
33+
}))
34+
defer server.Close()
35+
36+
// fetch response
37+
fn := func(ctx context.Context) Result[*http.Request, error] {
38+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
39+
if err != nil {
40+
return Err[*http.Request, error](err)
41+
}
42+
return Ok[*http.Request, error](req)
43+
}
44+
45+
var result Test
46+
err := retrier.Do(ctx, fn, &result, http.StatusOK)
47+
if err != nil {
48+
panic(err)
49+
}
50+
fmt.Printf("Response: %+v\n", result)
51+
52+
// `Retrier` configuration is copy and so the base `Retrier` can be used and even customized for one-off requests.
53+
// eg for this request we change the max attempts from the default configuration.
54+
err = retrier.MaxAttempts(errorsext.MaxAttempts, 2).Do(ctx, fn, &result, http.StatusOK)
55+
if err != nil {
56+
panic(err)
57+
}
58+
fmt.Printf("Response: %+v\n", result)
59+
}

ascii/helpers.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package asciiext
2+
3+
// IsAlphanumeric returns true if the byte is an ASCII letter or digit.
4+
func IsAlphanumeric(c byte) bool {
5+
return IsLower(c) || IsUpper(c) || IsDigit(c)
6+
}
7+
8+
// IsUpper returns true if the byte is an ASCII uppercase letter.
9+
func IsUpper(c byte) bool {
10+
return c >= 'A' && c <= 'Z'
11+
}
12+
13+
// IsLower returns true if the byte is an ASCII lowercase letter.
14+
func IsLower(c byte) bool {
15+
return c >= 'a' && c <= 'z'
16+
}
17+
18+
// IsDigit returns true if the byte is an ASCII digit.
19+
func IsDigit(c byte) bool {
20+
return c >= '0' && c <= '9'
21+
}

errors/do.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package errorsext
55

66
import (
77
"context"
8+
89
optionext "github.com/go-playground/pkg/v5/values/option"
910
resultext "github.com/go-playground/pkg/v5/values/result"
1011
)
@@ -21,6 +22,8 @@ type IsRetryableFn[E any] func(err E) (reason string, isRetryable bool)
2122
type OnRetryFn[E any] func(ctx context.Context, originalErr E, reason string, attempt int) optionext.Option[E]
2223

2324
// DoRetryable will execute the provided functions code and automatically retry using the provided retry function.
25+
//
26+
// Deprecated: use `errorsext.Retrier` instead which corrects design issues with the current implementation.
2427
func DoRetryable[T, E any](ctx context.Context, isRetryFn IsRetryableFn[E], onRetryFn OnRetryFn[E], fn RetryableFn[T, E]) resultext.Result[T, E] {
2528
var attempt int
2629
for {

errors/retrier.go

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
//go:build go1.18
2+
// +build go1.18
3+
4+
package errorsext
5+
6+
import (
7+
"context"
8+
"time"
9+
10+
. "github.com/go-playground/pkg/v5/values/result"
11+
)
12+
13+
// MaxAttemptsMode is used to set the mode for the maximum number of attempts.
14+
//
15+
// eg. Should the max attempts apply to all errors, just ones not determined to be retryable, reset on retryable errors, etc.
16+
type MaxAttemptsMode uint8
17+
18+
const (
19+
// MaxAttemptsNonRetryableReset will apply the max attempts to all errors not determined to be retryable, but will
20+
// reset the attempts if a retryable error is encountered after a non-retryable error.
21+
MaxAttemptsNonRetryableReset MaxAttemptsMode = iota
22+
23+
// MaxAttemptsNonRetryable will apply the max attempts to all errors not determined to be retryable.
24+
MaxAttemptsNonRetryable
25+
26+
// MaxAttempts will apply the max attempts to all errors, even those determined to be retryable.
27+
MaxAttempts
28+
29+
// MaxAttemptsUnlimited will not apply a maximum number of attempts.
30+
MaxAttemptsUnlimited
31+
)
32+
33+
// BackoffFn is a function used to apply a backoff strategy to the retryable function.
34+
//
35+
// It accepts `E` in cases where the amount of time to backoff is dynamic, for example when and http request fails
36+
// with a 429 status code, the `Retry-After` header can be used to determine how long to backoff. It is not required
37+
// to use or handle `E` and can be ignored if desired.
38+
type BackoffFn[E any] func(ctx context.Context, attempt int, e E)
39+
40+
// IsRetryableFn2 is called to determine if the type E is retryable.
41+
type IsRetryableFn2[E any] func(ctx context.Context, e E) (isRetryable bool)
42+
43+
// EarlyReturnFn is the function that can be used to bypass all retry logic, no matter the MaxAttemptsMode, for when the
44+
// type of `E` will never succeed and should not be retried.
45+
//
46+
// eg. If retrying an HTTP request and getting 400 Bad Request, it's unlikely to ever succeed and should not be retried.
47+
type EarlyReturnFn[E any] func(ctx context.Context, e E) (earlyReturn bool)
48+
49+
// Retryer is used to retry any fallible operation.
50+
type Retryer[T, E any] struct {
51+
isRetryableFn IsRetryableFn2[E]
52+
isEarlyReturnFn EarlyReturnFn[E]
53+
maxAttemptsMode MaxAttemptsMode
54+
maxAttempts uint8
55+
bo BackoffFn[E]
56+
timeout time.Duration
57+
}
58+
59+
// NewRetryer returns a new `Retryer` with sane default values.
60+
//
61+
// The default values are:
62+
// - `MaxAttemptsMode` is `MaxAttemptsNonRetryableReset`.
63+
// - `MaxAttempts` is 5.
64+
// - `Timeout` is 0 no context timeout.
65+
// - `IsRetryableFn` will always return false as `E` is unknown until defined.
66+
// - `BackoffFn` will sleep for 200ms. It's recommended to use exponential backoff for production.
67+
// - `EarlyReturnFn` will be None.
68+
func NewRetryer[T, E any]() Retryer[T, E] {
69+
return Retryer[T, E]{
70+
isRetryableFn: func(_ context.Context, _ E) bool { return false },
71+
maxAttemptsMode: MaxAttemptsNonRetryableReset,
72+
maxAttempts: 5,
73+
bo: func(ctx context.Context, attempt int, _ E) {
74+
t := time.NewTimer(time.Millisecond * 200)
75+
defer t.Stop()
76+
select {
77+
case <-ctx.Done():
78+
case <-t.C:
79+
}
80+
},
81+
}
82+
}
83+
84+
// IsRetryableFn sets the `IsRetryableFn` for the `Retryer`.
85+
func (r Retryer[T, E]) IsRetryableFn(fn IsRetryableFn2[E]) Retryer[T, E] {
86+
if fn == nil {
87+
fn = func(_ context.Context, _ E) bool { return false }
88+
}
89+
r.isRetryableFn = fn
90+
return r
91+
}
92+
93+
// IsEarlyReturnFn sets the `EarlyReturnFn` for the `Retryer`.
94+
//
95+
// NOTE: If the `EarlyReturnFn` and `IsRetryableFn` are both set and a conflicting `IsRetryableFn` will take precedence.
96+
func (r Retryer[T, E]) IsEarlyReturnFn(fn EarlyReturnFn[E]) Retryer[T, E] {
97+
r.isEarlyReturnFn = fn
98+
return r
99+
}
100+
101+
// MaxAttempts sets the maximum number of attempts for the `Retryer`.
102+
//
103+
// NOTE: Max attempts is optional and if not set will retry indefinitely on retryable errors.
104+
func (r Retryer[T, E]) MaxAttempts(mode MaxAttemptsMode, maxAttempts uint8) Retryer[T, E] {
105+
r.maxAttemptsMode, r.maxAttempts = mode, maxAttempts
106+
return r
107+
}
108+
109+
// Backoff sets the backoff function for the `Retryer`.
110+
func (r Retryer[T, E]) Backoff(fn BackoffFn[E]) Retryer[T, E] {
111+
if fn == nil {
112+
fn = func(_ context.Context, _ int, _ E) {}
113+
}
114+
r.bo = fn
115+
return r
116+
}
117+
118+
// Timeout sets the timeout for the `Retryer`. This is the timeout per `RetyableFn` attempt and not the entirety
119+
// of the `Retryer` execution.
120+
//
121+
// A timeout of 0 will disable the timeout and is the default.
122+
func (r Retryer[T, E]) Timeout(timeout time.Duration) Retryer[T, E] {
123+
r.timeout = timeout
124+
return r
125+
}
126+
127+
// Do will execute the provided functions code and automatically retry using the provided retry function.
128+
func (r Retryer[T, E]) Do(ctx context.Context, fn RetryableFn[T, E]) Result[T, E] {
129+
var attempt int
130+
remaining := r.maxAttempts
131+
for {
132+
var result Result[T, E]
133+
if r.timeout == 0 {
134+
result = fn(ctx)
135+
} else {
136+
ctx, cancel := context.WithTimeout(ctx, r.timeout)
137+
result = fn(ctx)
138+
cancel()
139+
}
140+
if result.IsErr() {
141+
err := result.Err()
142+
isRetryable := r.isRetryableFn(ctx, err)
143+
if !isRetryable && r.isEarlyReturnFn != nil && r.isEarlyReturnFn(ctx, err) {
144+
return result
145+
}
146+
147+
switch r.maxAttemptsMode {
148+
case MaxAttemptsUnlimited:
149+
goto RETRY
150+
case MaxAttemptsNonRetryableReset:
151+
if isRetryable {
152+
remaining = r.maxAttempts
153+
goto RETRY
154+
} else if remaining > 0 {
155+
remaining--
156+
}
157+
case MaxAttemptsNonRetryable:
158+
if isRetryable {
159+
goto RETRY
160+
} else if remaining > 0 {
161+
remaining--
162+
}
163+
case MaxAttempts:
164+
if remaining > 0 {
165+
remaining--
166+
}
167+
}
168+
if remaining == 0 {
169+
return result
170+
}
171+
172+
RETRY:
173+
r.bo(ctx, attempt, err)
174+
attempt++
175+
continue
176+
}
177+
return result
178+
}
179+
}

0 commit comments

Comments
 (0)