reset retries after stable playback
This commit is contained in:
@@ -20,8 +20,21 @@ func TestStatusPreservesValues(t *testing.T) {
|
|||||||
if status.Unit != UnitVideo {
|
if status.Unit != UnitVideo {
|
||||||
t.Errorf("Unit = %v, want %v", status.Unit, UnitVideo)
|
t.Errorf("Unit = %v, want %v", status.Unit, UnitVideo)
|
||||||
}
|
}
|
||||||
// Check the remaining fields similarly.
|
|
||||||
if !errors.Is(status.Err, wantErr) {
|
if !errors.Is(status.Err, wantErr) {
|
||||||
t.Errorf("Err = %v, want %v", status.Err, wantErr)
|
t.Errorf("Err = %v, want %v", status.Err, wantErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if status.State != StateReconnecting {
|
||||||
|
t.Errorf("State = %v, want %v", status.State, StateReconnecting)
|
||||||
|
}
|
||||||
|
if status.Attempt != 2 {
|
||||||
|
t.Errorf("Attempt = %d, want 2", status.Attempt)
|
||||||
|
}
|
||||||
|
if status.FailedAttempts != 1 {
|
||||||
|
t.Errorf("FailedAttempts = %d, want 1", status.FailedAttempts)
|
||||||
|
}
|
||||||
|
if status.RetryIn != time.Second {
|
||||||
|
t.Errorf("RetryIn = %s, want %s", status.RetryIn, time.Second)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type attemptFunc func(context.Context) error
|
// attemptFunc returns whether useful media was received before the attempt ended.
|
||||||
|
type attemptFunc func(context.Context) (becameStable bool, err error)
|
||||||
type retryDecider func(error) bool
|
type retryDecider func(error) bool
|
||||||
type waitFunc func(context.Context, time.Duration) error
|
type waitFunc func(context.Context, time.Duration) error
|
||||||
|
|
||||||
@@ -32,7 +33,7 @@ func runWithRetry(
|
|||||||
failedAttempts := 0
|
failedAttempts := 0
|
||||||
|
|
||||||
for {
|
for {
|
||||||
err := attempt(ctx)
|
becameStable, err := attempt(ctx)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -40,6 +41,10 @@ func runWithRetry(
|
|||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if becameStable {
|
||||||
|
failedAttempts = 0
|
||||||
|
}
|
||||||
|
|
||||||
failedAttempts++
|
failedAttempts++
|
||||||
willRetry := shouldRetry(err) && policy.canRetry(failedAttempts)
|
willRetry := shouldRetry(err) && policy.canRetry(failedAttempts)
|
||||||
if !willRetry {
|
if !willRetry {
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ func TestRunWithRetryFirstAttemptSucceeds(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(3),
|
testRetryPolicy(3),
|
||||||
func(context.Context) error {
|
func(context.Context) (bool, error) {
|
||||||
attempts++
|
attempts++
|
||||||
return nil
|
return false, nil
|
||||||
},
|
},
|
||||||
func(error) bool {
|
func(error) bool {
|
||||||
t.Fatal("shouldRetry called after successful attempt")
|
t.Fatal("shouldRetry called after successful attempt")
|
||||||
@@ -52,12 +52,12 @@ func TestRunWithRetryFailuresThenSuccess(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(3),
|
testRetryPolicy(3),
|
||||||
func(context.Context) error {
|
func(context.Context) (bool, error) {
|
||||||
attempts++
|
attempts++
|
||||||
if attempts < 3 {
|
if attempts < 3 {
|
||||||
return attemptErr
|
return false, attemptErr
|
||||||
}
|
}
|
||||||
return nil
|
return false, nil
|
||||||
},
|
},
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(_ context.Context, delay time.Duration) error {
|
func(_ context.Context, delay time.Duration) error {
|
||||||
@@ -87,9 +87,9 @@ func TestRunWithRetryFiniteAttemptsExhausted(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(3),
|
testRetryPolicy(3),
|
||||||
func(context.Context) error {
|
func(context.Context) (bool, error) {
|
||||||
attempts++
|
attempts++
|
||||||
return attemptErr
|
return false, attemptErr
|
||||||
},
|
},
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(context.Context, time.Duration) error {
|
func(context.Context, time.Duration) error {
|
||||||
@@ -117,12 +117,12 @@ func TestRunWithRetryUnlimitedEventuallySucceeds(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(0),
|
testRetryPolicy(0),
|
||||||
func(context.Context) error {
|
func(context.Context) (bool, error) {
|
||||||
attempts++
|
attempts++
|
||||||
if attempts < 20 {
|
if attempts < 20 {
|
||||||
return attemptErr
|
return false, attemptErr
|
||||||
}
|
}
|
||||||
return nil
|
return false, nil
|
||||||
},
|
},
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(context.Context, time.Duration) error { return nil },
|
func(context.Context, time.Duration) error { return nil },
|
||||||
@@ -144,9 +144,9 @@ func TestRunWithRetryStopsWhenErrorIsNotRetryable(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(0),
|
testRetryPolicy(0),
|
||||||
func(context.Context) error {
|
func(context.Context) (bool, error) {
|
||||||
attempts++
|
attempts++
|
||||||
return attemptErr
|
return false, attemptErr
|
||||||
},
|
},
|
||||||
func(error) bool { return false },
|
func(error) bool { return false },
|
||||||
func(context.Context, time.Duration) error {
|
func(context.Context, time.Duration) error {
|
||||||
@@ -171,9 +171,9 @@ func TestRunWithRetryReturnsCancellationFromAttempt(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
ctx,
|
ctx,
|
||||||
testRetryPolicy(0),
|
testRetryPolicy(0),
|
||||||
func(context.Context) error {
|
func(context.Context) (bool, error) {
|
||||||
cancel()
|
cancel()
|
||||||
return attemptErr
|
return false, attemptErr
|
||||||
},
|
},
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(context.Context, time.Duration) error {
|
func(context.Context, time.Duration) error {
|
||||||
@@ -195,7 +195,7 @@ func TestRunWithRetryReturnsCancellationDuringBackoff(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
ctx,
|
ctx,
|
||||||
testRetryPolicy(0),
|
testRetryPolicy(0),
|
||||||
func(context.Context) error { return attemptErr },
|
func(context.Context) (bool, error) { return false, attemptErr },
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(ctx context.Context, _ time.Duration) error {
|
func(ctx context.Context, _ time.Duration) error {
|
||||||
cancel()
|
cancel()
|
||||||
@@ -216,7 +216,7 @@ func TestRunWithRetryReturnsWaitError(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(0),
|
testRetryPolicy(0),
|
||||||
func(context.Context) error { return attemptErr },
|
func(context.Context) (bool, error) { return false, attemptErr },
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(context.Context, time.Duration) error { return waitErr },
|
func(context.Context, time.Duration) error { return waitErr },
|
||||||
nil,
|
nil,
|
||||||
@@ -245,12 +245,12 @@ func TestRetryObserverReportsFailuresBeforeSuccess(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(3),
|
testRetryPolicy(3),
|
||||||
func(context.Context) error {
|
func(context.Context) (bool, error) {
|
||||||
attempts++
|
attempts++
|
||||||
if attempts < 3 {
|
if attempts < 3 {
|
||||||
return attemptErr
|
return false, attemptErr
|
||||||
}
|
}
|
||||||
return nil
|
return false, nil
|
||||||
},
|
},
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(context.Context, time.Duration) error { return nil },
|
func(context.Context, time.Duration) error { return nil },
|
||||||
@@ -291,7 +291,7 @@ func TestRetryObserverReportsExhaustion(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(2),
|
testRetryPolicy(2),
|
||||||
func(context.Context) error { return attemptErr },
|
func(context.Context) (bool, error) { return false, attemptErr },
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(context.Context, time.Duration) error { return nil },
|
func(context.Context, time.Duration) error { return nil },
|
||||||
func(event retryEvent) {
|
func(event retryEvent) {
|
||||||
@@ -328,7 +328,7 @@ func TestRetryObserverNotCalledOnImmediateSuccess(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
testRetryPolicy(3),
|
testRetryPolicy(3),
|
||||||
func(context.Context) error { return nil },
|
func(context.Context) (bool, error) { return false, nil },
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(context.Context, time.Duration) error { return nil },
|
func(context.Context, time.Duration) error { return nil },
|
||||||
func(retryEvent) { observerCalls++ },
|
func(retryEvent) { observerCalls++ },
|
||||||
@@ -349,9 +349,9 @@ func TestRetryObserverNotCalledWhenAttemptCancelsContext(t *testing.T) {
|
|||||||
err := runWithRetry(
|
err := runWithRetry(
|
||||||
ctx,
|
ctx,
|
||||||
testRetryPolicy(0),
|
testRetryPolicy(0),
|
||||||
func(context.Context) error {
|
func(context.Context) (bool, error) {
|
||||||
cancel()
|
cancel()
|
||||||
return errors.New("attempt interrupted")
|
return false, errors.New("attempt interrupted")
|
||||||
},
|
},
|
||||||
func(error) bool { return true },
|
func(error) bool { return true },
|
||||||
func(context.Context, time.Duration) error { return nil },
|
func(context.Context, time.Duration) error { return nil },
|
||||||
@@ -365,3 +365,61 @@ func TestRetryObserverNotCalledWhenAttemptCancelsContext(t *testing.T) {
|
|||||||
t.Fatalf("observer call count = %d, want 0", observerCalls)
|
t.Fatalf("observer call count = %d, want 0", observerCalls)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunWithRetryResetsFailuresAfterStableAttempt(t *testing.T) {
|
||||||
|
attemptErr := errors.New("attempt failed")
|
||||||
|
attempts := 0
|
||||||
|
var events []retryEvent
|
||||||
|
|
||||||
|
err := runWithRetry(
|
||||||
|
context.Background(),
|
||||||
|
testRetryPolicy(2),
|
||||||
|
func(context.Context) (bool, error) {
|
||||||
|
attempts++
|
||||||
|
switch attempts {
|
||||||
|
case 1:
|
||||||
|
return false, attemptErr
|
||||||
|
case 2:
|
||||||
|
return true, attemptErr
|
||||||
|
default:
|
||||||
|
return false, attemptErr
|
||||||
|
}
|
||||||
|
},
|
||||||
|
func(error) bool { return true },
|
||||||
|
func(context.Context, time.Duration) error { return nil },
|
||||||
|
func(event retryEvent) {
|
||||||
|
events = append(events, event)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
if !errors.Is(err, attemptErr) {
|
||||||
|
t.Fatalf("runWithRetry() error = %v, want %v", err, attemptErr)
|
||||||
|
}
|
||||||
|
if attempts != 3 {
|
||||||
|
t.Fatalf("attempt count = %d, want 3", attempts)
|
||||||
|
}
|
||||||
|
if len(events) != 3 {
|
||||||
|
t.Fatalf("event count = %d, want 3", len(events))
|
||||||
|
}
|
||||||
|
|
||||||
|
wantFailedAttempts := []int{1, 1, 2}
|
||||||
|
wantWillRetry := []bool{true, true, false}
|
||||||
|
for i, event := range events {
|
||||||
|
if event.FailedAttempts != wantFailedAttempts[i] {
|
||||||
|
t.Errorf(
|
||||||
|
"event %d failed attempts = %d, want %d",
|
||||||
|
i,
|
||||||
|
event.FailedAttempts,
|
||||||
|
wantFailedAttempts[i],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if event.WillRetry != wantWillRetry[i] {
|
||||||
|
t.Errorf(
|
||||||
|
"event %d WillRetry = %t, want %t",
|
||||||
|
i,
|
||||||
|
event.WillRetry,
|
||||||
|
wantWillRetry[i],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user