From 89a0522e2097914f1c4165e83d11e6c9219269ac Mon Sep 17 00:00:00 2001 From: Dmitry Sergeev Date: Thu, 27 Aug 2026 08:47:53 +0300 Subject: [PATCH] report retry progress --- internal/playback/supervisor.go | 32 ++++++- internal/playback/supervisor_test.go | 137 +++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/internal/playback/supervisor.go b/internal/playback/supervisor.go index bbe9f4d..0f00d94 100644 --- a/internal/playback/supervisor.go +++ b/internal/playback/supervisor.go @@ -27,6 +27,7 @@ func runWithRetry( attempt attemptFunc, shouldRetry retryDecider, wait waitFunc, + observer retryObserver, ) error { failedAttempts := 0 @@ -40,11 +41,29 @@ func runWithRetry( } failedAttempts++ - if !shouldRetry(err) || !policy.canRetry(failedAttempts) { + willRetry := shouldRetry(err) && policy.canRetry(failedAttempts) + if !willRetry { + if observer != nil { + observer(retryEvent{ + FailedAttempts: failedAttempts, + Err: err, + WillRetry: false, + }) + } return err } - if err := wait(ctx, policy.retryDelay(failedAttempts)); err != nil { + delay := policy.retryDelay(failedAttempts) + if observer != nil { + observer(retryEvent{ + FailedAttempts: failedAttempts, + Err: err, + RetryIn: delay, + WillRetry: true, + }) + } + + if err := wait(ctx, delay); err != nil { if ctx.Err() != nil { return ctx.Err() } @@ -52,3 +71,12 @@ func runWithRetry( } } } + +type retryEvent struct { + FailedAttempts int + Err error + RetryIn time.Duration + WillRetry bool +} + +type retryObserver func(retryEvent) diff --git a/internal/playback/supervisor_test.go b/internal/playback/supervisor_test.go index 9466696..9b852a6 100644 --- a/internal/playback/supervisor_test.go +++ b/internal/playback/supervisor_test.go @@ -33,6 +33,7 @@ func TestRunWithRetryFirstAttemptSucceeds(t *testing.T) { t.Fatal("wait called after successful attempt") return nil }, + nil, ) if err != nil { @@ -63,6 +64,7 @@ func TestRunWithRetryFailuresThenSuccess(t *testing.T) { delays = append(delays, delay) return nil }, + nil, ) if err != nil { @@ -94,6 +96,7 @@ func TestRunWithRetryFiniteAttemptsExhausted(t *testing.T) { waits++ return nil }, + nil, ) if !errors.Is(err, attemptErr) { @@ -123,6 +126,7 @@ func TestRunWithRetryUnlimitedEventuallySucceeds(t *testing.T) { }, func(error) bool { return true }, func(context.Context, time.Duration) error { return nil }, + nil, ) if err != nil { @@ -149,6 +153,7 @@ func TestRunWithRetryStopsWhenErrorIsNotRetryable(t *testing.T) { t.Fatal("wait called for non-retryable error") return nil }, + nil, ) if !errors.Is(err, attemptErr) { @@ -175,6 +180,7 @@ func TestRunWithRetryReturnsCancellationFromAttempt(t *testing.T) { t.Fatal("wait called after cancellation") return nil }, + nil, ) if !errors.Is(err, context.Canceled) { @@ -195,6 +201,7 @@ func TestRunWithRetryReturnsCancellationDuringBackoff(t *testing.T) { cancel() return ctx.Err() }, + nil, ) if !errors.Is(err, context.Canceled) { @@ -212,6 +219,7 @@ func TestRunWithRetryReturnsWaitError(t *testing.T) { func(context.Context) error { return attemptErr }, func(error) bool { return true }, func(context.Context, time.Duration) error { return waitErr }, + nil, ) if !errors.Is(err, waitErr) { @@ -228,3 +236,132 @@ func TestWaitForRetryReturnsCancellation(t *testing.T) { t.Fatalf("waitForRetry() error = %v, want context.Canceled", err) } } + +func TestRetryObserverReportsFailuresBeforeSuccess(t *testing.T) { + attemptErr := errors.New("attempt failed") + attempts := 0 + var events []retryEvent + + err := runWithRetry( + context.Background(), + testRetryPolicy(3), + func(context.Context) error { + attempts++ + if attempts < 3 { + return attemptErr + } + return nil + }, + func(error) bool { return true }, + func(context.Context, time.Duration) error { return nil }, + func(event retryEvent) { + events = append(events, event) + }, + ) + + if err != nil { + t.Fatalf("runWithRetry() error = %v, want nil", err) + } + if len(events) != 2 { + t.Fatalf("event count = %d, want 2", len(events)) + } + + wantDelays := []time.Duration{500 * time.Millisecond, time.Second} + for i, event := range events { + wantAttempts := i + 1 + if event.FailedAttempts != wantAttempts { + t.Errorf("event %d failed attempts = %d, want %d", i, event.FailedAttempts, wantAttempts) + } + if !errors.Is(event.Err, attemptErr) { + t.Errorf("event %d error = %v, want %v", i, event.Err, attemptErr) + } + if event.RetryIn != wantDelays[i] { + t.Errorf("event %d retry delay = %s, want %s", i, event.RetryIn, wantDelays[i]) + } + if !event.WillRetry { + t.Errorf("event %d WillRetry = false, want true", i) + } + } +} + +func TestRetryObserverReportsExhaustion(t *testing.T) { + attemptErr := errors.New("attempt failed") + var events []retryEvent + + err := runWithRetry( + context.Background(), + testRetryPolicy(2), + func(context.Context) error { return attemptErr }, + func(error) bool { return true }, + func(context.Context, time.Duration) error { return nil }, + func(event retryEvent) { + events = append(events, event) + }, + ) + + if !errors.Is(err, attemptErr) { + t.Fatalf("runWithRetry() error = %v, want %v", err, attemptErr) + } + if len(events) != 2 { + t.Fatalf("event count = %d, want 2", len(events)) + } + if !events[0].WillRetry || events[0].RetryIn != 500*time.Millisecond { + t.Errorf("first event = %+v, want retry after 500ms", events[0]) + } + final := events[1] + if final.FailedAttempts != 2 { + t.Errorf("final failed attempts = %d, want 2", final.FailedAttempts) + } + if final.WillRetry { + t.Error("final WillRetry = true, want false") + } + if final.RetryIn != 0 { + t.Errorf("final retry delay = %s, want 0", final.RetryIn) + } + if !errors.Is(final.Err, attemptErr) { + t.Errorf("final error = %v, want %v", final.Err, attemptErr) + } +} + +func TestRetryObserverNotCalledOnImmediateSuccess(t *testing.T) { + observerCalls := 0 + err := runWithRetry( + context.Background(), + testRetryPolicy(3), + func(context.Context) error { return nil }, + func(error) bool { return true }, + func(context.Context, time.Duration) error { return nil }, + func(retryEvent) { observerCalls++ }, + ) + + if err != nil { + t.Fatalf("runWithRetry() error = %v, want nil", err) + } + if observerCalls != 0 { + t.Fatalf("observer call count = %d, want 0", observerCalls) + } +} + +func TestRetryObserverNotCalledWhenAttemptCancelsContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + observerCalls := 0 + + err := runWithRetry( + ctx, + testRetryPolicy(0), + func(context.Context) error { + cancel() + return errors.New("attempt interrupted") + }, + func(error) bool { return true }, + func(context.Context, time.Duration) error { return nil }, + func(retryEvent) { observerCalls++ }, + ) + + if !errors.Is(err, context.Canceled) { + t.Fatalf("runWithRetry() error = %v, want context.Canceled", err) + } + if observerCalls != 0 { + t.Fatalf("observer call count = %d, want 0", observerCalls) + } +}