diff --git a/packages/base-data-service/package.json b/packages/base-data-service/package.json index 5a793d8841..c22c5c35f1 100644 --- a/packages/base-data-service/package.json +++ b/packages/base-data-service/package.json @@ -58,6 +58,7 @@ "@metamask/messenger": "^1.2.0", "@metamask/utils": "^11.11.0", "@tanstack/query-core": "^4.43.0", + "cockatiel": "^3.1.2", "fast-deep-equal": "^3.1.3" }, "devDependencies": { diff --git a/packages/base-data-service/src/BaseDataService.ts b/packages/base-data-service/src/BaseDataService.ts index fdd2896e2b..888144e6fa 100644 --- a/packages/base-data-service/src/BaseDataService.ts +++ b/packages/base-data-service/src/BaseDataService.ts @@ -1,8 +1,3 @@ -import { - createServicePolicy, - CreateServicePolicyOptions, - ServicePolicy, -} from '@metamask/controller-utils'; import { Messenger, ActionConstraint, @@ -26,6 +21,12 @@ import { } from '@tanstack/query-core'; import deepEqual from 'fast-deep-equal'; +import { + createServicePolicy, + CreateServicePolicyOptions, + ServicePolicy, +} from './createServicePolicy'; + // Data service queries use the following format: ['ServiceActionName', ...params] export type QueryKey = [string, ...Json[]]; diff --git a/packages/base-data-service/src/createServicePolicy.test.ts b/packages/base-data-service/src/createServicePolicy.test.ts new file mode 100644 index 0000000000..2b4ec35d2c --- /dev/null +++ b/packages/base-data-service/src/createServicePolicy.test.ts @@ -0,0 +1,3701 @@ +import { CircuitState, handleWhen } from 'cockatiel'; + +import { + createServicePolicy, + DEFAULT_CIRCUIT_BREAK_DURATION, + DEFAULT_DEGRADED_THRESHOLD, + DEFAULT_MAX_CONSECUTIVE_FAILURES, + DEFAULT_MAX_RETRIES, +} from './createServicePolicy'; + +describe('createServicePolicy', () => { + beforeEach(() => { + jest.useFakeTimers({ doNotFake: ['nextTick', 'queueMicrotask'] }); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + describe('wrapping a service that succeeds on the first try', () => { + it('returns a policy that returns what the service returns', async () => { + const mockService = jest.fn(() => ({ some: 'data' })); + const policy = createServicePolicy(); + + const returnValue = await policy.execute(mockService); + + expect(returnValue).toStrictEqual({ some: 'data' }); + }); + + it('only calls the service once before returning', async () => { + const mockService = jest.fn(); + const policy = createServicePolicy(); + + await policy.execute(mockService); + + expect(mockService).toHaveBeenCalledTimes(1); + }); + + it('does not call onBreak listeners, since the circuit never opens', async () => { + const mockService = jest.fn(); + const onBreakListener = jest.fn(); + const policy = createServicePolicy(); + + policy.onBreak(onBreakListener); + + await policy.execute(mockService); + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + describe.each([ + { + desc: `the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, + threshold: DEFAULT_DEGRADED_THRESHOLD, + options: {}, + }, + { + desc: 'a custom degraded threshold', + threshold: 2000, + options: { degradedThreshold: 2000 }, + }, + ])('using $desc', ({ threshold, options }) => { + describe('if the service execution time is below the threshold', () => { + it('does not call onDegraded listeners', async () => { + const mockService = jest.fn(); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy(options); + policy.onDegraded(onDegradedListener); + + await policy.execute(mockService); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const mockService = jest.fn(); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy(options); + policy.onAvailable(onAvailableListener); + + await policy.execute(mockService); + await policy.execute(mockService); + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the service execution time is beyond the threshold', () => { + it('calls onDegraded listeners once with the execution time', async () => { + const delay = threshold + 1; + const mockService = jest.fn(() => { + return new Promise((resolve) => { + setTimeout(() => resolve(), delay); + }); + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy(options); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + jest.advanceTimersByTime(delay); + await promise; + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ + duration: delay, + }); + }); + + it('does not call onAvailable listeners', async () => { + const delay = threshold + 1; + const mockService = jest.fn(() => { + return new Promise((resolve) => { + setTimeout(() => resolve(), delay); + }); + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy(options); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + jest.advanceTimersByTime(delay); + await promise; + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('wrapping a service that always fails', () => { + describe('if a custom retry filter policy is given and the retry filter policy filters out the thrown error', () => { + it('throws what the service throws', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); + + const promise = policy.execute(mockService); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls the service once and only once', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); + + const promise = policy.execute(mockService); + await ignoreRejection(promise); + + expect(mockService).toHaveBeenCalledTimes(1); + }); + + it('does not call onRetry listeners', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onRetryListener = jest.fn(); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); + policy.onRetry(onRetryListener); + + const promise = policy.execute(mockService); + await ignoreRejection(promise); + + expect(onRetryListener).not.toHaveBeenCalled(); + }); + + it('does not call onBreak listeners', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise queue + // is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + it('does not call onDegraded listeners', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise queue + // is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + retryFilterPolicy: handleWhen( + (caughtError) => caughtError.message !== 'failure', + ), + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise queue + // is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + + describe('using the default retry filter policy (which retries all errors)', () => { + describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { + it(`calls the service a total of ${ + 1 + DEFAULT_MAX_RETRIES + } times, delaying each retry using a backoff formula`, async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy(); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // It's safe not to await these promises; adding them to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(0); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(176.27932892814937); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(186.8886145345685); + await ignoreRejection(promise); + + expect(mockService).toHaveBeenCalledTimes(1 + DEFAULT_MAX_RETRIES); + }); + + it('calls onRetry listeners once per retry', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onRetryListener = jest.fn(); + const policy = createServicePolicy(); + policy.onRetry(onRetryListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise queue is + // enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onRetryListener).toHaveBeenCalledTimes(DEFAULT_MAX_RETRIES); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + it('throws what the service throws', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy(); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call onBreak listeners, since the max number of consecutive failures is never reached', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy(); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + it('calls onDegraded listeners once with the error, since the circuit is still closed', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy(); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ error }); + }); + + it('does not call onAvailable listeners', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy(); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call onBreak listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + it('calls onDegraded listeners once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ error }); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('never calls onDegraded listeners, since the circuit is open', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the service is executed again', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('never calls onDegraded listeners, since the circuit is open', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('using a custom max number of retries', () => { + it(`calls the service a total of 1 + times, delaying each retry using a backoff formula`, async () => { + const maxRetries = 5; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // It's safe not to await these promises; adding them to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(0); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(176.27932892814937); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(186.8886145345685); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(366.8287823691078); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(731.8792783578953); + await ignoreRejection(promise); + + expect(mockService).toHaveBeenCalledTimes(1 + maxRetries); + }); + + it('calls onRetry listeners once per retry', async () => { + const maxRetries = 5; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onRetryListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + }); + policy.onRetry(onRetryListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise queue is + // enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onRetryListener).toHaveBeenCalledTimes(maxRetries); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call onBreak listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + it('calls onDegraded listeners once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ error }); + }); + + it('does not call onAvailable listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('never calls onDegraded listeners, since the circuit is open', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const mockService = jest.fn(() => { + throw new Error('failure'); + }); + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('never calls onDegraded listeners, since the circuit is open', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow(error); + }); + + it('does not call onBreak listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + it('calls onDegraded listeners once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ error }); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('throws what the service throws', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow(error); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('never calls onDegraded listeners, since the circuit is open', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('never calls onAvailable listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + + it('throws a BrokenCircuitError instead of whatever error the service produces if the policy is executed again', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + + const secondExecution = policy.execute(mockService); + await expect(secondExecution).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError instead of whatever error the service produces', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('never calls onDegraded listeners, since the circuit is open', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + }); + }); + + describe('wrapping a service that fails continuously and then succeeds on the final try', () => { + // NOTE: Using a custom retry filter policy is not tested here since the + // same thing would happen as above if the error is filtered out + + describe(`using the default max retries (${DEFAULT_MAX_RETRIES})`, () => { + it(`calls the service a total of ${ + 1 + DEFAULT_MAX_RETRIES + } times, delaying each retry using a backoff formula`, async () => { + let invocationCounter = 0; + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }); + const policy = createServicePolicy(); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // It's safe not to await these promises; adding them to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(0); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(176.27932892814937); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(186.8886145345685); + await promise; + + expect(mockService).toHaveBeenCalledTimes(1 + DEFAULT_MAX_RETRIES); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + it('returns what the service returns', async () => { + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const policy = createServicePolicy(); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise queue + // is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call onBreak listeners, since the max number of consecutive failures is never reached', async () => { + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy(); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise queue + // is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + describe.each([ + { + desc: `the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, + threshold: DEFAULT_DEGRADED_THRESHOLD, + options: {}, + }, + { + desc: 'a custom degraded threshold', + threshold: 2000, + options: { degradedThreshold: 2000 }, + }, + ])('using $desc', ({ threshold, options }) => { + describe('if the service execution time is below the threshold', () => { + it('does not call onDegraded listeners', async () => { + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy(options); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if ( + invocationCounter > 0 && + invocationCounter % (DEFAULT_MAX_RETRIES + 1) === 0 + ) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy(options); + policy.onAvailable(onAvailableListener); + + const promise1 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise1; + const promise2 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise2; + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the service execution time is beyond the threshold', () => { + it('calls onDegraded listeners once with the execution time', async () => { + let invocationCounter = 0; + const delay = threshold + 1; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy(options); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ + duration: delay, + }); + }); + + it('does not call onAvailable listeners', async () => { + let invocationCounter = 0; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy(options); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call onBreak listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + describe.each([ + { + desc: `the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, + threshold: DEFAULT_DEGRADED_THRESHOLD, + options: {}, + }, + { + desc: 'a custom degraded threshold', + threshold: 2000, + options: { degradedThreshold: 2000 }, + }, + ])('using $desc', ({ threshold, options }) => { + describe('if the service execution time is below the threshold', () => { + it('does not call onDegraded listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter >= DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const promise1 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise1; + const promise2 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise2; + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the service execution time is beyond the threshold', () => { + it('calls onDegraded listeners once with the execution time', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ + duration: delay, + }); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 2; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }; + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call onBreak listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + describe.each([ + { + desc: `the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, + threshold: DEFAULT_DEGRADED_THRESHOLD, + options: {}, + }, + { + desc: 'a custom degraded threshold', + threshold: 2000, + options: { degradedThreshold: 2000 }, + }, + ])('using $desc', ({ threshold, options }) => { + describe('if the service execution time is below the threshold', () => { + it('does not call onDegraded listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter >= DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const promise1 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise1; + const promise2 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise2; + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the service execution time is beyond the threshold', () => { + it('calls onDegraded listeners once with the execution time', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ + duration: delay, + }); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('does not call onDegraded listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + + describe('after the circuit break duration has elapsed', () => { + describe.each([ + { + desc: `the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, + duration: DEFAULT_CIRCUIT_BREAK_DURATION, + options: {}, + }, + { + desc: 'a custom circuit break duration', + duration: 5_000, + options: { + // This has to be high enough to exceed the exponential backoff + circuitBreakDuration: 5_000, + }, + }, + ])('using $desc', ({ duration, options }) => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + jest.advanceTimersByTime(duration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter >= DEFAULT_MAX_RETRIES + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + jest.advanceTimersByTime(duration); + await policy.execute(mockService); + await policy.execute(mockService); + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + }); + }); + }); + }); + + describe('using a custom max number of retries', () => { + it(`calls the service a total of 1 + times, delaying each retry using a backoff formula`, async () => { + const maxRetries = 5; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }); + const policy = createServicePolicy({ maxRetries }); + // Each retry delay is randomized using a decorrelated jitter formula, + // so we need to prevent that + jest.spyOn(Math, 'random').mockReturnValue(0); + + const promise = policy.execute(mockService); + // It's safe not to await these promises; adding them to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(0); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(176.27932892814937); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(186.8886145345685); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(366.8287823691078); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.advanceTimersByTimeAsync(731.8792783578953); + await promise; + + expect(mockService).toHaveBeenCalledTimes(1 + maxRetries); + }); + + describe(`using the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call onBreak listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + describe.each([ + { + desc: `the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, + threshold: DEFAULT_DEGRADED_THRESHOLD, + options: {}, + }, + { + desc: 'a custom degraded threshold', + threshold: 2000, + options: { degradedThreshold: 2000 }, + }, + ])('using $desc', ({ threshold, options }) => { + describe('if the service execution time is below the threshold', () => { + it('does not call onDegraded listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ ...options, maxRetries }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter >= maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ ...options, maxRetries }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + await policy.execute(mockService); + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the service execution time is beyond the threshold', () => { + it('calls onDegraded listeners once with the execution time', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ ...options, maxRetries }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ + duration: delay, + }); + }); + + it('does not call onAvailable listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 2; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ ...options, maxRetries }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call onBreak listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + describe.each([ + { + desc: `the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, + threshold: DEFAULT_DEGRADED_THRESHOLD, + options: {}, + }, + { + desc: 'a custom degraded threshold', + threshold: 2000, + options: { degradedThreshold: 2000 }, + }, + ])('using $desc', () => { + describe('if the service execution time is below the threshold', () => { + it('does not call onDegraded listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter >= maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + await policy.execute(mockService); + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the service execution time is beyond the threshold', () => { + it('calls onDegraded listeners once with the execution time', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ + duration: delay, + }); + }); + + it('does not call onAvailable listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES - 1; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('does not call onDegraded listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ maxRetries }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + + describe('after the circuit break duration has elapsed', () => { + describe.each([ + { + desc: `the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, + duration: DEFAULT_CIRCUIT_BREAK_DURATION, + options: {}, + }, + { + desc: 'a custom circuit break duration', + duration: 5_000, + options: { + // This has to be high enough to exceed the exponential backoff + circuitBreakDuration: 50_000, + }, + }, + ])('using $desc', ({ duration, options }) => { + it('returns what the service returns', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ maxRetries, ...options }); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + jest.advanceTimersByTime(duration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxRetries = DEFAULT_MAX_CONSECUTIVE_FAILURES; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter >= maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ maxRetries, ...options }); + policy.onAvailable(onAvailableListener); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + jest.advanceTimersByTime(duration); + await policy.execute(mockService); + await policy.execute(mockService); + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + }); + }); + }); + + describe('using a custom max number of consecutive failures', () => { + describe('if the initial run + retries is less than the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call onBreak listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + describe.each([ + { + desc: `the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, + threshold: DEFAULT_DEGRADED_THRESHOLD, + options: {}, + }, + { + desc: 'a custom degraded threshold', + threshold: 2000, + options: { degradedThreshold: 2000 }, + }, + ])('using $desc', ({ threshold, options }) => { + describe('if the service execution time is below the threshold', () => { + it('does not call onDegraded listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter >= maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const promise1 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise1; + const promise2 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise2; + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the service execution time is beyond the threshold', () => { + it('calls onDegraded listeners once with the execution time', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const delay = DEFAULT_DEGRADED_THRESHOLD + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ + duration: delay, + }); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 2; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('if the initial run + retries is equal to the max number of consecutive failures', () => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + + expect(await promise).toStrictEqual({ some: 'data' }); + }); + + it('does not call onBreak listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + describe.each([ + { + desc: `the default degraded threshold (${DEFAULT_DEGRADED_THRESHOLD})`, + threshold: DEFAULT_DEGRADED_THRESHOLD, + options: {}, + }, + { + desc: 'a custom degraded threshold', + threshold: 2000, + options: { degradedThreshold: 2000 }, + }, + ])('using $desc', ({ threshold, options }) => { + describe('if the service execution time is below the threshold', () => { + it('does not call onDegraded listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter % (maxRetries + 1) === 0) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const promise1 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise1; + const promise2 = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise2; + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + + describe('if the service execution time is beyond the threshold', () => { + it('calls onDegraded listeners once with the execution time', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onDegradedListener).toHaveBeenCalledTimes(1); + expect(onDegradedListener).toHaveBeenCalledWith({ + duration: delay, + }); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures - 1; + const delay = threshold + 1; + let invocationCounter = 0; + const mockService = (): Promise<{ some: string }> => { + invocationCounter += 1; + return new Promise((resolve, reject) => { + if (invocationCounter === DEFAULT_MAX_RETRIES + 1) { + setTimeout(() => resolve({ some: 'data' }), delay); + } else { + reject(new Error('failure')); + } + }); + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await promise; + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + }); + }); + }); + + describe('if the initial run + retries is greater than the max number of consecutive failures', () => { + it('throws a BrokenCircuitError before the service can succeed', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + await expect(promise).rejects.toThrow( + new Error( + 'Execution prevented because the circuit breaker is open', + ), + ); + }); + + it('calls onBreak listeners once with the error', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('does not call onDegraded listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onDegradedListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + policy.onDegraded(onDegradedListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onDegradedListener).not.toHaveBeenCalled(); + }); + + it('does not call onAvailable listeners', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + }); + policy.onAvailable(onAvailableListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onAvailableListener).not.toHaveBeenCalled(); + }); + + describe('after the circuit break duration has elapsed', () => { + describe.each([ + { + desc: `the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, + duration: DEFAULT_CIRCUIT_BREAK_DURATION, + options: {}, + }, + { + desc: 'a custom circuit break duration', + duration: 5_000, + options: { + // This has to be high enough to exceed the exponential backoff + circuitBreakDuration: 5_000, + }, + }, + ])('using $desc', ({ duration, options }) => { + it('returns what the service returns', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter === maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + jest.advanceTimersByTime(duration); + const result = await policy.execute(mockService); + + expect(result).toStrictEqual({ some: 'data' }); + }); + + it('calls onAvailable listeners once, even if the service is called more than once', async () => { + const maxConsecutiveFailures = 5; + const maxRetries = maxConsecutiveFailures; + let invocationCounter = 0; + const error = new Error('failure'); + const mockService = (): { some: string } => { + invocationCounter += 1; + if (invocationCounter >= maxRetries + 1) { + return { some: 'data' }; + } + throw error; + }; + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + maxRetries, + maxConsecutiveFailures, + ...options, + }); + policy.onAvailable(onAvailableListener); + + const firstExecution = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(firstExecution); + jest.advanceTimersByTime(duration); + await policy.execute(mockService); + await policy.execute(mockService); + + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }); + }); + }); + }); + }); + }); + + describe('wrapping a service that succeeds at first and then fails enough to break the circuit', () => { + describe.each([ + { + desc: `the default max number of consecutive failures (${DEFAULT_MAX_CONSECUTIVE_FAILURES})`, + maxConsecutiveFailures: DEFAULT_MAX_CONSECUTIVE_FAILURES, + optionsWithMaxConsecutiveFailures: {}, + }, + { + desc: 'a custom max number of consecutive failures', + maxConsecutiveFailures: DEFAULT_MAX_RETRIES + 1, + optionsWithMaxConsecutiveFailures: { + maxConsecutiveFailures: DEFAULT_MAX_RETRIES + 1, + }, + }, + ])( + 'using $desc', + ({ maxConsecutiveFailures, optionsWithMaxConsecutiveFailures }) => { + describe.each([ + { + desc: `the default circuit break duration (${DEFAULT_CIRCUIT_BREAK_DURATION})`, + circuitBreakDuration: DEFAULT_CIRCUIT_BREAK_DURATION, + optionsWithCircuitBreakDuration: {}, + }, + { + desc: 'a custom circuit break duration', + circuitBreakDuration: DEFAULT_CIRCUIT_BREAK_DURATION, + optionsWithCircuitBreakDuration: { + // This has to be high enough to exceed the exponential backoff + circuitBreakDuration: 5_000, + }, + }, + ])( + 'using $desc', + ({ circuitBreakDuration, optionsWithCircuitBreakDuration }) => { + it('calls onAvailable listeners if the service finally succeeds', async () => { + let invocationCounter = 0; + const mockService = jest.fn(() => { + invocationCounter += 1; + if ( + invocationCounter === 1 || + invocationCounter === maxConsecutiveFailures + 2 + ) { + return { some: 'data' }; + } + throw new Error('failure'); + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + ...optionsWithMaxConsecutiveFailures, + ...optionsWithCircuitBreakDuration, + }); + policy.onRetry(() => { + jest.advanceTimersToNextTimer(); + }); + policy.onAvailable(onAvailableListener); + + // Execute the service successfully once + await policy.execute(mockService); + expect(onAvailableListener).toHaveBeenCalledTimes(1); + + // Execute and retry until we break the circuit + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + jest.advanceTimersByTime(circuitBreakDuration); + + await policy.execute(mockService); + expect(onAvailableListener).toHaveBeenCalledTimes(2); + }); + + it('does not call onAvailable listeners if the service finally fails', async () => { + let invocationCounter = 0; + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy({ + ...optionsWithMaxConsecutiveFailures, + ...optionsWithCircuitBreakDuration, + }); + policy.onRetry(() => { + jest.advanceTimersToNextTimer(); + }); + policy.onAvailable(onAvailableListener); + + // Execute the service successfully once + await policy.execute(mockService); + expect(onAvailableListener).toHaveBeenCalledTimes(1); + + // Execute and retry until we break the circuit + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + jest.advanceTimersByTime(circuitBreakDuration); + + await ignoreRejection(policy.execute(mockService)); + expect(onAvailableListener).toHaveBeenCalledTimes(1); + }); + }, + ); + }, + ); + }); + + describe('getRemainingCircuitOpenDuration', () => { + it('returns the number of milliseconds before the circuit will transition from open to half-open', async () => { + const mockService = (): never => { + throw new Error('failure'); + }; + const policy = createServicePolicy(); + policy.onRetry(() => { + jest.advanceTimersToNextTimer(); + }); + // Retry until we break the circuit + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + jest.advanceTimersByTime(1000); + + expect(policy.getRemainingCircuitOpenDuration()).toBe( + DEFAULT_CIRCUIT_BREAK_DURATION - 1000, + ); + }); + + it('returns null if the circuit is closed', () => { + const policy = createServicePolicy(); + + expect(policy.getRemainingCircuitOpenDuration()).toBeNull(); + }); + }); + + describe('getCircuitState', () => { + it('returns the state of the circuit', async () => { + const mockService = (): never => { + throw new Error('failure'); + }; + const policy = createServicePolicy(); + policy.onRetry(() => { + jest.advanceTimersToNextTimer(); + }); + + expect(policy.getCircuitState()).toBe(CircuitState.Closed); + + // Retry until we break the circuit + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + expect(policy.getCircuitState()).toBe(CircuitState.Open); + + jest.advanceTimersByTime(DEFAULT_CIRCUIT_BREAK_DURATION); + const promise = ignoreRejection(policy.execute(mockService)); + expect(policy.getCircuitState()).toBe(CircuitState.HalfOpen); + await promise; + expect(policy.getCircuitState()).toBe(CircuitState.Open); + }); + }); + + describe('reset', () => { + it('resets the state of the circuit to "closed"', async () => { + let invocationCounter = 0; + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_CONSECUTIVE_FAILURES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }); + const policy = createServicePolicy(); + policy.onRetry(() => { + jest.advanceTimersToNextTimer(); + }); + // Retry until we break the circuit + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + expect(policy.getCircuitState()).toBe(CircuitState.Open); + + policy.reset(); + + expect(policy.getCircuitState()).toBe(CircuitState.Closed); + }); + + it('allows the service to be executed successfully again if its circuit has broken after resetting', async () => { + let invocationCounter = 0; + const mockService = jest.fn(() => { + invocationCounter += 1; + if (invocationCounter === DEFAULT_MAX_CONSECUTIVE_FAILURES + 1) { + return { some: 'data' }; + } + throw new Error('failure'); + }); + const policy = createServicePolicy(); + policy.onRetry(() => { + jest.advanceTimersToNextTimer(); + }); + // Retry until we break the circuit + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + + policy.reset(); + + expect(await policy.execute(mockService)).toStrictEqual({ some: 'data' }); + }); + + it('calls onAvailable listeners if the service was executed successfully, its circuit broke, it was reset, and executes again, successfully', async () => { + let invocationCounter = 0; + const mockService = jest.fn(() => { + invocationCounter += 1; + if ( + invocationCounter === 1 || + invocationCounter === DEFAULT_MAX_CONSECUTIVE_FAILURES + 2 + ) { + return { some: 'data' }; + } + throw new Error('failure'); + }); + const onAvailableListener = jest.fn(); + const policy = createServicePolicy(); + policy.onRetry(() => { + jest.advanceTimersToNextTimer(); + }); + policy.onAvailable(onAvailableListener); + + // Execute the service successfully once + await policy.execute(mockService); + expect(onAvailableListener).toHaveBeenCalledTimes(1); + + // Execute and retry until we break the circuit + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + + policy.reset(); + + await policy.execute(mockService); + expect(onAvailableListener).toHaveBeenCalledTimes(2); + }); + + it('allows the service to be executed unsuccessfully again if its circuit has broken after resetting', async () => { + const mockService = jest.fn(() => { + throw new Error('failure'); + }); + const policy = createServicePolicy(); + policy.onRetry(() => { + jest.advanceTimersToNextTimer(); + }); + // Retry until we break the circuit + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + await ignoreRejection(policy.execute(mockService)); + + policy.reset(); + + await expect(policy.execute(mockService)).rejects.toThrow('failure'); + }); + }); + + describe('using a custom isServiceFailure predicate', () => { + it('opens the circuit when the predicate treats the error as a service failure', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + isServiceFailure: () => true, + }); + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // It's safe not to await this promise; adding it to the promise + // queue is enough to prevent this test from running indefinitely. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + expect(onBreakListener).toHaveBeenCalledWith({ error }); + }); + + it('never opens the circuit when the predicate does not treat the error as a service failure', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ + maxConsecutiveFailures, + isServiceFailure: () => false, + }); + policy.onBreak(onBreakListener); + + // Execute more times than the max consecutive failures so that the + // circuit would open if these errors were counted as failures. + for (let i = 0; i < maxConsecutiveFailures + 1; i++) { + const promise = policy.execute(mockService); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + } + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + + it('calls the predicate with the error thrown by the service', async () => { + const error = new Error('failure'); + const mockService = jest.fn(() => { + throw error; + }); + const isServiceFailure = jest.fn(() => true); + const policy = createServicePolicy({ + maxConsecutiveFailures: DEFAULT_MAX_RETRIES + 1, + isServiceFailure, + }); + + const promise = policy.execute(mockService); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(isServiceFailure).toHaveBeenCalledWith(error); + }); + }); + + describe('using the default isServiceFailure predicate', () => { + it('opens the circuit for an error with an HTTP status >= 500', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = Object.assign(new Error('failure'), { httpStatus: 500 }); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ maxConsecutiveFailures }); + policy.onBreak(onBreakListener); + + const promise = policy.execute(mockService); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + + expect(onBreakListener).toHaveBeenCalledTimes(1); + }); + + it('never opens the circuit for an error with an HTTP status < 500', async () => { + const maxConsecutiveFailures = DEFAULT_MAX_RETRIES + 1; + const error = Object.assign(new Error('failure'), { httpStatus: 400 }); + const mockService = jest.fn(() => { + throw error; + }); + const onBreakListener = jest.fn(); + const policy = createServicePolicy({ maxConsecutiveFailures }); + policy.onBreak(onBreakListener); + + // Execute more times than the max consecutive failures so that the + // circuit would open if these errors were counted as failures. + for (let i = 0; i < maxConsecutiveFailures + 1; i++) { + const promise = policy.execute(mockService); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + jest.runAllTimersAsync(); + await ignoreRejection(promise); + } + + expect(onBreakListener).not.toHaveBeenCalled(); + }); + }); +}); + +/** + * Some tests involve a rejected promise that is not necessarily the focus of + * the test. In these cases we don't want to ignore the error in case the + * promise _isn't_ rejected, but we don't want to highlight the assertion, + * either. + * + * @param promise - A promise that rejects. + */ +async function ignoreRejection(promise: Promise): Promise { + await expect(promise).rejects.toThrow(expect.any(Error)); +} diff --git a/packages/base-data-service/src/createServicePolicy.ts b/packages/base-data-service/src/createServicePolicy.ts new file mode 100644 index 0000000000..906f85229a --- /dev/null +++ b/packages/base-data-service/src/createServicePolicy.ts @@ -0,0 +1,403 @@ +import { + CircuitState, + EventEmitter as CockatielEventEmitter, + ConsecutiveBreaker, + ExponentialBackoff, + circuitBreaker, + handleAll, + handleWhen, + retry, + wrap, +} from 'cockatiel'; +import type { + CircuitBreakerPolicy, + Event as CockatielEvent, + FailureReason, + IBackoffFactory, + IPolicy, + Policy, + RetryPolicy, +} from 'cockatiel'; + +/** + * The options for `createServicePolicy`. + */ +export type CreateServicePolicyOptions = { + /** + * The backoff strategy to use. Mainly useful for testing so that a constant + * backoff can be used when mocking timers. Defaults to an instance of + * ExponentialBackoff. + */ + backoff?: IBackoffFactory; + /** + * The length of time (in milliseconds) to pause retries of the action after + * the number of failures reaches `maxConsecutiveFailures`. + */ + circuitBreakDuration?: number; + /** + * The length of time (in milliseconds) that governs when the service is + * regarded as degraded (affecting when `onDegraded` is called). + */ + degradedThreshold?: number; + /** + * Predicate function for when an error should be considered a service failure. + */ + isServiceFailure?: (error: unknown) => boolean; + /** + * The maximum number of times that the service is allowed to fail before + * pausing further retries. + */ + maxConsecutiveFailures?: number; + /** + * The maximum number of times that a failing service should be re-invoked + * before giving up. + */ + maxRetries?: number; + /** + * The policy used to control when the service should be retried based on + * either the result of the service or an error that it throws. For instance, + * you could use this to retry only certain errors. See `handleWhen` and + * friends from Cockatiel for more. + */ + retryFilterPolicy?: Policy; +}; + +/** + * The service policy object. + */ +export type ServicePolicy = IPolicy & { + /** + * The Cockatiel circuit breaker policy that the service policy uses + * internally. + */ + circuitBreakerPolicy: CircuitBreakerPolicy; + /** + * The amount of time to pause requests to the service if the number of + * maximum consecutive failures is reached. + */ + circuitBreakDuration: number; + /** + * @returns The state of the underlying circuit. + */ + getCircuitState: () => CircuitState; + /** + * If the circuit is open and ongoing requests are paused, returns the number + * of milliseconds before the requests will be attempted again. If the circuit + * is not open, returns null. + */ + getRemainingCircuitOpenDuration: () => number | null; + /** + * Resets the internal circuit breaker policy (if it is open, it will now be + * closed). + */ + reset: () => void; + /** + * The Cockatiel retry policy that the service policy uses internally. + */ + retryPolicy: RetryPolicy; + /** + * A function which is called when the number of times that the service fails + * in a row meets the set maximum number of consecutive failures. + */ + onBreak: CircuitBreakerPolicy['onBreak']; + /** + * A function which is called in two circumstances: 1) when the service + * succeeds before the maximum number of consecutive failures is reached, but + * takes more time than the `degradedThreshold` to run, or 2) if the service + * never succeeds before the retry policy gives up and before the maximum + * number of consecutive failures has been reached. + */ + onDegraded: CockatielEvent | { duration: number }>; + /** + * A function which is called when the service succeeds for the first time, + * or when the service fails enough times to cause the circuit to break and + * then recovers. + */ + onAvailable: CockatielEvent; + /** + * A function which will be called by the retry policy each time the service + * fails and the policy kicks off a timer to re-run the service. This is + * primarily useful in tests where we are mocking timers. + */ + onRetry: RetryPolicy['onRetry']; +}; + +/** + * Parts of the circuit breaker's internal and external state as necessary in + * order to compute the time remaining before the circuit will reopen. + */ +type InternalCircuitState = + | { + state: CircuitState.Open; + openedAt: number; + } + | { state: Exclude }; + +/** + * Availability statuses that the service can be in. + * + * Used to keep track of whether the `onAvailable` event should be fired. + */ +const AVAILABILITY_STATUSES = { + Available: 'available', + Degraded: 'degraded', + Unavailable: 'unavailable', + Unknown: 'unknown', +} as const; + +/** + * Availability statuses that the service can be in. + * + * Used to keep track of whether the `onAvailable` event should be fired. + */ +type AvailabilityStatus = + (typeof AVAILABILITY_STATUSES)[keyof typeof AVAILABILITY_STATUSES]; + +/** + * The maximum number of times that a failing service should be re-run before + * giving up. + */ +export const DEFAULT_MAX_RETRIES = 3; + +/** + * The maximum number of times that the service is allowed to fail before + * pausing further retries. This is set to a value such that if given a + * service that continually fails, the policy needs to be executed 3 times + * before further retries are paused. + */ +export const DEFAULT_MAX_CONSECUTIVE_FAILURES = (1 + DEFAULT_MAX_RETRIES) * 3; + +/** + * The default length of time (in milliseconds) to temporarily pause retries of + * the service after enough consecutive failures. + */ +export const DEFAULT_CIRCUIT_BREAK_DURATION = 30 * 60 * 1000; + +/** + * The default length of time (in milliseconds) that governs when the service is + * regarded as degraded (affecting when `onDegraded` is called). + */ +export const DEFAULT_DEGRADED_THRESHOLD = 5_000; + +const defaultIsServiceFailure = (error: unknown): boolean => { + if ( + typeof error === 'object' && + error !== null && + 'httpStatus' in error && + typeof error.httpStatus === 'number' + ) { + return error.httpStatus >= 500; + } + + // If the error is not an object, or doesn't have a numeric httpStatus + // property, consider it a service failure (e.g., network errors, timeouts, + // etc.) + return true; +}; + +/** + * The circuit breaker policy inside of the Cockatiel library exposes some of + * its state, but not all of it. Notably, the time that the circuit opened is + * not publicly accessible. So we have to record this ourselves. + * + * This function therefore allows us to obtain the circuit breaker state that we + * wish we could access. + * + * @param state - The public state of a circuit breaker policy. + * @returns if the circuit is open, the state of the circuit breaker policy plus + * the time that it opened, otherwise just the circuit state. + */ +function getInternalCircuitState(state: CircuitState): InternalCircuitState { + if (state === CircuitState.Open) { + return { state, openedAt: Date.now() }; + } + return { state }; +} + +/** + * Constructs an object exposing an `execute` method which, given a function — + * hereafter called the "service" — will retry that service with ever increasing + * delays until it succeeds. If the policy detects too many consecutive + * failures, it will block further retries until a designated time period has + * passed; this particular behavior is primarily designed for services that wrap + * API calls so as not to make needless HTTP requests when the API is down and + * to be able to recover when the API comes back up. In addition, hooks allow + * for responding to certain events, one of which can be used to detect when an + * HTTP request is performing slowly. + * + * Internally, this function makes use of the retry and circuit breaker policies + * from the [Cockatiel](https://www.npmjs.com/package/cockatiel) library; see + * there for more. + * + * @param options - The options to this function. See + * {@link CreateServicePolicyOptions}. + * @returns The service policy. + * @example + * This function is designed to be used in the context of a service class like + * this: + * ``` ts + * class Service { + * constructor() { + * this.#policy = createServicePolicy({ + * maxRetries: 3, + * retryFilterPolicy: handleWhen((error) => { + * return error.message.includes('oops'); + * }), + * maxConsecutiveFailures: 3, + * circuitBreakDuration: 5000, + * degradedThreshold: 2000, + * onBreak: () => { + * console.log('Circuit broke'); + * }, + * onDegraded: () => { + * console.log('Service is degraded'); + * }, + * }); + * } + * + * async fetch() { + * return await this.#policy.execute(async () => { + * const response = await fetch('https://some/url'); + * return await response.json(); + * }); + * } + * } + * ``` + */ +export function createServicePolicy( + options: CreateServicePolicyOptions = {}, +): ServicePolicy { + const { + maxRetries = DEFAULT_MAX_RETRIES, + retryFilterPolicy = handleAll, + maxConsecutiveFailures = DEFAULT_MAX_CONSECUTIVE_FAILURES, + circuitBreakDuration = DEFAULT_CIRCUIT_BREAK_DURATION, + degradedThreshold = DEFAULT_DEGRADED_THRESHOLD, + backoff = new ExponentialBackoff(), + isServiceFailure = defaultIsServiceFailure, + } = options; + + let availabilityStatus: AvailabilityStatus = AVAILABILITY_STATUSES.Unknown; + + const retryPolicy = retry(retryFilterPolicy, { + // Note that although the option here is called "max attempts", it's really + // maximum number of *retries* (attempts past the initial attempt). + maxAttempts: maxRetries, + // Retries of the service will be executed following ever increasing delays, + // determined by a backoff formula. + backoff, + }); + const onRetry = retryPolicy.onRetry.bind(retryPolicy); + + const consecutiveBreaker = new ConsecutiveBreaker(maxConsecutiveFailures); + const circuitBreakerPolicy = circuitBreaker(handleWhen(isServiceFailure), { + // While the circuit is open, any additional invocations of the service + // passed to the policy (either via automatic retries or by manually + // executing the policy again) will result in a BrokenCircuitError. This + // will remain the case until `circuitBreakDuration` passes, after which the + // service will be allowed to run again. If the service succeeds, the + // circuit will close, otherwise it will remain open. + halfOpenAfter: circuitBreakDuration, + breaker: consecutiveBreaker, + }); + + let internalCircuitState: InternalCircuitState = getInternalCircuitState( + circuitBreakerPolicy.state, + ); + circuitBreakerPolicy.onStateChange((state) => { + internalCircuitState = getInternalCircuitState(state); + }); + + circuitBreakerPolicy.onBreak(() => { + availabilityStatus = AVAILABILITY_STATUSES.Unavailable; + }); + const onBreak = circuitBreakerPolicy.onBreak.bind(circuitBreakerPolicy); + + const onDegradedEventEmitter = new CockatielEventEmitter< + FailureReason | { duration: number } + >(); + const onDegraded = onDegradedEventEmitter.addListener; + + const onAvailableEventEmitter = new CockatielEventEmitter(); + const onAvailable = onAvailableEventEmitter.addListener; + + retryPolicy.onGiveUp((data) => { + if (circuitBreakerPolicy.state === CircuitState.Closed) { + availabilityStatus = AVAILABILITY_STATUSES.Degraded; + onDegradedEventEmitter.emit(data); + } + }); + retryPolicy.onSuccess(({ duration }) => { + if (circuitBreakerPolicy.state === CircuitState.Closed) { + if (duration > degradedThreshold) { + availabilityStatus = AVAILABILITY_STATUSES.Degraded; + onDegradedEventEmitter.emit({ duration }); + } else if (availabilityStatus !== AVAILABILITY_STATUSES.Available) { + availabilityStatus = AVAILABILITY_STATUSES.Available; + onAvailableEventEmitter.emit(); + } + } + }); + + // Every time the retry policy makes an attempt, it executes the circuit + // breaker policy, which executes the service. + // + // Calling: + // + // policy.execute(() => { + // // do what the service does + // }) + // + // is equivalent to: + // + // retryPolicy.execute(() => { + // circuitBreakerPolicy.execute(() => { + // // do what the service does + // }); + // }); + // + // So if the retry policy succeeds or fails, it is because the circuit breaker + // policy succeeded or failed. And if there are any event listeners registered + // on the retry policy, by the time they are called, the state of the circuit + // breaker will have already changed. + const policy = wrap(retryPolicy, circuitBreakerPolicy); + + const getRemainingCircuitOpenDuration = (): number | null => { + if (internalCircuitState.state === CircuitState.Open) { + return internalCircuitState.openedAt + circuitBreakDuration - Date.now(); + } + return null; + }; + + const getCircuitState = (): CircuitState => { + return circuitBreakerPolicy.state; + }; + + const reset = (): void => { + // Set the state of the policy to "isolated" regardless of its current state + const { dispose } = circuitBreakerPolicy.isolate(); + // Reset the state to "closed" + dispose(); + + // Reset the counter on the breaker as well + consecutiveBreaker.success(); + + // Re-initialize the availability status so that if the service is executed + // successfully, onAvailable listeners will be called again + availabilityStatus = AVAILABILITY_STATUSES.Unknown; + }; + + return { + ...policy, + circuitBreakerPolicy, + circuitBreakDuration, + getCircuitState, + getRemainingCircuitOpenDuration, + reset, + retryPolicy, + onBreak, + onDegraded, + onAvailable, + onRetry, + }; +} diff --git a/packages/controller-utils/CHANGELOG.md b/packages/controller-utils/CHANGELOG.md index daa6ce7768..0762892e1d 100644 --- a/packages/controller-utils/CHANGELOG.md +++ b/packages/controller-utils/CHANGELOG.md @@ -11,6 +11,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add optional `startTime` to `TraceRequest` to allow backdating a span's start time ([#9315](https://github.com/MetaMask/core/pull/9315)) +### Deprecated + +- Deprecate `createServicePolicy` and supporting symbols + - Full set of deprecated enums, constants, and functions: + - `BrokenCircuitError` + - `CircuitState` + - `CockatielEventEmitter` + - `ConstantBackoff` + - `DEFAULT_CIRCUIT_BREAK_DURATION` + - `DEFAULT_DEGRADED_THRESHOLD` + - `DEFAULT_MAX_CONSECUTIVE_FAILURES` + - `DEFAULT_MAX_RETRIES` + - `ExponentialBackoff` + - `createServicePolicy` + - `handleAll` + - `handleWhen` + - Full set of deprecated types: + - `CockatielEvent` + - `CreateServicePolicyOptions` + - `CockatielFailureReason` + - `ServicePolicy` + - These will be removed in a future major version. Please use equivalent implementations from `@metamask/base-data-service` going forward. + ## [12.3.0] ### Added diff --git a/packages/controller-utils/src/create-service-policy.ts b/packages/controller-utils/src/create-service-policy.ts index b7e5a91a14..17e2bf3bae 100644 --- a/packages/controller-utils/src/create-service-policy.ts +++ b/packages/controller-utils/src/create-service-policy.ts @@ -22,19 +22,63 @@ import type { } from 'cockatiel'; export { + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent type from `@metamask/base-data-service`. + */ BrokenCircuitError, + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent type from `@metamask/base-data-service`. + */ CockatielEventEmitter, + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent type from `@metamask/base-data-service`. + */ CircuitState, + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent type from `@metamask/base-data-service`. + */ ConstantBackoff, + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent type from `@metamask/base-data-service`. + */ ExponentialBackoff, + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent function from + * `@metamask/base-data-service`. + */ handleAll, + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent function from + * `@metamask/base-data-service`. + */ handleWhen, }; -export type { CockatielEvent, FailureReason as CockatielFailureReason }; +export type { + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent type from `@metamask/base-data-service`. + */ + CockatielEvent, + /** + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent type from `@metamask/base-data-service`. + */ + FailureReason as CockatielFailureReason, +}; /** * The options for `createServicePolicy`. + * + * @deprecated This is deprecated and will be removed in a future major version. + * Please use the equivalent type from `@metamask/base-data-service`. */ export type CreateServicePolicyOptions = { /** @@ -78,6 +122,9 @@ export type CreateServicePolicyOptions = { /** * The service policy object. + * + * @deprecated This is deprecated and will be removed in a future major + * version. Please use the equivalent type from `@metamask/base-data-service`. */ export type ServicePolicy = IPolicy & { /** @@ -170,6 +217,9 @@ type AvailabilityStatus = /** * The maximum number of times that a failing service should be re-run before * giving up. + * + * @deprecated This is deprecated and will be removed in a future major version. + * Please use the equivalent variable from `@metamask/base-data-service`. */ export const DEFAULT_MAX_RETRIES = 3; @@ -178,18 +228,27 @@ export const DEFAULT_MAX_RETRIES = 3; * pausing further retries. This is set to a value such that if given a * service that continually fails, the policy needs to be executed 3 times * before further retries are paused. + * + * @deprecated This is deprecated and will be removed in a future major version. + * Please use the equivalent variable from `@metamask/base-data-service`. */ export const DEFAULT_MAX_CONSECUTIVE_FAILURES = (1 + DEFAULT_MAX_RETRIES) * 3; /** * The default length of time (in milliseconds) to temporarily pause retries of * the service after enough consecutive failures. + * + * @deprecated This is deprecated and will be removed in a future major version. + * Please use the equivalent variable from `@metamask/base-data-service`. */ export const DEFAULT_CIRCUIT_BREAK_DURATION = 30 * 60 * 1000; /** * The default length of time (in milliseconds) that governs when the service is * regarded as degraded (affecting when `onDegraded` is called). + * + * @deprecated This is deprecated and will be removed in a future major version. + * Please use the equivalent variable from `@metamask/base-data-service`. */ export const DEFAULT_DEGRADED_THRESHOLD = 5_000; @@ -277,6 +336,9 @@ function getInternalCircuitState(state: CircuitState): InternalCircuitState { * } * } * ``` + * + * @deprecated This is deprecated and will be removed in a future major version. + * Please use the equivalent function from `@metamask/base-data-service`. */ export function createServicePolicy( options: CreateServicePolicyOptions = {}, diff --git a/yarn.lock b/yarn.lock index 9a0b0651f0..abae119f0c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5894,6 +5894,7 @@ __metadata: "@tanstack/query-core": "npm:^4.43.0" "@ts-bridge/cli": "npm:^0.6.4" "@types/jest": "npm:^29.5.14" + cockatiel: "npm:^3.1.2" deepmerge: "npm:^4.2.2" fast-deep-equal: "npm:^3.1.3" jest: "npm:^29.7.0"