Circuit Breaker Pattern with Polly and Azure

Introduction

The Circuit Breaker pattern prevents cascading failures in distributed systems by stopping requests to failing services. Combined with Polly and Azure services, you can build highly resilient applications. This guide demonstrates practical implementations for .NET applications on Azure.

Circuit Breaker States

        ┌────────────────────────────────────────────────────┐
        │                                                     │
        │  ┌──────────┐                                      │
        │  │  Closed  │◄──────────────┐                      │
        │  │ (Normal) │               │                      │
        │  └────┬─────┘               │                      │
        │       │                     │                      │
        │  Failure threshold          │ Success in           │
        │  exceeded                   │ half-open state      │
        │       │                     │                      │
        │       ▼                     │                      │
        │  ┌──────────┐          ┌────┴─────┐               │
        │  │   Open   │──Timer──►│Half-Open │               │
        │  │(Failing) │ expires  │ (Testing)│               │
        │  └──────────┘          └────┬─────┘               │
        │       ▲                     │                      │
        │       │                     │                      │
        │       └─────────────────────┘                      │
        │         Failure in half-open state                 │
        └────────────────────────────────────────────────────┘

Basic Circuit Breaker with Polly

public static class CircuitBreakerPolicies
{
    public static IAsyncPolicy<HttpResponseMessage> CreateHttpCircuitBreaker()
    {
        return HttpPolicyExtensions
            .HandleTransientHttpError()  // 5xx and 408
            .OrResult(msg => msg.StatusCode == HttpStatusCode.TooManyRequests)
            .CircuitBreakerAsync(
                handledEventsAllowedBeforeBreaking: 5,
                durationOfBreak: TimeSpan.FromSeconds(30),
                onBreak: (outcome, breakDelay) =>
                {
                    Log.Warning(
                        "Circuit breaker opened for {BreakDelay}s due to {Exception}",
                        breakDelay.TotalSeconds,
                        outcome.Exception?.Message ?? outcome.Result.StatusCode.ToString());
                },
                onReset: () =>
                {
                    Log.Information("Circuit breaker reset");
                },
                onHalfOpen: () =>
                {
                    Log.Information("Circuit breaker half-open, testing...");
                });
    }
}

Advanced Circuit Breaker Configuration

Combining with Retry and Timeout

public static class ResiliencePolicies
{
    public static IAsyncPolicy<HttpResponseMessage> CreateResiliencePolicy()
    {
        // Timeout policy (innermost)
        var timeoutPolicy = Policy.TimeoutAsync<HttpResponseMessage>(
            TimeSpan.FromSeconds(10),
            TimeoutStrategy.Optimistic,
            onTimeoutAsync: (context, timespan, task) =>
            {
                Log.Warning("Request timed out after {Timeout}s", timespan.TotalSeconds);
                return Task.CompletedTask;
            });

        // Retry policy
        var retryPolicy = HttpPolicyExtensions
            .HandleTransientHttpError()
            .Or<TimeoutRejectedException>()
            .WaitAndRetryAsync(
                retryCount: 3,
                sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)),
                onRetry: (outcome, timespan, attempt, context) =>
                {
                    Log.Warning(
                        "Retry {Attempt} after {Delay}s",
                        attempt,
                        timespan.TotalSeconds);
                });

        // Circuit breaker policy (outermost)
        var circuitBreakerPolicy = HttpPolicyExtensions
            .HandleTransientHttpError()
            .Or<TimeoutRejectedException>()
            .AdvancedCircuitBreakerAsync(
                failureThreshold: 0.5,  // 50% failure rate
                samplingDuration: TimeSpan.FromSeconds(60),
                minimumThroughput: 10,
                durationOfBreak: TimeSpan.FromSeconds(30),
                onBreak: (outcome, state, breakDelay, context) =>
                {
                    Log.Error(
                        "Circuit OPEN. Failure rate: {FailureRate}%. Breaking for {BreakDelay}s",
                        state.FailureRate * 100,
                        breakDelay.TotalSeconds);
                },
                onReset: context =>
                {
                    Log.Information("Circuit RESET");
                },
                onHalfOpen: () =>
                {
                    Log.Information("Circuit HALF-OPEN");
                });

        // Wrap policies: CircuitBreaker -> Retry -> Timeout
        return Policy.WrapAsync(circuitBreakerPolicy, retryPolicy, timeoutPolicy);
    }
}

Service Registration

HttpClient with Polly

public static class ServiceCollectionExtensions
{
    public static IServiceCollection AddResilientHttpClients(
        this IServiceCollection services,
        IConfiguration configuration)
    {
        // Create policies
        var circuitBreakerPolicy = ResiliencePolicies.CreateResiliencePolicy();

        // Register named HTTP clients
        services.AddHttpClient("OrderService", client =>
        {
            client.BaseAddress = new Uri(configuration["Services:Orders:BaseUrl"]);
            client.DefaultRequestHeaders.Add("Accept", "application/json");
        })
        .AddPolicyHandler(circuitBreakerPolicy);

        services.AddHttpClient("PaymentService", client =>
        {
            client.BaseAddress = new Uri(configuration["Services:Payments:BaseUrl"]);
        })
        .AddPolicyHandler(circuitBreakerPolicy);

        // Or use typed clients
        services.AddHttpClient<IInventoryClient, InventoryClient>(client =>
        {
            client.BaseAddress = new Uri(configuration["Services:Inventory:BaseUrl"]);
        })
        .AddPolicyHandler(circuitBreakerPolicy);

        return services;
    }
}

Circuit Breaker Registry

Sharing Circuit Breakers Across Services

public class CircuitBreakerRegistry
{
    private readonly ConcurrentDictionary<string, AsyncCircuitBreakerPolicy> _breakers = new();

    public AsyncCircuitBreakerPolicy GetOrCreate(string name, CircuitBreakerOptions options)
    {
        return _breakers.GetOrAdd(name, _ => CreateCircuitBreaker(name, options));
    }

    private AsyncCircuitBreakerPolicy CreateCircuitBreaker(
        string name,
        CircuitBreakerOptions options)
    {
        return Policy
            .Handle<Exception>()
            .AdvancedCircuitBreakerAsync(
                failureThreshold: options.FailureThreshold,
                samplingDuration: options.SamplingDuration,
                minimumThroughput: options.MinimumThroughput,
                durationOfBreak: options.BreakDuration,
                onBreak: (exception, state, breakDuration, context) =>
                {
                    Log.Error(
                        "Circuit {Name} OPEN: {Exception}. Breaking for {Duration}s",
                        name,
                        exception.Message,
                        breakDuration.TotalSeconds);
                },
                onReset: context =>
                {
                    Log.Information("Circuit {Name} RESET", name);
                },
                onHalfOpen: () =>
                {
                    Log.Information("Circuit {Name} HALF-OPEN", name);
                });
    }

    public CircuitState GetState(string name)
    {
        return _breakers.TryGetValue(name, out var breaker)
            ? breaker.CircuitState
            : CircuitState.Closed;
    }

    public void Isolate(string name)
    {
        if (_breakers.TryGetValue(name, out var breaker))
        {
            breaker.Isolate();
            Log.Warning("Circuit {Name} manually ISOLATED", name);
        }
    }

    public void Reset(string name)
    {
        if (_breakers.TryGetValue(name, out var breaker))
        {
            breaker.Reset();
            Log.Information("Circuit {Name} manually RESET", name);
        }
    }
}

public record CircuitBreakerOptions
{
    public double FailureThreshold { get; init; } = 0.5;
    public TimeSpan SamplingDuration { get; init; } = TimeSpan.FromSeconds(60);
    public int MinimumThroughput { get; init; } = 10;
    public TimeSpan BreakDuration { get; init; } = TimeSpan.FromSeconds(30);
}

Fallback Strategies

Implementing Fallbacks

public class ResilientOrderService
{
    private readonly HttpClient _httpClient;
    private readonly IAsyncPolicy<HttpResponseMessage> _resiliencePolicy;
    private readonly IAsyncPolicy<HttpResponseMessage> _fallbackPolicy;
    private readonly ICache _cache;

    public ResilientOrderService(
        HttpClient httpClient,
        ICache cache)
    {
        _httpClient = httpClient;
        _cache = cache;

        _resiliencePolicy = ResiliencePolicies.CreateResiliencePolicy();

        _fallbackPolicy = Policy<HttpResponseMessage>
            .Handle<BrokenCircuitException>()
            .Or<HttpRequestException>()
            .FallbackAsync(
                fallbackAction: async (context, cancellationToken) =>
                {
                    // Return cached data or default response
                    var cachedData = await _cache.GetAsync<OrderResponse>("last-orders");
                    if (cachedData != null)
                    {
                        return new HttpResponseMessage(HttpStatusCode.OK)
                        {
                            Content = JsonContent.Create(cachedData)
                        };
                    }

                    return new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)
                    {
                        Content = new StringContent("Service temporarily unavailable")
                    };
                },
                onFallbackAsync: (result, context) =>
                {
                    Log.Warning("Fallback activated due to {Exception}", result.Exception?.Message);
                    return Task.CompletedTask;
                });
    }

    public async Task<OrderResponse?> GetOrdersAsync()
    {
        var policy = Policy.WrapAsync(_fallbackPolicy, _resiliencePolicy);

        var response = await policy.ExecuteAsync(async () =>
        {
            var result = await _httpClient.GetAsync("/api/orders");
            result.EnsureSuccessStatusCode();
            return result;
        });

        if (response.IsSuccessStatusCode)
        {
            var orders = await response.Content.ReadFromJsonAsync<OrderResponse>();

            // Cache successful response
            await _cache.SetAsync("last-orders", orders, TimeSpan.FromMinutes(5));

            return orders;
        }

        return null;
    }
}

Health Monitoring

Exposing Circuit Breaker State

public class CircuitBreakerHealthCheck : IHealthCheck
{
    private readonly CircuitBreakerRegistry _registry;
    private readonly string[] _criticalCircuits;

    public CircuitBreakerHealthCheck(
        CircuitBreakerRegistry registry,
        IConfiguration configuration)
    {
        _registry = registry;
        _criticalCircuits = configuration
            .GetSection("CircuitBreakers:Critical")
            .Get<string[]>() ?? Array.Empty<string>();
    }

    public Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context,
        CancellationToken cancellationToken = default)
    {
        var states = _criticalCircuits
            .Select(name => new { Name = name, State = _registry.GetState(name) })
            .ToList();

        var openCircuits = states.Where(s => s.State == CircuitState.Open).ToList();
        var halfOpenCircuits = states.Where(s => s.State == CircuitState.HalfOpen).ToList();

        if (openCircuits.Any())
        {
            return Task.FromResult(HealthCheckResult.Unhealthy(
                $"Circuit breakers open: {string.Join(", ", openCircuits.Select(c => c.Name))}",
                data: states.ToDictionary(s => s.Name, s => (object)s.State.ToString())));
        }

        if (halfOpenCircuits.Any())
        {
            return Task.FromResult(HealthCheckResult.Degraded(
                $"Circuit breakers testing: {string.Join(", ", halfOpenCircuits.Select(c => c.Name))}",
                data: states.ToDictionary(s => s.Name, s => (object)s.State.ToString())));
        }

        return Task.FromResult(HealthCheckResult.Healthy(
            "All circuit breakers closed",
            data: states.ToDictionary(s => s.Name, s => (object)s.State.ToString())));
    }
}

// Registration
services.AddHealthChecks()
    .AddCheck<CircuitBreakerHealthCheck>("circuit-breakers");

Metrics and Telemetry

public class TelemetryCircuitBreakerPolicy
{
    private readonly TelemetryClient _telemetryClient;

    public IAsyncPolicy<HttpResponseMessage> CreatePolicy(string serviceName)
    {
        return HttpPolicyExtensions
            .HandleTransientHttpError()
            .AdvancedCircuitBreakerAsync(
                failureThreshold: 0.5,
                samplingDuration: TimeSpan.FromSeconds(60),
                minimumThroughput: 10,
                durationOfBreak: TimeSpan.FromSeconds(30),
                onBreak: (outcome, state, breakDuration, context) =>
                {
                    _telemetryClient.TrackEvent("CircuitBreakerOpened", new Dictionary<string, string>
                    {
                        ["ServiceName"] = serviceName,
                        ["FailureRate"] = (state.FailureRate * 100).ToString("F2"),
                        ["BreakDuration"] = breakDuration.TotalSeconds.ToString()
                    });

                    _telemetryClient.GetMetric("CircuitBreakerState", "ServiceName")
                        .TrackValue(1, serviceName);  // 1 = Open
                },
                onReset: context =>
                {
                    _telemetryClient.TrackEvent("CircuitBreakerReset", new Dictionary<string, string>
                    {
                        ["ServiceName"] = serviceName
                    });

                    _telemetryClient.GetMetric("CircuitBreakerState", "ServiceName")
                        .TrackValue(0, serviceName);  // 0 = Closed
                },
                onHalfOpen: () =>
                {
                    _telemetryClient.GetMetric("CircuitBreakerState", "ServiceName")
                        .TrackValue(0.5, serviceName);  // 0.5 = Half-Open
                });
    }
}

API Endpoint for Manual Control

app.MapGet("/api/circuits", (CircuitBreakerRegistry registry) =>
{
    var states = new[] { "OrderService", "PaymentService", "InventoryService" }
        .Select(name => new { Name = name, State = registry.GetState(name).ToString() });

    return Results.Ok(states);
});

app.MapPost("/api/circuits/{name}/isolate", (string name, CircuitBreakerRegistry registry) =>
{
    registry.Isolate(name);
    return Results.Ok(new { Name = name, Action = "Isolated" });
}).RequireAuthorization("Admin");

app.MapPost("/api/circuits/{name}/reset", (string name, CircuitBreakerRegistry registry) =>
{
    registry.Reset(name);
    return Results.Ok(new { Name = name, Action = "Reset" });
}).RequireAuthorization("Admin");

Conclusion

The Circuit Breaker pattern is essential for building resilient microservices. Polly provides a mature implementation with advanced features like sampling-based thresholds. Combined with fallback strategies, health checks, and monitoring, you can build systems that gracefully handle failures and recover automatically.

References

• Circuit Breaker Pattern
• Polly Documentation