diff --git a/docs/ActiveActive.md b/docs/ActiveActive.md index be926a44f..daa49f826 100644 --- a/docs/ActiveActive.md +++ b/docs/ActiveActive.md @@ -2,8 +2,19 @@ ## Overview -The Active:Active feature provides automatic failover and intelligent routing across multiple Redis deployments. The library -automatically selects the best available endpoint based on: +The Active:Active feature provides automatic failover and intelligent routing across multiple Redis deployments. It is built from several cooperating pieces: + +1. **Connecting to multiple servers or regions at once**, giving a deployment redundant endpoints to fall back on. +2. **Health checks** that *actively* probe each endpoint on a timer to monitor its availability. +3. **Circuit breakers** that *passively* monitor availability from the observed success and failure of the traffic already flowing. +4. **Automatic retries** that let straightforward operations ride out a possibly-unstable connection — including silently transitioning to another endpoint during a failover, with no change to your calling code. + +The features for Active:Active are available in the `Availability` sub-namespace: + +``` csharp +using StackExchange.Redis.Availability; +``` +The library automatically selects the best available endpoint based on: 1. **Availability** - Connected endpoints are always preferred over disconnected ones 2. **Weight** - User-defined preference values (higher is better) @@ -354,93 +365,9 @@ var healthCheck = new HealthCheck ProbeCount = 3, ProbePolicy = HealthCheckProbePolicy.MajoritySuccess }; -// Healthy if 2 or more of probes succeed -``` - -### Custom Health Check Probes - -You can implement custom health check logic by extending `HealthCheckProbe`. Note that care must be used -if the probe involves talking to data via a `RedisKey`, as on "cluster" configurations, it must be ensured that the -key used resolves to the correct server; for this purpose, the `server.InventKey` method can be used: - -```csharp -public abstract class CustomProbe : HealthCheckProbe -{ - public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) - { - // create a random key that routes to the correct server, using - // the specified prefix - RedisKey key = server.InventKey("health-check/"); - // ... - } -} -```` - -Or more conveniently, the key-specific `KeyWriteHealthCheckProbe` encapsulates this logic: - -```csharp -public class CustomWriteProbe : KeyWriteHealthCheckProbe -{ - public override async Task CheckHealthAsync( - HealthCheck healthCheck, - IDatabaseAsync database, - RedisKey key) - { - try - { - var value = Guid.NewGuid().ToString(); - await database.StringSetAsync(key, value, expiry: healthCheck.ProbeTimeout); - bool isMatch = value == await database.StringGetAsync(key); - - return isMatch ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; - } - catch - { - return HealthCheckResult.Unhealthy; - } - } -} -``` - -### Custom Probe Policies - -In addition to the inbuilt policies, custom policies can be implemented by extending `HealthCheckProbePolicy`. -By checking the properties of the `HealthCheckProbeContext` parameter, your policy can make a determination -about the health of the server - returning `HealthCheckResult.Healthy` or `HealthCheckResult.Unhealthy` as -appropriate. If you return `HealthCheckResult.Inconclusive`, the health check will continue with additional probes. - -#### Example: Require at Least N Successes - -This example demonstrates a policy that requires at least a specified number of successful probes before declaring the endpoint healthy: - -```csharp -public class AtLeastPolicy(int requiredSuccesses) : HealthCheckProbePolicy -{ - public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) - { - // Success if we have at least the required number of successful probes - if (context.Success >= requiredSuccesses) return HealthCheckResult.Healthy; - - // If no more probes remaining, we haven't met our threshold; otherwise: keep trying - return context.Remaining == 0 ? HealthCheckResult.Unhealthy : HealthCheckResult.Inconclusive; - } -} - -// Use the custom policy requiring at least 2 successes -var healthCheck = new HealthCheck -{ - ProbeCount = 5, // Need enough probes to allow for the required successes - ProbePolicy = new AtLeastPolicy(2) -}; - -var options = new MultiGroupOptions -{ - HealthCheck = healthCheck -}; +// Healthy if 2 or more of 3 probes succeed ``` -This policy ensures that transient successes don't immediately mark an endpoint as healthy. It requires at least the specified number of successful probes, which provides better confidence in the endpoint's stability while still being more lenient than `AllSuccess`. - ### Health Check Behavior When a health check fails for a member: @@ -509,7 +436,6 @@ var options = new MultiGroupOptions FailureRateThreshold = 25, // trip above 25% failures MinimumNumberOfFailures = 100, // ...but only after 100 failures in the window MetricsWindowSize = TimeSpan.FromSeconds(5), // rolling window to measure over - TrackedExceptions = [typeof(RedisTimeoutException)], // which faults count as failures } }; ``` @@ -521,9 +447,8 @@ A `Builder` converts implicitly to a `CircuitBreaker`, so it can be assigned dir | `FailureRateThreshold` | 10 | Percentage of failures within the window that trips the breaker | | `MinimumNumberOfFailures` | 1000 | Minimum tracked failures in the window before the breaker can trip (avoids acting on tiny samples) | | `MetricsWindowSize` | 2 seconds | Rolling window over which successes and failures are counted | -| `TrackedExceptions` | `null` | Exception types that count as failures; when `null`, `RedisConnectionException` and `RedisTimeoutException` are tracked | -`TrackedExceptions` controls which faults count against the breaker; anything not tracked is treated as a success for circuit-breaking purposes (for example, a `RedisServerException` for a bad command is not a *connection* problem). Passing an array containing `typeof(Exception)` tracks everything; passing an empty array tracks nothing. +Which faults count against the breaker is decided by *classification*, not by exception type: transient and connection-level errors (and timeouts) count as failures, while application-level errors — for example a `RedisServerException` for a bad command, or a `WRONGTYPE` — are treated as a success for circuit-breaking purposes, since they are not a sign of an unhealthy *connection*. This is derived from the fault's `RedisErrorKind`; to change what counts, override `IsFailure(in FaultContext)` on a custom accumulator (see *Custom circuit breakers* below). ### Disabling the Circuit Breaker @@ -536,54 +461,87 @@ var options = new MultiGroupOptions }; ``` -### Custom Circuit Breakers (Advanced) +## Automatic Retries -> This is an advanced extension point; most applications should use `CircuitBreaker.Default` (optionally tuned via `CircuitBreaker.Builder`) or `CircuitBreaker.None`. - -You can implement your own policy by extending `CircuitBreaker` and its `Accumulator`. `CreateAccumulator()` is called once per underlying connection, so each accumulator holds the state for a single connection. -For every completed message the connection calls `ObserveResult`, passing a `CircuitBreakerContext` that exposes whether the operation succeeded (`Success`) and the associated `Fault` (if any). -Return `true` from `IsHealthy()` while the connection should be considered **healthy**, or `false` to **trip** the breaker and tear the connection down: +Health checks and circuit breakers keep the *group* pointed at a healthy member; **automatic retries** deal with the *individual operation* that was in flight when something went wrong. Wrapping a database with `WithRetry(...)` returns a database that transparently re-issues failed operations according to a `RetryPolicy` — riding out transient faults, and (in an Active:Active group) following a failover across to another member, without the caller having to catch-and-retry by hand. ```csharp -public sealed class ConsecutiveFailureBreaker(int limit) : CircuitBreaker -{ - public override Accumulator CreateAccumulator() => new Acc(limit); +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); - private sealed class Acc(int limit) : Accumulator - { - private int _consecutiveFailures; +// wrap the database once; reuse the wrapper like any other IDatabaseAsync +IDatabaseAsync db = conn.GetDatabase().WithRetry(new RetryPolicy()); - public override bool ObserveResult(in CircuitBreakerContext context) - { - if (context.Success) - { - _consecutiveFailures = 0; - } - else - { - _consecutiveFailures++; - } +// a transient fault (e.g. the active member briefly returning LOADING) is retried +// automatically; if the group fails over in the meantime, the retry lands on the new member +var value = await db.StringGetAsync("mykey"); +``` - // if Evaluate is specified, we should also compute IsHealthy(), - // often this can reuse local state needed by ObserveResult - return context.Evaluate ? IsHealthy() : true; - } +> You can call `WithRetry` on any database (`IDatabase` or `IDatabaseAsync`), but the wrapper it returns exposes only the **async** API — there is no synchronous form, since retrying may inherently have delays. It cannot wrap a batch or transaction, nor an already-retrying database. - // healthy until we hit the configured run of consecutive failures - public override bool IsHealthy() => _consecutiveFailures < limit; +### RetryPolicy settings - // called if the connection wants to discard accumulated history - public override void Reset() => _consecutiveFailures = 0; - } -} +`RetryPolicy` controls how many times, how often, and how far an operation is retried: -var options = new MultiGroupOptions +| Property | Default | Description | +|----------|---------|-------------| +| `MaxAttempts` | 3 | Total attempts (including the first) before giving up | +| `MaxAttemptsBeforeFailover` | 1 | Attempts against the current member before a retry is allowed to wait for / move to a failover member (only meaningful for multi-group connections) | +| `RetryDelay` | 1 second | Delay between same-server retries | +| `JitterMax` | 0.5 seconds | Upper bound of the additional random delay added to each retry, to avoid stampedes | +| `FailoverDelay` | 5 seconds | Maximum time to wait for a failover, when a retry is gated on one happening | +| `MaxCommandRetryCategory` | `CommandRetryWriteLastWins` | The most side-effecting command category that will be retried (see below) | + +```csharp +var policy = new RetryPolicy { - CircuitBreaker = new ConsecutiveFailureBreaker(limit: 5) + MaxAttempts = 5, + RetryDelay = TimeSpan.FromMilliseconds(200), + JitterMax = TimeSpan.FromMilliseconds(100), }; +IDatabaseAsync db = conn.GetDatabase().WithRetry(policy); ``` -Keep `ObserveResult` cheap and thread-safe: it runs on the hot path for every completed message, and may be called concurrently. +Only *transient* faults are retried — the same `RedisErrorKind`-based classification the default circuit breaker uses. An application-level error such as `WRONGTYPE`, or an unknown command, is not transient and is never retried, no matter what the policy says. + +### Which operations are safe to retry + +Retrying is not free of consequence: replaying `INCR` after an ambiguous failure could double-count, whereas replaying `GET` is harmless; `SET` is "last wins", so: *usally* fine. Every command therefore carries a **retry category** describing its side-effects, and a policy only retries commands at or below its `MaxCommandRetryCategory`. + +For the built-in typed methods (`StringGet`, `StringSet`, `HashSet`, ...) the library assigns the appropriate category automatically, so retries "just work" within the default policy. + +### Custom commands: `Execute` and `ScriptEvaluate` + +The library cannot infer the side-effects of a command it doesn't recognise — and that includes arbitrary commands issued via `Execute`/`ExecuteAsync`, and Lua run via `ScriptEvaluate`/`ScriptEvaluateAsync` (whose effect depends entirely on the script). Such commands are therefore treated **pessimistically**: an uncategorised command defaults to `CommandRetryNever` and is *not* retried. + +The categories, from safest to most dangerous, are: + +| `CommandFlags` value | Meaning | +|----------------------|---------| +| `CommandRetryAlways` | Always safe to retry, regardless of connection/server state | +| `CommandRetryConnection` | Connection-level or safe metadata (e.g. `CLIENT SETNAME`, `CONFIG GET`) | +| `CommandRetryReadOnly` | Pure reads (e.g. `GET`) | +| `CommandRetryWriteChecked` | Conditional writes (e.g. `SETNX`, `SET ... IFEQ`) | +| `CommandRetryWriteLastWins` | Unconditional overwrite — last-writer-wins (e.g. `SET`) | +| `CommandRetryWriteAccumulating` | Cumulative writes where a retry can double-apply (e.g. `INCR`, `LPUSH`) | +| `CommandRetryServerAdmin` | Server administration (e.g. `CONFIG SET`) | +| `CommandRetryNever` | Never retry | + + +When possible when using ad-hoc commands or script, callers should supply the most appropriate `CommandRetry*` category in the command's `CommandFlags`: + +```csharp +// an arbitrary read-only command: safe to retry +var result = await db.ExecuteAsync("LOLWUT", args: [], flags: CommandFlags.CommandRetryReadOnly); + +// a Lua script that only reads: opt into retries +var value = await db.ScriptEvaluateAsync( + "return redis.call('GET', KEYS[1])", + keys: [key], + flags: CommandFlags.CommandRetryReadOnly); +``` + +Choose the category honestly — it describes what a *replay* would do. If a retry could double-apply a side-effect, use `CommandRetryWriteAccumulating` (or leave it uncategorised) rather than claiming it's a read. +Conversely, if you want more-side-effecting operations retried across the board, raise the policy's `MaxCommandRetryCategory` instead of tagging each call. ## Unhealthy State and Failback @@ -825,3 +783,165 @@ if (newDC.IsConnected) } } ``` + +## Advanced Customization + +The building blocks above cover the common cases. The extension points below let you replace the default health-check, retry, and circuit-breaker behavior with your own logic; most applications will not need them. + +### Custom Health Check Probes + +You can implement custom health check logic by extending `HealthCheckProbe`. Note that care must be used +if the probe involves talking to data via a `RedisKey`, as on "cluster" configurations, it must be ensured that the +key used resolves to the correct server; for this purpose, the `server.InventKey` method can be used: + +```csharp +public abstract class CustomProbe : HealthCheckProbe +{ + public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) + { + // create a random key that routes to the correct server, using + // the specified prefix + RedisKey key = server.InventKey("health-check/"); + // ... + } +} +``` + +Or more conveniently, the key-specific `KeyWriteHealthCheckProbe` encapsulates this logic: + +```csharp +public class CustomWriteProbe : KeyWriteHealthCheckProbe +{ + public override async Task CheckHealthAsync( + HealthCheck healthCheck, + IDatabaseAsync database, + RedisKey key) + { + try + { + var value = Guid.NewGuid().ToString(); + await database.StringSetAsync(key, value, expiry: healthCheck.ProbeTimeout); + bool isMatch = value == await database.StringGetAsync(key); + + return isMatch ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; + } + catch + { + return HealthCheckResult.Unhealthy; + } + } +} +``` + +### Custom Probe Policies + +In addition to the inbuilt policies, custom policies can be implemented by extending `HealthCheckProbePolicy`. +By checking the properties of the `HealthCheckProbeContext` parameter, your policy can make a determination +about the health of the server - returning `HealthCheckResult.Healthy` or `HealthCheckResult.Unhealthy` as +appropriate. If you return `HealthCheckResult.Inconclusive`, the health check will continue with additional probes. + +#### Example: Require at Least N Successes + +This example demonstrates a policy that requires at least a specified number of successful probes before declaring the endpoint healthy: + +```csharp +public class AtLeastPolicy(int requiredSuccesses) : HealthCheckProbePolicy +{ + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + // Success if we have at least the required number of successful probes + if (context.Success >= requiredSuccesses) return HealthCheckResult.Healthy; + + // If no more probes remaining, we haven't met our threshold; otherwise: keep trying + return context.Remaining == 0 ? HealthCheckResult.Unhealthy : HealthCheckResult.Inconclusive; + } +} + +// Use the custom policy requiring at least 2 successes +var healthCheck = new HealthCheck +{ + ProbeCount = 5, // Need enough probes to allow for the required successes + ProbePolicy = new AtLeastPolicy(2) +}; + +var options = new MultiGroupOptions +{ + HealthCheck = healthCheck +}; +``` + +This policy ensures that transient successes don't immediately mark an endpoint as healthy. It requires at least the specified number of successful probes, which provides better confidence in the endpoint's stability while still being more lenient than `AllSuccess`. + +### Custom Circuit Breakers + +> This is an advanced extension point; most applications should use `CircuitBreaker.Default` (optionally tuned via `CircuitBreaker.Builder`) or `CircuitBreaker.None`. + +You can implement your own policy by extending `CircuitBreaker` and its `Accumulator`. `CreateAccumulator()` is called once per underlying connection, so each accumulator holds the state for a single connection. +For every completed message the connection calls `ObserveResult`, passing a `FaultContext` that describes the outcome: `IsFault` indicates whether a fault occurred, with the associated `Fault`, `ErrorKind` and `ConnectionFailureType` available for inspection. `IsHealthy()` is consulted separately: return `true` while the connection should be considered **healthy**, or `false` to **trip** the breaker and tear the connection down. + +By default only faults that `IsFailure` regards as genuine failures reach `ObserveResult` *as* failures (transient/connection errors, timeouts, and similar - classified from the fault's `ErrorKind`); everything else - including application-level errors such as a bad command - is passed as a success. Override `IsFailure(in FaultContext)` if you need different rules for what counts: + +```csharp +public sealed class ConsecutiveFailureBreaker(int limit) : CircuitBreaker +{ + public override Accumulator CreateAccumulator() => new Acc(limit); + + private sealed class Acc(int limit) : Accumulator + { + private int _consecutiveFailures; + + // a fault is present iff context.IsFault; a success resets the run + public override void ObserveResult(in FaultContext context) + { + if (context.IsFault) + { + _consecutiveFailures++; + } + else + { + _consecutiveFailures = 0; + } + } + + // healthy until we hit the configured run of consecutive failures + public override bool IsHealthy() => _consecutiveFailures < limit; + + // called if the connection wants to discard accumulated history + public override void Reset() => _consecutiveFailures = 0; + + // (optional) override to change what counts as a failure; the default classifies from ErrorKind + // protected override bool IsFailure(in FaultContext fault) => fault.IsFault; + } +} + +var options = new MultiGroupOptions +{ + CircuitBreaker = new ConsecutiveFailureBreaker(limit: 5) +}; +``` + +Keep `ObserveResult` cheap and thread-safe: it runs on the hot path for every completed message, and may be called concurrently. + +### Custom Retry Policies + +`RetryPolicy` is itself extensible: override `CanRetry(in FaultContext fault)` to make the retry decision yourself. It returns a `RetryPolicyResult` — `None` to give up, or a combination of `SameServer` and `FailoverServer` to indicate where a retry may be attempted. +The `FaultContext` gives you the classified `ErrorKind`, the `ConnectionFailureType`, and the command `Flags` (including its retry category) to base the decision on: + +```csharp +public sealed class ReadOnlyOnlyRetryPolicy : RetryPolicy +{ + public override RetryPolicyResult CanRetry(in FaultContext fault) + { + // only ever retry pure reads, and only on the same server + if (fault.ErrorKind == RedisErrorKind.Loading + && (fault.Flags & CommandFlags.CommandRetryReadOnly) != 0) + { + return RetryPolicyResult.SameServer; + } + // note: base.CanRetry(fault) would apply the default logic + return RetryPolicyResult.None; + } +} + +IDatabaseAsync db = conn.GetDatabase().WithRetry(new ReadOnlyOnlyRetryPolicy()); +``` diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs new file mode 100644 index 000000000..356e72619 --- /dev/null +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -0,0 +1,483 @@ +using System.Collections.Immutable; +using System.Reflection.Metadata.Ecma335; +using System.Text; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +// readable names for the info we gather; kept as tuples (+ BasicArray) so they have +// value equality and play nicely with incremental generator caching. +using ParamInfo = (string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default); +using MethodInfo = (string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters, StackExchange.Redis.Build.BasicArray TypeArgs); +using InterfaceInfo = (string Name, string Namespace, StackExchange.Redis.Build.AutoDatabaseGenerator.KnownInterfaces KnownType, StackExchange.Redis.Build.BasicArray<(string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters, StackExchange.Redis.Build.BasicArray TypeArgs)> Methods); +using ClassInfo = (string Name, string Namespace, StackExchange.Redis.Build.AutoDatabaseGenerator.KnownInterfaces Interfaces); + +namespace StackExchange.Redis.Build; + +[Generator(LanguageNames.CSharp)] +public class AutoDatabaseGenerator : IIncrementalGenerator +{ + public void Initialize(IncrementalGeneratorInitializationContext ctx) + { + var interfaces = ctx.SyntaxProvider + .CreateSyntaxProvider( + static (node, _) => node is InterfaceDeclarationSyntax decl && FastIndexFilter(decl), ExtractInterfaceMethods) + .Where(pair => pair.Name is { Length: > 0 }) + .Collect(); + + var classes = ctx.SyntaxProvider + .CreateSyntaxProvider( + static (node, _) => node is ClassDeclarationSyntax decl && FastClassFilter(decl), ExtractClasses) + .Where(pair => pair.Name is { Length: > 0 }) + .Collect(); + + ctx.RegisterSourceOutput(interfaces.Combine(classes), static (ctx, content) => Generate(ctx, content.Left, content.Right)); + } + + static KnownInterfaces Identify(string type) => type switch + { + "IDatabase" => KnownInterfaces.IDatabase, + "IDatabaseAsync" => KnownInterfaces.IDatabaseAsync, + "IRedis" => KnownInterfaces.IRedis, + "IRedisAsync" => KnownInterfaces.IRedisAsync, + _ => KnownInterfaces.None, + }; + + private static bool FastIndexFilter(InterfaceDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync + => Identify(decl.Identifier.ValueText) is not 0; + + private static bool FastClassFilter(ClassDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync + => decl.AttributeLists.Any(x => x.Attributes.Any(x => x.Name.ToString() is "AutoDatabase" or "AutoDatabaseAttribute")); + + private static bool IsOurInterface(INamedTypeSymbol symbol) => + symbol is + { + TypeKind: TypeKind.Interface, + Name: "IDatabase" or "IDatabaseAsync" or "IRedis" or "IRedisAsync", + ContainingType: null, + ContainingNamespace: + { + Name: "Redis", + ContainingNamespace: + { + Name: "StackExchange", + ContainingNamespace.IsGlobalNamespace: true + } + } + }; + + private static InterfaceInfo ExtractInterfaceMethods(GeneratorSyntaxContext context, CancellationToken cancel) + { + // note: we deliberately do NOT interpret anything here - just capture the raw shape of every + // method (name, return type, and per-parameter name/type/modifiers/optionality/default) so that + // later passes have everything they might need. + if (context.SemanticModel.GetDeclaredSymbol(context.Node, cancel) is not INamedTypeSymbol iface + || !IsOurInterface(iface)) + { + return default; + } + + var knownType = Identify(iface.Name); + if (knownType is KnownInterfaces.None) return default; + + var methods = new List(); + foreach (var member in iface.GetMembers()) + { + cancel.ThrowIfCancellationRequested(); + if (member is not IMethodSymbol { MethodKind: MethodKind.Ordinary } method) continue; + + var parameters = BasicArray.From(method.Parameters, static p => ( + Name: p.Name, + Type: p.Type.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), + RefKind: p.RefKind, + IsParams: p.IsParams, + IsOptional: p.IsOptional, + HasDefault: p.HasExplicitDefaultValue, + Default: p.HasExplicitDefaultValue ? FormatDefault(p.ExplicitDefaultValue) : null)); + + BasicArray typeArgs = default; + if (method.IsGenericMethod) + { + typeArgs = BasicArray.From(method.TypeParameters, p => p.Name); + } + methods.Add(( + Name: method.Name, + ReturnType: method.ReturnType.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), + Parameters: parameters, + TypeArgs: typeArgs)); + } + + var ns = iface.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + return (iface.Name, ns, knownType, BasicArray.From(methods)); + } + + private ClassInfo ExtractClasses(GeneratorSyntaxContext context, CancellationToken cancel) + { + // note: we deliberately do NOT interpret anything here - just capture the raw shape of every + // method (name, return type, and per-parameter name/type/modifiers/optionality/default) so that + // later passes have everything they might need. + if (context.SemanticModel.GetDeclaredSymbol(context.Node, cancel) is not INamedTypeSymbol cls + || !HasAutoDatabaseAttrib(cls)) + { + return default; + } + + static bool HasAutoDatabaseAttrib(INamedTypeSymbol symbol) + { + var attribs = symbol.GetAttributes(); + foreach (var attrib in attribs) + { + if (attrib.AttributeClass is + { + Name: "AutoDatabaseAttribute", + ContainingType: null, + ContainingNamespace: + { + Name: "Redis", + ContainingNamespace: + { + Name: "StackExchange", + ContainingNamespace.IsGlobalNamespace: true + } + } + }) + { + return true; + } + } + + return false; + } + + KnownInterfaces known = 0; + foreach (var iFace in cls.Interfaces) + { + if (IsOurInterface(iFace)) + { + switch (iFace.Name) + { + case "IDatabase": + known |= KnownInterfaces.IDatabase | KnownInterfaces.IDatabaseAsync | + KnownInterfaces.IRedis | KnownInterfaces.IRedisAsync; + break; + case "IDatabaseAsync": + known |= KnownInterfaces.IDatabaseAsync | KnownInterfaces.IRedisAsync; + break; + case "IRedis": + known |= KnownInterfaces.IRedis | KnownInterfaces.IRedisAsync; + break; + case "IRedisAsync": + known |= KnownInterfaces.IRedisAsync; + break; + } + } + } + + if (known is KnownInterfaces.None) return default; // nothing to do! + + var ns = cls.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + return (cls.Name, ns, known); + } + + [Flags] + internal enum KnownInterfaces + { + None = 0, + IDatabase = 1, + IDatabaseAsync = 2, + IRedis = 4, + IRedisAsync = 8, + } + + // methods that don't fit the capture-and-replay shape are left for the caller to implement manually: + // - generic methods (open type args can't be captured into a concrete state struct) + // - the Wait family (synchronization over caller-supplied Tasks, not server calls) + // - IsConnected: a synchronous status probe that IDatabaseAsync carries; it would route through the + // sync Execute funnel, which an async-only database (e.g. RetryDatabase) doesn't provide - and a + // connection check shouldn't be retried in any case + // - streaming returns (IEnumerable / IAsyncEnumerable) whose execution is deferred + private static bool SkipMethod(MethodInfo method) + => !method.TypeArgs.IsEmpty + || method.Name.Contains("Wait") + || method.Name == "IsConnected" + || method.ReturnType.StartsWith("System.Collections.Generic.IEnumerable<", StringComparison.Ordinal) + || method.ReturnType.StartsWith("System.Collections.Generic.IAsyncEnumerable<", StringComparison.Ordinal); + + private const string TaskType = "System.Threading.Tasks.Task"; + + // Task / Task returns route through ExecuteAsync (its own retry policy); everything else + // uses Execute. + private static bool IsAsync(string returnType) + => returnType.StartsWith(TaskType, StringComparison.Ordinal); + + // the retry machinery unwraps the Task and only ever sees the inner result, so key/channel + // (un)mapping must be decided on T, not Task; this strips the Task<...> wrapper from an async + // return type. A bare (non-generic) Task has no result and is returned unchanged (as is any + // non-async return type), which is harmless since neither matches NeedsMap. + private static string StripTask(string returnType) + => IsAsync(returnType) && returnType.StartsWith(TaskType + "<", StringComparison.Ordinal) + ? returnType.Substring(TaskType.Length + 1, returnType.Length - TaskType.Length - 2) + : returnType; + + private static string FormatDefault(object? value) => value switch + { + null => "null", + string s => $"\"{s}\"", + bool b => b ? "true" : "false", + _ => value.ToString() ?? "null", + }; + + private static void Generate(SourceProductionContext ctx, ImmutableArray interfaces, ImmutableArray classes) + { + if (interfaces.IsDefaultOrEmpty | classes.IsDefaultOrEmpty) return; // nothing to do + + // each interface is declared across several partial files, so we get one (identical) entry per + // partial declaration; structural equality lets us collapse those down to one per interface. + var iKeyed = new Dictionary(); + foreach (var t in interfaces) + { + if (t.KnownType is KnownInterfaces.None) continue; + if (!iKeyed.ContainsKey(t.KnownType)) iKeyed.Add(t.KnownType, t); + } + + var sb = new StringBuilder(); + var writer = new CodeWriter(sb); + writer.NewLine().Append("// "); + writer.NewLine().Append("// AutoDatabaseGenerator - explicit interface implementations that funnel every"); + writer.NewLine().Append("// call through Execute(state, projection), capturing the arguments in a generated"); + writer.NewLine().Append("// state struct (one per unique parameter-type signature) to avoid per-call closures."); + writer.NewLine().Append("#nullable enable"); // this needs to be explicit for code-gen + writer.NewLine(); + foreach (var cls in classes.Distinct()) + { + if (!string.IsNullOrWhiteSpace(cls.Namespace)) + { + writer.NewLine().Append("namespace ").Append(cls.Namespace).NewLine().Append("{").Indent(); + } + + // unique parameter-type signatures encountered while emitting this class's methods; + // keyed on the '|'-joined parameter types so distinct methods with the same shape share + // one state struct. tupleDefs[i] holds a representative parameter list for _tuple{i}. + var tupleIndex = new Dictionary(); + var tupleDefs = new List>(); + + // every unique key/channel-bearing result type across all shapes; a single shared + // singleton (emitted after the tuples) implements IRedisArgsResult for each, so + // tuples expose unmap dispatch via UnMapper without ever being cast to an interface + var allReturns = new HashSet(); + + bool isFirst = true; + writer.NewLine().Append("partial class ").Append(cls.Name); + AppendInterfaceDeclaration(KnownInterfaces.IDatabase); + AppendInterfaceDeclaration(KnownInterfaces.IDatabaseAsync); + AppendInterfaceDeclaration(KnownInterfaces.IRedis); + AppendInterfaceDeclaration(KnownInterfaces.IRedisAsync); + writer.NewLine().Append("{").Indent(); + AppendInterfaceMethods(KnownInterfaces.IDatabase); + AppendInterfaceMethods(KnownInterfaces.IDatabaseAsync); + AppendInterfaceMethods(KnownInterfaces.IRedis); + AppendInterfaceMethods(KnownInterfaces.IRedisAsync); + AppendTupleTypes(); + + writer.Outdent().NewLine().Append("}"); + + if (!string.IsNullOrWhiteSpace(cls.Namespace)) + { + writer.Outdent().NewLine().Append("}"); + } + writer.NewLine(); + + + void AppendInterfaceDeclaration(KnownInterfaces knownType) + { + if ((cls.Interfaces & knownType) is 0 | !iKeyed.TryGetValue(knownType, out var iType)) return; + writer.Append(isFirst ? " : " : ", ").Append("global::") + .Append(iType.Namespace).Append('.') .Append(iType.Name); + isFirst = false; + } + + void AppendInterfaceMethods(KnownInterfaces knownType) + { + if ((cls.Interfaces & knownType) is 0 | !iKeyed.TryGetValue(knownType, out var iType)) return; + foreach (var method in iType.Methods) + { + if (SkipMethod(method)) continue; // wonky by nature - left for the caller to implement manually + + writer.NewLine().Append(method.ReturnType).Append(" global::") + .Append(iType.Namespace).Append('.').Append(iType.Name).Append('.').Append(method.Name).Append("("); + bool firstParam = true; + foreach (var p in method.Parameters.Span) + { + if (firstParam) firstParam = false; + else writer.Append(", "); + writer.Append(p.Type).Append(" ").Append(p.Name); + } + writer.Append(')').Indent().NewLine(); + + // async methods route to ExecuteAsync (its own retry policy); everything else uses + // Execute. The state struct is shared regardless of return type - keyed on parameter + // types only - so e.g. ArraySet and ArraySetAsync land on the same _tupleN. The return + // type is Task-stripped so unmapping is keyed on the inner result the retry machinery + // actually sees, not the Task wrapper. + bool isAsync = IsAsync(method.ReturnType); + var simpleResult = StripTask(method.ReturnType); + bool needsMap = NeedsMap(simpleResult); + if (needsMap) allReturns.Add(simpleResult); + int tuple = GetTupleIndex(method.Parameters, needsMap); + writer.Append(isAsync ? "=> ExecuteAsync(new _tuple" : "=> Execute(new _tuple").Append(tuple).Append("("); + firstParam = true; + foreach (var p in method.Parameters.Span) + { + if (firstParam) firstParam = false; + else writer.Append(", "); + writer.Append(p.Name); + } + writer.Append("), static (state, inner) => inner.").Append(method.Name).Append("("); + for (int i = 0; i < method.Parameters.Length; i++) + { + if (i != 0) writer.Append(", "); + writer.Append("state.Arg").Append(i); + } + writer.Append("));").Outdent().NewLine(); + } + } + + static string GetTupleKey(BasicArray parameters) + { + var keyBuilder = new StringBuilder(); + foreach (var p in parameters.Span) + { + keyBuilder.Append(p.Type).Append('|'); // types-only key; no ref/out on this surface, so type is sufficient + } + + return keyBuilder.ToString(); + } + + bool TupleNeedsMap(BasicArray parameters) => + tupleIndex.TryGetValue(GetTupleKey(parameters), out var found) && found.NeedsMap; + + int GetTupleIndex(BasicArray parameters, bool needsMap) + { + var key = GetTupleKey(parameters); + int index; + if (tupleIndex.TryGetValue(key, out var found)) + { + index = found.Index; + if (needsMap && !found.NeedsMap) + { + found.NeedsMap = true; + tupleIndex[key] = found; + } + } + else + { + index = tupleDefs.Count; + tupleIndex.Add(key, (index, needsMap)); + tupleDefs.Add(parameters); + } + + return index; + } + + // const string CommandFlagsType = "StackExchange.Redis.CommandFlags"; + const string RedisKeyType = "StackExchange.Redis.RedisKey"; + const string RedisChannelType = "StackExchange.Redis.RedisChannel"; + // types that carry key(s) internally without "RedisKey" in their name, so the + // substring check below won't catch them; they rely on a Map extension method + const string StreamPositionType = "StackExchange.Redis.StreamPosition"; + // the Execute/ExecuteAsync escape hatch boxes keys/channels inside a loosely-typed + // arg list; these route through a Map extension that unboxes and rewrites matches + const string ScriptArgArrayType = "object[]"; + const string ScriptArgCollectionType = "System.Collections.Generic.ICollection"; + + static bool NeedsMap(string name) => + name.IndexOf(RedisKeyType, StringComparison.Ordinal) >= 0 + || name.IndexOf(RedisChannelType, StringComparison.Ordinal) >= 0 + || name.IndexOf(StreamPositionType, StringComparison.Ordinal) >= 0 + || name == ScriptArgArrayType + || name == ScriptArgCollectionType + || name.IndexOf("ListPopResult", StringComparison.Ordinal) >= 0 + || name.IndexOf("SortedSetPopResult", StringComparison.Ordinal) >= 0 + || name == ScriptArgCollectionType + "?"; + + void AppendTupleTypes() + { + for (int i = 0; i < tupleDefs.Count; i++) + { + var raw = tupleDefs[i]; + bool needsMap = TupleNeedsMap(raw); + var parameters = raw.Span; + + // fields are mutable: Map rewrites key/channel fields and Flags has a setter + writer.NewLine().NewLine().Append("private struct _tuple").Append(i).Append("("); + for (int p = 0; p < parameters.Length; p++) + { + if (p != 0) writer.Append(", "); + writer.Append(parameters[p].Type).Append(" arg").Append(p); + } + + writer.Append(") : global::StackExchange.Redis.IRedisArgs"); + writer.NewLine().Append("{").Indent(); + for (int p = 0; p < parameters.Length; p++) + { + writer.NewLine().Append("public ").Append(NeedsMap(parameters[p].Type) ? "" : "readonly ").Append(parameters[p].Type) + .Append(" Arg").Append(p).Append(" = arg").Append(p).Append(";"); + } + + // Map rewrites each scalar key/channel field directly, and defers container or + // loosely-typed fields to a matching Map extension method + writer.NewLine().Append("public void Map(global::StackExchange.Redis.IRedisArgsMutator mutator)") + .NewLine().Append("{").Indent(); + for (int p = 0; p < parameters.Length; p++) + { + if (NeedsMap(parameters[p].Type)) + { + // key/channel-bearing container (or the loosely-typed script arg list); it + // is the library's job to ensure a suitable Map extension method exists + writer.NewLine().Append("Arg").Append(p).Append(" = mutator.Map(Arg").Append(p).Append(");"); + } + } + writer.Outdent().NewLine().Append("}"); + + // tuples with key/channel-bearing results point UnMapper at the shared singleton + // (which knows how to unmap every such result type); the rest return null so the + // Execute helper skips unmapping entirely - and neither path boxes the struct + writer.NewLine().Append("public readonly object? UnMapper => ") + .Append(needsMap ? "_UnMapper.Instance;" : "null;"); + + writer.Outdent().NewLine().Append("}"); + } + + if (allReturns.Count is not 0) + { + // sorted for deterministic (cache-stable) output + var ordered = new List(allReturns); + ordered.Sort(StringComparer.Ordinal); + + writer.NewLine().NewLine().Append("private sealed class _UnMapper").Indent(); + bool firstIface = true; + foreach (var retType in ordered) + { + writer.NewLine().Append(firstIface ? ": " : ", ") + .Append("global::StackExchange.Redis.IRedisArgsResult<").Append(retType).Append(">"); + firstIface = false; + } + writer.Outdent().NewLine().Append("{").Indent(); + writer.NewLine().Append("public static readonly _UnMapper Instance = new();"); + foreach (var retType in ordered) + { + writer.NewLine().NewLine().Append(retType).Append(' ') + .Append("global::StackExchange.Redis.IRedisArgsResult<") + .Append(retType) + .Append(">.UnMap(global::StackExchange.Redis.IRedisArgsMutator mutator, ") + .Append(retType).Append(" value)") + .Indent().NewLine().Append("=> mutator.UnMap(value);").Outdent(); + } + writer.Outdent().NewLine().Append("}"); + } + } + } + writer.NewLine(); + + ctx.AddSource("AutoDatabase.generated.cs", sb.ToString()); + } +} diff --git a/eng/StackExchange.Redis.Build/BasicArray.cs b/eng/StackExchange.Redis.Build/BasicArray.cs index dc7984c75..0c2df6f92 100644 --- a/eng/StackExchange.Redis.Build/BasicArray.cs +++ b/eng/StackExchange.Redis.Build/BasicArray.cs @@ -38,7 +38,7 @@ public bool Equals(BasicArray other) int i = 0; foreach (ref readonly T el in this.Span) { - if (!_comparer.Equals(el, y[i])) return false; + if (!_comparer.Equals(el, y[i++])) return false; } return true; @@ -58,7 +58,7 @@ public override int GetHashCode() var hash = Length; foreach (ref readonly T el in this.Span) { - _ = (hash * -37) + _comparer.GetHashCode(el); + hash = (hash * -37) + _comparer.GetHashCode(el); } return hash; @@ -82,4 +82,25 @@ public void Add(in T value) public BasicArray Build() => new(elements, Count); } + + public static BasicArray From(ICollection collection) + { + if (collection.Count is 0) return default; + var arr = new T[collection.Count]; + collection.CopyTo(arr, 0); + return new(arr, arr.Length); + } + + public static BasicArray From(ICollection collection, Func selector) + { + if (collection.Count is 0) return default; + var arr = new T[collection.Count]; + int i = 0; + foreach (var item in collection) + { + arr[i++] = selector(item); + } + + return new(arr, i); + } } diff --git a/eng/StackExchange.Redis.Build/CodeWriter.cs b/eng/StackExchange.Redis.Build/CodeWriter.cs index 94d30ef47..8d3091322 100644 --- a/eng/StackExchange.Redis.Build/CodeWriter.cs +++ b/eng/StackExchange.Redis.Build/CodeWriter.cs @@ -14,7 +14,8 @@ internal sealed class CodeWriter(StringBuilder buffer) /// Starts a new line, applying the current indent. public CodeWriter NewLine() { - buffer.AppendLine().Append(' ', _indent * 4); + buffer.AppendLine(); + _lineHasContent = false; return this; } @@ -32,26 +33,41 @@ public CodeWriter Outdent() return this; } + private bool _lineHasContent; + + private void IndentIfNeeded() + { + if (!_lineHasContent) + { + buffer.Append(' ', _indent * 4); + _lineHasContent = true; + } + } + public CodeWriter Append(string? value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(char value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(int value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(long value) { + IndentIfNeeded(); buffer.Append(value); return this; } diff --git a/src/RESPite/Messages/RespReader.cs b/src/RESPite/Messages/RespReader.cs index 413d0174d..2137946d8 100644 --- a/src/RESPite/Messages/RespReader.cs +++ b/src/RESPite/Messages/RespReader.cs @@ -722,6 +722,29 @@ public readonly unsafe bool TryParseScalar( return TryGetSpan(out var span) ? parser(span, out value) : TryParseSlow(parser, out value); } + // same thing as ^^^, but as a utility method for text values, paying UTF8 encode + internal static unsafe bool TryParseScalar( + ReadOnlySpan source, + delegate* managed, out T, bool> parser, + out T value) + { + const int MAX_STACK = 128; + byte[]? lease = null; + int maxLen = RespConstants.UTF8.GetMaxByteCount(source.Length); + Span buffer = maxLen <= MAX_STACK + ? stackalloc byte[MAX_STACK] // prefer fixed size for perf + : (lease = ArrayPool.Shared.Rent(maxLen)); + try + { + var len = RespConstants.UTF8.GetBytes(source, buffer); + return parser(buffer.Slice(0, len), out value); + } + finally + { + if (lease is not null) ArrayPool.Shared.Return(lease); + } + } + [MethodImpl(MethodImplOptions.NoInlining)] private readonly unsafe bool TryParseSlow( delegate* managed, out T, bool> parser, diff --git a/src/StackExchange.Redis/AutoDatabase.cs b/src/StackExchange.Redis/AutoDatabase.cs new file mode 100644 index 000000000..395c5f70c --- /dev/null +++ b/src/StackExchange.Redis/AutoDatabase.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace StackExchange.Redis; + +[Conditional("DEBUG")] +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, AllowMultiple = false)] +internal sealed class AutoDatabaseAttribute : Attribute +{ +} + +internal interface IRedisArgs +{ + void Map(IRedisArgsMutator mutator); + object? UnMapper { get; } +} + +internal interface IRedisArgsMutator +{ + RedisKey Map(RedisKey key); + RedisChannel Map(RedisChannel channel); + + RedisKey UnMap(RedisKey key); + RedisChannel UnMap(RedisChannel channel); +} + +internal interface IRedisArgsResult +{ + T UnMap(IRedisArgsMutator mutator, T value); +} + +internal static class RedisArgsMutatorExtensions +{ + // these are used by the generated tuple-types via auto-database: each maps the key-bearing parts + // of an argument through the supplied mutator. They hang off IRedisArgsMutator (rather than the + // argument) so a call site reads consistently with the interface's own MapKey/MapChannel. + public static KeyValuePair Map( + this IRedisArgsMutator mutator, + KeyValuePair value) => + new(mutator.Map(value.Key), value.Value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TResult UnMap(this IRedisArgsMutator mutator, in TState state, TResult result) + where TState : struct, IRedisArgs + { + if (typeof(TResult) == typeof(RedisKey)) + { + var tmp = mutator.UnMap(Unsafe.As(ref result)); + return Unsafe.As(ref tmp); + } + else if (typeof(TResult) == typeof(RedisChannel)) + { + var tmp = mutator.UnMap(Unsafe.As(ref result)); + return Unsafe.As(ref tmp); + } + else if (typeof(TResult) == typeof(RedisValue)) + { + return result; // never mapped + } + else + { + return state.UnMapper is IRedisArgsResult unmap ? unmap.UnMap(mutator, result) : result; + } + } + + [return: NotNullIfNotNull("keys")] + public static RedisKey[]? Map(this IRedisArgsMutator mutator, RedisKey[]? keys) + { + if (keys is null || keys.Length is 0) return keys; + var arr = new RedisKey[keys.Length]; + for (int i = 0; i < arr.Length; i++) + { + arr[i] = mutator.Map(keys[i]); + } + return arr; + } + + [return: NotNullIfNotNull("pairs")] + public static KeyValuePair[]? Map( + this IRedisArgsMutator mutator, + KeyValuePair[]? pairs) + { + if (pairs is null || pairs.Length is 0) return pairs; + var arr = new KeyValuePair[pairs.Length]; + for (int i = 0; i < arr.Length; i++) + { + ref readonly KeyValuePair pair = ref pairs[i]; + arr[i] = new(mutator.Map(pair.Key), pair.Value); + } + return arr; + } + + public static StreamPosition Map(this IRedisArgsMutator mutator, StreamPosition value) => + new(mutator.Map(value.Key), value.Position); + + [return: NotNullIfNotNull("positions")] + public static StreamPosition[]? Map(this IRedisArgsMutator mutator, StreamPosition[]? positions) + { + if (positions is null || positions.Length is 0) return positions; + var arr = new StreamPosition[positions.Length]; + for (int i = 0; i < arr.Length; i++) + { + ref readonly StreamPosition position = ref positions[i]; + arr[i] = new(mutator.Map(position.Key), position.Position); + } + return arr; + } + + // the Execute/ExecuteAsync escape hatch takes a loosely-typed arg list in which any element may + // be a boxed RedisKey or RedisChannel; these rewrite just those entries (mirroring + // KeyPrefixed.ToInner), copying only when there is something to rewrite so the common call allocates nothing. + [return: NotNullIfNotNull("args")] + public static object[]? Map(this IRedisArgsMutator mutator, object[]? args) + { + if (args is null || args.Length is 0) return args; + object[]? copy = null; + for (int i = 0; i < args.Length; i++) + { + if (args[i] is RedisKey key) + { + (copy ??= (object[])args.Clone())[i] = mutator.Map(key); + } + else if (args[i] is RedisChannel channel) + { + (copy ??= (object[])args.Clone())[i] = mutator.Map(channel); + } + } + return copy ?? args; + } + + [return: NotNullIfNotNull("args")] + public static ICollection? Map(this IRedisArgsMutator mutator, ICollection? args) + { + if (args is null || args.Count is 0) return args; + bool any = false; + foreach (var arg in args) + { + if (arg is RedisKey or RedisChannel) + { + any = true; + break; + } + } + if (!any) return args; + + var copy = new object[args.Count]; + int i = 0; + foreach (var arg in args) + { + copy[i++] = arg switch + { + RedisKey key => mutator.Map(key), + RedisChannel channel => mutator.Map(channel), + _ => arg, + }; + } + return copy; + } + + public static SortedSetPopResult UnMap(this IRedisArgsMutator mutator, SortedSetPopResult value) => + value.IsNull ? SortedSetPopResult.Null : new(mutator.Map(value.Key), value.Entries); + + public static ListPopResult UnMap(this IRedisArgsMutator mutator, ListPopResult value) => + value.IsNull ? ListPopResult.Null : new(mutator.Map(value.Key), value.Values); +} diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index feaa65399..07e72d3b5 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -35,13 +35,14 @@ public class Builder private static readonly TimeSpan DefaultMetricsWindowSize = TimeSpan.FromSeconds(2); internal static CircuitBreaker DefaultInstance = new DefaultCircuitBreaker( - DefaultFailureRateThreshold, - DefaultMinimumNumberOfFailures, - DefaultMetricsWindowSize, +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list #if NET8_0_OR_GREATER - timeProvider: null, + null, #endif - trackedExceptions: null); +#pragma warning restore SA1114 + DefaultFailureRateThreshold, + DefaultMinimumNumberOfFailures, + DefaultMetricsWindowSize); /// /// Percentage of failures to trigger circuit breaker. @@ -73,34 +74,28 @@ public class Builder public CircuitBreaker Create() { if ((FailureRateThreshold is DefaultFailureRateThreshold - & MinimumNumberOfFailures is DefaultMinimumNumberOfFailures #if NET8_0_OR_GREATER & TimeProvider is null #endif - & TrackedExceptions is null) + & MinimumNumberOfFailures is DefaultMinimumNumberOfFailures) && MetricsWindowSize == DefaultMetricsWindowSize) return DefaultInstance; return new DefaultCircuitBreaker( - FailureRateThreshold, - MinimumNumberOfFailures, - MetricsWindowSize, +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list #if NET8_0_OR_GREATER TimeProvider, #endif - TrackedExceptions); +#pragma warning restore SA1114 + FailureRateThreshold, + MinimumNumberOfFailures, + MetricsWindowSize); } /// /// Create a new circuit-breaker instance. /// public static implicit operator CircuitBreaker(Builder builder) => builder.Create(); - - /// - /// Exceptions that count as failures. When null, - /// and are assumed. - /// - public Type[]? TrackedExceptions { get; set; } } /// @@ -108,30 +103,40 @@ public CircuitBreaker Create() /// public abstract Accumulator CreateAccumulator(); + internal static bool DefaultIsFailure(in FaultContext fault) + { + if (fault.ConnectionFailureType is not ConnectionFailureType.None) return true; + switch (fault.ErrorKind) + { + // what things *don't* trip the breaker? + case RedisErrorKind.None: // not even flagged + case RedisErrorKind.UnknownCommand: // application failure + case RedisErrorKind.ExecAbort: // transient to one command + case RedisErrorKind.WrongType: // application failure + case RedisErrorKind.NoPermission: // using the wrong keys? + case RedisErrorKind.UnknownError: // not sure what it is, but it starts ERR + case RedisErrorKind.Unknown: // pretty much anything we don't recognize; should we assume this is BAD? + return false; + default: + return true; + } + } + /// /// Collates observations for a connection. /// - public abstract class Accumulator + public abstract class Accumulator() { /// - /// Record a message outcome, and indicate whether the connection is considered healthy. + /// Record a message outcome. /// - /// True if the connection is still considered healthy. /// struct arg here is in case we want to add more things later - public abstract bool ObserveResult(in CircuitBreakerContext context); + public abstract void ObserveResult(in FaultContext fault); - internal bool ObserveResult(Exception? fault) - { - // only evaluate state upon failure; don't pay that overhead for success, just increment the counters - bool evaluate = fault is not null; - var ctx = new CircuitBreakerContext(fault, evaluate: evaluate); - bool healthy = ObserveResult(in ctx); - - // when we didn't ask for an evaluation, the returned verdict is meaningless: a custom - // implementation might return default(false) without having actually computed anything. - // never treat a non-evaluating observation as unhealthy - only a genuine evaluation may trip. - return !evaluate || healthy; - } + /// + /// Indicate whether a given fault should be considered a failure for the . + /// + protected virtual bool IsFailure(in FaultContext fault) => DefaultIsFailure(in fault); /// /// Indicate whether the connection is currently considered healthy, without recording an observation. @@ -143,34 +148,23 @@ internal bool ObserveResult(Exception? fault) /// Discard all accumulated observations, returning to a clean state. /// public abstract void Reset(); - } - /// - /// Provides information about a circuit-breaker test. - /// - public readonly struct CircuitBreakerContext(Exception? fault, bool evaluate = true) - { - /// - /// Was the operation a success. - /// - [MemberNotNullWhen(false, nameof(Fault))] - public bool Success => fault is null; - - /// - /// The fault associated with the operation. - /// - public Exception? Fault => fault; - - internal bool Evaluate => evaluate; - } + internal bool Trip(Exception? fault) + { + if (fault is not null) + { + var ctx = new FaultContext(fault); + if (IsFailure(ctx)) + { + ObserveResult(ctx); + return !IsHealthy(); + } + // otherwise, treat as success for the purposes of counting + } - private enum ExceptionStrategy - { - Default, - Any, - None, - CustomOpen, - CustomSealed, + ObserveResult(in FaultContext.Success); + return false; // never trip through success + } } private sealed class NulCircuitBreaker : CircuitBreaker @@ -183,15 +177,15 @@ private sealed class NulAccumulator : Accumulator { public static readonly NulAccumulator AccumulatorInstance = new(); private NulAccumulator() { } - public override bool ObserveResult(in CircuitBreakerContext context) => true; + public override void ObserveResult(in FaultContext context) { } public override bool IsHealthy() => true; public override void Reset() { } } + + // note we leave IsConnectionFault alone - that would impact RetryDatabase, where this is the key } private sealed class DefaultCircuitBreaker : CircuitBreaker { - private readonly ExceptionStrategy _trackingStrategy; - private readonly Type[] _trackedExceptions; private readonly double _failureRateThreshold; private readonly int _minimumNumberOfFailures; @@ -217,15 +211,13 @@ private long GetEpoch() => / _bucketTicks; public DefaultCircuitBreaker( - double failureRateThreshold, - int minimumNumberOfFailures, - TimeSpan metricsWindowSize, #if NET8_0_OR_GREATER TimeProvider? timeProvider, #endif - Type[]? trackedExceptions) + double failureRateThreshold, + int minimumNumberOfFailures, + TimeSpan metricsWindowSize) { - _trackedExceptions = CheckExceptions(trackedExceptions, out _trackingStrategy); _failureRateThreshold = failureRateThreshold; _minimumNumberOfFailures = minimumNumberOfFailures; @@ -302,11 +294,8 @@ public void Reset() } } - public override bool ObserveResult(in CircuitBreakerContext context) + public override void ObserveResult(in FaultContext result) { - // not-tracked failures (based on exception type) count as "success" for the purposes of circuit breaking - bool countAsSuccess = context.Success || !breaker.IsTracked(context.Fault); - // which time-slice are we in, and where does it live in the ring? long epoch = breaker.GetEpoch(); int index = (int)(epoch % BucketCount); @@ -318,9 +307,7 @@ public override bool ObserveResult(in CircuitBreakerContext context) // only getting results one at a time, so we shouldn't be over-stomping much *anyway* Span buckets = _buckets; // *not* a payload copy; this is in-place over the data ref Bucket bucket = ref buckets[index]; - bucket.Count(epoch, countAsSuccess); - - return !context.Evaluate || Evaluate(epoch); + bucket.Count(epoch, success: !result.IsFault); } public override bool IsHealthy() => Evaluate(breaker.GetEpoch()); @@ -362,90 +349,5 @@ public override void Reset() } } } - - private static Type[] CheckExceptions(Type[]? tracked, out ExceptionStrategy strategy) - { - if (tracked is null) - { - strategy = ExceptionStrategy.Default; - return []; - } - if (tracked.Length is 0) - { - strategy = ExceptionStrategy.None; - return []; - } - strategy = ExceptionStrategy.CustomOpen; - - static bool Contains(Type[] array, Type type) - { - for (int i = 0; i < array.Length; i++) - { - if (array[i] == type) return true; - } - - return false; - } - - // if we have Exception anywhere: we'll track everything - if (Contains(tracked, typeof(Exception))) - { - strategy = ExceptionStrategy.Any; - return []; - } - - if (tracked.Length is 2 - && Contains(tracked, typeof(RedisConnectionException)) - && Contains(tracked, typeof(RedisTimeoutException))) - { - strategy = ExceptionStrategy.Default; - return []; - } - - // finally, see if they're all sealed types - strategy = ExceptionStrategy.CustomSealed; - foreach (var exception in tracked) - { - if (!exception.IsSealed) - { - strategy = ExceptionStrategy.CustomOpen; - break; - } - } - return tracked; - } - - private bool IsTracked(Exception? fault) - { - if (fault is null) return false; - switch (_trackingStrategy) - { - case ExceptionStrategy.Any: - return true; - case ExceptionStrategy.Default: - return fault is RedisTimeoutException or RedisConnectionException; - case ExceptionStrategy.None: - return false; - } - - var span = _trackedExceptions.AsSpan(); - var actualType = fault.GetType(); - // check for exact matches - foreach (var testType in span) - { - if (ReferenceEquals(testType, actualType)) return true; - } - - if (_trackingStrategy is ExceptionStrategy.CustomOpen) - { - // we need to check for subclasses (more expensive) - foreach (var testType in span) - { - if (!testType.IsSealed && testType.IsAssignableFrom(actualType)) return true; - } - } - - return false; - } } } diff --git a/src/StackExchange.Redis/Availability/DatabaseExtensions.cs b/src/StackExchange.Redis/Availability/DatabaseExtensions.cs new file mode 100644 index 000000000..9733bf485 --- /dev/null +++ b/src/StackExchange.Redis/Availability/DatabaseExtensions.cs @@ -0,0 +1,20 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability +{ + /// + /// Provides availability-related extension methods (such as ) to database instances. + /// + public static class DatabaseExtensions + { + /// + /// Automatically retry operations when connection failure occurs. This has deep integration with + /// SE.Redis concepts, so can respond to server failover events, apply circuit-breaker rules, and + /// respect command effect categorization. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + public static IDatabaseAsync WithRetry(this IDatabaseAsync database, RetryPolicy retryPolicy) + => new RetryDatabase(database, retryPolicy); + } +} diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs new file mode 100644 index 000000000..4321630af --- /dev/null +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -0,0 +1,100 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Provides information about a circuit-breaker test. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public readonly struct FaultContext +{ + private readonly Exception? _fault; + private readonly ConnectionFailureType _connectionFailureType; + private readonly CommandFlags _flags; + + internal static readonly FaultContext Success = default; + + /// + /// Create a new . + /// + /// The fault associated with the operation, or null on success. + public FaultContext(Exception fault) + { + _fault = fault; + + var kind = RedisErrorKind.None; + _connectionFailureType = ConnectionFailureType.None; + var flags = CommandFlags.None; + switch (fault) + { + case RedisServerException server: + kind = server.Kind; + flags = server.Flags; + break; + case RedisConnectionException connection: + _connectionFailureType = connection.FailureType; + kind = RedisErrorKind.ConnectionFault; + flags = connection.Flags; + break; + case RedisTimeoutException timeout: + kind = RedisErrorKind.Timeout; + flags = timeout.Flags; + break; + case TimeoutException: + kind = RedisErrorKind.Timeout; + break; + } + + if (kind is not RedisErrorKind.None & _connectionFailureType is ConnectionFailureType.None) + { + // fill in some blanks + switch (kind) + { + case RedisErrorKind.Loading: + _connectionFailureType = ConnectionFailureType.Loading; + break; + case RedisErrorKind.NoAuth: + case RedisErrorKind.WrongPass: + _connectionFailureType = ConnectionFailureType.AuthenticationFailure; + break; + } + } + + flags &= Message.UserSelectableFlags; + if ((flags & Message.MaskRetryCategory) is 0) + { + // if no retry category found: assume the worst + flags |= CommandFlags.CommandRetryNever; + } + _flags = flags; + ErrorKind = kind; + } + + /// + /// Indicates whether a fault is present. + /// + [MemberNotNullWhen(true, nameof(Fault))] + public bool IsFault => _fault is not null; // excludes: default(FaultContext) + + /// + /// The fault associated with the operation. + /// + public Exception? Fault => _fault; + + /// + /// Get any command-flags associated with the operation. + /// + public CommandFlags Flags => _flags; + + /// + /// The classified server error condition associated with the fault, if any. + /// + public RedisErrorKind ErrorKind { get; } + + /// + /// The connection failure type associated with the fault, if any. + /// + public ConnectionFailureType ConnectionFailureType => _connectionFailureType; +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs index 77e0fe4d5..7bfc18033 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs @@ -1,15 +1,31 @@ using System; +using System.Threading; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.Availability; internal sealed partial class MultiGroupDatabase(MultiGroupMultiplexer parent, int database, object? asyncState) - : IDatabase + : IDatabase, IInternalDatabaseAsync { public object? AsyncState => asyncState; public int Database => database < 0 ? GetActiveDatabase().Database : database; public IConnectionMultiplexer Multiplexer => parent; + CancellationToken IInternalDatabaseAsync.GetNextFailover() => parent.GetNextFailover(); + + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + { + // fold in the currently-active member's features (when there is one) and label with its name + var active = TryGetActiveDatabase(); + var features = active is null ? DatabaseFeatureFlags.None : active.GetFeatures(out _); + name = ((IConnectionGroup)parent).ActiveMember?.Name ?? ""; + return features | DatabaseFeatureFlags.ConnectionGroup | DatabaseFeatureFlags.Failover; + } + + // uses the active member's name (via GetFeatures) when a member is active + public override string ToString() => this.BuildString(); + // for high DB numbers this might allocate even for null async-state scenarios; unavoidable for now private IDatabase GetActiveDatabase() => parent.Active.GetDatabase(database, asyncState); diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs index 378489cd4..4ff4eb872 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -320,12 +320,46 @@ internal void UpdateLatency() internal sealed partial class MultiGroupMultiplexer : IConnectionGroup { - private ConnectionMultiplexer? _active; + private ActiveStub _activeStub = new(null); + + private void SetActive(ConnectionMultiplexer? active) + { + ActiveStub? newObj = null; + while (true) + { + var oldObj = Volatile.Read(ref _activeStub); + + // is it already the same? + if (ReferenceEquals(oldObj.Active, active)) + { + newObj?.Dispose(); // never actually released to the world + return; // nothing to do! + } + + newObj ??= new(active); + if (ReferenceEquals(Interlocked.CompareExchange(ref _activeStub, newObj, oldObj), oldObj)) + { + // successful swap; flag the old one as failed-over + oldObj.Cancel(false); + return; + } + + // race? redo from start, but we can keep our newObj if we created one + } + } + + public CancellationToken GetNextFailover() => _activeStub.Token; + + private sealed class ActiveStub(ConnectionMultiplexer? active) : CancellationTokenSource + { + public ConnectionMultiplexer? Active => active; + } + private ConnectionGroupMember[] _members; public override string ToString() { - var muxer = _active; + var muxer = _activeStub.Active; ConnectionGroupMember? member = null; if (muxer is not null) { @@ -348,7 +382,7 @@ internal ConnectionMultiplexer Active { get { - return _active ?? Throw(); + return _activeStub.Active ?? Throw(); [DoesNotReturn] static ConnectionMultiplexer Throw() => @@ -357,12 +391,12 @@ static ConnectionMultiplexer Throw() => } // non-throwing twin of Active, for callers that have a trivial answer when the group is fully down - internal ConnectionMultiplexer? TryGetActive() => _active; + internal ConnectionMultiplexer? TryGetActive() => _activeStub.Active; // a completed "no endpoint" result, shared by the database/subscriber facades when the group is fully down internal static readonly Task NoEndpoint = Task.FromResult(null); - private ConnectionGroupMember? GetActiveMember() => GetMember(_active); + private ConnectionGroupMember? GetActiveMember() => GetMember(_activeStub.Active); private ConnectionGroupMember? GetMember(ConnectionMultiplexer? muxer) { @@ -433,7 +467,7 @@ private MultiGroupMultiplexer(ConnectionGroupMember[] members, MultiGroupOptions { _options = options; _members = members; - _active = null; + SetActive(null); _connectionFailedWithCircuitBreaker = OnMemberConnectionFailed; // multiplexers should already be attached (ConnectAsync sets them before constructing us) @@ -575,7 +609,7 @@ private long GetFailbackFailureCutoff() internal void SelectPreferredGroup() { if (_disposed) return; - var existingActive = _active; + var existingActive = _activeStub.Active; ConnectionGroupMember? preferredMember = null, previousMember = null; var members = _members; foreach (var member in members) @@ -594,7 +628,7 @@ internal void SelectPreferredGroup() } } - _active = preferredMember?.Multiplexer; + SetActive(preferredMember?.Multiplexer); if (preferredMember is not null && !ReferenceEquals(preferredMember, previousMember)) { @@ -610,7 +644,7 @@ internal void SelectPreferredGroup() private List DropAll() { _pollCancellation.Cancel(); // stop the polling loop promptly (idempotent) - _active = null; + SetActive(null); var members = Interlocked.Exchange(ref _members, []); if (members.Length is 0) return []; var muxers = new List(members.Length); @@ -671,8 +705,8 @@ public bool PreserveAsyncOrder // Unlike most members, these intentionally do *not* go via Active (which throws when no member is // available); callers routinely use IsConnected/IsConnecting as a pre-flight check and expect a // 'false' result - not an exception - when the entire group is down. - public bool IsConnected => _active?.IsConnected ?? false; - public bool IsConnecting => _active?.IsConnecting ?? false; + public bool IsConnected => _activeStub.Active?.IsConnected ?? false; + public bool IsConnecting => _activeStub.Active?.IsConnecting ?? false; [Obsolete] public bool IncludeDetailInExceptions diff --git a/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs index eb560004d..a029eef56 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs @@ -60,7 +60,7 @@ private static Action FilteredHandler(MultiGroupMultip // invoke a handler only if the active member is the one we expect return (channel, value) => { - if (ReferenceEquals(parent._active, active.Multiplexer)) + if (ReferenceEquals(parent._activeStub.Active, active.Multiplexer)) { handler(channel, value); } @@ -89,7 +89,7 @@ static async Task ForwardFilteredMessagesAsync( { while (readFrom.TryRead(out var message)) { - if (ReferenceEquals(parent._active, active.Multiplexer)) + if (ReferenceEquals(parent._activeStub.Active, active.Multiplexer)) { // Because of the switchover being imperfect, we can't guarantee exactly one writer, so // we need to be synchronized; in reality, it will *almost never* be contended, so diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs new file mode 100644 index 000000000..8dfa132a1 --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -0,0 +1,216 @@ +using System; +using System.Diagnostics; +using System.IO.Pipes; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; + +namespace StackExchange.Redis.Availability; + +[AutoDatabase] +internal partial class RetryDatabase : IDatabaseAsync, IRedisArgsMutator, IInternalDatabaseAsync +{ + // Note: we very deliberately do not include synchronous support for retry; it is inherently delay-ish + + // Note that only transient faults result in retries; this is defined by the RetryPolicy, along with + // understanding the category. The default RetryPolicy works the same as the default CircuitBreaker. + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + => _inner.GetFeatures(out name) | DatabaseFeatureFlags.Retry; + + /// + public override string ToString() => this.BuildString(); + + private readonly IDatabaseAsync _inner; + private readonly int _maxBeforeFailover, _maxAttempts, _delayMillis, _jitterMillis, _failoverMillis; + private readonly RetryPolicy _policy; + + public CancellationToken GetNextFailover() + => _maxAttempts > 1 & _maxBeforeFailover < _maxAttempts ? _inner.GetNextFailover() : CancellationToken.None; + + public RetryDatabase(IDatabaseAsync inner, RetryPolicy policy) + // cannot nest retry, and cannot issue retries *inside* a batch/transaction + : this(inner, policy, inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction | DatabaseFeatureFlags.Retry)) + { + } + + // test-only: supply the inner database's feature set directly (in particular whether failover is + // available), instead of probing a live inner - so that failover behaviour can be exercised over a + // null inner without a full IDatabaseAsync double. + internal RetryDatabase(IDatabaseAsync inner, RetryPolicy policy, DatabaseFeatureFlags features) + { + _policy = policy; + + // capture config locally rather than constant cross-object lookups (plus: mutability) + _maxBeforeFailover = (features & DatabaseFeatureFlags.Failover) == 0 ? int.MaxValue : policy.MaxAttemptsBeforeFailover; + _maxAttempts = policy.MaxAttempts; + if (_maxBeforeFailover == _maxAttempts) _maxBeforeFailover = int.MaxValue; // then we'll never look + + // guard the failover threshold: values < 1 can never be hit by the loop counter (which starts at 1), + // so they would *silently* disable failover rather than erroring; validate the raw policy value + if (policy.MaxAttemptsBeforeFailover < 1) throw new ArgumentOutOfRangeException(nameof(policy.MaxAttemptsBeforeFailover)); + _delayMillis = policy.DelayMilliseconds; + _failoverMillis = policy.FailoverMilliseconds; + _jitterMillis = policy.JitterMilliseconds; + if (_delayMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.RetryDelay)); + if (_jitterMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.JitterMax)); + if (_failoverMillis < 0) throw new ArgumentOutOfRangeException(nameof(policy.FailoverDelay)); + _inner = inner; + } + + public int Database => _inner.Database; + + public IConnectionMultiplexer Multiplexer => _inner.Multiplexer; + + // the generated explicit interface implementations funnel every call through these two + // overloads: the arguments are captured in a generated state struct and replayed against + // the inner database via a cacheable static projection (no per-call closure). Retry/failover + // policy will live here in due course; for now it is a straight pass-through. + + // async counterparts (Task / Task); these get their own retry/failover policy in due course. + private async Task ExecuteAsync(TState state, Func> operation) + where TState : struct, IRedisArgs + { + state.Map(this); + + int attempt = 0; + TResult result; + // note we need to capture this *before* the attempt - otherwise the failover could happen + // between the failed attempt and fetching this, and we'd miss it + CancellationToken failover = GetNextFailover(); + while (true) + { + try + { + result = await operation(state, _inner).ConfigureAwait(false); + break; + } + catch (Exception ex) when (CanRetry(++attempt, ex, ref failover, out var delay)) + { + await FailoverOrDelayAsync(delay).ConfigureAwait(false); + } + } + // post-process results outside the loop + return this.UnMap(state, result); + } + + private async Task ExecuteAsync(TState state, Func operation) + where TState : struct, IRedisArgs + { + state.Map(this); + + int attempt = 0; + // note we need to capture this *before* the attempt - otherwise the failover could happen + // between the failed attempt and fetching this, and we'd miss it + CancellationToken failover = GetNextFailover(); + while (true) + { + try + { + await operation(state, _inner).ConfigureAwait(false); + break; + } + catch (Exception ex) when (CanRetry(++attempt, ex, ref failover, out var delay)) + { + await FailoverOrDelayAsync(delay).ConfigureAwait(false); + } + } + // (nothing to post-process) + } + + internal bool CanRetry( + int attempt, + Exception fault, + ref CancellationToken failover, + out CancellationToken delay) + { + delay = CancellationToken.None; + if (attempt >= _maxAttempts) + { + // all used up + return false; + } + + // ask the retry policy for advice, and mask off the bits we know about + FaultContext ctx = new(fault); + var policy = _policy.CanRetry(ctx) & + (RetryPolicy.RetryPolicyResult.FailoverServer | RetryPolicy.RetryPolicyResult.SameServer); + if (policy is 0) + { + // retry policy says: nope + return false; + } + + if (policy is RetryPolicy.RetryPolicyResult.FailoverServer) + { + // we can *only* retry on a different server; is failover available? + delay = failover; + failover = CancellationToken.None; // only failover once + return delay.CanBeCanceled; + } + + if (attempt == _maxBeforeFailover) + { + // by count, we should really switch over to the failover now; is failover available *and* are we allowed? + delay = failover; + failover = CancellationToken.None; // only failover once + return delay.CanBeCanceled & (policy & RetryPolicy.RetryPolicyResult.FailoverServer) != 0; + } + + // can we pause and retry on the same server? + return (policy & RetryPolicy.RetryPolicyResult.SameServer) != 0; + } + + private Task FailoverOrDelayAsync(CancellationToken delay) + { + if (delay.CanBeCanceled) + { + return AwaitFailover(delay); + } + + // this is just a routine wait between operations; await delay+jitter + return Task.Delay(_delayMillis + ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None); + } + private async Task AwaitFailover(CancellationToken failover) + { + if (!failover.IsCancellationRequested) + { + // failover hasn't happened yet; allow up to "delay" time for that + try + { + await Task.Delay(_failoverMillis, failover).ConfigureAwait(false); + } + catch (OperationCanceledException) when (failover.IsCancellationRequested) + { + // we observed a failover, nice! + } + } + + // either way, we need to add jitter onto that; we can't add in the original delay, because if the failover + // happened before the timeout+jitter, all the awaiters would stampede + await Task.Delay(ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None).ConfigureAwait(false); + } + + void IRedisAsync.Wait(Task task) => _inner.Wait(task); + T IRedisAsync.Wait(Task task) => _inner.Wait(task); + void IRedisAsync.WaitAll(Task[] tasks) => _inner.WaitAll(tasks); + bool IRedisAsync.TryWait(Task task) => _inner.TryWait(task); + + // Methods the generator deliberately skips (see AutoDatabaseGenerator.SkipMethod): the Wait + // family, the synchronous IsConnected probe, and the streaming IEnumerable/IAsyncEnumerable scans + // don't fit the capture-and-replay shape, so they are implemented by hand. + // IsConnected is a straight pass-through: it is a cheap status check, not a server round-trip to retry. + bool IDatabaseAsync.IsConnected(RedisKey key, CommandFlags flags) => _inner.IsConnected(key, flags); + + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.HashScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.HashScanNoValuesAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.SetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.VectorSetRangeEnumerateAsync(RedisKey key, RedisValue start, RedisValue end, long count, Exclude exclude, CommandFlags flags) => throw new NotImplementedException(); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.SortedSetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotImplementedException(); + + RedisKey IRedisArgsMutator.Map(RedisKey key) => key; + + RedisChannel IRedisArgsMutator.Map(RedisChannel channel) => channel; + + RedisKey IRedisArgsMutator.UnMap(RedisKey key) => key; + RedisChannel IRedisArgsMutator.UnMap(RedisChannel channel) => channel; +} diff --git a/src/StackExchange.Redis/Availability/RetryPolicy.cs b/src/StackExchange.Redis/Availability/RetryPolicy.cs new file mode 100644 index 000000000..447424b2f --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryPolicy.cs @@ -0,0 +1,134 @@ +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Configures how messages can be retried due to connection / transient faults. Other faults (such as invalid +/// usage) are not retried. +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public class RetryPolicy +{ + /// + /// The maximum number of times an operation can be attempted. Defaults to 3. + /// + public int MaxAttempts { get; set; } = 3; + + /// + /// The maximum number of times to retry an operation before waiting for failover; this only currently + /// applies to multi-group connections created via ConnectionMultiplexer.ConnectGroupAsync. + /// Defaults to 1. + /// + public int MaxAttemptsBeforeFailover { get; set; } = 1; + + private int _delayMillis = 1000, _jitterMillis = 500, _failoverMillis = 5000; + + /// + /// Gets the time to wait between retries that are *not* dependent on a failover happening. Defaults to 1 second. + /// + public TimeSpan RetryDelay + { + get => TimeSpan.FromMilliseconds(_delayMillis); + set => _delayMillis = checked((int)value.TotalMilliseconds); + } + + /// + /// Gets the time to wait for a failover, after attempts. Only one + /// failover attempt is awaited. When this applies, is ignored, + /// but is still respected. Defaults to 5 seconds. + /// + public TimeSpan FailoverDelay + { + get => TimeSpan.FromMilliseconds(_failoverMillis); + set => _failoverMillis = checked((int)value.TotalMilliseconds); + } + + /// + /// Gets or sets the upper bound for jitter - additional random delay between retries to prevent stampedes. + /// Defaults to 0.5 seconds, meaning between 0 and 0.5 *additional* seconds on top of . + /// + public TimeSpan JitterMax + { + get => TimeSpan.FromMilliseconds(_jitterMillis); + set => _jitterMillis = checked((int)value.TotalMilliseconds); + } + + internal int DelayMilliseconds => _delayMillis; + internal int JitterMilliseconds => _jitterMillis; + internal int FailoverMilliseconds => _failoverMillis; + + /// + /// Gets or sets the max side-effect category that will be retried; defaults to . + /// + public CommandFlags MaxCommandRetryCategory + { + get => _maxCommandRetryCategory; + set + { + if ((value & Message.MaskRetryCategory) is 0 | (value & ~Message.MaskRetryCategory) is not 0) + throw new InvalidOperationException("Valid CommandRetry* flags should be specified"); + _maxCommandRetryCategory = value; + } + } + + private CommandFlags _maxCommandRetryCategory = CommandFlags.CommandRetryWriteLastWins; + + /// + /// Controls which operations can be repeated, optionally indicating that this should progress to + /// a new server. + /// + public virtual RetryPolicyResult CanRetry(in FaultContext fault) + { + var actual = fault.Flags & Message.MaskRetryCategory; + if (actual is 0) actual = CommandFlags.CommandRetryWriteAccumulating; // if not set, assume similar to INCR + + if (actual is CommandFlags.CommandRetryNever) + { + // explicitly disabled at command level + return RetryPolicyResult.None; + } + + if (actual > MaxCommandRetryCategory) // note this also covers CommandRetryAlways + { + // side-effects are beyond what the policy allows + return RetryPolicyResult.None; + } + + if (CircuitBreaker.DefaultIsFailure(in fault)) + { + // assume we can send it everywhere + var result = RetryPolicyResult.SameServer | RetryPolicyResult.FailoverServer; + if ((fault.Flags & Message.CommandServerSpecific) != 0) + result &= ~RetryPolicyResult.FailoverServer; + return result; + } + + // do not retry + return RetryPolicyResult.None; + } + + /// + /// Indicates the result of a query. + /// + [Flags] + public enum RetryPolicyResult + { + /// + /// None; the operation should not be retried. + /// + None = 0, + + /// + /// The operation can be retried on the same server. + /// + SameServer = 1, + + /// + /// The operation can be retried on a different server after a failover operation. + /// + FailoverServer = 2, + } +} diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs index 6f11fa2f3..658f6920c 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs @@ -169,6 +169,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, "Sentinel: The ConnectionMultiplexer is not a Sentinel connection. Detected as: " + ServerSelectionStrategy.ServerType); } @@ -204,6 +205,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, $"Sentinel: Failed connecting to configured primary for service: {config.ServiceName}"); } @@ -271,6 +273,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, $"Sentinel: Failed connecting to configured primary for service: {config.ServiceName}"); } @@ -418,7 +421,7 @@ internal void SwitchPrimary(EndPoint? switchBlame, ConnectionMultiplexer connect // Get new primary - try twice EndPoint newPrimaryEndPoint = GetConfiguredPrimaryForService(serviceName) ?? GetConfiguredPrimaryForService(serviceName) - ?? throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, $"Sentinel: Failed connecting to switch primary for service: {serviceName}"); + ?? throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, CommandFlags.None, $"Sentinel: Failed connecting to switch primary for service: {serviceName}"); connection.currentSentinelPrimaryEndPoint = newPrimaryEndPoint; diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 769a64b50..86fd7783d 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2036,7 +2036,7 @@ private WriteResult TryPushMessageToBridgeSync(Message message, ResultProcess WriteResult.Success => throw new ArgumentOutOfRangeException(nameof(result), "Be sure to check result isn't successful before calling GetException."), WriteResult.NoConnectionAvailable => ExceptionFactory.NoConnectionAvailable(this, message, server), WriteResult.TimeoutBeforeWrite => ExceptionFactory.Timeout(this, null, message, server, result, bridge), - _ => ExceptionFactory.ConnectionFailure(RawConfig.IncludeDetailInExceptions, ConnectionFailureType.ProtocolFailure, "An unknown error occurred when writing the message", server), + _ => ExceptionFactory.ConnectionFailure(RawConfig.IncludeDetailInExceptions, ConnectionFailureType.ProtocolFailure, message.Flags, "An unknown error occurred when writing the message", server), }; [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA1816:Dispose methods should call SuppressFinalize", Justification = "Intentional observation")] diff --git a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs new file mode 100644 index 000000000..02abc3b8b --- /dev/null +++ b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs @@ -0,0 +1,373 @@ +namespace StackExchange.Redis; + +internal static class CommandFlagsExtensions +{ + public static CommandFlags WithCategory(this CommandFlags flags, CommandFlags category) + // if the user hasn't already specified a category: use the category supplied + => ((flags & Message.MaskRetryCategory) is 0) ? flags | (category & Message.MaskRetryCategory) : flags; + + public static CommandFlags WithDefaultCategory(this CommandFlags flags, RedisCommand command) + { + if ((flags & Message.MaskRetryCategory) is 0) + { + // Get the suggested flags; note that the user might have included CommandServerSpecific, + // but we *also* suggest that below - we'll live with it, additively. + // Note also that some commands may have *conditionally* included their category based on + // rules specific to the parameters, for example SCAN 0 is not server specific, + // but SCAN 12341234 *is*. + flags |= DefaultCategory(command); + } + + return flags; + + static CommandFlags DefaultCategory(RedisCommand command) + { + // This is *not* using switch expressions very deliberately, because there are a *lot* of + // options in each; let's keep things vertical rather than horizontal. + switch (command) + { + // ========================================================================== + // CONNECTION / SESSION — node-agnostic, no keyspace side effects, safe to + // replay on a fresh connection. + // ========================================================================== + case RedisCommand.PING: + case RedisCommand.ECHO: + case RedisCommand.AUTH: + case RedisCommand.HELLO: + case RedisCommand.SELECT: + case RedisCommand.QUIT: + case RedisCommand.SUBSCRIBE: + case RedisCommand.UNSUBSCRIBE: + case RedisCommand.PSUBSCRIBE: + case RedisCommand.PUNSUBSCRIBE: + case RedisCommand.SSUBSCRIBE: + case RedisCommand.SUNSUBSCRIBE: + case RedisCommand.INFO: + case RedisCommand.TIME: + case RedisCommand.DBSIZE: + case RedisCommand.LASTSAVE: + case RedisCommand.COMMAND: + return CommandFlags.CommandRetryConnection; + + // CLIENT etc often use server-specific IDs + case RedisCommand.CLIENT: // note some can be considered admin, overridden locally + return CommandFlags.CommandRetryConnection | Message.CommandServerSpecific; + + // ========================================================================== + // READ-ONLY — no mutation, always safe to retry. + // ========================================================================== + case RedisCommand.GET: + case RedisCommand.MGET: + case RedisCommand.STRLEN: + case RedisCommand.GETRANGE: + case RedisCommand.EXISTS: + case RedisCommand.TYPE: + case RedisCommand.TTL: + case RedisCommand.PTTL: + case RedisCommand.EXPIRETIME: + case RedisCommand.PEXPIRETIME: + case RedisCommand.KEYS: + case RedisCommand.RANDOMKEY: + case RedisCommand.DUMP: + case RedisCommand.TOUCH: // technically bumps LRU/LFU state, but that's not a "real" side effect worth blocking retries over + case RedisCommand.OBJECT: + case RedisCommand.MEMORY: + case RedisCommand.SORT_RO: + case RedisCommand.SORT: // ignoring the STORE variant + case RedisCommand.LCS: + case RedisCommand.GETEX: // ignoring the TTL-mutating option variants + case RedisCommand.HGET: + case RedisCommand.HMGET: + case RedisCommand.HGETALL: + case RedisCommand.HKEYS: + case RedisCommand.HVALS: + case RedisCommand.HLEN: + case RedisCommand.HEXISTS: + case RedisCommand.HSTRLEN: + case RedisCommand.HRANDFIELD: + case RedisCommand.HPTTL: + case RedisCommand.HEXPIRETIME: + case RedisCommand.HPEXPIRETIME: + case RedisCommand.HGETEX: // ignoring TTL-mutating option variants + case RedisCommand.LLEN: + case RedisCommand.LRANGE: + case RedisCommand.LINDEX: + case RedisCommand.LPOS: + case RedisCommand.SMEMBERS: + case RedisCommand.SCARD: + case RedisCommand.SISMEMBER: + case RedisCommand.SMISMEMBER: + case RedisCommand.SRANDMEMBER: + case RedisCommand.SDIFF: + case RedisCommand.SINTER: + case RedisCommand.SINTERCARD: + case RedisCommand.SUNION: + case RedisCommand.ZCARD: + case RedisCommand.ZSCORE: + case RedisCommand.ZMSCORE: + case RedisCommand.ZRANK: + case RedisCommand.ZREVRANK: + case RedisCommand.ZCOUNT: + case RedisCommand.ZLEXCOUNT: + case RedisCommand.ZRANGE: + case RedisCommand.ZREVRANGE: + case RedisCommand.ZRANGEBYSCORE: + case RedisCommand.ZREVRANGEBYSCORE: + case RedisCommand.ZRANGEBYLEX: + case RedisCommand.ZREVRANGEBYLEX: + case RedisCommand.ZRANDMEMBER: + case RedisCommand.ZDIFF: + case RedisCommand.ZINTER: + case RedisCommand.ZUNION: + case RedisCommand.ZINTERCARD: + case RedisCommand.BITCOUNT: + case RedisCommand.BITPOS: + case RedisCommand.GETBIT: + case RedisCommand.PFCOUNT: + case RedisCommand.GEOPOS: + case RedisCommand.GEODIST: + case RedisCommand.GEOHASH: + case RedisCommand.GEOSEARCH: + case RedisCommand.XLEN: + case RedisCommand.XRANGE: + case RedisCommand.XREVRANGE: + case RedisCommand.XREAD: // group-less read only; XREADGROUP is handled separately below + case RedisCommand.XPENDING: + case RedisCommand.XINFO: + case RedisCommand.EVAL_RO: + case RedisCommand.EVALSHA_RO: + return CommandFlags.CommandRetryReadOnly; + + // PUBSUB/SCAN/etc are *basically* read-only, but make limited sense between nodes + case RedisCommand.PUBSUB: + case RedisCommand.SCAN: + case RedisCommand.ZSCAN: + case RedisCommand.SSCAN: + case RedisCommand.HSCAN: + return CommandFlags.CommandRetryReadOnly | Message.CommandServerSpecific; + + // ========================================================================== + // WRITE - CHECKED — inherently conditional/idempotent; a retry either + // no-ops or fails in a way that leaves the end-state identical. + // ========================================================================== + case RedisCommand.SETNX: + case RedisCommand.MSETNX: + case RedisCommand.HSETNX: + case RedisCommand.DEL: + case RedisCommand.UNLINK: + case RedisCommand.PERSIST: + case RedisCommand.RENAMENX: + case RedisCommand.COPY: // ignoring REPLACE — default behavior fails if dest exists + case RedisCommand.MOVE: + case RedisCommand.RESTORE: // ignoring REPLACE + case RedisCommand.GETDEL: + case RedisCommand.HGETDEL: + case RedisCommand.HPERSIST: + case RedisCommand.LTRIM: + case RedisCommand.SADD: + case RedisCommand.SREM: + case RedisCommand.SMOVE: + case RedisCommand.ZREM: + case RedisCommand.ZREMRANGEBYRANK: + case RedisCommand.ZREMRANGEBYSCORE: + case RedisCommand.ZREMRANGEBYLEX: + case RedisCommand.HDEL: + case RedisCommand.PFADD: + case RedisCommand.XDEL: + case RedisCommand.XTRIM: + case RedisCommand.XACK: + case RedisCommand.XGROUP: + case RedisCommand.XNACK: + return CommandFlags.CommandRetryWriteChecked; + + // ========================================================================== + // WRITE - LAST WINS — unconditional overwrite of a specific value/state; + // repeating with the same args always converges to the same final value. + // ========================================================================== + case RedisCommand.SET: + case RedisCommand.GETSET: + case RedisCommand.MSET: + case RedisCommand.SETEX: + case RedisCommand.PSETEX: + case RedisCommand.SETRANGE: + case RedisCommand.SETBIT: + case RedisCommand.BITOP: + case RedisCommand.RENAME: + case RedisCommand.EXPIRE: + case RedisCommand.PEXPIRE: + case RedisCommand.EXPIREAT: + case RedisCommand.PEXPIREAT: + case RedisCommand.HSET: + case RedisCommand.HMSET: + case RedisCommand.HEXPIRE: + case RedisCommand.HPEXPIRE: + case RedisCommand.HEXPIREAT: + case RedisCommand.HPEXPIREAT: + case RedisCommand.LSET: + case RedisCommand.ZADD: // ignoring INCR option + case RedisCommand.ZRANGESTORE: + case RedisCommand.ZUNIONSTORE: + case RedisCommand.ZINTERSTORE: + case RedisCommand.ZDIFFSTORE: + case RedisCommand.SDIFFSTORE: + case RedisCommand.SINTERSTORE: + case RedisCommand.SUNIONSTORE: + case RedisCommand.GEOADD: + case RedisCommand.GEOSEARCHSTORE: + case RedisCommand.GEORADIUS: // because of store scenarios + case RedisCommand.GEORADIUSBYMEMBER: + case RedisCommand.XCLAIM: + case RedisCommand.XAUTOCLAIM: + return CommandFlags.CommandRetryWriteLastWins; + + // ========================================================================== + // WRITE - ACCUMULATING — effect compounds with every additional call. + // ========================================================================== + case RedisCommand.INCR: + case RedisCommand.DECR: + case RedisCommand.INCRBY: + case RedisCommand.DECRBY: + case RedisCommand.INCRBYFLOAT: + case RedisCommand.APPEND: + case RedisCommand.HINCRBY: + case RedisCommand.HINCRBYFLOAT: + case RedisCommand.ZINCRBY: + case RedisCommand.LPUSH: + case RedisCommand.RPUSH: + case RedisCommand.LPUSHX: + case RedisCommand.RPUSHX: + case RedisCommand.LINSERT: + case RedisCommand.LREM: + case RedisCommand.LPOP: + case RedisCommand.RPOP: + case RedisCommand.RPOPLPUSH: + case RedisCommand.LMOVE: + case RedisCommand.LMPOP: + case RedisCommand.SPOP: + case RedisCommand.ZPOPMIN: + case RedisCommand.ZPOPMAX: + case RedisCommand.ZMPOP: + case RedisCommand.XADD: + case RedisCommand.PFMERGE: // destination accumulates union each call + return CommandFlags.CommandRetryWriteAccumulating; + + // ========================================================================== + // SERVER ADMIN / NODE-SPECIFIC + // ========================================================================== + case RedisCommand.REPLICAOF: + case RedisCommand.SLAVEOF: + case RedisCommand.BGSAVE: + case RedisCommand.BGREWRITEAOF: + case RedisCommand.SAVE: + case RedisCommand.SHUTDOWN: + case RedisCommand.FLUSHALL: + case RedisCommand.FLUSHDB: + case RedisCommand.SWAPDB: + case RedisCommand.MIGRATE: + case RedisCommand.DEBUG: + case RedisCommand.MONITOR: + case RedisCommand.CONFIG: // note CONFIG GET con be considered more safe + case RedisCommand.SLOWLOG: + case RedisCommand.LATENCY: + case RedisCommand.SCRIPT: + case RedisCommand.CLUSTER: // note: some like MYID can be considered more safe + return CommandFlags.CommandRetryServerAdmin | Message.CommandServerSpecific; + + // ========================================================================== + // NEVER — transactions, arbitrary scripts, and blocking/destructive or + // fire-and-forget commands where a lost ack makes blind retry dangerous. + // ========================================================================== + case RedisCommand.MULTI: + case RedisCommand.EXEC: + case RedisCommand.DISCARD: + case RedisCommand.WATCH: + case RedisCommand.UNWATCH: + case RedisCommand.PUBLISH: + case RedisCommand.SPUBLISH: + case RedisCommand.XREADGROUP: + case RedisCommand.BLPOP: + case RedisCommand.BRPOP: + case RedisCommand.BRPOPLPUSH: + return CommandFlags.CommandRetryNever; + + // ========================================================================== + // scripts / modules / functions; we're going to assume nothing too weird, + // at worst similar to INCR; but it is *hoped* that callers will supply hints. + // ========================================================================== + case RedisCommand.EVAL: + case RedisCommand.EVALSHA: + return CommandFlags.CommandRetryWriteAccumulating; + + // ---- CONNECTION / SESSION ---- + case RedisCommand.ASKING: // cluster ASK redirection flag - per-connection state + case RedisCommand.READONLY: // cluster client read-routing flag - per-connection state + case RedisCommand.READWRITE: // cluster client read-routing flag - per-connection state + case RedisCommand.ROLE: // replication role report, like INFO - no keyspace effect + return CommandFlags.CommandRetryConnection; + + // ---- READ-ONLY ---- + case RedisCommand.ARCOUNT: + case RedisCommand.ARINFO: // structure metadata, like INFO + case RedisCommand.ARGET: + case RedisCommand.ARGETRANGE: + case RedisCommand.ARGREP: + case RedisCommand.ARLASTITEMS: + case RedisCommand.ARLEN: + case RedisCommand.ARMGET: + case RedisCommand.ARSCAN: + case RedisCommand.AROP: + case RedisCommand.DIGEST: // computes a hash of existing data, doesn't mutate it + case RedisCommand.VCARD: + case RedisCommand.VDIM: + case RedisCommand.VEMB: + case RedisCommand.VGETATTR: + case RedisCommand.VINFO: + case RedisCommand.VISMEMBER: + case RedisCommand.VLINKS: + case RedisCommand.VRANDMEMBER: + case RedisCommand.VRANGE: + case RedisCommand.VSIM: + return CommandFlags.CommandRetryReadOnly; + + // ---- WRITE - CHECKED ---- + case RedisCommand.DELEX: // conditional/expiry-aware delete - converges same as DEL + case RedisCommand.VADD: // idempotent add-member, like SADD + case RedisCommand.VREM: // idempotent remove-member, like SREM/ZREM + case RedisCommand.XACKDEL: // ack+delete, converges like XACK/XDEL combined + case RedisCommand.XDELEX: + return CommandFlags.CommandRetryWriteChecked; + + // ---- WRITE - LAST WINS ---- + case RedisCommand.ARDEL: + case RedisCommand.ARDELRANGE: + case RedisCommand.ARMSET: + case RedisCommand.ARSEEK: // repositions a cursor to an explicit point - overwrite semantics + case RedisCommand.ARSET: + case RedisCommand.HSETEX: // HSET + expiry, unconditional overwrite + case RedisCommand.MSETEX: + case RedisCommand.VSETATTR: // unconditional attribute overwrite + case RedisCommand.XCFGSET: + return CommandFlags.CommandRetryWriteLastWins; + + // ---- WRITE - ACCUMULATING ---- + case RedisCommand.ARRING: // presumed create/configure-ring, unconditional define + case RedisCommand.ARINSERT: // ring-buffer insert, compounds like a push + case RedisCommand.ARNEXT: // advances a cursor/position with each call + case RedisCommand.INCREX: // counter semantics + expiry, still accumulating + return CommandFlags.CommandRetryWriteAccumulating; + + // ---- SERVER ADMIN / NODE-SPECIFIC ---- + case RedisCommand.HOTKEYS: // diagnostic/introspection, node-local + case RedisCommand.SENTINEL: + case RedisCommand.SYNC: // replication stream handshake + return CommandFlags.CommandRetryServerAdmin | Message.CommandServerSpecific; + + // if we don't recognize it: default to the most pessimistic + case RedisCommand.NONE: + case RedisCommand.UNKNOWN: + default: + return CommandFlags.CommandRetryNever; + } + } + } +} diff --git a/src/StackExchange.Redis/Enums/CommandFlags.cs b/src/StackExchange.Redis/Enums/CommandFlags.cs index ff69fb750..510510c6b 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.cs @@ -1,5 +1,7 @@ using System; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using RESPite; namespace StackExchange.Redis { @@ -111,5 +113,79 @@ public enum CommandFlags // 2048: Use subscription connection type; never user-specified, so not visible on the public API // 4096: Identifies handshake completion messages; never user-specified, so not visible on the public API + + /* + Command side-effect/retry logic: + The values below reserve a chunk of bits (bits 13-17, i.e. << 13) for command + retry logic/categorization; by default, nothing in this chunk will be passed and + the library will supply a value based on the command being issued. The caller can + override, though - ultimately it is their data/server! They will need to supply a + suitable value for Execute[Async] etc, otherwise we'll assume the worst. + + The math here is intended so that we can specify a numeric "max" that we'll + retry, and can filter with <=, so the higher numbers have more side-effects. + As such, this region is not flags per-se, and we are intentionally leaving gaps + to slide other things in later. Note that 0 is the implicit "not specified" value + (distinct from CommandRetryAlways), and must be resolved to a concrete category + downstream before any <= comparison. CommandServerSpecific (bit 18) is a genuine + single-bit flag, orthogonal to the severity ladder. + */ + + /// + /// The command is always safe to retry, regardless of connection or server state. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CommandRetryAlways = 1 << 13, // pre-shift value 1 + + /// + /// The command relates to the connection or to safe metadata (for example CLIENT SETNAME, + /// COMMAND, or CONFIG GET) and can be retried at the connection level. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CommandRetryConnection = 4 << 13, // pre-shift value 4 + + /// + /// The command only reads data (for example GET) and can be safely retried. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CommandRetryReadOnly = 8 << 13, // pre-shift value 8 + + /// + /// The command writes data conditionally (for example SETNX or SET ... IFEQ), so a retry + /// is checked against server state. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CommandRetryWriteChecked = 12 << 13, // pre-shift value 12 + + /// + /// The command writes data such that a retry simply overwrites (last-writer-wins, for example SET). + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CommandRetryWriteLastWins = 16 << 13, // pre-shift value 16 + + /// + /// The command writes data cumulatively (for example INCR or LPOP), so a retry can + /// double-apply and change the result. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CommandRetryWriteAccumulating = 20 << 13, // pre-shift value 20 + + /// + /// The command performs server administration (for example REPLICAOF or CONFIG SET); these + /// are commonly also endpoint-specific (the internal server-specific flag). + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CommandRetryServerAdmin = 24 << 13, // pre-shift value 24 + + /// + /// The command should never be retried. + /// + [Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] + CommandRetryNever = 31 << 13, // pre-shift value 31 (the full retry-category region) + + // 262144 (bit 18): "server specific" - the command is tied to a specific endpoint and must never be + // retried on a different endpoint (for example cursor-based operations); orthogonal to the retry-category + // region. Internal-only (see Message.CommandServerSpecific): the wrapper database cannot yet express + // endpoint-stickiness over a *range* of operations, so this is not (currently) on the public API. } } diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index 380ac0c0f..8428d41af 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -33,9 +33,9 @@ internal static Exception TooManyArgs(string command, int argCount) internal static Exception CommandHasWhitespace(string command) => new RedisCommandException($"The command '{command}' contains whitespace and would be sent as a single unknown token; pass each word as a separate argument, for example Execute(\"ACL\", \"SETUSER\", \"x\") rather than Execute(\"ACL SETUSER x\")."); - internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint? server) + internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, CommandFlags flags, string message, ServerEndPoint? server) { - var ex = new RedisConnectionException(failureType, message); + var ex = new RedisConnectionException(failureType, flags, message); if (includeDetail) AddExceptionDetail(ex, null, server, null); return ex; } @@ -154,7 +154,7 @@ internal static Exception NoConnectionAvailable( data = new List>(); AddCommonDetail(data, sb, message, multiplexer, server); } - var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); + var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, message?.Flags ?? CommandFlags.None, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); if (multiplexer.RawConfig.IncludeDetailInExceptions) { CopyDataToException(data, ex); @@ -283,12 +283,13 @@ internal static Exception Timeout(ConnectionMultiplexer multiplexer, string? bas // If we're from a backlog timeout scenario, we log a more intuitive connection exception for the timeout...because the timeout was a symptom // and we have a more direct cause: we had no connection to send it on. + var msgFlags = message?.Flags ?? CommandFlags.CommandRetryNever; Exception ex = logConnectionException && lastConnectionException is not null - ? new RedisConnectionException(lastConnectionException.FailureType, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) + ? new RedisConnectionException(lastConnectionException.FailureType, msgFlags, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, } - : new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) + : new RedisTimeoutException(msgFlags, sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, }; @@ -448,7 +449,7 @@ internal static Exception UnableToConnect(ConnectionMultiplexer muxer, string? f sb.Append(' ').Append(failureMessage.Trim()); } - return new RedisConnectionException(failureType, sb.ToString(), inner); + return new RedisConnectionException(failureType, CommandFlags.None, sb.ToString(), inner); } } } diff --git a/src/StackExchange.Redis/Exceptions.cs b/src/StackExchange.Redis/Exceptions.cs index 1f1c973ce..405a0ea4e 100644 --- a/src/StackExchange.Redis/Exceptions.cs +++ b/src/StackExchange.Redis/Exceptions.cs @@ -25,7 +25,6 @@ public RedisCommandException(string message, Exception innerException) : base(me #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisCommandException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } } @@ -41,8 +40,19 @@ public sealed partial class RedisTimeoutException : TimeoutException /// /// The message for the exception. /// The command status, as of when the timeout happened. - public RedisTimeoutException(string message, CommandStatus commandStatus) : base(message) + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisTimeoutException(string message, CommandStatus commandStatus) : this(CommandFlags.CommandRetryNever, message, commandStatus) { } + + /// + /// Creates a new . + /// + /// The command-flags associated with the faulting operation. + /// The message for the exception. + /// The command status, as of when the timeout happened. + public RedisTimeoutException(CommandFlags flags, string message, CommandStatus commandStatus) : base(message) { + Flags = flags; Commandstatus = commandStatus; } @@ -51,9 +61,13 @@ public RedisTimeoutException(string message, CommandStatus commandStatus) : base /// public CommandStatus Commandstatus { get; } + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public CommandFlags Flags { get; } + #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisTimeoutException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { @@ -67,7 +81,7 @@ private RedisTimeoutException(SerializationInfo info, StreamingContext ctx) : ba /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] #endif public override void GetObjectData(SerializationInfo info, StreamingContext context) { @@ -87,7 +101,9 @@ public sealed partial class RedisConnectionException : RedisException /// /// The type of connection failure. /// The message for the exception. - public RedisConnectionException(ConnectionFailureType failureType, string message) : this(failureType, message, null, CommandStatus.Unknown) { } + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message) : this(failureType, CommandFlags.CommandRetryNever, message, null, CommandStatus.Unknown) { } /// /// Creates a new . @@ -95,18 +111,33 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag /// The type of connection failure. /// The message for the exception. /// The inner exception. - public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException) : this(failureType, message, innerException, CommandStatus.Unknown) { } + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException) : this(failureType, CommandFlags.CommandRetryNever, message, innerException, CommandStatus.Unknown) { } + + /// + /// Creates a new . + /// + /// The type of connection failure. + /// The message for the exception. + /// The inner exception. + /// The status of the command. + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException, CommandStatus commandStatus) : this(failureType, CommandFlags.CommandRetryNever, message, innerException, commandStatus) { } /// /// Creates a new . /// /// The type of connection failure. + /// The command-flags associated with the faulting operation. /// The message for the exception. /// The inner exception. /// The status of the command. - public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException, CommandStatus commandStatus) : base(message, innerException) + public RedisConnectionException(ConnectionFailureType failureType, CommandFlags flags, string message, Exception? innerException = null, CommandStatus commandStatus = CommandStatus.Unknown) : base(message, innerException) { FailureType = failureType; + Flags = flags; CommandStatus = commandStatus; } @@ -115,6 +146,11 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag /// public ConnectionFailureType FailureType { get; } + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public CommandFlags Flags { get; } + /// /// Status of the command while communicating with Redis. /// @@ -122,7 +158,6 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisConnectionException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { @@ -137,7 +172,7 @@ private RedisConnectionException(SerializationInfo info, StreamingContext ctx) : /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] #endif public override void GetObjectData(SerializationInfo info, StreamingContext context) { @@ -173,7 +208,6 @@ public RedisException(string message, Exception? innerException) : base(message, /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif protected RedisException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } } @@ -188,12 +222,35 @@ public sealed partial class RedisServerException : RedisException /// Creates a new . /// /// The message for the exception. - public RedisServerException(string message) : base(message) { } + [Obsolete("Specify Kind and CommandFlags when possible")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisServerException(string message) : this(RedisErrorKind.Unknown, CommandFlags.CommandRetryNever, message) { } + + /// + /// Creates a new . + /// + /// The categorized meaning of the error. + /// The command-flags associated with the faulting operation. + /// The message for the exception. + public RedisServerException(RedisErrorKind kind, CommandFlags flags, string message) : base(message) + { + Kind = kind; + Flags = flags; + } #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisServerException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } + + /// + /// Identifies the kind of error received. + /// + public RedisErrorKind Kind { get; } + + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public CommandFlags Flags { get; } } } diff --git a/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs b/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs index 96b4ce8f6..f9a4238d2 100644 --- a/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs @@ -4,6 +4,7 @@ using System.IO; using System.Net; using System.Threading.Tasks; +using StackExchange.Redis.Availability; using StackExchange.Redis.Maintenance; using StackExchange.Redis.Profiling; using static StackExchange.Redis.ConnectionMultiplexer; diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.cs b/src/StackExchange.Redis/Interfaces/IDatabase.cs index c0761b9a7..dbaa70d42 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.cs @@ -16,7 +16,7 @@ public partial interface IDatabase : IRedis, IDatabaseAsync /// /// The numeric identifier of this database. /// - int Database { get; } + new int Database { get; } /// /// Allows creation of a group of operations that will be sent to the server as a single unit, diff --git a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs index 40db55cfd..b7f43848b 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs @@ -14,6 +14,9 @@ namespace StackExchange.Redis /// public partial interface IDatabaseAsync : IRedisAsync { + /// + int Database { get; } + /// /// Indicates whether the instance can communicate with the server (resolved using the supplied key and optional flags). /// diff --git a/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs b/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs new file mode 100644 index 000000000..7ac83afda --- /dev/null +++ b/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs @@ -0,0 +1,64 @@ +using System; +using System.Threading; + +namespace StackExchange.Redis.Interfaces; + +[Flags] +internal enum DatabaseFeatureFlags +{ + None = 0, + Cluster = 1 << 0, + ConnectionGroup = 1 << 1, + Batch = 1 << 2, + Transaction = 1 << 3, + KeyPrefix = 1 << 4, + Retry = 1 << 5, + Unknown = 1 << 6, + Failover = 1 << 7, +} + +internal interface IInternalDatabaseAsync : IDatabaseAsync +{ + DatabaseFeatureFlags GetFeatures(out string name); + CancellationToken GetNextFailover(); +} + +internal static class InternalDatabaseExtension +{ + internal static DatabaseFeatureFlags GetFeatures(this IDatabaseAsync database, out string name) + { + if (database is IInternalDatabaseAsync idb) + { + return idb.GetFeatures(out name); + } + + name = ""; + return DatabaseFeatureFlags.Unknown; + } + + internal static string BuildString(this IDatabaseAsync database) + { + var features = database.GetFeatures(out string name); + return string.IsNullOrEmpty(name) ? features.ToString() : $"{name}: {features}"; + } + + internal static DatabaseFeatureFlags RejectFlags(this IDatabaseAsync database, DatabaseFeatureFlags incompatible) + { + // note: returns *all* the features of the database provided + var features = database.GetFeatures(out _); + var overlap = features & incompatible; + if (overlap is not 0) Throw(overlap); + return features; + + static void Throw(DatabaseFeatureFlags overlap) => throw new InvalidOperationException( + $"This operation is not compatible with feature(s): {overlap}"); + } + + internal static CancellationToken GetNextFailover(this IDatabaseAsync database) + { + // get a CT that represents the next failover; you might be asking "shouldn't that be a Task getter?" - no, + // because Task *does* have ContinueWith, but it doesn't have any mechanism to *undo* that; conversely, + // CancellationToken is expressly designed with that intent, with Register(..) being scoped. + return database is IInternalDatabaseAsync ida ? ida.GetNextFailover() : CancellationToken.None; + } +} diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs index 952fe0ed3..877ffb0b5 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs @@ -3,11 +3,13 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Net; +using System.Threading; using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { - internal partial class KeyPrefixed : IDatabaseAsync where TInner : IDatabaseAsync + internal partial class KeyPrefixed : IDatabaseAsync, IInternalDatabaseAsync where TInner : IDatabaseAsync { internal KeyPrefixed(TInner inner, byte[] keyPrefix) { @@ -17,10 +19,25 @@ internal KeyPrefixed(TInner inner, byte[] keyPrefix) public IConnectionMultiplexer Multiplexer => Inner.Multiplexer; + public int Database => Inner.Database; + internal TInner Inner { get; } internal byte[] Prefix { get; } + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + => Inner.GetFeatures(out name) | GetDatabaseFeatures(); + + CancellationToken IInternalDatabaseAsync.GetNextFailover() => Inner.GetNextFailover(); + + // the flags contributed by this wrapper itself (on top of the inner database); the batch and + // transaction subclasses override to fold in their own flag, mirroring RedisDatabase/RedisBatch/ + // RedisTransaction rather than relying on the inner instance to carry it. + private protected virtual DatabaseFeatureFlags GetDatabaseFeatures() + => DatabaseFeatureFlags.KeyPrefix; + + public override string ToString() => this.BuildString(); + public Task DebugObjectAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => Inner.DebugObjectAsync(ToInner(key), flags); diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs index 6f5679a66..a5e07a011 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs @@ -1,8 +1,16 @@ -namespace StackExchange.Redis.KeyspaceIsolation +using StackExchange.Redis.Interfaces; + +namespace StackExchange.Redis.KeyspaceIsolation { internal sealed class KeyPrefixedBatch : KeyPrefixed, IBatch { - public KeyPrefixedBatch(IBatch inner, byte[] prefix) : base(inner, prefix) { } + public KeyPrefixedBatch(IBatch inner, byte[] prefix) : base(inner, prefix) + { + inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Batch; public void Execute() => Inner.Execute(); } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs index 2ca7ca384..15deebb89 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs @@ -16,8 +16,6 @@ public IBatch CreateBatch(object? asyncState = null) => public ITransaction CreateTransaction(object? asyncState = null) => new KeyPrefixedTransaction(Inner.CreateTransaction(asyncState), Prefix); - public int Database => Inner.Database; - public RedisValue DebugObject(RedisKey key, CommandFlags flags = CommandFlags.None) => Inner.DebugObject(ToInner(key), flags); diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs index 89703ba6a..7b7c6f0ce 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs @@ -1,10 +1,17 @@ using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { internal sealed class KeyPrefixedTransaction : KeyPrefixed, ITransaction { - public KeyPrefixedTransaction(ITransaction inner, byte[] prefix) : base(inner, prefix) { } + public KeyPrefixedTransaction(ITransaction inner, byte[] prefix) : base(inner, prefix) + { + inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Transaction; public ConditionResult AddCondition(Condition condition) => Inner.AddCondition(condition.MapKeys(GetMapFunction())); diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index b37995b78..b36f3e082 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -58,7 +58,10 @@ internal abstract partial class Message : ICompletable internal const CommandFlags InternalCallFlag = (CommandFlags)128, - NoFlushFlag = (CommandFlags)1024; + NoFlushFlag = (CommandFlags)1024, + // "server specific" (bit 18): tied to a specific endpoint, never retry elsewhere. Not (yet) a + // public CommandFlags member - see the note on the hidden bit-18 value in CommandFlags.cs. + CommandServerSpecific = (CommandFlags)(1 << 18); protected RedisCommand command; @@ -72,7 +75,12 @@ internal const CommandFlags | CommandFlags.PreferMaster | CommandFlags.PreferReplica; - private const CommandFlags UserSelectableFlags = CommandFlags.None + // the 5-bit retry-category severity region (bits 13-17); numerically equal to CommandRetryNever. + // deliberately excludes CommandServerSpecific (bit 18), which is an orthogonal flag, not part of + // the <=-comparable severity ladder. + internal const CommandFlags MaskRetryCategory = CommandFlags.CommandRetryNever; + + internal const CommandFlags UserSelectableFlags = CommandFlags.None | CommandFlags.DemandMaster | CommandFlags.DemandReplica | CommandFlags.PreferMaster @@ -83,6 +91,8 @@ internal const CommandFlags | CommandFlags.FireAndForget | CommandFlags.NoRedirect | CommandFlags.NoScriptCache + | MaskRetryCategory // caller may override the retry category... + | CommandServerSpecific // ...and the server-specific flag | NoFlushFlag; // we'll allow this one even though not advertised private IResultBox? resultBox; @@ -119,7 +129,9 @@ protected Message(int db, CommandFlags flags, RedisCommand command) bool primaryOnly = command.IsPrimaryOnly(); Db = db; this.command = command; - Flags = flags & UserSelectableFlags; + // apply the user-selectable flags, then fill in the default retry-category for this command + // (WithDefaultCategory is a no-op if the caller already specified a CommandRetry* category) + Flags = (flags & UserSelectableFlags).WithDefaultCategory(command); if (primaryOnly) SetPrimaryOnly(); CreatedDateTime = DateTime.UtcNow; @@ -599,6 +611,12 @@ internal static CommandFlags GetPrimaryReplicaFlags(CommandFlags flags) return flags & MaskPrimaryServerPreference; } + internal static CommandFlags GetRetryCategory(CommandFlags flags) + { + // isolate the retry-category region; 0 here means "not specified" (resolved downstream) + return flags & MaskRetryCategory; + } + internal static bool RequiresDatabase(RedisCommand command) { switch (command) diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index ca70965bb..a1ac6e2cf 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -1354,7 +1354,7 @@ private async ValueTask CompleteWriteAndReleaseLockAsync( private WriteResult HandleWriteException(PhysicalConnection? physical, Message message, Exception ex) { - var inner = new RedisConnectionException(ConnectionFailureType.InternalFailure, "Failed to write", ex); + var inner = new RedisConnectionException(ConnectionFailureType.InternalFailure, message.Flags, "Failed to write", ex); message.SetExceptionAndComplete(inner, physical); // Tear down the physical connection. A write that throws may have left a partial frame on the // wire, and continuing to use the same socket would let the next reply match the wrong message diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index c4cba0312..ab32d8ae2 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -485,7 +485,7 @@ void AddData(string? lk, string? sk, string? v) AddData("Version", "v", Utils.GetLibVersion()); - outerException = new RedisConnectionException(failureType, exMessage.ToString(), innerException); + outerException = new RedisConnectionException(failureType, CommandFlags.None, exMessage.ToString(), innerException); foreach (var kv in data) { @@ -1108,7 +1108,7 @@ private void OnDebugAbort() var bridge = BridgeCouldBeNull; if (bridge == null || !bridge.Multiplexer.AllowConnect) { - throw new RedisConnectionException(ConnectionFailureType.InternalFailure, "Aborting (AllowConnect: False)"); + throw new RedisConnectionException(ConnectionFailureType.InternalFailure, CommandFlags.None, "Aborting (AllowConnect: False)"); } } @@ -1183,7 +1183,7 @@ public void ObserveMessageResult(Exception? fault) // returns true while healthy); if it trips and we're the *first* to notice, hand off the // teardown rather than doing it inline: RecordConnectionFailed can fail the whole backlog and // build a detailed exception, which we don't want to pay for on the completion thread. - if (circuitBreaker is { } cb && !cb.ObserveResult(fault) + if (circuitBreaker is { } cb && cb.Trip(fault) && Interlocked.CompareExchange(ref _circuitBreakerState, CircuitBreakerTripped, CircuitBreakerHealthy) is CircuitBreakerHealthy) { // hand off to a worker; the heartbeat (see OnBridgeHeartbeat) is a backstop in case the diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 9f2483eb9..47cf16bcf 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,9 +1,22 @@ #nullable enable +StackExchange.Redis.IDatabaseAsync.Database.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy +[SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.JitterMax.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryDelay.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = default(StackExchange.Redis.RedisKey)) -> StackExchange.Redis.RedisKey -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsHealthy() -> bool -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext context) -> bool -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Reset() -> void -[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.CreateAccumulator() -> StackExchange.Redis.Availability.CircuitBreaker.Accumulator! +StackExchange.Redis.Availability.DatabaseExtensions +[SER007]static StackExchange.Redis.Availability.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy! retryPolicy) -> StackExchange.Redis.IDatabaseAsync! [SER007]StackExchange.Redis.Availability.CircuitBreaker [SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator [SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Accumulator() -> void @@ -16,15 +29,7 @@ StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = defa [SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MetricsWindowSize.set -> void [SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MinimumNumberOfFailures.get -> int [SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MinimumNumberOfFailures.set -> void -[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.TrackedExceptions.get -> System.Type![]? -[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.TrackedExceptions.set -> void [SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreaker() -> void -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.CircuitBreakerContext() -> void -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.CircuitBreakerContext(System.Exception? fault, bool evaluate = true) -> void -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.Fault.get -> System.Exception? -[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreakerContext.Success.get -> bool -[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Builder.implicit operator StackExchange.Redis.Availability.CircuitBreaker!(StackExchange.Redis.Availability.CircuitBreaker.Builder! builder) -> StackExchange.Redis.Availability.CircuitBreaker! [SER007]StackExchange.Redis.Availability.ConnectionGroupMember [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(StackExchange.Redis.ConfigurationOptions! configuration, string! name = "") -> void [SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(string! configuration, string! name = "") -> void @@ -97,31 +102,92 @@ StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = defa [SER007]StackExchange.Redis.Availability.MultiGroupOptions.MultiGroupOptions() -> void [SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember! member0, StackExchange.Redis.Availability.ConnectionGroupMember! member1, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! [SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember![]! members, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? +[SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.set -> void +[SER007]StackExchange.Redis.ConfigurationOptions.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +[SER007]StackExchange.Redis.ConfigurationOptions.HealthCheck.set -> void +[SER007]StackExchange.Redis.ConnectionFailureType.CircuitBreaker = 11 -> StackExchange.Redis.ConnectionFailureType +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.IsUnhealthy.get -> bool +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ResetIsUnhealthy() -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.set -> void +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsHealthy() -> bool +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Reset() -> void +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.CreateAccumulator() -> StackExchange.Redis.Availability.CircuitBreaker.Accumulator! [SER007]static StackExchange.Redis.Availability.CircuitBreaker.Default.get -> StackExchange.Redis.Availability.CircuitBreaker! [SER007]static StackExchange.Redis.Availability.CircuitBreaker.None.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Builder.implicit operator StackExchange.Redis.Availability.CircuitBreaker!(StackExchange.Redis.Availability.CircuitBreaker.Builder! builder) -> StackExchange.Redis.Availability.CircuitBreaker! [SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext context) -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult -[SER007]abstract StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! -[SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! -[SER007]override StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.ToString() -> string! -[SER007]override StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! -[SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! [SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.HealthyTask.get -> System.Threading.Tasks.Task! [SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.InconclusiveTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! [SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.IsConnected.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! [SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.Ping.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! [SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.StringSet.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe! -[SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! +[SER007]abstract StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext context) -> StackExchange.Redis.Availability.HealthCheck.HealthCheckResult [SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AllSuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! [SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.AnySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! [SER007]static StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy.MajoritySuccess.get -> StackExchange.Redis.Availability.HealthCheck.HealthCheckProbePolicy! +[SER007]abstract StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! +[SER007]override StackExchange.Redis.Availability.HealthCheck.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheck! healthCheck, StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! +[SER007]override StackExchange.Redis.Availability.HealthCheck.HealthCheckProbeContext.ToString() -> string! +[SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! [SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Default.get -> StackExchange.Redis.Availability.MultiGroupOptions! -[SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? -[SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.set -> void -[SER007]StackExchange.Redis.ConfigurationOptions.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? -[SER007]StackExchange.Redis.ConfigurationOptions.HealthCheck.set -> void -[SER007]StackExchange.Redis.ConnectionFailureType.CircuitBreaker = 11 -> StackExchange.Redis.ConnectionFailureType -[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.IsUnhealthy.get -> bool -[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ResetIsUnhealthy() -> void -[SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.get -> System.TimeSpan -[SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.set -> void +[SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! +[SER007]StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.None = 0 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Unknown = 1 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.Availability.FaultContext +[SER007]StackExchange.Redis.Availability.FaultContext.ConnectionFailureType.get -> StackExchange.Redis.ConnectionFailureType +[SER007]StackExchange.Redis.Availability.FaultContext.ErrorKind.get -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.Availability.FaultContext.Fault.get -> System.Exception? +[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext() -> void +[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception! fault) -> void +[SER007]StackExchange.Redis.Availability.FaultContext.Flags.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.Availability.FaultContext.IsFault.get -> bool +[SER007]StackExchange.Redis.RedisErrorKind.Ask = 15 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Busy = 12 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ClusterDown = 6 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ConnectionFault = 3 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.CrossSlot = 16 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ExecAbort = 23 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Loading = 5 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.MasterDown = 7 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.MaxClients = 13 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Misconfigured = 10 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Moved = 14 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoAuth = 17 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoPermission = 19 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoReplicas = 9 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoScript = 25 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NotPermitted = 20 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.OutOfMemory = 11 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ReadOnly = 24 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Timeout = 4 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.TryAgain = 8 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.UnknownCommand = 21 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.UnknownError = 2 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.WrongPass = 18 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.WrongType = 22 -> StackExchange.Redis.RedisErrorKind +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.FaultContext fault) -> void +[SER007]virtual StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsFailure(in StackExchange.Redis.Availability.FaultContext fault) -> bool +[SER007]virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault) -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.None = 0 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.SameServer = 1 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult.FailoverServer = 2 -> StackExchange.Redis.Availability.RetryPolicy.RetryPolicyResult +[SER007]StackExchange.Redis.CommandFlags.CommandRetryAlways = 8192 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryConnection = 32768 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryReadOnly = 65536 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteChecked = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryReadOnly -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins = 131072 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteAccumulating = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryServerAdmin = StackExchange.Redis.CommandFlags.CommandRetryReadOnly | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryNever = 253952 -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisConnectionException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisConnectionException.RedisConnectionException(StackExchange.Redis.ConnectionFailureType failureType, StackExchange.Redis.CommandFlags flags, string! message, System.Exception? innerException = null, StackExchange.Redis.CommandStatus commandStatus = StackExchange.Redis.CommandStatus.Unknown) -> void +StackExchange.Redis.RedisServerException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisServerException.Kind.get -> StackExchange.Redis.RedisErrorKind +StackExchange.Redis.RedisServerException.RedisServerException(StackExchange.Redis.RedisErrorKind kind, StackExchange.Redis.CommandFlags flags, string! message) -> void +StackExchange.Redis.RedisTimeoutException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisTimeoutException.RedisTimeoutException(StackExchange.Redis.CommandFlags flags, string! message, StackExchange.Redis.CommandStatus commandStatus) -> void diff --git a/src/StackExchange.Redis/RedisBatch.cs b/src/StackExchange.Redis/RedisBatch.cs index c827ba1dc..5b9e527ea 100644 --- a/src/StackExchange.Redis/RedisBatch.cs +++ b/src/StackExchange.Redis/RedisBatch.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { @@ -8,7 +9,14 @@ internal sealed class RedisBatch : RedisDatabase, IBatch { private List? pending; - public RedisBatch(RedisDatabase wrapped, object? asyncState) : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) { } + public RedisBatch(RedisDatabase wrapped, object? asyncState) + : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) + { + wrapped.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Batch; public void Execute() { diff --git a/src/StackExchange.Redis/RedisDatabase.cs b/src/StackExchange.Redis/RedisDatabase.cs index 21518eaa7..aadaf5abe 100644 --- a/src/StackExchange.Redis/RedisDatabase.cs +++ b/src/StackExchange.Redis/RedisDatabase.cs @@ -4,12 +4,14 @@ using System.Diagnostics; using System.Net; using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using RESPite.Messages; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { - internal partial class RedisDatabase : RedisBase, IDatabase + internal partial class RedisDatabase : RedisBase, IDatabase, IInternalDatabaseAsync { internal RedisDatabase(ConnectionMultiplexer multiplexer, int db, object? asyncState) : base(multiplexer, asyncState) @@ -21,6 +23,20 @@ internal RedisDatabase(ConnectionMultiplexer multiplexer, int db, object? asyncS public int Database { get; } + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + { + name = multiplexer.ClientName; + return GetDatabaseFeatures(); + } + + CancellationToken IInternalDatabaseAsync.GetNextFailover() => CancellationToken.None; + + // the base feature set for this database; wrapping databases (batch, transaction, ...) override + // to fold in their own flag. Cluster is intrinsic to the underlying connection, so it lives here. + private protected virtual DatabaseFeatureFlags GetDatabaseFeatures() + => multiplexer.ServerSelectionStrategy.ServerType == ServerType.Cluster + ? DatabaseFeatureFlags.Cluster : DatabaseFeatureFlags.None; + public IBatch CreateBatch(object? asyncState) { if (this is IBatch) throw new NotSupportedException("Nested batches are not supported"); diff --git a/src/StackExchange.Redis/RedisErrorKind.cs b/src/StackExchange.Redis/RedisErrorKind.cs new file mode 100644 index 000000000..4fdebf33f --- /dev/null +++ b/src/StackExchange.Redis/RedisErrorKind.cs @@ -0,0 +1,235 @@ +using System; +using System.Buffers; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using RESPite; +using RESPite.Messages; + +namespace StackExchange.Redis; + +/// +/// Well-known server error conditions, identified from the error-reply prefix (and in some cases the +/// message text). Used to classify faults consistently - in particular to decide whether a fault is +/// transient (worth retrying / awaiting failover) or permanent (retrying will never help). +/// +[Experimental(Experiments.ActiveActive, UrlFormat = Experiments.UrlFormat)] +public enum RedisErrorKind +{ + /// + /// No error condition; the reply was not an error. + /// + [AsciiHash("")] + None = 0, + + /// + /// The error was not recognized as one of the well-known conditions and did not start with ERR. + /// + [AsciiHash("")] + Unknown, + + /// + /// The error was not recognized as one of the well-known conditions, but started with ERR. + /// + [AsciiHash("")] + UnknownError, + + /// + /// The error was due to a connection fault, categorized separately by . + /// + [AsciiHash("")] + ConnectionFault, + + /// + /// The operation timed out; this is a client-side condition and does not correspond to a server reply. + /// + [AsciiHash("")] + Timeout, + + // --- availability / typically transient (retry or failover may recover) --- + + /// + /// LOADING - the server is still loading its dataset into memory and is not yet ready. + /// + Loading, + + /// + /// CLUSTERDOWN - the cluster is down; the hash slot is not currently being served. + /// + ClusterDown, + + /// + /// MASTERDOWN - the link with the primary is down and replica-serve-stale-data is no. + /// + MasterDown, + + /// + /// TRYAGAIN - a multi-key operation spans slots that are being migrated; the client should retry. + /// + TryAgain, + + /// + /// NOREPLICAS - not enough healthy replicas to satisfy the configured min-replicas-to-write. + /// + NoReplicas, + + /// + /// MISCONF - RDB/AOF persistence is misconfigured and writes are currently refused. + /// + [AsciiHash("MISCONF")] + Misconfigured, + + /// + /// OOM - the command was refused because used memory is above the maxmemory limit. + /// + [AsciiHash("OOM")] + OutOfMemory, + + /// + /// BUSY - a script or function is running and blocking the server (needs SCRIPT KILL etc.). + /// + Busy, + + /// + /// MAXCLIENTS - the configured maximum number of client connections has been reached. + /// + MaxClients, + + // --- cluster slot routing --- + + /// + /// MOVED - the hash slot has permanently moved to another endpoint. + /// + Moved, + + /// + /// ASK - the hash slot is temporarily served by another endpoint during migration. + /// + Ask, + + /// + /// CROSSSLOT - the keys in a multi-key operation do not all hash to the same slot. + /// + CrossSlot, + + // --- authentication / authorization (will not recover without a credential or ACL change) --- + + /// + /// NOAUTH - authentication is required before commands can be issued. + /// + NoAuth, + + /// + /// WRONGPASS - the supplied username/password pair was rejected. + /// + WrongPass, + + /// + /// NOPERM - the authenticated ACL user is not permitted to run this command/key/channel. + /// + [AsciiHash("NOPERM")] + NoPermission, + + /// + /// ERR operation not permitted - the operation is not permitted in the current context. + /// + [AsciiHash("")] // matched via the "ERR " branch, not the first token + NotPermitted, + + // --- client / usage errors (permanent - retry and failover will never help) --- + + /// + /// ERR unknown command - the command is not known to the server (typo, disabled, or unsupported version). + /// + [AsciiHash("ERR")] // matched via the "ERR " branch, not the first token + UnknownCommand, + + /// + /// WRONGTYPE - the operation was applied against a key holding an incompatible value type. + /// + WrongType, + + /// + /// EXECABORT - the transaction was discarded because of an earlier error while queuing commands. + /// + ExecAbort, + + /// + /// READONLY - a write was attempted against a read-only replica. + /// + ReadOnly, + + /// + /// NOSCRIPT - no matching script for EVALSHA; the script must be re-loaded. + /// + NoScript, +} + +internal static partial class RedisErrorKindMetadata +{ + [AsciiHash(CaseSensitive = false)] + private static partial bool TryParseFirstTokenCI(ReadOnlySpan value, out RedisErrorKind kind); + + /// + /// Classifies the error currently held by . The caller is assumed to have + /// already established that this is an error, so an unrecognized reply yields + /// (never ). + /// + internal static unsafe RedisErrorKind Classify(in RespReader reader) + => reader.TryParseScalar(&TryParse, out RedisErrorKind kind) ? kind : RedisErrorKind.Unknown; + + internal static unsafe RedisErrorKind Classify(string message) + => RespReader.TryParseScalar(message.AsSpan(), &TryParse, out RedisErrorKind kind) ? kind : RedisErrorKind.Unknown; + + internal static bool TryParse(ReadOnlySpan value, out RedisErrorKind kind) + { + var space = value.IndexOf((byte)' '); + // Deal with the exact matches on the first token first - many errors may or may not + // have descriptive text after the leading token. + var firstToken = space > 0 ? value.Slice(0, space) : value; + if (TryParseFirstTokenCI(firstToken, out kind)) + { + // check for more specific "ERR ..." scenarios + if (kind is RedisErrorKind.UnknownError & space > 0) + { + // get the message text after "ERR " + value = value.Slice(space + 1); + + // some ERR conditions can be identified further, noting that the text may or + // may not have some tokens - sometimes we need partial match. + var valueHash = AsciiHash.HashUC(value); + if (value.Length is OperationNotPermitted.Length && OperationNotPermitted.IsCI(value, valueHash)) + { + kind = RedisErrorKind.NotPermitted; + } + else if (value.Length >= UnknownCommand.Length && + AsciiHash.SequenceEqualsCI(value.Slice(0, UnknownCommand.Length), UnknownCommand.U8)) + { + kind = RedisErrorKind.UnknownCommand; + } + } + return true; + } + + kind = value.IsEmpty ? RedisErrorKind.None : RedisErrorKind.Unknown; + return true; + } + + [AsciiHash("operation not permitted")] + private static partial class OperationNotPermitted { } + + [AsciiHash("unknown command")] + private static partial class UnknownCommand { } + + /* while these are recognizable, we issue SELECT *on behalf* of the user + and react internally, so reporting them seems... unnecessary. + + [AsciiHash("DB index is out of range")] + private static partial class DbIndexOutOfRange { } + + [AsciiHash("invalid DB index")] + private static partial class InvalidDbIndex { } + + [AsciiHash("SELECT is not allowed in cluster mode")] + private static partial class SelectNotAllowedInClusterMode { } + */ +} diff --git a/src/StackExchange.Redis/RedisResult.cs b/src/StackExchange.Redis/RedisResult.cs index 78ed649bc..1d496468f 100644 --- a/src/StackExchange.Redis/RedisResult.cs +++ b/src/StackExchange.Redis/RedisResult.cs @@ -558,7 +558,10 @@ internal override RedisValue AsRedisValue() private sealed class ErrorRedisResult : RedisResult { private readonly string value; + private RedisErrorKind kind; // lazily parsed from the value + private RedisErrorKind Kind => kind is RedisErrorKind.None ? GetKind() : kind; + private RedisErrorKind GetKind() => kind = RedisErrorKindMetadata.Classify(value); public ErrorRedisResult(string? value, ResultType type) : base(type) { this.value = value ?? throw new ArgumentNullException(nameof(value)); @@ -570,30 +573,30 @@ public ErrorRedisResult(string? value, ResultType type) : base(type) type = null; return value; } - internal override bool AsBoolean() => throw new RedisServerException(value); - internal override bool[] AsBooleanArray() => throw new RedisServerException(value); - internal override byte[] AsByteArray() => throw new RedisServerException(value); - internal override byte[][] AsByteArrayArray() => throw new RedisServerException(value); - internal override double AsDouble() => throw new RedisServerException(value); - internal override double[] AsDoubleArray() => throw new RedisServerException(value); - internal override int AsInt32() => throw new RedisServerException(value); - internal override int[] AsInt32Array() => throw new RedisServerException(value); - internal override long AsInt64() => throw new RedisServerException(value); - internal override ulong AsUInt64() => throw new RedisServerException(value); - internal override long[] AsInt64Array() => throw new RedisServerException(value); - internal override ulong[] AsUInt64Array() => throw new RedisServerException(value); - internal override bool? AsNullableBoolean() => throw new RedisServerException(value); - internal override double? AsNullableDouble() => throw new RedisServerException(value); - internal override int? AsNullableInt32() => throw new RedisServerException(value); - internal override long? AsNullableInt64() => throw new RedisServerException(value); - internal override ulong? AsNullableUInt64() => throw new RedisServerException(value); - internal override RedisKey AsRedisKey() => throw new RedisServerException(value); - internal override RedisKey[] AsRedisKeyArray() => throw new RedisServerException(value); - internal override RedisResult[] AsRedisResultArray() => throw new RedisServerException(value); - internal override RedisValue AsRedisValue() => throw new RedisServerException(value); - internal override RedisValue[] AsRedisValueArray() => throw new RedisServerException(value); - internal override string? AsString() => throw new RedisServerException(value); - internal override string?[]? AsStringArray() => throw new RedisServerException(value); + internal override bool AsBoolean() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override bool[] AsBooleanArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override byte[] AsByteArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override byte[][] AsByteArrayArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double AsDouble() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double[] AsDoubleArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int AsInt32() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int[] AsInt32Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long AsInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong AsUInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long[] AsInt64Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong[] AsUInt64Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override bool? AsNullableBoolean() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double? AsNullableDouble() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int? AsNullableInt32() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long? AsNullableInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong? AsNullableUInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisKey AsRedisKey() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisKey[] AsRedisKeyArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisResult[] AsRedisResultArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisValue AsRedisValue() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisValue[] AsRedisValueArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override string? AsString() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override string?[]? AsStringArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); } private sealed class SingleRedisResult : RedisResult, IConvertible diff --git a/src/StackExchange.Redis/RedisTransaction.cs b/src/StackExchange.Redis/RedisTransaction.cs index 5d5430671..c1c39b3c1 100644 --- a/src/StackExchange.Redis/RedisTransaction.cs +++ b/src/StackExchange.Redis/RedisTransaction.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using RESPite.Messages; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { @@ -14,8 +15,12 @@ internal sealed class RedisTransaction : RedisDatabase, ITransaction private List? _pending; private object SyncLock => this; + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Transaction; + public RedisTransaction(RedisDatabase wrapped, object? asyncState) : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) { + wrapped.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); // need to check we can reliably do this... var commandMap = multiplexer.CommandMap; commandMap.AssertAvailable(RedisCommand.MULTI); @@ -480,11 +485,12 @@ public override bool SetResult(PhysicalConnection connection, Message message, r reader.MovePastBof(); if (reader.IsError && message is TransactionMessage tran) { + var errorKind = RedisErrorKindMetadata.Classify(reader); string error = reader.ReadString()!; foreach (var op in tran.InnerOperations) { var inner = op.Wrapped; - ServerFail(inner, error); + ServerFail(inner, errorKind, error); inner.Complete(connection); } } diff --git a/src/StackExchange.Redis/ResultProcessor.Literals.cs b/src/StackExchange.Redis/ResultProcessor.Literals.cs index 7d50cc09e..3deaa7027 100644 --- a/src/StackExchange.Redis/ResultProcessor.Literals.cs +++ b/src/StackExchange.Redis/ResultProcessor.Literals.cs @@ -8,16 +8,6 @@ internal partial class Literals { #pragma warning disable CS8981, SA1300, SA1134 // forgive naming etc // ReSharper disable InconsistentNaming - [AsciiHash] internal static partial class NOAUTH { } - [AsciiHash] internal static partial class WRONGPASS { } - [AsciiHash] internal static partial class NOSCRIPT { } - [AsciiHash] internal static partial class MOVED { } - [AsciiHash] internal static partial class ASK { } - [AsciiHash] internal static partial class READONLY { } - [AsciiHash] internal static partial class LOADING { } - [AsciiHash("ERR operation not permitted")] - internal static partial class ERR_not_permitted { } - // Result processor literals [AsciiHash] internal static partial class OK diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index c4ad5e801..45a09cfbc 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -210,15 +210,15 @@ public void ConnectionFail(Message message, ConnectionFailureType fail, Exceptio sb.Append(", "); sb.Append(annotation); } - var ex = new RedisConnectionException(fail, sb.ToString(), innerException); + var ex = new RedisConnectionException(fail, message?.Flags ?? CommandFlags.None, sb.ToString(), innerException); SetException(message, ex); } public static void ConnectionFail(Message message, ConnectionFailureType fail, string errorMessage) => - SetException(message, new RedisConnectionException(fail, errorMessage)); + SetException(message, new RedisConnectionException(fail, message.Flags, errorMessage)); - public static void ServerFail(Message message, string errorMessage) => - SetException(message, new RedisServerException(errorMessage)); + public static void ServerFail(Message message, RedisErrorKind kind, string errorMessage) => + SetException(message, new RedisServerException(kind, message.Flags, errorMessage)); public static void SetException(Message? message, Exception ex) { @@ -263,22 +263,24 @@ private bool HandleCommonError(Message message, RespReader reader, PhysicalConne { connection.OnDetailLog($"applying common error-handling: {reader.GetOverview()}"); var bridge = connection.BridgeCouldBeNull; - if (reader.StartsWith(Literals.NOAUTH.U8)) + var errorKind = RedisErrorKindMetadata.Classify(reader); + switch (errorKind) { - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException("NOAUTH Returned - connection has not yet authenticated")); - } - else if (reader.StartsWith(Literals.WRONGPASS.U8)) - { - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(reader.GetOverview())); + case RedisErrorKind.NoAuth: + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, message.Flags, "NOAUTH Returned - connection has not yet authenticated")); + break; + case RedisErrorKind.WrongPass: + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, message.Flags, reader.GetOverview())); + break; } var server = bridge?.ServerEndPoint; bool log = !message.IsInternalCall; - bool isMoved = reader.StartsWith(Literals.MOVED.U8); + bool isMoved = errorKind == RedisErrorKind.Moved; bool wasNoRedirect = (message.Flags & CommandFlags.NoRedirect) != 0; string? err = string.Empty; bool unableToConnectError = false; - if (isMoved || reader.StartsWith(Literals.ASK.U8)) + if (isMoved || errorKind == RedisErrorKind.Ask) { connection.OnDetailLog($"redirect via {(isMoved ? "MOVED" : "ASK")} to '{reader.ReadString()}'"); message.SetResponseReceived(); @@ -361,7 +363,7 @@ private bool HandleCommonError(Message message, RespReader reader, PhysicalConne } else { - ServerFail(message, err); + ServerFail(message, errorKind, err); } return true; @@ -852,7 +854,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r { var copy = reader; reader.MovePastBof(); - if (reader.IsError && reader.StartsWith(Literals.READONLY.U8)) + if (reader.IsError && RedisErrorKindMetadata.Classify(reader) == RedisErrorKind.ReadOnly) { var bridge = connection.BridgeCouldBeNull; if (bridge != null) @@ -2249,7 +2251,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r { var copy = reader; reader.MovePastBof(); - if (reader.IsError && reader.StartsWith(Literals.NOSCRIPT.U8)) + if (reader.IsError && RedisErrorKindMetadata.Classify(reader) == RedisErrorKind.NoScript) { // scripts are not flushed individually, so assume the entire script cache is toast ("SCRIPT FLUSH") connection.BridgeCouldBeNull?.ServerEndPoint?.FlushScriptCache(); message.SetScriptUnavailable(); @@ -3167,17 +3169,18 @@ public override bool SetResult(PhysicalConnection connection, Message message, r if (isError) { reader = copy; // rewind and re-parse - if (reader.StartsWith(Literals.ERR_not_permitted.U8) || reader.StartsWith(Literals.NOAUTH.U8)) + var errorKind = RedisErrorKindMetadata.Classify(reader); + if (errorKind is RedisErrorKind.NotPermitted or RedisErrorKind.NoAuth) { connection.RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure, new Exception(reader.GetOverview() + " Verify if the Redis password provided is correct. Attempted command: " + message.Command)); } - else if (reader.StartsWith(Literals.LOADING.U8)) + else if (errorKind == RedisErrorKind.Loading) { connection.RecordConnectionFailed(ConnectionFailureType.Loading); } else { - connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, new RedisServerException(reader.GetOverview())); + connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, new RedisServerException(RedisErrorKind.ConnectionFault, message.Flags, reader.GetOverview())); } } diff --git a/src/StackExchange.Redis/ServerSelectionStrategy.cs b/src/StackExchange.Redis/ServerSelectionStrategy.cs index 27358e8c7..80c9b9efe 100644 --- a/src/StackExchange.Redis/ServerSelectionStrategy.cs +++ b/src/StackExchange.Redis/ServerSelectionStrategy.cs @@ -52,9 +52,9 @@ internal sealed partial class ServerSelectionStrategy private int anyStartOffset = SharedRandom.Next(); // initialize to a random value so routing isn't uniform #if NET - private static Random SharedRandom => Random.Shared; + internal static Random SharedRandom => Random.Shared; #else - private static Random SharedRandom { get; } = new(); + internal static Random SharedRandom { get; } = new(); #endif private ServerEndPoint[]? map; diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs index 074dafcd9..54783b8b0 100644 --- a/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs @@ -60,19 +60,17 @@ private sealed class CountingCircuitBreaker : CircuitBreaker private sealed class CountingAccumulator(CountingCircuitBreaker owner) : Accumulator { - public override bool ObserveResult(in CircuitBreakerContext context) + public override void ObserveResult(in FaultContext context) { - if (context.Success) + if (context.IsFault) { - Interlocked.Increment(ref owner._successes); + Interlocked.Increment(ref owner._failures); + owner.LastFault = context.Fault; } else { - Interlocked.Increment(ref owner._failures); - owner.LastFault = context.Fault; + Interlocked.Increment(ref owner._successes); } - - return true; // stay healthy; we're only here to observe, not to trip the connection } public override bool IsHealthy() => true; diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs index dad3dd015..ab0b15545 100644 --- a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs @@ -48,33 +48,7 @@ public void None_IsAlwaysHealthy() { var acc = CircuitBreaker.None.CreateAccumulator(); // even a solid wall of tracked failures never trips the no-op breaker - Assert.True(Record(acc, 10_000, new RedisTimeoutException("boom", CommandStatus.Unknown))); - } - - [Fact] - public void NonEvaluatingObservation_IsNeverTreatedAsUnhealthy() - { - // a deliberately naive breaker whose accumulator *always* reports unhealthy - e.g. one that - // returns default(false) whether or not it was actually asked to evaluate. A success is - // observed without evaluation, so that bogus verdict must be ignored (not read as a trip); - // only a genuine evaluation (a fault) is allowed to report unhealthy. - var acc = new AlwaysUnhealthyBreaker().CreateAccumulator(); - - Assert.True(acc.ObserveResult((Exception?)null)); // success -> not evaluating -> verdict ignored - Assert.False(acc.ObserveResult(Timeout())); // fault -> evaluating -> verdict honoured - } - - // never reports healthy, even when not evaluating; stands in for a buggy/naive custom breaker - private sealed class AlwaysUnhealthyBreaker : CircuitBreaker - { - public override Accumulator CreateAccumulator() => new Acc(); - - private sealed class Acc : Accumulator - { - public override bool ObserveResult(in CircuitBreakerContext context) => false; - public override bool IsHealthy() => false; - public override void Reset() { } - } + Assert.True(Record(acc, 10_000, new RedisTimeoutException(CommandFlags.None, "boom", CommandStatus.Unknown))); } #if NET8_0_OR_GREATER @@ -128,23 +102,6 @@ public void UntrackedExceptions_CountAsSuccess() Assert.False(Record(acc, 100, Timeout())); } - [Fact] - public void CustomTrackedExceptions_AreHonoured() - { - var time = new ManualTimeProvider(); - var acc = Build( - time, - failureRateThreshold: 1, - minimumNumberOfFailures: 1, - trackedExceptions: [typeof(InvalidOperationException)]).CreateAccumulator(); - - // the default Redis faults are now *un*tracked, so they read as success - Assert.True(Record(acc, 100, Timeout())); - - // whereas our nominated type trips it - Assert.False(Record(acc, 100, new InvalidOperationException("tracked"))); - } - [Fact] public void OldFailures_AgeOutOfTheWindow() { @@ -194,14 +151,12 @@ public void Reset_DiscardsHistory() private static CircuitBreaker Build( TimeProvider time, double failureRateThreshold, - int minimumNumberOfFailures, - Type[]? trackedExceptions = null) + int minimumNumberOfFailures) => new CircuitBreaker.Builder { FailureRateThreshold = failureRateThreshold, MinimumNumberOfFailures = minimumNumberOfFailures, MetricsWindowSize = TimeSpan.FromSeconds(10), - TrackedExceptions = trackedExceptions, TimeProvider = time, }.Create(); @@ -222,18 +177,17 @@ private sealed class ManualTimeProvider : TimeProvider } #endif - private static RedisTimeoutException Timeout() => new("timeout", CommandStatus.Unknown); + private static RedisTimeoutException Timeout() => new(CommandFlags.None, "timeout", CommandStatus.Unknown); private static bool Record(CircuitBreaker.Accumulator accumulator, int count, Exception? fault = null) { - // success/failure is derived from the presence of a fault; pass a fault for a failure, none for a success - var context = new CircuitBreaker.CircuitBreakerContext(fault); - bool healthy = true; + // Trip applies the IsFailure gate (a fault the breaker doesn't consider a failure is counted as a + // success), then records via ObserveResult; call it rather than ObserveResult directly so the gate runs for (int i = 0; i < count; i++) { - healthy = accumulator.ObserveResult(in context); + accumulator.Trip(fault); } - return healthy; + return accumulator.IsHealthy(); } } diff --git a/tests/StackExchange.Redis.Tests/ControllableProbe.cs b/tests/StackExchange.Redis.Tests/ControllableProbe.cs new file mode 100644 index 000000000..4d8e47dc7 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ControllableProbe.cs @@ -0,0 +1,18 @@ +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; + +namespace StackExchange.Redis.Tests; + +// A health-check probe whose verdict is driven by the test: nominated endpoints report unhealthy on +// demand, everything else is healthy. This keeps a specific member deselected deterministically, even +// after its physical connection reconnects underneath us. +internal sealed class ControllableProbe : HealthCheck.HealthCheckProbe +{ + private volatile EndPoint? _down; + + public void MarkDown(EndPoint endpoint) => _down = endpoint; + + public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) + => Equals(server.EndPoint, _down) ? UnhealthyTask : HealthyTask; +} diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs index 18436db46..da739ae8f 100644 --- a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs @@ -8,7 +8,7 @@ namespace StackExchange.Redis.Tests.MultiGroupTests; [RunPerProtocol] -public class CircuitBreakerRerouteTests(ITestOutputHelper log) +public class CircuitBreakerRerouteTests(ITestOutputHelper log) : TestBase(log) { // A circuit-breaker trip must steer the group away from the affected member *promptly* - i.e. via // the shim's ConnectionFailed(CircuitBreaker) fast-path, not by waiting for the next health-check @@ -23,9 +23,9 @@ public async Task CircuitBreakerTrip_ReroutesAwayFromMember() EndPoint bravo = new DnsEndPoint("bravo", 6379); EndPoint charlie = new DnsEndPoint("charlie", 6379); - using var serverA = new InProcessTestServer(log, endpoint: alpha); - using var serverB = new InProcessTestServer(log, endpoint: bravo); - using var serverC = new InProcessTestServer(log, endpoint: charlie); + using var serverA = new InProcessTestServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + using var serverC = new InProcessTestServer(Output, endpoint: charlie); var breaker = new FlipBreaker(); var probe = new ControllableProbe(); @@ -60,7 +60,7 @@ public async Task CircuitBreakerTrip_ReroutesAwayFromMember() int circuitBreakerEvents = 0; typed.ConnectionFailed += (_, e) => { - log.WriteLine($"ConnectionFailed: {e.FailureType} @ {e.EndPoint}"); + Log($"ConnectionFailed: {e.FailureType} @ {e.EndPoint}"); if (e.FailureType == ConnectionFailureType.CircuitBreaker) { Interlocked.Increment(ref circuitBreakerEvents); @@ -77,7 +77,7 @@ public async Task CircuitBreakerTrip_ReroutesAwayFromMember() breaker.Trip(); var db = conn.GetDatabase(); var fault = await Assert.ThrowsAnyAsync(() => db.ExecuteAsync("nonesuch")); - log.WriteLine($"observed fault: {fault.GetType().Name}: {fault.Message}"); + Log($"observed fault: {fault.GetType().Name}: {fault.Message}"); // wait (briefly) for the fast-path to react; nothing else can move us within this window var moved = await WaitForActiveAsync(conn, notMember: members[0], timeout: TimeSpan.FromSeconds(5)); @@ -113,20 +113,13 @@ private sealed class FlipBreaker : CircuitBreaker private sealed class Acc(FlipBreaker owner) : Accumulator { - public override bool ObserveResult(in CircuitBreakerContext context) => !owner._tripped; + // this breaker trips on demand regardless of *what* faulted, so treat every observed fault as a + // failure - otherwise Trip's IsFailure gate would filter out non-failure faults (e.g. an unknown + // command) before the tripped state is ever consulted + protected override bool IsFailure(in FaultContext fault) => true; + public override void ObserveResult(in FaultContext context) { } public override bool IsHealthy() => !owner._tripped; public override void Reset() { } } } - - // reports the nominated endpoints unhealthy on demand; everything else is healthy. This keeps the - // tripped member deselected even after its physical connection reconnects. - private sealed class ControllableProbe : HealthCheck.HealthCheckProbe - { - private volatile EndPoint? _down; - public void MarkDown(EndPoint endpoint) => _down = endpoint; - - public override Task CheckHealthAsync(HealthCheck healthCheck, IServer server) - => Equals(server.EndPoint, _down) ? UnhealthyTask : HealthyTask; - } } diff --git a/tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs similarity index 99% rename from tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs rename to tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs index 78036e635..b3ea620f5 100644 --- a/tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs @@ -11,7 +11,7 @@ namespace StackExchange.Redis.Tests; -public class RetryPolicyUnitTests(ITestOutputHelper log) +public class ReconnectRetryPolicyUnitTests(ITestOutputHelper log) { [Theory] [InlineData(FailureMode.Success)] diff --git a/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs new file mode 100644 index 000000000..b8e630da1 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs @@ -0,0 +1,191 @@ +using System.Threading; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Interfaces; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +public class CommandRetryPolicyUnitTests +{ + // --- RetryPolicy.CanRetry: spoofed fault scenarios ------------------------------- + + // Builds a FaultContext for a spoofed server error of the given kind, carrying the given + // command-flags, and asks the policy whether it may be retried. + private static RetryPolicy.RetryPolicyResult CanRetry(RedisErrorKind kind, CommandFlags flags, RetryPolicy? policy = null) + { + // the exception carries both the Kind and the command-flags; FaultContext reads them back + var fault = new FaultContext(new RedisServerException(kind, flags, kind.ToString())); + return (policy ?? new RetryPolicy()).CanRetry(in fault); + } + + // The command's retry-category is checked against the policy's max category: the default max is + // CommandRetryWriteLastWins, so anything at-or-below that is in-range, and anything with more + // side-effects is not. Using a transient LOADING fault so the error-kind check permits a retry + // whenever the category is in-range - isolating the category logic. + [Theory] + [InlineData(CommandFlags.CommandRetryAlways, true)] + [InlineData(CommandFlags.CommandRetryConnection, true)] + [InlineData(CommandFlags.CommandRetryReadOnly, true)] + [InlineData(CommandFlags.CommandRetryWriteChecked, true)] + [InlineData(CommandFlags.CommandRetryWriteLastWins, true)] // == default max + [InlineData(CommandFlags.CommandRetryWriteAccumulating, false)] // beyond default max + [InlineData(CommandFlags.CommandRetryServerAdmin, false)] + [InlineData(CommandFlags.CommandRetryNever, false)] + [InlineData(CommandFlags.None, false)] // unspecified => assume worst (accumulating) => beyond default max + public void CanRetry_CategoryVersusDefaultMax(CommandFlags category, bool expectRetry) + { + var result = CanRetry(RedisErrorKind.Loading, category); + Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + } + + // With an in-range category (== default max), the outcome is decided purely by whether the error + // is transient: LOADING is worth retrying, WRONGTYPE is an application error that will not fix itself. + [Theory] + [InlineData(RedisErrorKind.Loading, true)] // still loading the dataset - transient + [InlineData(RedisErrorKind.ClusterDown, true)] // slot temporarily unserved - transient + [InlineData(RedisErrorKind.WrongType, false)] // wrong data type - application error + [InlineData(RedisErrorKind.NoPermission, false)] // ACL - application error + public void CanRetry_ErrorKindGatesRetry_WhenInRange(RedisErrorKind kind, bool expectRetry) + { + var result = CanRetry(kind, CommandFlags.CommandRetryWriteLastWins); + Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + } + + // "never" and "always" adjust only the category range - they do not override the error-kind check: + // an "always" command still won't retry an application error, and a "never" command won't retry even + // a transient one. + [Theory] + [InlineData(CommandFlags.CommandRetryAlways, RedisErrorKind.Loading, true)] + [InlineData(CommandFlags.CommandRetryAlways, RedisErrorKind.WrongType, false)] + [InlineData(CommandFlags.CommandRetryNever, RedisErrorKind.Loading, false)] + [InlineData(CommandFlags.CommandRetryNever, RedisErrorKind.WrongType, false)] + public void CanRetry_NeverAndAlwaysAffectRangeNotErrorKind(CommandFlags category, RedisErrorKind kind, bool expectRetry) + { + var result = CanRetry(kind, category); + Assert.Equal(expectRetry, result != RetryPolicy.RetryPolicyResult.None); + } + + // When a retry is permitted, it normally offers both the same server and a failover server; but a + // "server specific" (sticky) command must not move endpoints, so only the same-server option remains. + [Theory] + [InlineData(CommandFlags.None, RetryPolicy.RetryPolicyResult.SameServer | RetryPolicy.RetryPolicyResult.FailoverServer)] + [InlineData(Message.CommandServerSpecific, RetryPolicy.RetryPolicyResult.SameServer)] + public void CanRetry_ServerSpecificRestrictsToSameServer(CommandFlags extra, RetryPolicy.RetryPolicyResult expected) + { + // in-range category (== default max) + transient error => a retry is offered; the sticky flag + // only changes *where* the retry may go, not *whether* it happens. + var result = CanRetry(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins | extra); + Assert.Equal(expected, result); + } + + // The sticky (server-specific) flag lives outside the retry-category region, so it is masked off + // before the category-vs-max comparison and must not change the range verdict (retry-at-all vs none) + // - it only affects the same/failover choice, covered above. + [Theory] + [InlineData(CommandFlags.CommandRetryReadOnly, true)] // in range + [InlineData(CommandFlags.CommandRetryServerAdmin, false)] // beyond default max + public void CanRetry_ServerSpecificDoesNotAffectRange(CommandFlags category, bool expectRetry) + { + var withoutFlag = CanRetry(RedisErrorKind.Loading, category); + var withFlag = CanRetry(RedisErrorKind.Loading, category | Message.CommandServerSpecific); + + Assert.Equal(expectRetry, withoutFlag != RetryPolicy.RetryPolicyResult.None); + Assert.Equal(expectRetry, withFlag != RetryPolicy.RetryPolicyResult.None); + } + + // --- RetryDatabase.CanRetry: attempt accounting ---------------------------------- + + // With max-attempts-before-failover pinned equal to max-attempts, the failover path is disabled, so + // this exercises pure same-server attempt counting. A transient LOADING fault on an in-range command + // means the policy would allow a retry; the only gate is the attempt counter: with MaxAttempts=3, + // attempts 1 and 2 may retry, attempt 3 is exhausted. Because we never fail over, the out "delay" is + // never cancellable (that is how "don't wait for failover" is expressed) and the ref "failover" is + // left untouched. + [Theory] + [InlineData(1, true)] + [InlineData(2, true)] + [InlineData(3, false)] + public void RetryDatabase_CanRetry_MaxAttempts_NoFailover(int attempt, bool expected) + { + var policy = new RetryPolicy { MaxAttempts = 3, MaxAttemptsBeforeFailover = 3 }; + var db = new RetryDatabase(null!, policy); // CanRetry never touches the inner database + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + + var fault = new RedisServerException(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins, "LOADING"); + var result = db.CanRetry(attempt, fault, ref failover, out var delay); + + Assert.Equal(expected, result); + Assert.False(delay.CanBeCanceled); // never waiting for a failover + Assert.Equal(token, failover); // ref failover untouched + } + + // MaxAttempts=4 with failover enabled after 2 attempts. A single "failover" token is threaded through + // the sequence to observe the state machine: attempts 1..3 return true, attempt 4 is exhausted. The + // interesting step is attempt 2 (== MaxAttemptsBeforeFailover): it still returns true, but now hands the + // failover token back as "delay" and clears the ref (we fail over only once); attempt 3 therefore sees + // no failover token and drops back to a plain same-server retry. + [Fact] + public void RetryDatabase_CanRetry_FailoverAtThreshold() + { + var policy = new RetryPolicy { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; + // failover is only armed when the inner database advertises the feature; supply it explicitly + var db = new RetryDatabase(null!, policy, DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + var fault = new RedisServerException(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins, "LOADING"); + + // attempt 1: plain same-server retry; failover token not yet consumed + Assert.True(db.CanRetry(1, fault, ref failover, out var delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(token, failover); + + // attempt 2 (== MaxAttemptsBeforeFailover): still a retry, but now fail over - "delay" becomes the + // failover token and the ref is cleared to None so it only fires once + Assert.True(db.CanRetry(2, fault, ref failover, out delay)); + Assert.Equal(token, delay); + Assert.True(delay.CanBeCanceled); + Assert.Equal(CancellationToken.None, failover); + + // attempt 3: failover already spent (ref is None) -> back to a same-server retry + Assert.True(db.CanRetry(3, fault, ref failover, out delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(CancellationToken.None, failover); + + // attempt 4: no retries left + Assert.False(db.CanRetry(4, fault, ref failover, out delay)); + Assert.False(delay.CanBeCanceled); + } + + // As above, but with the sticky (server-specific) flag set: the policy now permits only same-server + // retries, so there is no failover option at the threshold. Current behaviour: CanRetry returns *false* + // at attempt 2 - the command gives up rather than continuing on the same server - because the threshold + // branch (attempt == MaxAttemptsBeforeFailover) requires FailoverServer permission and does not fall + // back to a same-server retry. It also consumes the failover token as a side-effect of that branch. + [Fact] + public void RetryDatabase_CanRetry_ServerSpecific_CannotFailover() + { + var policy = new RetryPolicy { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; + var db = new RetryDatabase(null!, policy, DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + const CommandFlags flags = CommandFlags.CommandRetryWriteLastWins | Message.CommandServerSpecific; + var fault = new RedisServerException(RedisErrorKind.Loading, flags, "LOADING"); + + // attempt 1: same-server retry; failover token untouched + Assert.True(db.CanRetry(1, fault, ref failover, out var delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(token, failover); + + // attempt 2 (== MaxAttemptsBeforeFailover): sticky forbids failover -> gives up (false), even though + // attempts remain; the failover token is still consumed to None as a side-effect of the branch + Assert.False(db.CanRetry(2, fault, ref failover, out delay)); + Assert.Equal(CancellationToken.None, failover); + } +} diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs new file mode 100644 index 000000000..e2b9583f7 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -0,0 +1,153 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +[RunPerProtocol] +public class RetryEndToEndTests(ITestOutputHelper log) : TestBase(log) +{ + // End-to-end: a server that answers the first couple of GETs with a transient LOADING error, then + // serves normally. Wrapping the database with .WithRetry should transparently ride through the LOADING + // responses; we can then observe (via the server's counter) that it really did take three GETs before + // one succeeded. + [Fact] + public async Task WithRetry_RidesOutTransientLoading() + { + using var server = new LoadingServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + + RedisKey key = "retry:loading"; + Assert.True(await db.StringSetAsync(key, "hello")); // seed the value before we start failing GETs + + // queue up two LOADING responses; the third GET should succeed + server.LoadingOps = 2; + + // zero delay/jitter so the test isn't paying the default ~1s retry backoff between attempts + var policy = new RetryPolicy + { + MaxAttempts = 3, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var retryDb = db.WithRetry(policy); + + var value = await retryDb.StringGetAsync(key); + + Assert.Equal("hello", value); // retries rode out the LOADING responses + Assert.Equal(0, server.LoadingOps); // both LOADING responses were consumed + Assert.Equal(3, server.GetOpsReceived); // 2 x LOADING + 1 x success + } + + // Multi-group + WithRetry: two backends hold *different* values for the same key. The group is weighted + // towards A, so a normal read returns A's value. When A drops into LOADING, the faults trip A's + // (deliberately hair-trigger) circuit-breaker; the group reroutes to B, and the retry wrapper rides that + // failover transparently - the very same StringGet now returns B's value, with no caller intervention and + // no configuration change. + [Fact] + public async Task WithRetry_FailsOverBetweenGroupsOnLoading() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + using var serverA = new InProcessTestServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + + RedisKey key = "retry:multigroup"; + + // seed each backend with its own distinct value (direct connections, so each cache gets its own) + await using (var seedA = await serverA.ConnectAsync()) + { + Assert.True(await seedA.GetDatabase().StringSetAsync(key, "from-A")); + } + await using (var seedB = await serverB.ConnectAsync()) + { + Assert.True(await seedB.GetDatabase().StringSetAsync(key, "from-B")); + } + + var probe = new ControllableProbe(); + + // A carries a hair-trigger breaker (trips on the first fault); B keeps the default (never trips here) + var configA = serverA.GetClientConfig(); + configA.CircuitBreaker = new CircuitBreaker.Builder + { + MinimumNumberOfFailures = 1, + FailureRateThreshold = 1, + }; + + ConnectionGroupMember[] members = + [ + new(configA, "A") { Weight = 9 }, // highest weight -> initially active + new(serverB.GetClientConfig(), "B") { Weight = 1 }, // failover target + ]; + + var options = new MultiGroupOptions + { + HealthCheck = new HealthCheck + { + Probe = probe, + Interval = TimeSpan.FromMinutes(30), // huge: the breaker fast-path is what reroutes us + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + Assert.True(conn.IsConnected); + Assert.Same(members[0], conn.ActiveMember); // A is active (highest weight) + + // failover enabled, plenty of attempts, no artificial delay between them + var policy = new RetryPolicy + { + MaxAttempts = 20, + MaxAttemptsBeforeFailover = 1, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var db = conn.GetDatabase().WithRetry(policy); + + // normal read: routed to the active (weighted) member, A + string? before = await db.StringGetAsync(key); + Assert.Equal("from-A", before); + + // knock A into LOADING and hold it down; nothing else about the setup changes + serverA.IsLoading = true; + probe.MarkDown(alpha); + + // the *same* call now transparently rides the circuit-break failover across to B + string? after = await db.StringGetAsync(key); + Assert.Equal("from-B", after); + Assert.Same(members[1], conn.ActiveMember); // we really did move to B + } + + // An in-proc server that fails the first LoadingOps GET operations with a transient LOADING error + // (decrementing the counter each time), then serves normally. Every GET bumps GetOpsReceived so the + // test can confirm how many attempts actually reached the server. + private sealed class LoadingServer(ITestOutputHelper? log) : InProcessTestServer(log) + { + // the server core processes operations under a lock (single-threaded, like Redis), so plain fields + // are fine here + public int GetOpsReceived { get; private set; } + + public int LoadingOps { get; set; } + + protected override TypedRedisValue Get(RedisClient client, in RedisRequest request) + { + GetOpsReceived++; + + // while LOADING ops remain, consume one and reply with a transient LOADING error + if (LoadingOps > 0) + { + LoadingOps--; + return TypedRedisValue.Error("LOADING Redis is loading the dataset in memory"); + } + + return base.Get(client, in request); + } + } +} diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index 26f1a2fc3..77ad4f1ed 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -160,8 +160,18 @@ protected override void AppendStats(StringBuilder sb) public string Password { get; set; } = ""; + /// + /// When set, every command is rejected with a LOADING error, mimicking a server that is + /// still loading its dataset into memory. + /// + public bool IsLoading { get; set; } + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) { + if (IsLoading) + { + return TypedRedisValue.Error("LOADING"); + } var pw = Password; if (!string.IsNullOrEmpty(pw) & !client.IsAuthenticated) {