From 9ec4f2276ffe0be2e23ba36b8a7d0a47f9946b41 Mon Sep 17 00:00:00 2001 From: JKorf Date: Tue, 2 Jul 2024 16:13:10 +0200 Subject: [PATCH] Updated single endpoint limit configuration, added LongConverter, updated SystemTextJsonComparer logic --- CryptoExchange.Net/Clients/RestApiClient.cs | 4 +- .../SystemTextJson/LongConverter.cs | 40 ++++++++++ .../SystemTextJson/SerializerOptions.cs | 3 +- .../Objects/RequestDefinition.cs | 10 +-- .../Objects/RequestDefinitionCache.cs | 9 +-- .../RateLimiting/Guards/SingleLimitGuard.cs | 41 +++++++--- .../RateLimiting/Interfaces/IRateLimitGate.cs | 11 +-- .../RateLimiting/RateLimitGate.cs | 23 +++--- .../Comparers/SystemTextJsonComparer.cs | 80 +++++++++++-------- 9 files changed, 142 insertions(+), 79 deletions(-) create mode 100644 CryptoExchange.Net/Converters/SystemTextJson/LongConverter.cs diff --git a/CryptoExchange.Net/Clients/RestApiClient.cs b/CryptoExchange.Net/Clients/RestApiClient.cs index 3504c43..81d7c7f 100644 --- a/CryptoExchange.Net/Clients/RestApiClient.cs +++ b/CryptoExchange.Net/Clients/RestApiClient.cs @@ -309,14 +309,14 @@ namespace CryptoExchange.Net.Clients } // Endpoint specific rate limiting - if (definition.EndpointLimitCount != null && definition.EndpointLimitPeriod != null) + if (definition.LimitGuard != null && ClientOptions.RateLimiterEnabled) { if (definition.RateLimitGate == null) throw new Exception("Ratelimit gate not set when endpoint limit is specified"); if (ClientOptions.RateLimiterEnabled) { - var limitResult = await definition.RateLimitGate.ProcessSingleAsync(_logger, requestId, RateLimitItemType.Request, definition, baseAddress, AuthenticationProvider?._credentials.Key, requestWeight, ClientOptions.RateLimitingBehaviour, cancellationToken).ConfigureAwait(false); + var limitResult = await definition.RateLimitGate.ProcessSingleAsync(_logger, requestId, definition.LimitGuard, RateLimitItemType.Request, definition, baseAddress, AuthenticationProvider?._credentials.Key, ClientOptions.RateLimitingBehaviour, cancellationToken).ConfigureAwait(false); if (!limitResult) return new CallResult(limitResult.Error!); } diff --git a/CryptoExchange.Net/Converters/SystemTextJson/LongConverter.cs b/CryptoExchange.Net/Converters/SystemTextJson/LongConverter.cs new file mode 100644 index 0000000..96698e9 --- /dev/null +++ b/CryptoExchange.Net/Converters/SystemTextJson/LongConverter.cs @@ -0,0 +1,40 @@ +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CryptoExchange.Net.Converters.SystemTextJson +{ + /// + /// Int converter + /// + public class LongConverter : JsonConverter + { + /// + public override long? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return null; + + if (reader.TokenType == JsonTokenType.String) + { + var value = reader.GetString(); + if (string.IsNullOrEmpty(value)) + return null; + + return long.Parse(value, NumberStyles.Integer, CultureInfo.InvariantCulture); + } + + return reader.GetInt64(); + } + + /// + public override void Write(Utf8JsonWriter writer, long? value, JsonSerializerOptions options) + { + if (value == null) + writer.WriteNullValue(); + else + writer.WriteNumberValue(value.Value); + } + } +} diff --git a/CryptoExchange.Net/Converters/SystemTextJson/SerializerOptions.cs b/CryptoExchange.Net/Converters/SystemTextJson/SerializerOptions.cs index 5ff4000..71c867f 100644 --- a/CryptoExchange.Net/Converters/SystemTextJson/SerializerOptions.cs +++ b/CryptoExchange.Net/Converters/SystemTextJson/SerializerOptions.cs @@ -21,7 +21,8 @@ namespace CryptoExchange.Net.Converters.SystemTextJson new EnumConverter(), new BoolConverter(), new DecimalConverter(), - new IntConverter() + new IntConverter(), + new LongConverter() } }; } diff --git a/CryptoExchange.Net/Objects/RequestDefinition.cs b/CryptoExchange.Net/Objects/RequestDefinition.cs index 04549fb..a28f5b8 100644 --- a/CryptoExchange.Net/Objects/RequestDefinition.cs +++ b/CryptoExchange.Net/Objects/RequestDefinition.cs @@ -48,18 +48,16 @@ namespace CryptoExchange.Net.Objects /// Request weight /// public int Weight { get; set; } = 1; + /// /// Rate limit gate to use /// public IRateLimitGate? RateLimitGate { get; set; } + /// - /// Rate limit for this specific endpoint + /// Individual endpoint rate limit guard to use /// - public int? EndpointLimitCount { get; set; } - /// - /// Rate limit period for this specific endpoint - /// - public TimeSpan? EndpointLimitPeriod { get; set; } + public IRateLimitGuard? LimitGuard { get; set; } /// diff --git a/CryptoExchange.Net/Objects/RequestDefinitionCache.cs b/CryptoExchange.Net/Objects/RequestDefinitionCache.cs index 9e3ab47..45dde1b 100644 --- a/CryptoExchange.Net/Objects/RequestDefinitionCache.cs +++ b/CryptoExchange.Net/Objects/RequestDefinitionCache.cs @@ -41,8 +41,7 @@ namespace CryptoExchange.Net.Objects /// The HttpMethod /// Endpoint path /// The rate limit gate - /// The limit count for this specific endpoint - /// The period for the limit for this specific endpoint + /// The rate limit guard for this specific endpoint /// Request weight /// Endpoint is authenticated /// Request body format @@ -56,8 +55,7 @@ namespace CryptoExchange.Net.Objects IRateLimitGate? rateLimitGate, int weight, bool authenticated, - int? endpointLimitCount = null, - TimeSpan? endpointLimitPeriod = null, + IRateLimitGuard? limitGuard = null, RequestBodyFormat? requestBodyFormat = null, HttpMethodParameterPosition? parameterPosition = null, ArrayParametersSerialization? arraySerialization = null, @@ -69,8 +67,7 @@ namespace CryptoExchange.Net.Objects def = new RequestDefinition(path, method) { Authenticated = authenticated, - EndpointLimitCount = endpointLimitCount, - EndpointLimitPeriod = endpointLimitPeriod, + LimitGuard = limitGuard, RateLimitGate = rateLimitGate, Weight = weight, ArraySerialization = arraySerialization, diff --git a/CryptoExchange.Net/RateLimiting/Guards/SingleLimitGuard.cs b/CryptoExchange.Net/RateLimiting/Guards/SingleLimitGuard.cs index 95479ee..af11e68 100644 --- a/CryptoExchange.Net/RateLimiting/Guards/SingleLimitGuard.cs +++ b/CryptoExchange.Net/RateLimiting/Guards/SingleLimitGuard.cs @@ -12,9 +12,22 @@ namespace CryptoExchange.Net.RateLimiting.Guards /// public class SingleLimitGuard : IRateLimitGuard { + /// + /// Default endpoint limit + /// + public static Func Default { get; } = new Func((def, host, key) => def.Path + def.Method); + + /// + /// Endpoint limit per API key + /// + public static Func PerApiKey { get; } = new Func((def, host, key) => def.Path + def.Method); + private readonly Dictionary _trackers; private readonly RateLimitWindowType _windowType; private readonly double? _decayRate; + private readonly int _limit; + private readonly TimeSpan _period; + private readonly Func _keySelector; /// public string Name => "EndpointLimitGuard"; @@ -25,20 +38,28 @@ namespace CryptoExchange.Net.RateLimiting.Guards /// /// ctor /// - public SingleLimitGuard(RateLimitWindowType windowType, double? decayRate = null) + public SingleLimitGuard( + int limit, + TimeSpan period, + RateLimitWindowType windowType, + double? decayRate = null, + Func? keySelector = null) { + _limit = limit; + _period = period; _windowType = windowType; _decayRate = decayRate; + _keySelector = keySelector ?? Default; _trackers = new Dictionary(); } /// public LimitCheck Check(RateLimitItemType type, RequestDefinition definition, string host, SecureString? apiKey, int requestWeight) { - var key = definition.Path + definition.Method; + var key = _keySelector(definition, host, apiKey); if (!_trackers.TryGetValue(key, out var tracker)) { - tracker = CreateTracker(definition.EndpointLimitCount!.Value, definition.EndpointLimitPeriod!.Value); + tracker = CreateTracker(); _trackers.Add(key, tracker); } @@ -46,27 +67,27 @@ namespace CryptoExchange.Net.RateLimiting.Guards if (delay == default) return LimitCheck.NotNeeded; - return LimitCheck.Needed(delay, definition.EndpointLimitCount!.Value, definition.EndpointLimitPeriod!.Value, tracker.Current); + return LimitCheck.Needed(delay, _limit, _period, tracker.Current); } /// public RateLimitState ApplyWeight(RateLimitItemType type, RequestDefinition definition, string host, SecureString? apiKey, int requestWeight) { - var key = definition.Path + definition.Method; + var key = _keySelector(definition, host, apiKey); var tracker = _trackers[key]; tracker.ApplyWeight(requestWeight); - return RateLimitState.Applied(definition.EndpointLimitCount!.Value, definition.EndpointLimitPeriod!.Value, tracker.Current); + return RateLimitState.Applied(_limit, _period, tracker.Current); } /// /// Create a new WindowTracker /// /// - protected IWindowTracker CreateTracker(int limit, TimeSpan timeSpan) + protected IWindowTracker CreateTracker() { - return _windowType == RateLimitWindowType.Sliding ? new SlidingWindowTracker(limit, timeSpan) - : _windowType == RateLimitWindowType.Fixed ? new FixedWindowTracker(limit, timeSpan) : - new DecayWindowTracker(limit, timeSpan, _decayRate ?? throw new InvalidOperationException("Decay rate not provided")); + return _windowType == RateLimitWindowType.Sliding ? new SlidingWindowTracker(_limit, _period) + : _windowType == RateLimitWindowType.Fixed ? new FixedWindowTracker(_limit, _period) : + new DecayWindowTracker(_limit, _period, _decayRate ?? throw new InvalidOperationException("Decay rate not provided")); } } } diff --git a/CryptoExchange.Net/RateLimiting/Interfaces/IRateLimitGate.cs b/CryptoExchange.Net/RateLimiting/Interfaces/IRateLimitGate.cs index c4a1e67..38c1ff8 100644 --- a/CryptoExchange.Net/RateLimiting/Interfaces/IRateLimitGate.cs +++ b/CryptoExchange.Net/RateLimiting/Interfaces/IRateLimitGate.cs @@ -32,13 +32,6 @@ namespace CryptoExchange.Net.RateLimiting.Interfaces /// Task SetRetryAfterGuardAsync(DateTime retryAfter); - /// - /// Set the SingleLimitGuard for handling individual endpoint rate limits - /// - /// - /// - IRateLimitGate SetSingleLimitGuard(SingleLimitGuard guard); - /// /// Returns the 'retry after' timestamp if set /// @@ -65,14 +58,14 @@ namespace CryptoExchange.Net.RateLimiting.Interfaces /// /// Logger /// Id of the item to check + /// The guard /// The rate limit item type /// The request definition /// The host address /// The API key - /// Request weight /// Behaviour when rate limit is hit /// Cancelation token /// Error if RateLimitingBehaviour is Fail and rate limit is hit - Task ProcessSingleAsync(ILogger logger, int itemId, RateLimitItemType type, RequestDefinition definition, string baseAddress, SecureString? apiKey, int requestWeight, RateLimitingBehaviour behaviour, CancellationToken ct); + Task ProcessSingleAsync(ILogger logger, int itemId, IRateLimitGuard guard, RateLimitItemType type, RequestDefinition definition, string baseAddress, SecureString? apiKey, RateLimitingBehaviour behaviour, CancellationToken ct); } } diff --git a/CryptoExchange.Net/RateLimiting/RateLimitGate.cs b/CryptoExchange.Net/RateLimiting/RateLimitGate.cs index 900523d..afe91dd 100644 --- a/CryptoExchange.Net/RateLimiting/RateLimitGate.cs +++ b/CryptoExchange.Net/RateLimiting/RateLimitGate.cs @@ -16,7 +16,6 @@ namespace CryptoExchange.Net.RateLimiting /// public class RateLimitGate : IRateLimitGate { - private IRateLimitGuard _singleLimitGuard = new SingleLimitGuard(RateLimitWindowType.Sliding); private readonly ConcurrentBag _guards; private readonly SemaphoreSlim _semaphore; private readonly string _name; @@ -53,16 +52,23 @@ namespace CryptoExchange.Net.RateLimiting } /// - public async Task ProcessSingleAsync(ILogger logger, int itemId, RateLimitItemType type, RequestDefinition definition, string host, SecureString? apiKey, int requestWeight, RateLimitingBehaviour rateLimitingBehaviour, CancellationToken ct) + public async Task ProcessSingleAsync( + ILogger logger, + int itemId, + IRateLimitGuard guard, + RateLimitItemType type, + RequestDefinition definition, + string host, + SecureString? apiKey, + RateLimitingBehaviour rateLimitingBehaviour, + CancellationToken ct) { await _semaphore.WaitAsync(ct).ConfigureAwait(false); - if (requestWeight == 0) - requestWeight = 1; _waitingCount++; try { - return await CheckGuardsAsync(new IRateLimitGuard[] { _singleLimitGuard }, logger, itemId, type, definition, host, apiKey, requestWeight, rateLimitingBehaviour, ct).ConfigureAwait(false); + return await CheckGuardsAsync(new IRateLimitGuard[] { guard }, logger, itemId, type, definition, host, apiKey, 1, rateLimitingBehaviour, ct).ConfigureAwait(false); } finally { @@ -130,13 +136,6 @@ namespace CryptoExchange.Net.RateLimiting return this; } - /// - public IRateLimitGate SetSingleLimitGuard(SingleLimitGuard guard) - { - _singleLimitGuard = guard; - return this; - } - /// public async Task SetRetryAfterGuardAsync(DateTime retryAfter) { diff --git a/CryptoExchange.Net/Testing/Comparers/SystemTextJsonComparer.cs b/CryptoExchange.Net/Testing/Comparers/SystemTextJsonComparer.cs index a130eb0..7cd2ebf 100644 --- a/CryptoExchange.Net/Testing/Comparers/SystemTextJsonComparer.cs +++ b/CryptoExchange.Net/Testing/Comparers/SystemTextJsonComparer.cs @@ -70,48 +70,62 @@ namespace CryptoExchange.Net.Testing.Comparers else if (jsonObject!.Type == JTokenType.Array) { var jObjs = (JArray)jsonObject; - var list = (IEnumerable)resultData; - var enumerator = list.GetEnumerator(); - foreach (var jObj in jObjs) + if (resultData is IEnumerable list) { - enumerator.MoveNext(); - if (jObj.Type == JTokenType.Object) + var enumerator = list.GetEnumerator(); + foreach (var jObj in jObjs) { - foreach (var subProp in ((JObject)jObj).Properties()) + enumerator.MoveNext(); + if (jObj.Type == JTokenType.Object) { - if (ignoreProperties?.Contains(subProp.Name) == true) + foreach (var subProp in ((JObject)jObj).Properties()) + { + if (ignoreProperties?.Contains(subProp.Name) == true) + continue; + CheckObject(method, subProp, enumerator.Current, ignoreProperties!); + } + } + else if (jObj.Type == JTokenType.Array) + { + var resultObj = enumerator.Current; + if (resultObj is string) + // string list continue; - CheckObject(method, subProp, enumerator.Current, ignoreProperties!); + + var resultProps = resultObj.GetType().GetProperties().Select(p => (p, p.GetCustomAttributes(typeof(ArrayPropertyAttribute), true).Cast().SingleOrDefault())); + var arrayConverterProperty = resultObj.GetType().GetCustomAttributes(typeof(JsonConverterAttribute), true).FirstOrDefault(); + var jsonConverter = ((JsonConverterAttribute)arrayConverterProperty!).ConverterType; + if (jsonConverter != typeof(ArrayConverter)) + // Not array converter? + continue; + + int i = 0; + foreach (var item in jObj.Children()) + { + var arrayProp = resultProps.Where(p => p.Item2 != null).SingleOrDefault(p => p.Item2!.Index == i).p; + if (arrayProp != null) + CheckPropertyValue(method, item, arrayProp.GetValue(resultObj), arrayProp.PropertyType, arrayProp.Name, "Array index " + i, ignoreProperties!); + i++; + } } - } - else if (jObj.Type == JTokenType.Array) - { - var resultObj = enumerator.Current; - if (resultObj is string) - // string list - continue; - - var resultProps = resultObj.GetType().GetProperties().Select(p => (p, p.GetCustomAttributes(typeof(ArrayPropertyAttribute), true).Cast().SingleOrDefault())); - var arrayConverterProperty = resultObj.GetType().GetCustomAttributes(typeof(JsonConverterAttribute), true).FirstOrDefault(); - var jsonConverter = ((JsonConverterAttribute)arrayConverterProperty!).ConverterType; - if (jsonConverter != typeof(ArrayConverter)) - // Not array converter? - continue; - - int i = 0; - foreach (var item in jObj.Children()) + else { - var arrayProp = resultProps.Where(p => p.Item2 != null).SingleOrDefault(p => p.Item2!.Index == i).p; - if (arrayProp != null) - CheckPropertyValue(method, item, arrayProp.GetValue(resultObj), arrayProp.PropertyType, arrayProp.Name, "Array index " + i, ignoreProperties!); - i++; + var value = enumerator.Current; + if (value == default && ((JValue)jObj).Type != JTokenType.Null) + throw new Exception($"{method}: Array has no value while input json array has value {jObj}"); } } - else + } + else + { + var resultProps = resultData.GetType().GetProperties().Select(p => (p, p.GetCustomAttributes(typeof(ArrayPropertyAttribute), true).Cast().SingleOrDefault())); + int i = 0; + foreach (var item in jObjs.Children()) { - var value = enumerator.Current; - if (value == default && ((JValue)jObj).Type != JTokenType.Null) - throw new Exception($"{method}: Array has no value while input json array has value {jObj}"); + var arrayProp = resultProps.Where(p => p.Item2 != null).SingleOrDefault(p => p.Item2!.Index == i).p; + if (arrayProp != null) + CheckPropertyValue(method, item, arrayProp.GetValue(resultData), arrayProp.PropertyType, arrayProp.Name, "Array index " + i, ignoreProperties!); + i++; } } }