flywithu/CryptoExchange.Net
1
0
Fork 0
mirror of https://github.com/JKorf/CryptoExchange.Net synced 2025-06-07 16:06:15 +00:00
Code Issues Releases Wiki Activity

Updated single endpoint limit configuration, added LongConverter, updated SystemTextJsonComparer logic

Browse Source
This commit is contained in:
JKorf 2024-07-02 16:13:10 +02:00
parent 0a0c66541e
commit 9ec4f2276f
9 changed files with 142 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CryptoExchange.Net/Clients/RestApiClient.cs
View File

@ -309,14 +309,14 @@ namespace CryptoExchange.Net.Clients
}
// Endpoint specific rate limiting
if (definition.EndpointLimitCount != null && definition.EndpointLimitPeriod != null)
if (definition.LimitGuard != null && ClientOptions.RateLimiterEnabled)
{
if (definition.RateLimitGate == null)
throw new Exception("Ratelimit gate not set when endpoint limit is specified");
if (ClientOptions.RateLimiterEnabled)
{
var limitResult = await definition.RateLimitGate.ProcessSingleAsync(_logger, requestId, RateLimitItemType.Request, definition, baseAddress, AuthenticationProvider?._credentials.Key, requestWeight, ClientOptions.RateLimitingBehaviour, cancellationToken).ConfigureAwait(false);
var limitResult = await definition.RateLimitGate.ProcessSingleAsync(_logger, requestId, definition.LimitGuard, RateLimitItemType.Request, definition, baseAddress, AuthenticationProvider?._credentials.Key, ClientOptions.RateLimitingBehaviour, cancellationToken).ConfigureAwait(false);
if (!limitResult)
return new CallResult(limitResult.Error!);
}

40
CryptoExchange.Net/Converters/SystemTextJson/LongConverter.cs Normal file
View File

@ -0,0 +1,40 @@
using System;
using System.Globalization;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace CryptoExchange.Net.Converters.SystemTextJson
{
/// <summary>
/// Int converter
/// </summary>
public class LongConverter : JsonConverter<long?>
{
/// <inheritdoc />
public override long? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType == JsonTokenType.Null)
return null;
if (reader.TokenType == JsonTokenType.String)
{
var value = reader.GetString();
if (string.IsNullOrEmpty(value))
return null;
return long.Parse(value, NumberStyles.Integer, CultureInfo.InvariantCulture);
}
return reader.GetInt64();
}
/// <inheritdoc />
public override void Write(Utf8JsonWriter writer, long? value, JsonSerializerOptions options)
{
if (value == null)
writer.WriteNullValue();
else
writer.WriteNumberValue(value.Value);
}
}
}

3
CryptoExchange.Net/Converters/SystemTextJson/SerializerOptions.cs
View File

@ -21,7 +21,8 @@ namespace CryptoExchange.Net.Converters.SystemTextJson
new EnumConverter(),
new BoolConverter(),
new DecimalConverter(),
new IntConverter()
new IntConverter(),
new LongConverter()
}
};
}

10
CryptoExchange.Net/Objects/RequestDefinition.cs
View File

@ -48,18 +48,16 @@ namespace CryptoExchange.Net.Objects
/// Request weight
/// </summary>
public int Weight { get; set; } = 1;
/// <summary>
/// Rate limit gate to use
/// </summary>
public IRateLimitGate? RateLimitGate { get; set; }
/// <summary>
/// Rate limit for this specific endpoint
/// Individual endpoint rate limit guard to use
/// </summary>
public int? EndpointLimitCount { get; set; }
/// <summary>
/// Rate limit period for this specific endpoint
/// </summary>
public TimeSpan? EndpointLimitPeriod { get; set; }
public IRateLimitGuard? LimitGuard { get; set; }
/// <summary>

9
CryptoExchange.Net/Objects/RequestDefinitionCache.cs
View File

@ -41,8 +41,7 @@ namespace CryptoExchange.Net.Objects
/// <param name="method">The HttpMethod</param>
/// <param name="path">Endpoint path</param>
/// <param name="rateLimitGate">The rate limit gate</param>
/// <param name="endpointLimitCount">The limit count for this specific endpoint</param>
/// <param name="endpointLimitPeriod">The period for the limit for this specific endpoint</param>
/// <param name="limitGuard">The rate limit guard for this specific endpoint</param>
/// <param name="weight">Request weight</param>
/// <param name="authenticated">Endpoint is authenticated</param>
/// <param name="requestBodyFormat">Request body format</param>
@ -56,8 +55,7 @@ namespace CryptoExchange.Net.Objects
IRateLimitGate? rateLimitGate,
int weight,
bool authenticated,
int? endpointLimitCount = null,
TimeSpan? endpointLimitPeriod = null,
IRateLimitGuard? limitGuard = null,
RequestBodyFormat? requestBodyFormat = null,
HttpMethodParameterPosition? parameterPosition = null,
ArrayParametersSerialization? arraySerialization = null,
@ -69,8 +67,7 @@ namespace CryptoExchange.Net.Objects
def = new RequestDefinition(path, method)
{
Authenticated = authenticated,
EndpointLimitCount = endpointLimitCount,
EndpointLimitPeriod = endpointLimitPeriod,
LimitGuard = limitGuard,
RateLimitGate = rateLimitGate,
Weight = weight,
ArraySerialization = arraySerialization,

41
CryptoExchange.Net/RateLimiting/Guards/SingleLimitGuard.cs
View File

@ -12,9 +12,22 @@ namespace CryptoExchange.Net.RateLimiting.Guards
/// </summary>
public class SingleLimitGuard : IRateLimitGuard
{
/// <summary>
/// Default endpoint limit
/// </summary>
public static Func<RequestDefinition, string, SecureString?, string> Default { get; } = new Func<RequestDefinition, string, SecureString?, string>((def, host, key) => def.Path + def.Method);
/// <summary>
/// Endpoint limit per API key
/// </summary>
public static Func<RequestDefinition, string, SecureString?, string> PerApiKey { get; } = new Func<RequestDefinition, string, SecureString?, string>((def, host, key) => def.Path + def.Method);
private readonly Dictionary<string, IWindowTracker> _trackers;
private readonly RateLimitWindowType _windowType;
private readonly double? _decayRate;
private readonly int _limit;
private readonly TimeSpan _period;
private readonly Func<RequestDefinition, string, SecureString?, string> _keySelector;
/// <inheritdoc />
public string Name => "EndpointLimitGuard";
@ -25,20 +38,28 @@ namespace CryptoExchange.Net.RateLimiting.Guards
/// <summary>
/// ctor
/// </summary>
public SingleLimitGuard(RateLimitWindowType windowType, double? decayRate = null)
public SingleLimitGuard(
int limit,
TimeSpan period,
RateLimitWindowType windowType,
double? decayRate = null,
Func<RequestDefinition, string, SecureString?, string>? keySelector = null)
{
_limit = limit;
_period = period;
_windowType = windowType;
_decayRate = decayRate;
_keySelector = keySelector ?? Default;
_trackers = new Dictionary<string, IWindowTracker>();
}
/// <inheritdoc />
public LimitCheck Check(RateLimitItemType type, RequestDefinition definition, string host, SecureString? apiKey, int requestWeight)
{
var key = definition.Path + definition.Method;
var key = _keySelector(definition, host, apiKey);
if (!_trackers.TryGetValue(key, out var tracker))
{
tracker = CreateTracker(definition.EndpointLimitCount!.Value, definition.EndpointLimitPeriod!.Value);
tracker = CreateTracker();
_trackers.Add(key, tracker);
}
@ -46,27 +67,27 @@ namespace CryptoExchange.Net.RateLimiting.Guards
if (delay == default)
return LimitCheck.NotNeeded;
return LimitCheck.Needed(delay, definition.EndpointLimitCount!.Value, definition.EndpointLimitPeriod!.Value, tracker.Current);
return LimitCheck.Needed(delay, _limit, _period, tracker.Current);
}
/// <inheritdoc />
public RateLimitState ApplyWeight(RateLimitItemType type, RequestDefinition definition, string host, SecureString? apiKey, int requestWeight)
{
var key = definition.Path + definition.Method;
var key = _keySelector(definition, host, apiKey);
var tracker = _trackers[key];
tracker.ApplyWeight(requestWeight);
return RateLimitState.Applied(definition.EndpointLimitCount!.Value, definition.EndpointLimitPeriod!.Value, tracker.Current);
return RateLimitState.Applied(_limit, _period, tracker.Current);
}
/// <summary>
/// Create a new WindowTracker
/// </summary>
/// <returns></returns>
protected IWindowTracker CreateTracker(int limit, TimeSpan timeSpan)
protected IWindowTracker CreateTracker()
{
return _windowType == RateLimitWindowType.Sliding ? new SlidingWindowTracker(limit, timeSpan)
: _windowType == RateLimitWindowType.Fixed ? new FixedWindowTracker(limit, timeSpan) :
new DecayWindowTracker(limit, timeSpan, _decayRate ?? throw new InvalidOperationException("Decay rate not provided"));
return _windowType == RateLimitWindowType.Sliding ? new SlidingWindowTracker(_limit, _period)
: _windowType == RateLimitWindowType.Fixed ? new FixedWindowTracker(_limit, _period) :
new DecayWindowTracker(_limit, _period, _decayRate ?? throw new InvalidOperationException("Decay rate not provided"));
}
}
}

11
CryptoExchange.Net/RateLimiting/Interfaces/IRateLimitGate.cs
View File

@ -32,13 +32,6 @@ namespace CryptoExchange.Net.RateLimiting.Interfaces
/// <returns></returns>
Task SetRetryAfterGuardAsync(DateTime retryAfter);
/// <summary>
/// Set the SingleLimitGuard for handling individual endpoint rate limits
/// </summary>
/// <param name="guard"></param>
/// <returns></returns>
IRateLimitGate SetSingleLimitGuard(SingleLimitGuard guard);
/// <summary>
/// Returns the 'retry after' timestamp if set
/// </summary>
@ -65,14 +58,14 @@ namespace CryptoExchange.Net.RateLimiting.Interfaces
/// </summary>
/// <param name="logger">Logger</param>
/// <param name="itemId">Id of the item to check</param>
/// <param name="guard">The guard</param>
/// <param name="type">The rate limit item type</param>
/// <param name="definition">The request definition</param>
/// <param name="baseAddress">The host address</param>
/// <param name="apiKey">The API key</param>
/// <param name="requestWeight">Request weight</param>
/// <param name="behaviour">Behaviour when rate limit is hit</param>
/// <param name="ct">Cancelation token</param>
/// <returns>Error if RateLimitingBehaviour is Fail and rate limit is hit</returns>
Task<CallResult> ProcessSingleAsync(ILogger logger, int itemId, RateLimitItemType type, RequestDefinition definition, string baseAddress, SecureString? apiKey, int requestWeight, RateLimitingBehaviour behaviour, CancellationToken ct);
Task<CallResult> ProcessSingleAsync(ILogger logger, int itemId, IRateLimitGuard guard, RateLimitItemType type, RequestDefinition definition, string baseAddress, SecureString? apiKey, RateLimitingBehaviour behaviour, CancellationToken ct);
}
}

23
CryptoExchange.Net/RateLimiting/RateLimitGate.cs
View File

@ -16,7 +16,6 @@ namespace CryptoExchange.Net.RateLimiting
/// <inheritdoc />
public class RateLimitGate : IRateLimitGate
{
private IRateLimitGuard _singleLimitGuard = new SingleLimitGuard(RateLimitWindowType.Sliding);
private readonly ConcurrentBag<IRateLimitGuard> _guards;
private readonly SemaphoreSlim _semaphore;
private readonly string _name;
@ -53,16 +52,23 @@ namespace CryptoExchange.Net.RateLimiting
}
/// <inheritdoc />
public async Task<CallResult> ProcessSingleAsync(ILogger logger, int itemId, RateLimitItemType type, RequestDefinition definition, string host, SecureString? apiKey, int requestWeight, RateLimitingBehaviour rateLimitingBehaviour, CancellationToken ct)
public async Task<CallResult> ProcessSingleAsync(
ILogger logger,
int itemId,
IRateLimitGuard guard,
RateLimitItemType type,
RequestDefinition definition,
string host,
SecureString? apiKey,
RateLimitingBehaviour rateLimitingBehaviour,
CancellationToken ct)
{
await _semaphore.WaitAsync(ct).ConfigureAwait(false);
if (requestWeight == 0)
requestWeight = 1;
_waitingCount++;
try
{
return await CheckGuardsAsync(new IRateLimitGuard[] { _singleLimitGuard }, logger, itemId, type, definition, host, apiKey, requestWeight, rateLimitingBehaviour, ct).ConfigureAwait(false);
return await CheckGuardsAsync(new IRateLimitGuard[] { guard }, logger, itemId, type, definition, host, apiKey, 1, rateLimitingBehaviour, ct).ConfigureAwait(false);
}
finally
{
@ -130,13 +136,6 @@ namespace CryptoExchange.Net.RateLimiting
return this;
}
/// <inheritdoc />
public IRateLimitGate SetSingleLimitGuard(SingleLimitGuard guard)
{
_singleLimitGuard = guard;
return this;
}
/// <inheritdoc />
public async Task SetRetryAfterGuardAsync(DateTime retryAfter)
{

16
CryptoExchange.Net/Testing/Comparers/SystemTextJsonComparer.cs
View File

@ -70,7 +70,8 @@ namespace CryptoExchange.Net.Testing.Comparers
else if (jsonObject!.Type == JTokenType.Array)
{
var jObjs = (JArray)jsonObject;
var list = (IEnumerable)resultData;
if (resultData is IEnumerable list)
{
var enumerator = list.GetEnumerator();
foreach (var jObj in jObjs)
{
@ -116,6 +117,19 @@ namespace CryptoExchange.Net.Testing.Comparers
}
}
else
{
var resultProps = resultData.GetType().GetProperties().Select(p => (p, p.GetCustomAttributes(typeof(ArrayPropertyAttribute), true).Cast<ArrayPropertyAttribute>().SingleOrDefault()));
int i = 0;
foreach (var item in jObjs.Children())
{
var arrayProp = resultProps.Where(p => p.Item2 != null).SingleOrDefault(p => p.Item2!.Index == i).p;
if (arrayProp != null)
CheckPropertyValue(method, item, arrayProp.GetValue(resultData), arrayProp.PropertyType, arrayProp.Name, "Array index " + i, ignoreProperties!);
i++;
}
}
}
else
{
foreach (var item in jsonObject)
{