From c2105fe690c6b4a468d3d45e847a2e32172c702a Mon Sep 17 00:00:00 2001 From: Jkorf Date: Wed, 8 Dec 2021 16:20:44 +0100 Subject: [PATCH] Wip, support for time syncing, refactoring authentication --- .../Authentication/AuthenticationProvider.cs | 169 +++++++++++++++--- .../Authentication/SignOutputType.cs | 17 ++ CryptoExchange.Net/Clients/BaseRestClient.cs | 131 ++++++++++---- CryptoExchange.Net/Clients/RestApiClient.cs | 72 +++++++- CryptoExchange.Net/ExtensionMethods.cs | 44 +++++ .../Interfaces/IRequestFactory.cs | 2 +- CryptoExchange.Net/Objects/Options.cs | 8 +- CryptoExchange.Net/Objects/TimeSyncModel.cs | 21 +++ CryptoExchange.Net/Requests/RequestFactory.cs | 2 +- 9 files changed, 402 insertions(+), 64 deletions(-) create mode 100644 CryptoExchange.Net/Authentication/SignOutputType.cs create mode 100644 CryptoExchange.Net/Objects/TimeSyncModel.cs diff --git a/CryptoExchange.Net/Authentication/AuthenticationProvider.cs b/CryptoExchange.Net/Authentication/AuthenticationProvider.cs index ed266c6..5c54462 100644 --- a/CryptoExchange.Net/Authentication/AuthenticationProvider.cs +++ b/CryptoExchange.Net/Authentication/AuthenticationProvider.cs @@ -1,6 +1,9 @@ using CryptoExchange.Net.Objects; +using System; using System.Collections.Generic; using System.Net.Http; +using System.Security.Cryptography; +using System.Text; namespace CryptoExchange.Net.Authentication { @@ -14,45 +17,151 @@ namespace CryptoExchange.Net.Authentication /// public ApiCredentials Credentials { get; } + /// + /// + protected byte[] _sBytes; + /// /// ctor /// /// protected AuthenticationProvider(ApiCredentials credentials) { + if (credentials.Secret == null) + throw new ArgumentException("ApiKey/Secret needed"); + Credentials = credentials; + _sBytes = Encoding.UTF8.GetBytes(credentials.Secret.GetString()); } /// - /// Add authentication to the parameter list based on the provided credentials + /// Authenticate a request where the parameters need to be in the Uri /// - /// The uri the request is for - /// The HTTP method of the request - /// The provided parameters for the request - /// Wether or not the request needs to be signed. If not typically the parameters list can just be returned - /// Where parameters are placed, in the URI or in the request body - /// How array parameters are serialized - /// Should return the original parameter list including any authentication parameters needed - public virtual Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed, - HttpMethodParameterPosition parameterPosition, ArrayParametersSerialization arraySerialization) + /// The Api client sending the request + /// The uri for the request + /// The method of the request + /// The request parameters + /// The request headers + /// If the requests should be authenticated + /// Array serialization type + /// + public abstract void AuthenticateUriRequest( + RestApiClient apiClient, + ref Uri uri, + HttpMethod method, + SortedDictionary parameters, + Dictionary headers, + bool auth, + ArrayParametersSerialization arraySerialization); + + /// + /// Authenticate a request where the parameters need to be in the request body + /// + /// The Api client sending the request + /// The uri for the request + /// The method of the request + /// The request parameters + /// The request headers + /// If the requests should be authenticated + /// Array serialization type + public abstract void AuthenticateBodyRequest( + RestApiClient apiClient, + Uri uri, + HttpMethod method, + SortedDictionary parameters, + Dictionary headers, + bool auth, + ArrayParametersSerialization arraySerialization); + + /// + /// SHA256 sign the data and return the hash + /// + /// Data to sign + /// String type + /// + protected static string SignSHA256(string data, SignOutputType? outputType = null) { - return parameters; + using var encryptor = SHA256.Create(); + var resultBytes = encryptor.ComputeHash(Encoding.UTF8.GetBytes(data)); + return outputType == SignOutputType.Base64 ? BytesToBase64String(resultBytes): BytesToHexString(resultBytes); } /// - /// Add authentication to the header dictionary based on the provided credentials + /// SHA384 sign the data and return the hash /// - /// The uri the request is for - /// The HTTP method of the request - /// The provided parameters for the request - /// Wether or not the request needs to be signed. If not typically the parameters list can just be returned - /// Where post parameters are placed, in the URI or in the request body - /// How array parameters are serialized - /// Should return a dictionary containing any header key/value pairs needed for authenticating the request - public virtual Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed, - HttpMethodParameterPosition parameterPosition, ArrayParametersSerialization arraySerialization) + /// Data to sign + /// String type + /// + protected static string SignSHA384(string data, SignOutputType? outputType = null) { - return new Dictionary(); + using var encryptor = SHA384.Create(); + var resultBytes = encryptor.ComputeHash(Encoding.UTF8.GetBytes(data)); + return outputType == SignOutputType.Base64 ? BytesToBase64String(resultBytes) : BytesToHexString(resultBytes); + } + + /// + /// SHA512 sign the data and return the hash + /// + /// Data to sign + /// String type + /// + protected static string SignSHA512(string data, SignOutputType? outputType = null) + { + using var encryptor = SHA512.Create(); + var resultBytes = encryptor.ComputeHash(Encoding.UTF8.GetBytes(data)); + return outputType == SignOutputType.Base64 ? BytesToBase64String(resultBytes) : BytesToHexString(resultBytes); + } + + /// + /// MD5 sign the data and return the hash + /// + /// Data to sign + /// String type + /// + protected static string SignMD5(string data, SignOutputType? outputType = null) + { + using var encryptor = MD5.Create(); + var resultBytes = encryptor.ComputeHash(Encoding.UTF8.GetBytes(data)); + return outputType == SignOutputType.Base64 ? BytesToBase64String(resultBytes) : BytesToHexString(resultBytes); + } + + /// + /// HMACSHA256 sign the data and return the hash + /// + /// Data to sign + /// String type + /// + protected string SignHMACSHA256(string data, SignOutputType? outputType = null) + { + using var encryptor = new HMACSHA256(_sBytes); + var resultBytes = encryptor.ComputeHash(Encoding.UTF8.GetBytes(data)); + return outputType == SignOutputType.Base64 ? BytesToBase64String(resultBytes) : BytesToHexString(resultBytes); + } + + /// + /// HMACSHA384 sign the data and return the hash + /// + /// Data to sign + /// String type + /// + protected string SignHMACSHA384(string data, SignOutputType? outputType = null) + { + using var encryptor = new HMACSHA384(_sBytes); + var resultBytes = encryptor.ComputeHash(Encoding.UTF8.GetBytes(data)); + return outputType == SignOutputType.Base64 ? BytesToBase64String(resultBytes) : BytesToHexString(resultBytes); + } + + /// + /// HMACSHA512 sign the data and return the hash + /// + /// Data to sign + /// String type + /// + protected string SignHMACSHA512(string data, SignOutputType? outputType = null) + { + using var encryptor = new HMACSHA512(_sBytes); + var resultBytes = encryptor.ComputeHash(Encoding.UTF8.GetBytes(data)); + return outputType == SignOutputType.Base64 ? BytesToBase64String(resultBytes) : BytesToHexString(resultBytes); } /// @@ -76,16 +185,26 @@ namespace CryptoExchange.Net.Authentication } /// - /// Convert byte array to hex + /// Convert byte array to hex string /// /// /// - protected static string ByteToString(byte[] buff) + protected static string BytesToHexString(byte[] buff) { var result = string.Empty; foreach (var t in buff) - result += t.ToString("X2"); /* hex format */ + result += t.ToString("X2"); return result; } + + /// + /// Convert byte array to base64 string + /// + /// + /// + protected static string BytesToBase64String(byte[] buff) + { + return Convert.ToBase64String(buff); + } } } diff --git a/CryptoExchange.Net/Authentication/SignOutputType.cs b/CryptoExchange.Net/Authentication/SignOutputType.cs new file mode 100644 index 0000000..2c8ae5a --- /dev/null +++ b/CryptoExchange.Net/Authentication/SignOutputType.cs @@ -0,0 +1,17 @@ +namespace CryptoExchange.Net.Authentication +{ + /// + /// Output string type + /// + public enum SignOutputType + { + /// + /// Hex string + /// + Hex, + /// + /// Base64 string + /// + Base64 + } +} diff --git a/CryptoExchange.Net/Clients/BaseRestClient.cs b/CryptoExchange.Net/Clients/BaseRestClient.cs index be4f991..6a7ab4f 100644 --- a/CryptoExchange.Net/Clients/BaseRestClient.cs +++ b/CryptoExchange.Net/Clients/BaseRestClient.cs @@ -83,9 +83,9 @@ namespace CryptoExchange.Net ClientOptions = exchangeOptions; RequestFactory.Configure(exchangeOptions.RequestTimeout, exchangeOptions.Proxy, exchangeOptions.HttpClient); - } + /// /// Execute a request to the uri and deserialize the response into the provided type parameter /// @@ -118,6 +118,11 @@ namespace CryptoExchange.Net ) where T : class { var requestId = NextId(); + + var syncTimeResult = await apiClient.SyncTimeAsync().ConfigureAwait(false); + if (!syncTimeResult) + return syncTimeResult.As(default); + log.Write(LogLevel.Debug, $"[{requestId}] Creating request for " + uri); if (signed && apiClient.AuthenticationProvider == null) { @@ -145,6 +150,9 @@ namespace CryptoExchange.Net paramString += " with headers " + string.Join(", ", headers.Select(h => h.Key + $"=[{string.Join(",", h.Value)}]")); } + apiClient.TotalRequestsMade++; + TotalRequestsMade++; + log.Write(LogLevel.Debug, $"[{requestId}] Sending {method}{(signed ? " signed" : "")} request to {request.Uri}{paramString ?? " "}{(ClientOptions.Proxy == null ? "" : $" via proxy {ClientOptions.Proxy.Host}")}"); return await GetResponseAsync(request, deserializer, cancellationToken).ConfigureAwait(false); } @@ -160,7 +168,6 @@ namespace CryptoExchange.Net { try { - TotalRequestsMade++; var sw = Stopwatch.StartNew(); var response = await request.GetResponseAsync(cancellationToken).ConfigureAwait(false); sw.Stop(); @@ -278,81 +285,135 @@ namespace CryptoExchange.Net int requestId, Dictionary? additionalHeaders) { - parameters ??= new Dictionary(); + SortedDictionary sortedParameters = new SortedDictionary(GetParameterComparer()); + if (parameters != null) + sortedParameters = new SortedDictionary(parameters, GetParameterComparer()); - var uriString = uri.ToString(); - if (apiClient.AuthenticationProvider != null) - parameters = apiClient.AuthenticationProvider.AddAuthenticationToParameters(uriString, method, parameters, signed, parameterPosition, arraySerialization); - - if (parameterPosition == HttpMethodParameterPosition.InUri && parameters?.Any() == true) - uriString += "?" + parameters.CreateParamString(true, arraySerialization); - - var contentType = requestBodyFormat == RequestBodyFormat.Json ? Constants.JsonContentHeader : Constants.FormContentHeader; - var request = RequestFactory.Create(method, uriString, requestId); - request.Accept = Constants.JsonContentHeader; + if (parameterPosition == HttpMethodParameterPosition.InUri) + { + foreach (var parameter in sortedParameters) + uri = uri.AddQueryParmeter(parameter.Key, parameter.Value.ToString()); + } + var length = sortedParameters.Count; var headers = new Dictionary(); if (apiClient.AuthenticationProvider != null) - headers = apiClient.AuthenticationProvider.AddAuthenticationToHeaders(uriString, method, parameters!, signed, parameterPosition, arraySerialization); + { + if(parameterPosition == HttpMethodParameterPosition.InUri) + apiClient.AuthenticationProvider.AuthenticateUriRequest(apiClient, ref uri, method, sortedParameters, headers, signed, arraySerialization); + else + apiClient.AuthenticationProvider.AuthenticateBodyRequest(apiClient, uri, method, sortedParameters, headers, signed, arraySerialization); + } + + if (parameterPosition == HttpMethodParameterPosition.InUri) + { + // Add the auth parameters to the uri, start with a new URI to be able to sort the parameters including the auth parameters + if (sortedParameters.Count != length) + { + var uriBuilder = new UriBuilder(); + uriBuilder.Scheme = uri.Scheme; + uriBuilder.Host = uri.Host; + uriBuilder.Path = uri.AbsolutePath; + uri = uriBuilder.Uri; + foreach(var parameter in sortedParameters) + uri = uri.AddQueryParmeter(parameter.Key, parameter.Value.ToString()); + } + } + + var request = RequestFactory.Create(method, uri, requestId); + request.Accept = Constants.JsonContentHeader; foreach (var header in headers) request.AddHeader(header.Key, header.Value); - if (additionalHeaders != null) - { + if (additionalHeaders != null) + { foreach (var header in additionalHeaders) request.AddHeader(header.Key, header.Value); } - if(StandardRequestHeaders != null) + if (StandardRequestHeaders != null) { foreach (var header in StandardRequestHeaders) // Only add it if it isn't overwritten - if(additionalHeaders?.ContainsKey(header.Key) != true) + if (additionalHeaders?.ContainsKey(header.Key) != true) request.AddHeader(header.Key, header.Value); } if (parameterPosition == HttpMethodParameterPosition.InBody) { - if (parameters?.Any() == true) - WriteParamBody(request, parameters, contentType); + var contentType = requestBodyFormat == RequestBodyFormat.Json ? Constants.JsonContentHeader : Constants.FormContentHeader; + if (sortedParameters?.Any() == true) + WriteParamBody(request, sortedParameters, contentType); else request.SetContent(requestBodyEmptyContent, contentType); } return request; + + //var uriString = uri.ToString(); + //if (apiClient.AuthenticationProvider != null) + // parameters = apiClient.AuthenticationProvider.AddAuthenticationToParameters(uriString, method, parameters, signed, parameterPosition, arraySerialization); + + //if (parameterPosition == HttpMethodParameterPosition.InUri && parameters?.Any() == true) + // uriString += "?" + parameters.CreateParamString(true, arraySerialization); + + //var contentType = requestBodyFormat == RequestBodyFormat.Json ? Constants.JsonContentHeader : Constants.FormContentHeader; + //var request = RequestFactory.Create(method, uriString, requestId); + //request.Accept = Constants.JsonContentHeader; + + //var headers = new Dictionary(); + //if (apiClient.AuthenticationProvider != null) + // headers = apiClient.AuthenticationProvider.AddAuthenticationToHeaders(uriString, method, parameters!, signed, parameterPosition, arraySerialization); + + //foreach (var header in headers) + // request.AddHeader(header.Key, header.Value); + + //if (additionalHeaders != null) + //{ + // foreach (var header in additionalHeaders) + // request.AddHeader(header.Key, header.Value); + //} + + //if(StandardRequestHeaders != null) + //{ + // foreach (var header in StandardRequestHeaders) + // // Only add it if it isn't overwritten + // if(additionalHeaders?.ContainsKey(header.Key) != true) + // request.AddHeader(header.Key, header.Value); + //} + + //if (parameterPosition == HttpMethodParameterPosition.InBody) + //{ + // if (parameters?.Any() == true) + // WriteParamBody(request, parameters, contentType); + // else + // request.SetContent(requestBodyEmptyContent, contentType); + //} + + //return request; } + protected virtual IComparer GetParameterComparer() => null; + /// /// Writes the parameters of the request to the request object body /// /// The request to set the parameters on /// The parameters to set /// The content type of the data - protected virtual void WriteParamBody(IRequest request, Dictionary parameters, string contentType) + protected virtual void WriteParamBody(IRequest request, SortedDictionary parameters, string contentType) { if (requestBodyFormat == RequestBodyFormat.Json) { // Write the parameters as json in the body - var stringData = JsonConvert.SerializeObject(parameters.OrderBy(p => p.Key).ToDictionary(p => p.Key, p => p.Value)); + var stringData = JsonConvert.SerializeObject(parameters); request.SetContent(stringData, contentType); } else if (requestBodyFormat == RequestBodyFormat.FormData) { // Write the parameters as form data in the body - var formData = HttpUtility.ParseQueryString(string.Empty); - foreach (var kvp in parameters.OrderBy(p => p.Key)) - { - if (kvp.Value.GetType().IsArray) - { - var array = (Array)kvp.Value; - foreach (var value in array) - formData.Add(kvp.Key, value.ToString()); - } - else - formData.Add(kvp.Key, kvp.Value.ToString()); - } - var stringData = formData.ToString(); + var stringData = parameters.ToFormData(); request.SetContent(stringData, contentType); } } diff --git a/CryptoExchange.Net/Clients/RestApiClient.cs b/CryptoExchange.Net/Clients/RestApiClient.cs index f0e9550..1247425 100644 --- a/CryptoExchange.Net/Clients/RestApiClient.cs +++ b/CryptoExchange.Net/Clients/RestApiClient.cs @@ -1,6 +1,11 @@ +using System; using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using CryptoExchange.Net.Interfaces; +using CryptoExchange.Net.Logging; using CryptoExchange.Net.Objects; +using Microsoft.Extensions.Logging; namespace CryptoExchange.Net { @@ -9,16 +14,27 @@ namespace CryptoExchange.Net /// public abstract class RestApiClient: BaseApiClient { + protected abstract TimeSyncModel GetTimeSyncParameters(); + protected abstract void UpdateTimeOffset(TimeSpan offset); + public abstract TimeSpan GetTimeOffset(); + + /// + /// Total amount of requests made with this API client + /// + public int TotalRequestsMade { get; set; } + /// /// Options for this client /// - internal RestApiClientOptions Options { get; } + public RestApiClientOptions Options { get; } /// /// List of rate limiters /// internal IEnumerable RateLimiters { get; } + private Log _log; + /// /// ctor /// @@ -34,5 +50,59 @@ namespace CryptoExchange.Net RateLimiters = rateLimiters; } + /// + /// Retrieve the server time for the purpose of syncing time between client and server to prevent authentication issues + /// + /// Server time + protected abstract Task> GetServerTimestampAsync(); + + internal async Task> SyncTimeAsync() + { + var timeSyncParams = GetTimeSyncParameters(); + if (await timeSyncParams.Semaphore.WaitAsync(0).ConfigureAwait(false)) + { + if (!timeSyncParams.SyncTime || (DateTime.UtcNow - timeSyncParams.LastSyncTime < TimeSpan.FromHours(1))) + { + timeSyncParams.Semaphore.Release(); + return new WebCallResult(null, null, true, null); + } + + var localTime = DateTime.UtcNow; + var result = await GetServerTimestampAsync().ConfigureAwait(false); + if (!result) + { + timeSyncParams.Semaphore.Release(); + return result.As(false); + } + + if (TotalRequestsMade == 1) + { + // If this was the first request make another one to calculate the offset since the first one can be slower + localTime = DateTime.UtcNow; + result = await GetServerTimestampAsync().ConfigureAwait(false); + if (!result) + { + timeSyncParams.Semaphore.Release(); + return result.As(false); + } + } + + // Calculate time offset between local and server + var offset = result.Data - localTime; + if (offset.TotalMilliseconds >= 0 && offset.TotalMilliseconds < 500) + { + // Small offset, probably mainly due to ping. Don't adjust time + UpdateTimeOffset(offset); + timeSyncParams.Semaphore.Release(); + } + else + { + UpdateTimeOffset(offset); + timeSyncParams.Semaphore.Release(); + } + } + + return new WebCallResult(null, null, true, null); + } } } diff --git a/CryptoExchange.Net/ExtensionMethods.cs b/CryptoExchange.Net/ExtensionMethods.cs index 55a265e..5e39b28 100644 --- a/CryptoExchange.Net/ExtensionMethods.cs +++ b/CryptoExchange.Net/ExtensionMethods.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Runtime.InteropServices; using System.Security; using System.Text; +using System.Web; using CryptoExchange.Net.Logging; using CryptoExchange.Net.Objects; using Microsoft.Extensions.Logging; @@ -142,6 +143,49 @@ namespace CryptoExchange.Net return uriString; } + /// + /// Convert a dictionary to formdata string + /// + /// + /// + public static string ToFormData(this SortedDictionary parameters) + { + var formData = HttpUtility.ParseQueryString(string.Empty); + foreach (var kvp in parameters.OrderBy(p => p.Key)) + { + if (kvp.Value.GetType().IsArray) + { + var array = (Array)kvp.Value; + foreach (var value in array) + formData.Add(kvp.Key, value.ToString()); + } + else + formData.Add(kvp.Key, kvp.Value.ToString()); + } + return formData.ToString(); + } + + /// + /// Add parameter to URI + /// + /// + /// + /// + /// + public static Uri AddQueryParmeter(this Uri uri, string name, string value) + { + var httpValueCollection = HttpUtility.ParseQueryString(uri.Query); + + httpValueCollection.Remove(name); + httpValueCollection.Add(name, value); + + var ub = new UriBuilder(uri); + ub.Query = httpValueCollection.ToString(); + + return ub.Uri; + } + + /// /// Get the string the secure string is representing /// diff --git a/CryptoExchange.Net/Interfaces/IRequestFactory.cs b/CryptoExchange.Net/Interfaces/IRequestFactory.cs index 2779d92..72a25d6 100644 --- a/CryptoExchange.Net/Interfaces/IRequestFactory.cs +++ b/CryptoExchange.Net/Interfaces/IRequestFactory.cs @@ -16,7 +16,7 @@ namespace CryptoExchange.Net.Interfaces /// /// /// - IRequest Create(HttpMethod method, string uri, int requestId); + IRequest Create(HttpMethod method, Uri uri, int requestId); /// /// Configure the requests created by this factory diff --git a/CryptoExchange.Net/Objects/Options.cs b/CryptoExchange.Net/Objects/Options.cs index ae55a7d..9116da5 100644 --- a/CryptoExchange.Net/Objects/Options.cs +++ b/CryptoExchange.Net/Objects/Options.cs @@ -275,6 +275,11 @@ namespace CryptoExchange.Net.Objects /// public RateLimitingBehaviour RateLimitingBehaviour { get; set; } = RateLimitingBehaviour.Wait; + /// + /// Whether or not to automatically sync the local time with the server time + /// + public bool AutoTimestamp { get; set; } = true; + /// /// ctor /// @@ -312,12 +317,13 @@ namespace CryptoExchange.Net.Objects if(def.RateLimiters != null) input.RateLimiters = def.RateLimiters.ToList(); input.RateLimitingBehaviour = def.RateLimitingBehaviour; + input.AutoTimestamp = def.AutoTimestamp; } /// public override string ToString() { - return $"{base.ToString()}, RateLimiters: {RateLimiters?.Count}, RateLimitBehaviour: {RateLimitingBehaviour}"; + return $"{base.ToString()}, RateLimiters: {RateLimiters?.Count}, RateLimitBehaviour: {RateLimitingBehaviour}, AutoTimestamp: {AutoTimestamp}"; } } diff --git a/CryptoExchange.Net/Objects/TimeSyncModel.cs b/CryptoExchange.Net/Objects/TimeSyncModel.cs new file mode 100644 index 0000000..6bc1c64 --- /dev/null +++ b/CryptoExchange.Net/Objects/TimeSyncModel.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; + +namespace CryptoExchange.Net.Objects +{ + public class TimeSyncModel + { + public bool SyncTime { get; set; } + public SemaphoreSlim Semaphore { get; set; } + public DateTime LastSyncTime { get; set; } + + public TimeSyncModel(bool syncTime, SemaphoreSlim semaphore, DateTime lastSyncTime) + { + SyncTime = syncTime; + Semaphore = semaphore; + LastSyncTime = lastSyncTime; + } + } +} diff --git a/CryptoExchange.Net/Requests/RequestFactory.cs b/CryptoExchange.Net/Requests/RequestFactory.cs index 3da74c4..fb9487e 100644 --- a/CryptoExchange.Net/Requests/RequestFactory.cs +++ b/CryptoExchange.Net/Requests/RequestFactory.cs @@ -36,7 +36,7 @@ namespace CryptoExchange.Net.Requests } /// - public IRequest Create(HttpMethod method, string uri, int requestId) + public IRequest Create(HttpMethod method, Uri uri, int requestId) { if (httpClient == null) throw new InvalidOperationException("Cant create request before configuring http client");