using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using CryptoExchange.Net.Authentication; using CryptoExchange.Net.Interfaces; using CryptoExchange.Net.Objects; using CryptoExchange.Net.Requests; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace CryptoExchange.Net { /// /// Base rest client /// public abstract class BaseRestClient : BaseClient, IRestClient { /// /// The factory for creating requests. Used for unit testing /// public IRequestFactory RequestFactory { get; set; } = new RequestFactory(); /// /// Where to put the parameters for requests with different Http methods /// protected Dictionary ParameterPositions { get; set; } = new Dictionary { { HttpMethod.Get, HttpMethodParameterPosition.InUri }, { HttpMethod.Post, HttpMethodParameterPosition.InBody }, { HttpMethod.Delete, HttpMethodParameterPosition.InBody }, { HttpMethod.Put, HttpMethodParameterPosition.InBody } }; /// /// Request body content type /// protected RequestBodyFormat requestBodyFormat = RequestBodyFormat.Json; /// /// Whether or not we need to manually parse an error instead of relying on the http status code /// protected bool manualParseError = false; /// /// How to serialize array parameters when making requests /// protected ArrayParametersSerialization arraySerialization = ArrayParametersSerialization.Array; /// /// What request body should be set when no data is send (only used in combination with postParametersPosition.InBody) /// protected string requestBodyEmptyContent = "{}"; /// public int TotalRequestsMade => ApiClients.OfType().Sum(s => s.TotalRequestsMade); /// /// Request headers to be sent with each request /// protected Dictionary? StandardRequestHeaders { get; set; } /// /// Client options /// public new BaseRestClientOptions ClientOptions { get; } /// /// ctor /// /// The name of the API this client is for /// The options for this client protected BaseRestClient(string name, BaseRestClientOptions options) : base(name, options) { if (options == null) throw new ArgumentNullException(nameof(options)); ClientOptions = options; RequestFactory.Configure(options.RequestTimeout, options.Proxy, options.HttpClient); } /// public void SetApiCredentials(ApiCredentials credentials) { foreach (var apiClient in ApiClients) apiClient.SetApiCredentials(credentials); } /// /// Execute a request to the uri and deserialize the response into the provided type parameter /// /// The type to deserialize into /// The API client the request is for /// The uri to send the request to /// The method of the request /// Cancellation token /// The parameters of the request /// Whether or not the request should be authenticated /// Where the parameters should be placed, overwrites the value set in the client /// How array parameters should be serialized, overwrites the value set in the client /// Credits used for the request /// The JsonSerializer to use for deserialization /// Additional headers to send with the request /// Ignore rate limits for this request /// [return: NotNull] protected virtual async Task> SendRequestAsync( RestApiClient apiClient, Uri uri, HttpMethod method, CancellationToken cancellationToken, Dictionary? parameters = null, bool signed = false, HttpMethodParameterPosition? parameterPosition = null, ArrayParametersSerialization? arraySerialization = null, int requestWeight = 1, JsonSerializer? deserializer = null, Dictionary? additionalHeaders = null, bool ignoreRatelimit = false ) where T : class { var requestId = NextId(); if (signed) { var syncTimeResult = await apiClient.SyncTimeAsync().ConfigureAwait(false); if (!syncTimeResult) { log.Write(LogLevel.Debug, $"[{requestId}] Failed to sync time, aborting request: " + syncTimeResult.Error); return syncTimeResult.As(default); } } if (!ignoreRatelimit) { foreach (var limiter in apiClient.RateLimiters) { var limitResult = await limiter.LimitRequestAsync(log, uri.AbsolutePath, method, signed, apiClient.Options.ApiCredentials?.Key, apiClient.Options.RateLimitingBehaviour, requestWeight, cancellationToken).ConfigureAwait(false); if (!limitResult.Success) return new WebCallResult(limitResult.Error!); } } log.Write(LogLevel.Debug, $"[{requestId}] Creating request for " + uri); if (signed && apiClient.AuthenticationProvider == null) { log.Write(LogLevel.Warning, $"[{requestId}] Request {uri.AbsolutePath} failed because no ApiCredentials were provided"); return new WebCallResult(new NoApiCredentialsError()); } var paramsPosition = parameterPosition ?? ParameterPositions[method]; var request = ConstructRequest(apiClient, uri, method, parameters, signed, paramsPosition, arraySerialization ?? this.arraySerialization, requestId, additionalHeaders); string? paramString = ""; if (paramsPosition == HttpMethodParameterPosition.InBody) paramString = $" with request body '{request.Content}'"; if (log.Level == LogLevel.Trace) { var headers = request.GetHeaders(); if (headers.Any()) paramString += " with headers " + string.Join(", ", headers.Select(h => h.Key + $"=[{string.Join(",", h.Value)}]")); } apiClient.TotalRequestsMade++; log.Write(LogLevel.Debug, $"[{requestId}] Sending {method}{(signed ? " signed" : "")} request to {request.Uri}{paramString ?? " "}{(ClientOptions.Proxy == null ? "" : $" via proxy {ClientOptions.Proxy.Host}")}"); return await GetResponseAsync(request, deserializer, cancellationToken).ConfigureAwait(false); } /// /// Executes the request and returns the result deserialized into the type parameter class /// /// The request object to execute /// The JsonSerializer to use for deserialization /// Cancellation token /// protected virtual async Task> GetResponseAsync(IRequest request, JsonSerializer? deserializer, CancellationToken cancellationToken) { try { var sw = Stopwatch.StartNew(); var response = await request.GetResponseAsync(cancellationToken).ConfigureAwait(false); sw.Stop(); var statusCode = response.StatusCode; var headers = response.ResponseHeaders; var responseStream = await response.GetResponseStreamAsync().ConfigureAwait(false); if (response.IsSuccessStatusCode) { // If we have to manually parse error responses (can't rely on HttpStatusCode) we'll need to read the full // response before being able to deserialize it into the resulting type since we don't know if it an error response or data if (manualParseError) { using var reader = new StreamReader(responseStream); var data = await reader.ReadToEndAsync().ConfigureAwait(false); responseStream.Close(); response.Close(); log.Write(LogLevel.Debug, $"[{request.RequestId}] Response received in {sw.ElapsedMilliseconds}ms: {data}"); // Validate if it is valid json. Sometimes other data will be returned, 502 error html pages for example var parseResult = ValidateJson(data); if (!parseResult.Success) return new WebCallResult(response.StatusCode, response.ResponseHeaders, sw.Elapsed, ClientOptions.OutputOriginalData ? data : null, request.Uri.ToString(), request.Content, request.Method, request.GetHeaders(), default, parseResult.Error!); // Let the library implementation see if it is an error response, and if so parse the error var error = await TryParseErrorAsync(parseResult.Data).ConfigureAwait(false); if (error != null) return new WebCallResult(response.StatusCode, response.ResponseHeaders, sw.Elapsed, ClientOptions.OutputOriginalData ? data : null, request.Uri.ToString(), request.Content, request.Method, request.GetHeaders(), default, error!); // Not an error, so continue deserializing var deserializeResult = Deserialize(parseResult.Data, deserializer, request.RequestId); return new WebCallResult(response.StatusCode, response.ResponseHeaders, sw.Elapsed, ClientOptions.OutputOriginalData ? data: null, request.Uri.ToString(), request.Content, request.Method, request.GetHeaders(), deserializeResult.Data, deserializeResult.Error); } else { // Success status code, and we don't have to check for errors. Continue deserializing directly from the stream var desResult = await DeserializeAsync(responseStream, deserializer, request.RequestId, sw.ElapsedMilliseconds).ConfigureAwait(false); responseStream.Close(); response.Close(); return new WebCallResult(statusCode, headers, sw.Elapsed, ClientOptions.OutputOriginalData ? desResult.OriginalData : null, request.Uri.ToString(), request.Content, request.Method, request.GetHeaders(), desResult.Data, desResult.Error); } } else { // Http status code indicates error using var reader = new StreamReader(responseStream); var data = await reader.ReadToEndAsync().ConfigureAwait(false); log.Write(LogLevel.Debug, $"[{request.RequestId}] Error received: {data}"); responseStream.Close(); response.Close(); var parseResult = ValidateJson(data); var error = parseResult.Success ? ParseErrorResponse(parseResult.Data) : parseResult.Error!; if(error.Code == null || error.Code == 0) error.Code = (int)response.StatusCode; return new WebCallResult(statusCode, headers, sw.Elapsed, data, request.Uri.ToString(), request.Content, request.Method, request.GetHeaders(), default, error); } } catch (HttpRequestException requestException) { // Request exception, can't reach server for instance var exceptionInfo = requestException.ToLogString(); log.Write(LogLevel.Warning, $"[{request.RequestId}] Request exception: " + exceptionInfo); return new WebCallResult(null, null, null, null, request.Uri.ToString(), request.Content, request.Method, request.GetHeaders(), default, new WebError(exceptionInfo)); } catch (OperationCanceledException canceledException) { if (cancellationToken != default && canceledException.CancellationToken == cancellationToken) { // Cancellation token canceled by caller log.Write(LogLevel.Warning, $"[{request.RequestId}] Request canceled by cancellation token"); return new WebCallResult(null, null, null, null, request.Uri.ToString(), request.Content, request.Method, request.GetHeaders(), default, new CancellationRequestedError()); } else { // Request timed out log.Write(LogLevel.Warning, $"[{request.RequestId}] Request timed out: " + canceledException.ToLogString()); return new WebCallResult(null, null, null, null, request.Uri.ToString(), request.Content, request.Method, request.GetHeaders(), default, new WebError($"[{request.RequestId}] Request timed out")); } } } /// /// Can be used to parse an error even though response status indicates success. Some apis always return 200 OK, even though there is an error. /// When setting manualParseError to true this method will be called for each response to be able to check if the response is an error or not. /// If the response is an error this method should return the parsed error, else it should return null /// /// Received data /// Null if not an error, Error otherwise protected virtual Task TryParseErrorAsync(JToken data) { return Task.FromResult(null); } /// /// Creates a request object /// /// The API client the request is for /// The uri to send the request to /// The method of the request /// The parameters of the request /// Whether or not the request should be authenticated /// Where the parameters should be placed /// How array parameters should be serialized /// Unique id of a request /// Additional headers to send with the request /// protected virtual IRequest ConstructRequest( RestApiClient apiClient, Uri uri, HttpMethod method, Dictionary? parameters, bool signed, HttpMethodParameterPosition parameterPosition, ArrayParametersSerialization arraySerialization, int requestId, Dictionary? additionalHeaders) { parameters ??= new Dictionary(); for (var i = 0; i< parameters.Count; i++) { var kvp = parameters.ElementAt(i); if (kvp.Value is Func delegateValue) parameters[kvp.Key] = delegateValue(); } if (parameterPosition == HttpMethodParameterPosition.InUri) { foreach (var parameter in parameters) uri = uri.AddQueryParmeter(parameter.Key, parameter.Value.ToString()); } var headers = new Dictionary(); var uriParameters = parameterPosition == HttpMethodParameterPosition.InUri ? new SortedDictionary(parameters) : new SortedDictionary(); var bodyParameters = parameterPosition == HttpMethodParameterPosition.InBody ? new SortedDictionary(parameters) : new SortedDictionary(); if (apiClient.AuthenticationProvider != null) apiClient.AuthenticationProvider.AuthenticateRequest( apiClient, uri, method, parameters, signed, arraySerialization, parameterPosition, out uriParameters, out bodyParameters, out headers); // Sanity check foreach(var param in parameters) { if (!uriParameters.ContainsKey(param.Key) && !bodyParameters.ContainsKey(param.Key)) throw new Exception($"Missing parameter {param.Key} after authentication processing. AuthenticationProvider implementation " + $"should return provided parameters in either the uri or body parameters output"); } // Add the auth parameters to the uri, start with a new URI to be able to sort the parameters including the auth parameters uri = uri.SetParameters(uriParameters, arraySerialization); var request = RequestFactory.Create(method, uri, requestId); request.Accept = Constants.JsonContentHeader; foreach (var header in headers) request.AddHeader(header.Key, header.Value); if (additionalHeaders != null) { foreach (var header in additionalHeaders) request.AddHeader(header.Key, header.Value); } if (StandardRequestHeaders != null) { foreach (var header in StandardRequestHeaders) // Only add it if it isn't overwritten if (additionalHeaders?.ContainsKey(header.Key) != true) request.AddHeader(header.Key, header.Value); } if (parameterPosition == HttpMethodParameterPosition.InBody) { var contentType = requestBodyFormat == RequestBodyFormat.Json ? Constants.JsonContentHeader : Constants.FormContentHeader; if (bodyParameters.Any()) WriteParamBody(request, bodyParameters, contentType); else request.SetContent(requestBodyEmptyContent, contentType); } return request; } /// /// Writes the parameters of the request to the request object body /// /// The request to set the parameters on /// The parameters to set /// The content type of the data protected virtual void WriteParamBody(IRequest request, SortedDictionary parameters, string contentType) { if (requestBodyFormat == RequestBodyFormat.Json) { // Write the parameters as json in the body var stringData = JsonConvert.SerializeObject(parameters); request.SetContent(stringData, contentType); } else if (requestBodyFormat == RequestBodyFormat.FormData) { // Write the parameters as form data in the body var stringData = parameters.ToFormData(); request.SetContent(stringData, contentType); } } /// /// Parse an error response from the server. Only used when server returns a status other than Success(200) /// /// The string the request returned /// protected virtual Error ParseErrorResponse(JToken error) { return new ServerError(error.ToString()); } } }