diff --git a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs new file mode 100644 index 0000000..beefa97 --- /dev/null +++ b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs @@ -0,0 +1,69 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using CryptoExchange.Net.Logging; +using CryptoExchange.Net.Objects; +using CryptoExchange.Net.OrderBook; +using CryptoExchange.Net.Sockets; +using CryptoExchange.Net.UnitTests.TestImplementations; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +namespace CryptoExchange.Net.UnitTests +{ + [TestFixture] + public class SymbolOrderBookTests + { + private static OrderBookOptions defaultOrderBookOptions = new OrderBookOptions("Test", true, false); + + private class TestableSymbolOrderBook : SymbolOrderBook + { + public TestableSymbolOrderBook() : base("BTC/USD", defaultOrderBookOptions) + { + } + + public override void Dispose() {} + + protected override Task> DoResync() + { + throw new NotImplementedException(); + } + + protected override Task> DoStart() + { + throw new NotImplementedException(); + } + } + + [TestCase] + public void GivenEmptyBidList_WhenBestBid_ThenEmptySymbolOrderBookEntry() + { + var symbolOrderBook = new TestableSymbolOrderBook(); + Assert.IsNotNull(symbolOrderBook.BestBid); + Assert.AreEqual(0m, symbolOrderBook.BestBid.Price); + Assert.AreEqual(0m, symbolOrderBook.BestAsk.Quantity); + } + + [TestCase] + public void GivenEmptyAskList_WhenBestAsk_ThenEmptySymbolOrderBookEntry() + { + var symbolOrderBook = new TestableSymbolOrderBook(); + Assert.IsNotNull(symbolOrderBook.BestBid); + Assert.AreEqual(0m, symbolOrderBook.BestBid.Price); + Assert.AreEqual(0m, symbolOrderBook.BestAsk.Quantity); + } + + [TestCase] + public void GivenEmptyBidAndAskList_WhenBestOffers_ThenEmptySymbolOrderBookEntries() + { + var symbolOrderBook = new TestableSymbolOrderBook(); + Assert.IsNotNull(symbolOrderBook.BestOffers); + Assert.IsNotNull(symbolOrderBook.BestOffers.Bid); + Assert.IsNotNull(symbolOrderBook.BestOffers.Ask); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Bid.Price); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Bid.Quantity); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Ask.Price); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Ask.Quantity); + } + } +} diff --git a/CryptoExchange.Net.UnitTests/TestImplementations/TestBaseClient.cs b/CryptoExchange.Net.UnitTests/TestImplementations/TestBaseClient.cs index 9b36f42..d9e2e91 100644 --- a/CryptoExchange.Net.UnitTests/TestImplementations/TestBaseClient.cs +++ b/CryptoExchange.Net.UnitTests/TestImplementations/TestBaseClient.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Net.Http; using CryptoExchange.Net.Authentication; using CryptoExchange.Net.Logging; @@ -38,14 +39,14 @@ namespace CryptoExchange.Net.UnitTests { } - public override Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed) + public override Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed, PostParameters postParameters, ArrayParametersSerialization arraySerialization) { - return base.AddAuthenticationToHeaders(uri, method, parameters, signed); + return base.AddAuthenticationToHeaders(uri, method, parameters, signed, postParameters, arraySerialization); } - public override Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed) + public override Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed, PostParameters postParameters, ArrayParametersSerialization arraySerialization) { - return base.AddAuthenticationToParameters(uri, method, parameters, signed); + return base.AddAuthenticationToParameters(uri, method, parameters, signed, postParameters, arraySerialization); } public override string Sign(string toSign) diff --git a/CryptoExchange.Net/Authentication/AuthenticationProvider.cs b/CryptoExchange.Net/Authentication/AuthenticationProvider.cs index 8739fed..21857b1 100644 --- a/CryptoExchange.Net/Authentication/AuthenticationProvider.cs +++ b/CryptoExchange.Net/Authentication/AuthenticationProvider.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using CryptoExchange.Net.Objects; +using System.Collections.Generic; using System.Net.Http; namespace CryptoExchange.Net.Authentication @@ -29,8 +30,11 @@ namespace CryptoExchange.Net.Authentication /// /// /// + /// + /// /// - public virtual Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed) + public virtual Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed, + PostParameters postParameterPosition, ArrayParametersSerialization arraySerialization) { return parameters; } @@ -42,8 +46,11 @@ namespace CryptoExchange.Net.Authentication /// /// /// + /// + /// /// - public virtual Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed) + public virtual Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed, + PostParameters postParameterPosition, ArrayParametersSerialization arraySerialization) { return new Dictionary(); } diff --git a/CryptoExchange.Net/CryptoExchange.Net.csproj b/CryptoExchange.Net/CryptoExchange.Net.csproj index 6c4f1cb..dab827b 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.csproj +++ b/CryptoExchange.Net/CryptoExchange.Net.csproj @@ -6,12 +6,12 @@ CryptoExchange.Net JKorf A base package for implementing cryptocurrency exchange API's - 3.0.5 + 3.0.11 false https://github.com/JKorf/CryptoExchange.Net en true - 3.0.5 - Added PausedActivity events on socket subscriptions + 3.0.11 - Added support for checksum in SymbolOrderBook enable 8.0 MIT diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index 97639e7..abf449e 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -97,7 +97,7 @@ - + Add authentication to the parameter list @@ -105,9 +105,11 @@ + + - + Add authentication to the header dictionary @@ -115,6 +117,8 @@ + + @@ -837,6 +841,11 @@ The best ask currently in the order book + + + BestBid/BesAsk returned as a pair + + Start connecting and synchronizing the order book @@ -1144,7 +1153,6 @@ The proxy port - Create new settings for a proxy @@ -1154,7 +1162,6 @@ The proxy password - Create new settings for a proxy @@ -1326,6 +1333,11 @@ Connecting + + + Reconnecting + + Syncing data @@ -1538,11 +1550,20 @@ Whether each update should have a consecutive id number. Used to identify and reconnect when numbers are skipped. - + + + Whether or not a level should be removed from the book when it's pushed out of scope of the limit. For example with a book of limit 10, + when a new bid is added which makes the total amount of bids 11, should the last bid entry be removed + + + The name of the order book implementation Whether each update should have a consecutive id number. Used to identify and reconnect when numbers are skipped. + Whether or not a level should be removed from the book when it's pushed out of scope of the limit. For example with a book of limit 10, + when a new bid is added which makes the total amount of bids 11, should the last bid entry be removed + Amount of levels for this order book @@ -1659,41 +1680,6 @@ - - - Buffer entry for order book - - - - - List of asks - - - - - List of bids - - - - - Buffer entry with a single update id per update - - - - - First update id - - - - - List of asks - - - - - List of bids - - Buffer entry with a first and last update id @@ -1754,6 +1740,11 @@ If order book is set + + + The amount of levels for this book + + The status of the order book. Order book is up to date when the status is `Synced` @@ -1819,6 +1810,11 @@ The best ask currently in the order book + + + BestBid/BesAsk returned as a pair + + ctor @@ -1867,6 +1863,13 @@ + + + Validate a checksum with the current order book + + + + Set the initial data for the order book @@ -1883,6 +1886,12 @@ + + + Add a checksum to the process queue + + + Update the order book using a first/last update id @@ -2104,11 +2113,21 @@ Request body content type + + + Whether or not we need to manually parse an error instead of relying on the http status code + + How to serialize array parameters + + + What request body should be when no data is send + + Timeout for requests @@ -2159,7 +2178,7 @@ The roundtrip time of the ping request - + Execute a request @@ -2169,7 +2188,9 @@ Cancellation token The parameters of the request Whether or not the request should be authenticated - Whether or not the resulting object should be checked for missing properties in the mapping (only outputs if log verbosity is Debug) + Whether or not the resulting object should be checked for missing properties in the mapping (only outputs if log verbosity is Debug) + Where the post parameters should be placed + How array paramters should be serialized @@ -2180,7 +2201,15 @@ Cancellation token - + + + Can be used to parse an error even though response status indicates success. Some apis always return 200 OK, even though there is an error. + This can be used together with ManualParseError to check if it is an error before deserializing to an object + + Received data + Null if not an error, Error otherwise + + Creates a request object @@ -2188,6 +2217,8 @@ The method of the request The parameters of the request Whether or not the request should be authenticated + Where the post parameters should be placed + How array paramters should be serialized @@ -2918,5 +2949,148 @@ + + + Specifies that is allowed as an input even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that is disallowed as an input even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that a method that will never return under any circumstance. + + + + + Initializes a new instance of the class. + + + + + Specifies that the method will not return if the associated + parameter is passed the specified value. + + + + + Gets the condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Initializes a new instance of the + class with the specified parameter value. + + + The condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Specifies that an output may be even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that when a method returns , + the parameter may be even if the corresponding type disallows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter may be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter may be . + + + + + Specifies that an output is not even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that the output will be non- if the + named parameter is non-. + + + + + Gets the associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Initializes the attribute with the associated parameter name. + + + The associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Specifies that when a method returns , + the parameter will not be even if the corresponding type allows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter will not be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter will not be . + + diff --git a/CryptoExchange.Net/ExtensionMethods.cs b/CryptoExchange.Net/ExtensionMethods.cs index 1090cf9..1ac235b 100644 --- a/CryptoExchange.Net/ExtensionMethods.cs +++ b/CryptoExchange.Net/ExtensionMethods.cs @@ -7,6 +7,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Threading; using System.Threading.Tasks; +using System.Web; using CryptoExchange.Net.Logging; using CryptoExchange.Net.Objects; using Newtonsoft.Json; @@ -79,16 +80,16 @@ namespace CryptoExchange.Net foreach (var arrayEntry in arraysParameters) { if(serializationType == ArrayParametersSerialization.Array) - uriString += $"{string.Join("&", ((object[])(urlEncodeValues ? WebUtility.UrlEncode(arrayEntry.Value.ToString()) : arrayEntry.Value)).Select(v => $"{arrayEntry.Key}[]={v}"))}&"; + uriString += $"{string.Join("&", ((object[])(urlEncodeValues ? Uri.EscapeDataString(arrayEntry.Value.ToString()) : arrayEntry.Value)).Select(v => $"{arrayEntry.Key}[]={v}"))}&"; else { var array = (Array)arrayEntry.Value; - uriString += string.Join("&", array.OfType().Select(a => $"{arrayEntry.Key}={WebUtility.UrlEncode(a.ToString())}")); + uriString += string.Join("&", array.OfType().Select(a => $"{arrayEntry.Key}={Uri.EscapeDataString(a.ToString())}")); uriString += "&"; } } - uriString += $"{string.Join("&", parameters.Where(p => !p.Value.GetType().IsArray).Select(s => $"{s.Key}={(urlEncodeValues ? WebUtility.UrlEncode(s.Value.ToString()) : s.Value)}"))}"; + uriString += $"{string.Join("&", parameters.Where(p => !p.Value.GetType().IsArray).Select(s => $"{s.Key}={(urlEncodeValues ? Uri.EscapeDataString(s.Value.ToString()) : s.Value)}"))}"; uriString = uriString.TrimEnd('&'); return uriString; } diff --git a/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs b/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs index 9d0b502..ee24057 100644 --- a/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs +++ b/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs @@ -70,6 +70,11 @@ namespace CryptoExchange.Net.Interfaces /// ISymbolOrderBookEntry BestAsk { get; } + /// + /// BestBid/BesAsk returned as a pair + /// + (ISymbolOrderBookEntry Bid, ISymbolOrderBookEntry Ask) BestOffers { get; } + /// /// Start connecting and synchronizing the order book /// diff --git a/CryptoExchange.Net/Objects/ApiProxy.cs b/CryptoExchange.Net/Objects/ApiProxy.cs index bbbd65e..1e75156 100644 --- a/CryptoExchange.Net/Objects/ApiProxy.cs +++ b/CryptoExchange.Net/Objects/ApiProxy.cs @@ -1,5 +1,4 @@ -using System; -using System.Security; +using System.Security; namespace CryptoExchange.Net.Objects { @@ -36,7 +35,6 @@ namespace CryptoExchange.Net.Objects { } - /// /// /// Create new settings for a proxy /// @@ -48,7 +46,6 @@ namespace CryptoExchange.Net.Objects { } - /// /// /// Create new settings for a proxy /// @@ -58,9 +55,6 @@ namespace CryptoExchange.Net.Objects /// The proxy password public ApiProxy(string host, int port, string? login, SecureString? password) { - if (!host.StartsWith("http")) - throw new ArgumentException("Proxy host should start with either http:// or https://"); - Host = host; Port = port; Login = login; diff --git a/CryptoExchange.Net/Objects/Enums.cs b/CryptoExchange.Net/Objects/Enums.cs index 3af3694..785b8ff 100644 --- a/CryptoExchange.Net/Objects/Enums.cs +++ b/CryptoExchange.Net/Objects/Enums.cs @@ -59,6 +59,10 @@ /// Connecting, /// + /// Reconnecting + /// + Reconnecting, + /// /// Syncing data /// Syncing, diff --git a/CryptoExchange.Net/Objects/Options.cs b/CryptoExchange.Net/Objects/Options.cs index 18744ab..515494d 100644 --- a/CryptoExchange.Net/Objects/Options.cs +++ b/CryptoExchange.Net/Objects/Options.cs @@ -44,20 +44,30 @@ namespace CryptoExchange.Net.Objects /// public bool SequenceNumbersAreConsecutive { get; } + /// + /// Whether or not a level should be removed from the book when it's pushed out of scope of the limit. For example with a book of limit 10, + /// when a new bid is added which makes the total amount of bids 11, should the last bid entry be removed + /// + public bool StrictLevels { get; } + /// /// /// The name of the order book implementation /// Whether each update should have a consecutive id number. Used to identify and reconnect when numbers are skipped. - public OrderBookOptions(string name, bool sequencesAreConsecutive) + /// Whether or not a level should be removed from the book when it's pushed out of scope of the limit. For example with a book of limit 10, + /// when a new bid is added which makes the total amount of bids 11, should the last bid entry be removed + /// Amount of levels for this order book + public OrderBookOptions(string name, bool sequencesAreConsecutive, bool strictLevels) { OrderBookName = name; SequenceNumbersAreConsecutive = sequencesAreConsecutive; + StrictLevels = strictLevels; } /// public override string ToString() { - return $"{base.ToString()}, OrderBookName: {OrderBookName}, SequenceNumbersAreConsequtive: {SequenceNumbersAreConsecutive}"; + return $"{base.ToString()}, OrderBookName: {OrderBookName}, SequenceNumbersAreConsequtive: {SequenceNumbersAreConsecutive}, StrictLevels: {StrictLevels}"; } } diff --git a/CryptoExchange.Net/OrderBook/ProcessBufferEntry.cs b/CryptoExchange.Net/OrderBook/ProcessBufferEntry.cs index 3cf64bc..1e7c919 100644 --- a/CryptoExchange.Net/OrderBook/ProcessBufferEntry.cs +++ b/CryptoExchange.Net/OrderBook/ProcessBufferEntry.cs @@ -3,40 +3,6 @@ using System.Collections.Generic; namespace CryptoExchange.Net.OrderBook { - /// - /// Buffer entry for order book - /// - public class ProcessBufferEntry - { - /// - /// List of asks - /// - public IEnumerable Asks { get; set; } = new List(); - /// - /// List of bids - /// - public IEnumerable Bids { get; set; } = new List(); - } - - /// - /// Buffer entry with a single update id per update - /// - public class ProcessBufferSingleSequenceEntry - { - /// - /// First update id - /// - public long UpdateId { get; set; } - /// - /// List of asks - /// - public IEnumerable Asks { get; set; } = new List(); - /// - /// List of bids - /// - public IEnumerable Bids { get; set; } = new List(); - } - /// /// Buffer entry with a first and last update id /// diff --git a/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs b/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs new file mode 100644 index 0000000..14832a0 --- /dev/null +++ b/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs @@ -0,0 +1,26 @@ +using CryptoExchange.Net.Interfaces; +using System.Collections.Generic; + +namespace CryptoExchange.Net.OrderBook +{ + internal class ProcessQueueItem + { + public long StartUpdateId { get; set; } + public long EndUpdateId { get; set; } + public IEnumerable Bids { get; set; } = new List(); + public IEnumerable Asks { get; set; } = new List(); + } + + internal class InitialOrderBookItem + { + public long StartUpdateId { get; set; } + public long EndUpdateId { get; set; } + public IEnumerable Bids { get; set; } = new List(); + public IEnumerable Asks { get; set; } = new List(); + } + + internal class ChecksumItem + { + public int Checksum { get; set; } + } +} diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index 36e94b8..8f4e32f 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using CryptoExchange.Net.Interfaces; using CryptoExchange.Net.Logging; @@ -19,8 +21,7 @@ namespace CryptoExchange.Net.OrderBook /// /// The process buffer, used while syncing /// - protected readonly List processBuffer; - private readonly object bookLock = new object(); + protected readonly List processBuffer; /// /// The ask list /// @@ -28,11 +29,18 @@ namespace CryptoExchange.Net.OrderBook /// /// The bid list /// - protected SortedList bids; + + private readonly object bookLock = new object(); + private OrderBookStatus status; private UpdateSubscription? subscription; private readonly bool sequencesAreConsecutive; + private readonly bool strictLevels; + + private Task _processTask; + private AutoResetEvent _queueEvent; + private ConcurrentQueue _processQueue; /// /// Order book implementation id @@ -48,6 +56,11 @@ namespace CryptoExchange.Net.OrderBook /// protected bool bookSet; + /// + /// The amount of levels for this book + /// + protected int? Levels { get; set; } = null; + /// /// The status of the order book. Order book is up to date when the status is `Synced` /// @@ -127,6 +140,14 @@ namespace CryptoExchange.Net.OrderBook } } + private class EmptySymbolOrderBookEntry : ISymbolOrderBookEntry + { + public decimal Quantity { get { return 0m; } set {; } } + public decimal Price { get { return 0m; } set {; } } + } + + private static ISymbolOrderBookEntry emptySymbolOrderBookEntry = new EmptySymbolOrderBookEntry(); + /// /// The best bid currently in the order book /// @@ -135,7 +156,7 @@ namespace CryptoExchange.Net.OrderBook get { lock (bookLock) - return bids.FirstOrDefault().Value; + return bids.FirstOrDefault().Value ?? emptySymbolOrderBookEntry; } } @@ -147,7 +168,17 @@ namespace CryptoExchange.Net.OrderBook get { lock (bookLock) - return asks.FirstOrDefault().Value; + return asks.FirstOrDefault().Value ?? emptySymbolOrderBookEntry; + } + } + + /// + /// BestBid/BesAsk returned as a pair + /// + public (ISymbolOrderBookEntry Bid, ISymbolOrderBookEntry Ask) BestOffers { + get { + lock (bookLock) + return (BestBid,BestAsk); } } @@ -165,8 +196,12 @@ namespace CryptoExchange.Net.OrderBook throw new ArgumentNullException(nameof(options)); Id = options.OrderBookName; - processBuffer = new List(); + processBuffer = new List(); + _processQueue = new ConcurrentQueue(); + _queueEvent = new AutoResetEvent(false); + sequencesAreConsecutive = options.SequenceNumbersAreConsecutive; + strictLevels = options.StrictLevels; Symbol = symbol; Status = OrderBookStatus.Disconnected; @@ -190,7 +225,10 @@ namespace CryptoExchange.Net.OrderBook /// public async Task> StartAsync() { + log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} starting"); Status = OrderBookStatus.Connecting; + _processTask = Task.Run(ProcessQueue); + var startResult = await DoStart().ConfigureAwait(false); if (!startResult) return new CallResult(false, startResult.Error); @@ -205,7 +243,10 @@ namespace CryptoExchange.Net.OrderBook private void Reset() { log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} connection lost"); - Status = OrderBookStatus.Connecting; + Status = OrderBookStatus.Reconnecting; + _queueEvent.Set(); + // Clear queue + while(_processQueue.TryDequeue(out _)) processBuffer.Clear(); bookSet = false; DoReset(); @@ -240,7 +281,10 @@ namespace CryptoExchange.Net.OrderBook /// public async Task StopAsync() { + log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} stopping"); Status = OrderBookStatus.Disconnected; + _queueEvent.Set(); + _processTask.Wait(); if(subscription != null) await subscription.Close().ConfigureAwait(false); } @@ -263,12 +307,34 @@ namespace CryptoExchange.Net.OrderBook protected abstract Task> DoResync(); /// - /// Set the initial data for the order book + /// Validate a checksum with the current order book /// - /// The last update sequence number - /// List of asks - /// List of bids - protected void SetInitialOrderBook(long orderBookSequenceNumber, IEnumerable bidList, IEnumerable askList) + /// + /// + protected virtual bool DoChecksum(int checksum) => true; + + private void ProcessQueue() + { + while(Status != OrderBookStatus.Disconnected) + { + _queueEvent.WaitOne(); + + while (_processQueue.TryDequeue(out var item)) + { + if (Status == OrderBookStatus.Disconnected) + break; + + if (item is InitialOrderBookItem iobi) + ProcessInitialOrderBookItem(iobi); + if (item is ProcessQueueItem pqi) + ProcessQueueItem(pqi); + else if (item is ChecksumItem ci) + ProcessChecksum(ci); + } + } + } + + private void ProcessInitialOrderBookItem(InitialOrderBookItem item) { lock (bookLock) { @@ -276,31 +342,88 @@ namespace CryptoExchange.Net.OrderBook return; asks.Clear(); - foreach (var ask in askList) + foreach (var ask in item.Asks) asks.Add(ask.Price, ask); bids.Clear(); - foreach (var bid in bidList) + foreach (var bid in item.Bids) bids.Add(bid.Price, bid); - LastSequenceNumber = orderBookSequenceNumber; + LastSequenceNumber = item.EndUpdateId; AskCount = asks.Count; - BidCount = asks.Count; + BidCount = bids.Count; - bookSet = true; LastOrderBookUpdate = DateTime.UtcNow; - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} data set: {BidCount} bids, {AskCount} asks. #{orderBookSequenceNumber}"); + log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} data set: {BidCount} bids, {AskCount} asks. #{item.EndUpdateId}"); CheckProcessBuffer(); - OnOrderBookUpdate?.Invoke(bidList, askList); + OnOrderBookUpdate?.Invoke(item.Asks, item.Bids); OnBestOffersChanged?.Invoke(BestBid, BestAsk); } } - private void CheckBestOffersChanged(ISymbolOrderBookEntry prevBestBid, ISymbolOrderBookEntry prevBestAsk) + private void ProcessQueueItem(ProcessQueueItem item) { - if (BestBid.Price != prevBestBid.Price || BestBid.Quantity != prevBestBid.Quantity || - BestAsk.Price != prevBestAsk.Price || BestAsk.Quantity != prevBestAsk.Quantity) - OnBestOffersChanged?.Invoke(BestBid, BestAsk); + lock (bookLock) + { + if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected) + return; + + if (!bookSet) + { + processBuffer.Add(new ProcessBufferRangeSequenceEntry() + { + Asks = item.Asks, + Bids = item.Bids, + FirstUpdateId = item.StartUpdateId, + LastUpdateId = item.EndUpdateId, + }); + log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update buffered #{item.StartUpdateId}-#{item.EndUpdateId} [{Asks.Count()} asks, {Bids.Count()} bids]"); + } + else + { + CheckProcessBuffer(); + var (prevBestBid, prevBestAsk) = BestOffers; + ProcessRangeUpdates(item.StartUpdateId, item.EndUpdateId, item.Bids, item.Asks); + + if (asks.First().Key < bids.First().Key) + { + log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} detected out of sync order book. Resyncing"); + _ = subscription?.Reconnect(); + return; + } + + OnOrderBookUpdate?.Invoke(item.Bids, item.Asks); + CheckBestOffersChanged(prevBestBid, prevBestAsk); + } + } + } + + private void ProcessChecksum(ChecksumItem ci) + { + lock (bookLock) + { + var checksumResult = DoChecksum(ci.Checksum); + if(!checksumResult) + { + log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} out of sync. Resyncing"); + _ = subscription?.Reconnect(); + return; + } + } + } + + /// + /// Set the initial data for the order book + /// + /// The last update sequence number + /// List of asks + /// List of bids + protected void SetInitialOrderBook(long orderBookSequenceNumber, IEnumerable bidList, IEnumerable askList) + { + bookSet = true; + + _processQueue.Enqueue(new InitialOrderBookItem { StartUpdateId = orderBookSequenceNumber, EndUpdateId = orderBookSequenceNumber, Asks = askList, Bids = bidList }); + _queueEvent.Set(); } /// @@ -311,31 +434,18 @@ namespace CryptoExchange.Net.OrderBook /// protected void UpdateOrderBook(long rangeUpdateId, IEnumerable bids, IEnumerable asks) { - lock (bookLock) - { - if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected) - return; + _processQueue.Enqueue(new ProcessQueueItem { StartUpdateId = rangeUpdateId, EndUpdateId = rangeUpdateId, Asks = asks, Bids = bids }); + _queueEvent.Set(); + } - if (!bookSet) - { - processBuffer.Add(new ProcessBufferSingleSequenceEntry() - { - UpdateId = rangeUpdateId, - Asks = asks, - Bids = bids - }); - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update buffered #{rangeUpdateId}"); - } - else - { - CheckProcessBuffer(); - var prevBestBid = BestBid; - var prevBestAsk = BestAsk; - ProcessSingleSequenceUpdates(rangeUpdateId, bids, asks); - OnOrderBookUpdate?.Invoke(bids, asks); - CheckBestOffersChanged(prevBestBid, prevBestAsk); - } - } + /// + /// Add a checksum to the process queue + /// + /// + protected void AddChecksum(int checksum) + { + _processQueue.Enqueue(new ChecksumItem() { Checksum = checksum }); + _queueEvent.Set(); } /// @@ -347,32 +457,8 @@ namespace CryptoExchange.Net.OrderBook /// protected void UpdateOrderBook(long firstUpdateId, long lastUpdateId, IEnumerable bids, IEnumerable asks) { - lock (bookLock) - { - if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected) - return; - - if (!bookSet) - { - processBuffer.Add(new ProcessBufferRangeSequenceEntry() - { - Asks = asks, - Bids = bids, - FirstUpdateId = firstUpdateId, - LastUpdateId = lastUpdateId - }); - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update buffered #{firstUpdateId}-{lastUpdateId}"); - } - else - { - CheckProcessBuffer(); - var prevBestBid = BestBid; - var prevBestAsk = BestAsk; - ProcessRangeUpdates(firstUpdateId, lastUpdateId, bids, asks); - OnOrderBookUpdate?.Invoke(bids, asks); - CheckBestOffersChanged(prevBestBid, prevBestAsk); - } - } + _processQueue.Enqueue(new ProcessQueueItem { StartUpdateId = firstUpdateId, EndUpdateId = lastUpdateId, Asks = asks, Bids = bids }); + _queueEvent.Set(); } /// @@ -382,43 +468,11 @@ namespace CryptoExchange.Net.OrderBook /// List of asks protected void UpdateOrderBook(IEnumerable bids, IEnumerable asks) { - lock (bookLock) - { - if (!bookSet) - { - processBuffer.Add(new ProcessBufferEntry - { - Asks = asks, - Bids = bids - }); - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update buffered #{Math.Min(bids.Min(b => b.Sequence), asks.Min(a => a.Sequence))}-{Math.Max(bids.Max(b => b.Sequence), asks.Max(a => a.Sequence))}"); - } - else - { - CheckProcessBuffer(); - var prevBestBid = BestBid; - var prevBestAsk = BestAsk; - ProcessUpdates(bids, asks); - OnOrderBookUpdate?.Invoke(bids, asks); - CheckBestOffersChanged(prevBestBid, prevBestAsk); - } - } - } + var highest = Math.Max(bids.Any() ? bids.Max(b => b.Sequence) : 0, asks.Any() ? asks.Max(a => a.Sequence) : 0); + var lowest = Math.Min(bids.Any() ? bids.Min(b => b.Sequence) : long.MaxValue, asks.Any() ? asks.Min(a => a.Sequence) : long.MaxValue); - private void ProcessUpdates(IEnumerable bids, IEnumerable asks) - { - var entries = new Dictionary(); - foreach (var entry in asks.OrderBy(a => a.Sequence)) - entries.Add(entry, OrderBookEntryType.Ask); - foreach (var entry in bids.OrderBy(a => a.Sequence)) - entries.Add(entry, OrderBookEntryType.Bid); - - foreach (var entry in entries.OrderBy(e => e.Key.Sequence)) - { - if(ProcessUpdate(entry.Key.Sequence, entry.Value, entry.Key)) - LastSequenceNumber = entry.Key.Sequence; - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update #{LastSequenceNumber}"); - } + _processQueue.Enqueue(new ProcessQueueItem { StartUpdateId = lowest, EndUpdateId = highest , Asks = asks, Bids = bids }); + _queueEvent.Set(); } private void ProcessRangeUpdates(long firstUpdateId, long lastUpdateId, IEnumerable bids, IEnumerable asks) @@ -435,28 +489,25 @@ namespace CryptoExchange.Net.OrderBook foreach (var entry in asks) ProcessUpdate(LastSequenceNumber + 1, OrderBookEntryType.Ask, entry); + if (Levels.HasValue && strictLevels) + { + while (this.bids.Count() > Levels.Value) + { + BidCount--; + this.bids.Remove(this.bids.Last().Key); + } + + while (this.asks.Count() > Levels.Value) + { + AskCount--; + this.asks.Remove(this.asks.Last().Key); + } + } + LastSequenceNumber = lastUpdateId; log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update processed #{firstUpdateId}-{lastUpdateId}"); } - private void ProcessSingleSequenceUpdates(long updateId, IEnumerable bids, IEnumerable asks) - { - foreach (var entry in bids) - { - if (!ProcessUpdate(updateId, OrderBookEntryType.Bid, entry)) - return; - } - - foreach (var entry in asks) - { - if (!ProcessUpdate(updateId, OrderBookEntryType.Ask, entry)) - return; - } - - LastSequenceNumber = updateId; - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update processed #{LastSequenceNumber}"); - } - /// /// Check and empty the process buffer; see what entries to update the book with /// @@ -468,13 +519,7 @@ namespace CryptoExchange.Net.OrderBook foreach (var bufferEntry in pbList) { - if (bufferEntry is ProcessBufferEntry pbe) - ProcessUpdates(pbe.Bids, pbe.Asks); - else if(bufferEntry is ProcessBufferRangeSequenceEntry pbrse) - ProcessRangeUpdates(pbrse.FirstUpdateId, pbrse.LastUpdateId, pbrse.Bids, pbrse.Asks); - else if (bufferEntry is ProcessBufferSingleSequenceEntry pbsse) - ProcessSingleSequenceUpdates(pbsse.UpdateId, pbsse.Bids, pbsse.Asks); - + ProcessRangeUpdates(bufferEntry.FirstUpdateId, bufferEntry.LastUpdateId, bufferEntry.Bids, bufferEntry.Asks); processBuffer.Remove(bufferEntry); } } @@ -500,7 +545,6 @@ namespace CryptoExchange.Net.OrderBook { // Out of sync log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} out of sync (expected { LastSequenceNumber + 1}, was {sequence}), reconnecting"); - Status = OrderBookStatus.Connecting; subscription?.Reconnect(); return false; } @@ -526,7 +570,7 @@ namespace CryptoExchange.Net.OrderBook } else { - listToChange[entry.Price].Quantity = entry.Quantity; + listToChange[entry.Price] = entry; } } @@ -552,6 +596,14 @@ namespace CryptoExchange.Net.OrderBook return new CallResult(true, null); } + private void CheckBestOffersChanged(ISymbolOrderBookEntry prevBestBid, ISymbolOrderBookEntry prevBestAsk) + { + var (bestBid, bestAsk) = BestOffers; + if (bestBid.Price != prevBestBid.Price || bestBid.Quantity != prevBestBid.Quantity || + bestAsk.Price != prevBestAsk.Price || bestAsk.Quantity != prevBestAsk.Quantity) + OnBestOffersChanged?.Invoke(bestBid, bestAsk); + } + /// /// Dispose the order book /// diff --git a/CryptoExchange.Net/RestClient.cs b/CryptoExchange.Net/RestClient.cs index 048cf6c..d3a0f55 100644 --- a/CryptoExchange.Net/RestClient.cs +++ b/CryptoExchange.Net/RestClient.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net.Http; using System.Net.NetworkInformation; using System.Net.Sockets; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using System.Web; @@ -39,11 +40,21 @@ namespace CryptoExchange.Net /// protected RequestBodyFormat requestBodyFormat = RequestBodyFormat.Json; + /// + /// Whether or not we need to manually parse an error instead of relying on the http status code + /// + protected bool manualParseError = false; + /// /// How to serialize array parameters /// protected ArrayParametersSerialization arraySerialization = ArrayParametersSerialization.Array; + /// + /// What request body should be when no data is send + /// + protected string requestBodyEmptyContent = "{}"; + /// /// Timeout for requests /// @@ -153,11 +164,13 @@ namespace CryptoExchange.Net /// Cancellation token /// The parameters of the request /// Whether or not the request should be authenticated - /// Whether or not the resulting object should be checked for missing properties in the mapping (only outputs if log verbosity is Debug) + /// Whether or not the resulting object should be checked for missing properties in the mapping (only outputs if log verbosity is Debug) + /// Where the post parameters should be placed + /// How array paramters should be serialized /// [return: NotNull] protected virtual async Task> SendRequest(Uri uri, HttpMethod method, CancellationToken cancellationToken, - Dictionary? parameters = null, bool signed = false, bool checkResult = true) where T : class + Dictionary? parameters = null, bool signed = false, bool checkResult = true, PostParameters? postPosition = null, ArrayParametersSerialization? arraySerialization = null) where T : class { log.Write(LogVerbosity.Debug, "Creating request for " + uri); if (signed && authProvider == null) @@ -166,7 +179,7 @@ namespace CryptoExchange.Net return new WebCallResult(null, null, null, new NoApiCredentialsError()); } - var request = ConstructRequest(uri, method, parameters, signed); + var request = ConstructRequest(uri, method, parameters, signed, postPosition ?? postParametersPosition, arraySerialization ?? this.arraySerialization); foreach (var limiter in RateLimiters) { var limitResult = limiter.LimitRequest(this, uri.AbsolutePath, RateLimitBehaviour); @@ -205,16 +218,38 @@ namespace CryptoExchange.Net var responseStream = await response.GetResponseStream().ConfigureAwait(false); if (response.IsSuccessStatusCode) { - var desResult = await Deserialize(responseStream).ConfigureAwait(false); - responseStream.Close(); - response.Close(); + if (manualParseError) + { + using var reader = new StreamReader(responseStream); + var data = await reader.ReadToEndAsync().ConfigureAwait(false); + responseStream.Close(); + response.Close(); + log.Write(LogVerbosity.Debug, $"Data received: {data}"); - return new WebCallResult(statusCode, headers, desResult.Data, desResult.Error); + var parseResult = ValidateJson(data); + if (!parseResult.Success) + return WebCallResult.CreateErrorResult(response.StatusCode, response.ResponseHeaders, new ServerError(data)); + var error = await TryParseError(parseResult.Data); + if(error != null) + return WebCallResult.CreateErrorResult(response.StatusCode, response.ResponseHeaders, error); + + var deserializeResult = Deserialize(parseResult.Data); + return new WebCallResult(response.StatusCode, response.ResponseHeaders, deserializeResult.Data, deserializeResult.Error); + } + else + { + var desResult = await Deserialize(responseStream).ConfigureAwait(false); + responseStream.Close(); + response.Close(); + + return new WebCallResult(statusCode, headers, desResult.Data, desResult.Error); + } } else { using var reader = new StreamReader(responseStream); var data = await reader.ReadToEndAsync().ConfigureAwait(false); + log.Write(LogVerbosity.Debug, $"Error received: {data}"); responseStream.Close(); response.Close(); var parseResult = ValidateJson(data); @@ -243,6 +278,17 @@ namespace CryptoExchange.Net } } + /// + /// Can be used to parse an error even though response status indicates success. Some apis always return 200 OK, even though there is an error. + /// This can be used together with ManualParseError to check if it is an error before deserializing to an object + /// + /// Received data + /// Null if not an error, Error otherwise + protected virtual Task TryParseError(JToken data) + { + return Task.FromResult(null); + } + /// /// Creates a request object /// @@ -250,17 +296,19 @@ namespace CryptoExchange.Net /// The method of the request /// The parameters of the request /// Whether or not the request should be authenticated + /// Where the post parameters should be placed + /// How array paramters should be serialized /// - protected virtual IRequest ConstructRequest(Uri uri, HttpMethod method, Dictionary? parameters, bool signed) + protected virtual IRequest ConstructRequest(Uri uri, HttpMethod method, Dictionary? parameters, bool signed, PostParameters postPosition, ArrayParametersSerialization arraySerialization) { if (parameters == null) parameters = new Dictionary(); var uriString = uri.ToString(); if(authProvider != null) - parameters = authProvider.AddAuthenticationToParameters(uriString, method, parameters, signed); + parameters = authProvider.AddAuthenticationToParameters(uriString, method, parameters, signed, postPosition, arraySerialization); - if((method == HttpMethod.Get || method == HttpMethod.Delete || postParametersPosition == PostParameters.InUri) && parameters?.Any() == true) + if((method == HttpMethod.Get || method == HttpMethod.Delete || postPosition == PostParameters.InUri) && parameters?.Any() == true) uriString += "?" + parameters.CreateParamString(true, arraySerialization); var contentType = requestBodyFormat == RequestBodyFormat.Json ? Constants.JsonContentHeader : Constants.FormContentHeader; @@ -269,17 +317,17 @@ namespace CryptoExchange.Net var headers = new Dictionary(); if (authProvider != null) - headers = authProvider.AddAuthenticationToHeaders(uriString, method, parameters!, signed); + headers = authProvider.AddAuthenticationToHeaders(uriString, method, parameters!, signed, postPosition, arraySerialization); foreach (var header in headers) request.AddHeader(header.Key, header.Value); - if ((method == HttpMethod.Post || method == HttpMethod.Put) && postParametersPosition != PostParameters.InUri) + if ((method == HttpMethod.Post || method == HttpMethod.Put) && postPosition != PostParameters.InUri) { if(parameters?.Any() == true) WriteParamBody(request, parameters, contentType); else - request.SetContent("{}", contentType); + request.SetContent(requestBodyEmptyContent, contentType); } return request; @@ -302,7 +350,16 @@ namespace CryptoExchange.Net { var formData = HttpUtility.ParseQueryString(string.Empty); foreach (var kvp in parameters.OrderBy(p => p.Key)) - formData.Add(kvp.Key, kvp.Value.ToString()); + { + if (kvp.Value.GetType().IsArray) + { + var array = (Array)kvp.Value; + foreach(var value in array) + formData.Add(kvp.Key, value.ToString()); + } + else + formData.Add(kvp.Key, kvp.Value.ToString()); + } var stringData = formData.ToString(); request.SetContent(stringData, contentType); } diff --git a/CryptoExchange.Net/SocketClient.cs b/CryptoExchange.Net/SocketClient.cs index 77ea973..daf83c7 100644 --- a/CryptoExchange.Net/SocketClient.cs +++ b/CryptoExchange.Net/SocketClient.cs @@ -579,7 +579,7 @@ namespace CryptoExchange.Net periodicEvent?.Set(); periodicEvent?.Dispose(); log.Write(LogVerbosity.Debug, "Disposing socket client, closing all subscriptions"); - UnsubscribeAll().Wait(); + Task.Run(UnsubscribeAll).Wait(); semaphoreSlim?.Dispose(); base.Dispose(); } diff --git a/CryptoExchange.Net/Sockets/SocketConnection.cs b/CryptoExchange.Net/Sockets/SocketConnection.cs index 8b6e2c8..5d30915 100644 --- a/CryptoExchange.Net/Sockets/SocketConnection.cs +++ b/CryptoExchange.Net/Sockets/SocketConnection.cs @@ -136,9 +136,16 @@ namespace CryptoExchange.Net.Sockets private void ProcessMessage(string data) { log.Write(LogVerbosity.Debug, $"Socket {Socket.Id} received data: " + data); + if (string.IsNullOrEmpty(data)) return; + var tokenData = data.ToJToken(log); if (tokenData == null) - return; + { + data = $"\"{data}\""; + tokenData = data.ToJToken(log); + if (tokenData == null) + return; + } var handledResponse = false; foreach (var pendingRequest in pendingRequests.ToList()) diff --git a/README.md b/README.md index f7b667c..3d85219 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,27 @@ The order book will automatically reconnect when the connection is lost and resy To stop synchronizing an order book use the `Stop` method. ## Release notes +* Version 3.0.11 - 20 Jun 2020 + * Added support for checksum in SymbolOrderBook + +* Version 3.0.10 - 16 Jun 2020 + * Fix for order book synchronization + +* Version 3.0.9 - 07 Jun 2020 + * Added arraySerialization and postParameterPosition to AuthenticationProvider interface + * Fixed array serialization in request body + +* Version 3.0.8 - 02 Jun 2020 + * Added requestBodyEmptyContent setting for rest client + * Added TryParseError for rest implementations to check for error with success status code + +* Version 3.0.7 - 20 May 2020 + * Added error debug output + * Fix for unsubscribe causing possible deadlock + +* Version 3.0.6 - 03 Mar 2020 + * Added BestOffer to SymbolOrderBook, removed invalid check on proxy + * Version 3.0.5 - 05 Feb 2020 * Added PausedActivity events on socket subscriptions