From 08d7022815200aad6a6a32642c5af84417bd976d Mon Sep 17 00:00:00 2001 From: Ben Davison Date: Thu, 30 Jan 2020 16:55:47 +0000 Subject: [PATCH 01/22] introduce a default empty ISymbolOrderBookEntry that is returned buy BestBid or BestAsk if called when the bid or ask lists are empty. This resolves a null reference exception seen during syncronization (specifically when connecting to Kraken). I have also introduced BestOffers which returns both bid and ask in the scope of one lock allowing the caller to be sure that nothing has changed between BestBid and BestAsk request. --- .../SymbolOrderBookTests.cs | 72 +++++++++ CryptoExchange.Net/CryptoExchange.Net.xml | 148 ++++++++++++++++++ .../OrderBook/SymbolOrderBook.cs | 40 +++-- 3 files changed, 249 insertions(+), 11 deletions(-) create mode 100644 CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs diff --git a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs new file mode 100644 index 0000000..efbe657 --- /dev/null +++ b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs @@ -0,0 +1,72 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using CryptoExchange.Net.Logging; +using CryptoExchange.Net.Objects; +using CryptoExchange.Net.OrderBook; +using CryptoExchange.Net.Sockets; +using CryptoExchange.Net.UnitTests.TestImplementations; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +namespace CryptoExchange.Net.UnitTests +{ + [TestFixture] + public class SymbolOrderBookTests + { + private static OrderBookOptions defaultOrderBookOptions = new OrderBookOptions("Test", true); + + private class TestableSymbolOrderBook : SymbolOrderBook + { + public TestableSymbolOrderBook() : base("BTC/USD", defaultOrderBookOptions) + { + } + + public override void Dispose() + { + throw new NotImplementedException(); + } + + protected override Task> DoResync() + { + throw new NotImplementedException(); + } + + protected override Task> DoStart() + { + throw new NotImplementedException(); + } + } + + [TestCase] + public void GivenEmptyBidList_WhenBestBid_ThenEmptySymbolOrderBookEntry() + { + var symbolOrderBook = new TestableSymbolOrderBook(); + Assert.IsNotNull(symbolOrderBook.BestBid); + Assert.AreEqual(0m, symbolOrderBook.BestBid.Price); + Assert.AreEqual(0m, symbolOrderBook.BestAsk.Quantity); + } + + [TestCase] + public void GivenEmptyAskList_WhenBestAsk_ThenEmptySymbolOrderBookEntry() + { + var symbolOrderBook = new TestableSymbolOrderBook(); + Assert.IsNotNull(symbolOrderBook.BestBid); + Assert.AreEqual(0m, symbolOrderBook.BestBid.Price); + Assert.AreEqual(0m, symbolOrderBook.BestAsk.Quantity); + } + + [TestCase] + public void GivenEmptyBidAndAskList_WhenBestOffers_ThenEmptySymbolOrderBookEntries() + { + var symbolOrderBook = new TestableSymbolOrderBook(); + Assert.IsNotNull(symbolOrderBook.BestOffers); + Assert.IsNotNull(symbolOrderBook.BestOffers.Item1); + Assert.IsNotNull(symbolOrderBook.BestOffers.Item2); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Item1.Price); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Item1.Quantity); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Item2.Price); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Item2.Quantity); + } + } +} diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index 62e5487..0954f72 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -1819,6 +1819,11 @@ The best ask currently in the order book + + + BestBid/BesAsk returned as a pair + + ctor @@ -2898,5 +2903,148 @@ + + + Specifies that is allowed as an input even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that is disallowed as an input even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that a method that will never return under any circumstance. + + + + + Initializes a new instance of the class. + + + + + Specifies that the method will not return if the associated + parameter is passed the specified value. + + + + + Gets the condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Initializes a new instance of the + class with the specified parameter value. + + + The condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Specifies that an output may be even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that when a method returns , + the parameter may be even if the corresponding type disallows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter may be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter may be . + + + + + Specifies that an output is not even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that the output will be non- if the + named parameter is non-. + + + + + Gets the associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Initializes the attribute with the associated parameter name. + + + The associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Specifies that when a method returns , + the parameter will not be even if the corresponding type allows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter will not be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter will not be . + + diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index 36e94b8..73bd76f 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -127,6 +127,14 @@ namespace CryptoExchange.Net.OrderBook } } + private class EmptySymbolOrderBookEntry : ISymbolOrderBookEntry + { + public decimal Quantity { get { return 0m; } set {; } } + public decimal Price { get { return 0m; } set {; } } + } + + private static ISymbolOrderBookEntry emptySymbolOrderBookEntry = new EmptySymbolOrderBookEntry(); + /// /// The best bid currently in the order book /// @@ -135,7 +143,7 @@ namespace CryptoExchange.Net.OrderBook get { lock (bookLock) - return bids.FirstOrDefault().Value; + return bids.FirstOrDefault().Value ?? emptySymbolOrderBookEntry; } } @@ -147,7 +155,19 @@ namespace CryptoExchange.Net.OrderBook get { lock (bookLock) - return asks.FirstOrDefault().Value; + return asks.FirstOrDefault().Value ?? emptySymbolOrderBookEntry; + } + } + + /// + /// BestBid/BesAsk returned as a pair + /// + public Tuple BestOffers { + get { + lock (bookLock) + { + return new Tuple(BestBid,BestAsk); + } } } @@ -298,9 +318,10 @@ namespace CryptoExchange.Net.OrderBook private void CheckBestOffersChanged(ISymbolOrderBookEntry prevBestBid, ISymbolOrderBookEntry prevBestAsk) { - if (BestBid.Price != prevBestBid.Price || BestBid.Quantity != prevBestBid.Quantity || - BestAsk.Price != prevBestAsk.Price || BestAsk.Quantity != prevBestAsk.Quantity) - OnBestOffersChanged?.Invoke(BestBid, BestAsk); + var (bestBid, bestAsk) = BestOffers; + if (bestBid.Price != prevBestBid.Price || bestBid.Quantity != prevBestBid.Quantity || + bestAsk.Price != prevBestAsk.Price || bestAsk.Quantity != prevBestAsk.Quantity) + OnBestOffersChanged?.Invoke(bestBid, bestAsk); } /// @@ -329,8 +350,7 @@ namespace CryptoExchange.Net.OrderBook else { CheckProcessBuffer(); - var prevBestBid = BestBid; - var prevBestAsk = BestAsk; + var (prevBestBid, prevBestAsk) = BestOffers; ProcessSingleSequenceUpdates(rangeUpdateId, bids, asks); OnOrderBookUpdate?.Invoke(bids, asks); CheckBestOffersChanged(prevBestBid, prevBestAsk); @@ -366,8 +386,7 @@ namespace CryptoExchange.Net.OrderBook else { CheckProcessBuffer(); - var prevBestBid = BestBid; - var prevBestAsk = BestAsk; + var (prevBestBid, prevBestAsk) = BestOffers; ProcessRangeUpdates(firstUpdateId, lastUpdateId, bids, asks); OnOrderBookUpdate?.Invoke(bids, asks); CheckBestOffersChanged(prevBestBid, prevBestAsk); @@ -396,8 +415,7 @@ namespace CryptoExchange.Net.OrderBook else { CheckProcessBuffer(); - var prevBestBid = BestBid; - var prevBestAsk = BestAsk; + var (prevBestBid, prevBestAsk) = BestOffers; ProcessUpdates(bids, asks); OnOrderBookUpdate?.Invoke(bids, asks); CheckBestOffersChanged(prevBestBid, prevBestAsk); From 635ba1747c181ac6cbe522e57a30bb642fc9a28f Mon Sep 17 00:00:00 2001 From: Ben Davison Date: Thu, 30 Jan 2020 17:59:21 +0000 Subject: [PATCH 02/22] removed not implemented exception from the dispose override in the TestableSymbolOrderBook implementation --- CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs index efbe657..d0ac5c6 100644 --- a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs +++ b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs @@ -22,10 +22,7 @@ namespace CryptoExchange.Net.UnitTests { } - public override void Dispose() - { - throw new NotImplementedException(); - } + public override void Dispose() {} protected override Task> DoResync() { From 4a01c30f3441ebc4152c014c2ac4cd5a1964b668 Mon Sep 17 00:00:00 2001 From: Ben Davison Date: Thu, 30 Jan 2020 21:44:14 +0000 Subject: [PATCH 03/22] Expose BestOffers on ISymbolOrderBook interface --- CryptoExchange.Net/CryptoExchange.Net.xml | 5 +++++ CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index 0954f72..880dd15 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -837,6 +837,11 @@ The best ask currently in the order book + + + BestBid/BesAsk returned as a pair + + Start connecting and synchronizing the order book diff --git a/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs b/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs index 9d0b502..10275c3 100644 --- a/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs +++ b/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs @@ -70,6 +70,11 @@ namespace CryptoExchange.Net.Interfaces /// ISymbolOrderBookEntry BestAsk { get; } + /// + /// BestBid/BesAsk returned as a pair + /// + Tuple BestOffers { get; } + /// /// Start connecting and synchronizing the order book /// From 07d0a0159d850b973afb3886d9eae698bd9ff2ad Mon Sep 17 00:00:00 2001 From: Ben Davison Date: Thu, 30 Jan 2020 22:01:14 +0000 Subject: [PATCH 04/22] Use ValueTuple for BestOffers --- CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs | 12 ++++++------ CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs | 2 +- CryptoExchange.Net/OrderBook/SymbolOrderBook.cs | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs index d0ac5c6..4f1214f 100644 --- a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs +++ b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs @@ -58,12 +58,12 @@ namespace CryptoExchange.Net.UnitTests { var symbolOrderBook = new TestableSymbolOrderBook(); Assert.IsNotNull(symbolOrderBook.BestOffers); - Assert.IsNotNull(symbolOrderBook.BestOffers.Item1); - Assert.IsNotNull(symbolOrderBook.BestOffers.Item2); - Assert.AreEqual(0m, symbolOrderBook.BestOffers.Item1.Price); - Assert.AreEqual(0m, symbolOrderBook.BestOffers.Item1.Quantity); - Assert.AreEqual(0m, symbolOrderBook.BestOffers.Item2.Price); - Assert.AreEqual(0m, symbolOrderBook.BestOffers.Item2.Quantity); + Assert.IsNotNull(symbolOrderBook.BestOffers.BestBid); + Assert.IsNotNull(symbolOrderBook.BestOffers.BestAsk); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.BestBid.Price); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.BestBid.Quantity); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.BestAsk.Price); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.BestAsk.Quantity); } } } diff --git a/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs b/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs index 10275c3..06321d8 100644 --- a/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs +++ b/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs @@ -73,7 +73,7 @@ namespace CryptoExchange.Net.Interfaces /// /// BestBid/BesAsk returned as a pair /// - Tuple BestOffers { get; } + (ISymbolOrderBookEntry BestBid, ISymbolOrderBookEntry BestAsk) BestOffers { get; } /// /// Start connecting and synchronizing the order book diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index 73bd76f..143307b 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -162,12 +162,10 @@ namespace CryptoExchange.Net.OrderBook /// /// BestBid/BesAsk returned as a pair /// - public Tuple BestOffers { + public (ISymbolOrderBookEntry BestBid, ISymbolOrderBookEntry BestAsk) BestOffers { get { lock (bookLock) - { - return new Tuple(BestBid,BestAsk); - } + return (BestBid,BestAsk); } } From d384d1ee5babbfd098f27f3b53923627a1e9bf5e Mon Sep 17 00:00:00 2001 From: Ben Davison Date: Thu, 30 Jan 2020 22:03:44 +0000 Subject: [PATCH 05/22] Rename ValueTuple in BestOffers --- CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs | 12 ++++++------ CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs | 2 +- CryptoExchange.Net/OrderBook/SymbolOrderBook.cs | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs index 4f1214f..59917c6 100644 --- a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs +++ b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs @@ -58,12 +58,12 @@ namespace CryptoExchange.Net.UnitTests { var symbolOrderBook = new TestableSymbolOrderBook(); Assert.IsNotNull(symbolOrderBook.BestOffers); - Assert.IsNotNull(symbolOrderBook.BestOffers.BestBid); - Assert.IsNotNull(symbolOrderBook.BestOffers.BestAsk); - Assert.AreEqual(0m, symbolOrderBook.BestOffers.BestBid.Price); - Assert.AreEqual(0m, symbolOrderBook.BestOffers.BestBid.Quantity); - Assert.AreEqual(0m, symbolOrderBook.BestOffers.BestAsk.Price); - Assert.AreEqual(0m, symbolOrderBook.BestOffers.BestAsk.Quantity); + Assert.IsNotNull(symbolOrderBook.BestOffers.Bid); + Assert.IsNotNull(symbolOrderBook.BestOffers.Ask); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Bid.Price); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Bid.Quantity); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Ask.Price); + Assert.AreEqual(0m, symbolOrderBook.BestOffers.Ask.Quantity); } } } diff --git a/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs b/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs index 06321d8..ee24057 100644 --- a/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs +++ b/CryptoExchange.Net/Interfaces/ISymbolOrderBook.cs @@ -73,7 +73,7 @@ namespace CryptoExchange.Net.Interfaces /// /// BestBid/BesAsk returned as a pair /// - (ISymbolOrderBookEntry BestBid, ISymbolOrderBookEntry BestAsk) BestOffers { get; } + (ISymbolOrderBookEntry Bid, ISymbolOrderBookEntry Ask) BestOffers { get; } /// /// Start connecting and synchronizing the order book diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index 143307b..ea35512 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -162,7 +162,7 @@ namespace CryptoExchange.Net.OrderBook /// /// BestBid/BesAsk returned as a pair /// - public (ISymbolOrderBookEntry BestBid, ISymbolOrderBookEntry BestAsk) BestOffers { + public (ISymbolOrderBookEntry Bid, ISymbolOrderBookEntry Ask) BestOffers { get { lock (bookLock) return (BestBid,BestAsk); From 0141934ce0f4efe87c1b0ea09bec687f40b65c8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20=C3=96ner?= Date: Fri, 31 Jan 2020 23:32:49 +0300 Subject: [PATCH 06/22] Update SocketConnection.cs --- CryptoExchange.Net/Sockets/SocketConnection.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CryptoExchange.Net/Sockets/SocketConnection.cs b/CryptoExchange.Net/Sockets/SocketConnection.cs index e1704e0..1e4de7f 100644 --- a/CryptoExchange.Net/Sockets/SocketConnection.cs +++ b/CryptoExchange.Net/Sockets/SocketConnection.cs @@ -113,9 +113,16 @@ namespace CryptoExchange.Net.Sockets private void ProcessMessage(string data) { log.Write(LogVerbosity.Debug, $"Socket {Socket.Id} received data: " + data); + if (string.IsNullOrEmpty(data)) return; + var tokenData = data.ToJToken(log); if (tokenData == null) - return; + { + data = $"\"{data}\""; + tokenData = data.ToJToken(log); + if (tokenData == null) + return; + } var handledResponse = false; foreach (var pendingRequest in pendingRequests.ToList()) From ecb486862089cf2754839164a55d03423c3e18ed Mon Sep 17 00:00:00 2001 From: JKorf Date: Tue, 3 Mar 2020 09:24:00 +0100 Subject: [PATCH 07/22] removed invalid check on proxy --- CryptoExchange.Net/Objects/ApiProxy.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/CryptoExchange.Net/Objects/ApiProxy.cs b/CryptoExchange.Net/Objects/ApiProxy.cs index bbbd65e..1e75156 100644 --- a/CryptoExchange.Net/Objects/ApiProxy.cs +++ b/CryptoExchange.Net/Objects/ApiProxy.cs @@ -1,5 +1,4 @@ -using System; -using System.Security; +using System.Security; namespace CryptoExchange.Net.Objects { @@ -36,7 +35,6 @@ namespace CryptoExchange.Net.Objects { } - /// /// /// Create new settings for a proxy /// @@ -48,7 +46,6 @@ namespace CryptoExchange.Net.Objects { } - /// /// /// Create new settings for a proxy /// @@ -58,9 +55,6 @@ namespace CryptoExchange.Net.Objects /// The proxy password public ApiProxy(string host, int port, string? login, SecureString? password) { - if (!host.StartsWith("http")) - throw new ArgumentException("Proxy host should start with either http:// or https://"); - Host = host; Port = port; Login = login; From e31c0aa49308ce78929952a366ffc75a230f4df8 Mon Sep 17 00:00:00 2001 From: JKorf Date: Tue, 3 Mar 2020 10:28:21 +0100 Subject: [PATCH 08/22] Update CryptoExchange.Net.xml --- CryptoExchange.Net/CryptoExchange.Net.xml | 145 ---------------------- 1 file changed, 145 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index afa751c..d5c70b9 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -1149,7 +1149,6 @@ The proxy port - Create new settings for a proxy @@ -1159,7 +1158,6 @@ The proxy password - Create new settings for a proxy @@ -2928,148 +2926,5 @@ - - - Specifies that is allowed as an input even if the - corresponding type disallows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that is disallowed as an input even if the - corresponding type allows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that a method that will never return under any circumstance. - - - - - Initializes a new instance of the class. - - - - - Specifies that the method will not return if the associated - parameter is passed the specified value. - - - - - Gets the condition parameter value. - Code after the method is considered unreachable by diagnostics if the argument - to the associated parameter matches this value. - - - - - Initializes a new instance of the - class with the specified parameter value. - - - The condition parameter value. - Code after the method is considered unreachable by diagnostics if the argument - to the associated parameter matches this value. - - - - - Specifies that an output may be even if the - corresponding type disallows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that when a method returns , - the parameter may be even if the corresponding type disallows it. - - - - - Gets the return value condition. - If the method returns this value, the associated parameter may be . - - - - - Initializes the attribute with the specified return value condition. - - - The return value condition. - If the method returns this value, the associated parameter may be . - - - - - Specifies that an output is not even if the - corresponding type allows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that the output will be non- if the - named parameter is non-. - - - - - Gets the associated parameter name. - The output will be non- if the argument to the - parameter specified is non-. - - - - - Initializes the attribute with the associated parameter name. - - - The associated parameter name. - The output will be non- if the argument to the - parameter specified is non-. - - - - - Specifies that when a method returns , - the parameter will not be even if the corresponding type allows it. - - - - - Gets the return value condition. - If the method returns this value, the associated parameter will not be . - - - - - Initializes the attribute with the specified return value condition. - - - The return value condition. - If the method returns this value, the associated parameter will not be . - - From e7de98da645adb538dd7ae6b7653195cabeee15d Mon Sep 17 00:00:00 2001 From: JKorf Date: Tue, 3 Mar 2020 10:37:15 +0100 Subject: [PATCH 09/22] Updated version --- CryptoExchange.Net/CryptoExchange.Net.csproj | 4 ++-- README.md | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.csproj b/CryptoExchange.Net/CryptoExchange.Net.csproj index 6c4f1cb..0dd43fa 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.csproj +++ b/CryptoExchange.Net/CryptoExchange.Net.csproj @@ -6,12 +6,12 @@ CryptoExchange.Net JKorf A base package for implementing cryptocurrency exchange API's - 3.0.5 + 3.0.6 false https://github.com/JKorf/CryptoExchange.Net en true - 3.0.5 - Added PausedActivity events on socket subscriptions + 3.0.6 - Added BestOffer to SymbolOrderBook, removed invalid check on proxy enable 8.0 MIT diff --git a/README.md b/README.md index f7b667c..93cdf82 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,9 @@ The order book will automatically reconnect when the connection is lost and resy To stop synchronizing an order book use the `Stop` method. ## Release notes +* Version 3.0.6 - 03 Mar 2020 + * Added BestOffer to SymbolOrderBook, removed invalid check on proxy + * Version 3.0.5 - 05 Feb 2020 * Added PausedActivity events on socket subscriptions From 46e1fee3342ba3c9790dd73df1e4c1adf226e231 Mon Sep 17 00:00:00 2001 From: JKorf Date: Wed, 20 May 2020 09:14:57 +0200 Subject: [PATCH 10/22] Added error debug output, fix for freeze when unsubscribing --- CryptoExchange.Net/CryptoExchange.Net.xml | 145 ++++++++++++++++++++++ CryptoExchange.Net/RestClient.cs | 1 + CryptoExchange.Net/SocketClient.cs | 2 +- 3 files changed, 147 insertions(+), 1 deletion(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index d5c70b9..cf643e3 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -2928,3 +2928,148 @@ +System.Diagnostics.CodeAnalysis.AllowNullAttribute"> + + Specifies that is allowed as an input even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that is disallowed as an input even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that a method that will never return under any circumstance. + + + + + Initializes a new instance of the class. + + + + + Specifies that the method will not return if the associated + parameter is passed the specified value. + + + + + Gets the condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Initializes a new instance of the + class with the specified parameter value. + + + The condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Specifies that an output may be even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that when a method returns , + the parameter may be even if the corresponding type disallows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter may be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter may be . + + + + + Specifies that an output is not even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that the output will be non- if the + named parameter is non-. + + + + + Gets the associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Initializes the attribute with the associated parameter name. + + + The associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Specifies that when a method returns , + the parameter will not be even if the corresponding type allows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter will not be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter will not be . + + + + diff --git a/CryptoExchange.Net/RestClient.cs b/CryptoExchange.Net/RestClient.cs index 048cf6c..a36e3b7 100644 --- a/CryptoExchange.Net/RestClient.cs +++ b/CryptoExchange.Net/RestClient.cs @@ -215,6 +215,7 @@ namespace CryptoExchange.Net { using var reader = new StreamReader(responseStream); var data = await reader.ReadToEndAsync().ConfigureAwait(false); + log.Write(LogVerbosity.Debug, $"Error received: {data}"); responseStream.Close(); response.Close(); var parseResult = ValidateJson(data); diff --git a/CryptoExchange.Net/SocketClient.cs b/CryptoExchange.Net/SocketClient.cs index 77ea973..daf83c7 100644 --- a/CryptoExchange.Net/SocketClient.cs +++ b/CryptoExchange.Net/SocketClient.cs @@ -579,7 +579,7 @@ namespace CryptoExchange.Net periodicEvent?.Set(); periodicEvent?.Dispose(); log.Write(LogVerbosity.Debug, "Disposing socket client, closing all subscriptions"); - UnsubscribeAll().Wait(); + Task.Run(UnsubscribeAll).Wait(); semaphoreSlim?.Dispose(); base.Dispose(); } From 6e4df34aeaeec76b39e9692198db8c5fa86da573 Mon Sep 17 00:00:00 2001 From: JKorf Date: Wed, 20 May 2020 09:16:06 +0200 Subject: [PATCH 11/22] updated version --- CryptoExchange.Net/CryptoExchange.Net.csproj | 4 +- CryptoExchange.Net/CryptoExchange.Net.xml | 145 ------------------- README.md | 4 + 3 files changed, 6 insertions(+), 147 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.csproj b/CryptoExchange.Net/CryptoExchange.Net.csproj index 0dd43fa..b8e461c 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.csproj +++ b/CryptoExchange.Net/CryptoExchange.Net.csproj @@ -6,12 +6,12 @@ CryptoExchange.Net JKorf A base package for implementing cryptocurrency exchange API's - 3.0.6 + 3.0.7 false https://github.com/JKorf/CryptoExchange.Net en true - 3.0.6 - Added BestOffer to SymbolOrderBook, removed invalid check on proxy + 3.0.7 - Added error debug output, Fix for unsubscribe causing possible deadlock enable 8.0 MIT diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index cf643e3..d5c70b9 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -2928,148 +2928,3 @@ -System.Diagnostics.CodeAnalysis.AllowNullAttribute"> - - Specifies that is allowed as an input even if the - corresponding type disallows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that is disallowed as an input even if the - corresponding type allows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that a method that will never return under any circumstance. - - - - - Initializes a new instance of the class. - - - - - Specifies that the method will not return if the associated - parameter is passed the specified value. - - - - - Gets the condition parameter value. - Code after the method is considered unreachable by diagnostics if the argument - to the associated parameter matches this value. - - - - - Initializes a new instance of the - class with the specified parameter value. - - - The condition parameter value. - Code after the method is considered unreachable by diagnostics if the argument - to the associated parameter matches this value. - - - - - Specifies that an output may be even if the - corresponding type disallows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that when a method returns , - the parameter may be even if the corresponding type disallows it. - - - - - Gets the return value condition. - If the method returns this value, the associated parameter may be . - - - - - Initializes the attribute with the specified return value condition. - - - The return value condition. - If the method returns this value, the associated parameter may be . - - - - - Specifies that an output is not even if the - corresponding type allows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that the output will be non- if the - named parameter is non-. - - - - - Gets the associated parameter name. - The output will be non- if the argument to the - parameter specified is non-. - - - - - Initializes the attribute with the associated parameter name. - - - The associated parameter name. - The output will be non- if the argument to the - parameter specified is non-. - - - - - Specifies that when a method returns , - the parameter will not be even if the corresponding type allows it. - - - - - Gets the return value condition. - If the method returns this value, the associated parameter will not be . - - - - - Initializes the attribute with the specified return value condition. - - - The return value condition. - If the method returns this value, the associated parameter will not be . - - - - diff --git a/README.md b/README.md index 93cdf82..614612c 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,10 @@ The order book will automatically reconnect when the connection is lost and resy To stop synchronizing an order book use the `Stop` method. ## Release notes +* Version 3.0.7 - 20 May 2020 + * Added error debug output + * Fix for unsubscribe causing possible deadlock + * Version 3.0.6 - 03 Mar 2020 * Added BestOffer to SymbolOrderBook, removed invalid check on proxy From b95215866bc5b7ff0bdca47d98efcb51732c4b16 Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Tue, 2 Jun 2020 19:32:55 +0200 Subject: [PATCH 12/22] Added requestBodyEmptyContent setting for rest client, Added TryParseError for rest implementations to check for error with success status code --- CryptoExchange.Net/CryptoExchange.Net.csproj | 4 +- CryptoExchange.Net/CryptoExchange.Net.xml | 156 +++++++++++++++++++ CryptoExchange.Net/RestClient.cs | 53 ++++++- README.md | 4 + 4 files changed, 210 insertions(+), 7 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.csproj b/CryptoExchange.Net/CryptoExchange.Net.csproj index b8e461c..279d669 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.csproj +++ b/CryptoExchange.Net/CryptoExchange.Net.csproj @@ -6,12 +6,12 @@ CryptoExchange.Net JKorf A base package for implementing cryptocurrency exchange API's - 3.0.7 + 3.0.8 false https://github.com/JKorf/CryptoExchange.Net en true - 3.0.7 - Added error debug output, Fix for unsubscribe causing possible deadlock + 3.0.8 - Added empty body content setting, added TryParseError virtual method enable 8.0 MIT diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index d5c70b9..aa6e171 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -2112,6 +2112,11 @@ Request body content type + + + Whether or not we need to manually parse an error instead of relying on the http status code + + How to serialize array parameters @@ -2188,6 +2193,14 @@ Cancellation token + + + Can be used to parse an error even though response status indicates success. Some apis always return 200 OK, even though there is an error. + This can be used together with ManualParseError to check if it is an error before deserializing to an object + + Received data + Null if not an error, Error otherwise + Creates a request object @@ -2926,5 +2939,148 @@ + + + Specifies that is allowed as an input even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that is disallowed as an input even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that a method that will never return under any circumstance. + + + + + Initializes a new instance of the class. + + + + + Specifies that the method will not return if the associated + parameter is passed the specified value. + + + + + Gets the condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Initializes a new instance of the + class with the specified parameter value. + + + The condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Specifies that an output may be even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that when a method returns , + the parameter may be even if the corresponding type disallows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter may be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter may be . + + + + + Specifies that an output is not even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that the output will be non- if the + named parameter is non-. + + + + + Gets the associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Initializes the attribute with the associated parameter name. + + + The associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Specifies that when a method returns , + the parameter will not be even if the corresponding type allows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter will not be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter will not be . + + diff --git a/CryptoExchange.Net/RestClient.cs b/CryptoExchange.Net/RestClient.cs index a36e3b7..42043c4 100644 --- a/CryptoExchange.Net/RestClient.cs +++ b/CryptoExchange.Net/RestClient.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net.Http; using System.Net.NetworkInformation; using System.Net.Sockets; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using System.Web; @@ -39,11 +40,21 @@ namespace CryptoExchange.Net /// protected RequestBodyFormat requestBodyFormat = RequestBodyFormat.Json; + /// + /// Whether or not we need to manually parse an error instead of relying on the http status code + /// + protected bool manualParseError = false; + /// /// How to serialize array parameters /// protected ArrayParametersSerialization arraySerialization = ArrayParametersSerialization.Array; + /// + /// What request body should be when no data is send + /// + protected string requestBodyEmptyContent = "{}"; + /// /// Timeout for requests /// @@ -205,11 +216,32 @@ namespace CryptoExchange.Net var responseStream = await response.GetResponseStream().ConfigureAwait(false); if (response.IsSuccessStatusCode) { - var desResult = await Deserialize(responseStream).ConfigureAwait(false); - responseStream.Close(); - response.Close(); + if (manualParseError) + { + using var reader = new StreamReader(responseStream); + var data = await reader.ReadToEndAsync().ConfigureAwait(false); + responseStream.Close(); + response.Close(); + log.Write(LogVerbosity.Debug, $"Data received: {data}"); - return new WebCallResult(statusCode, headers, desResult.Data, desResult.Error); + var parseResult = ValidateJson(data); + if (!parseResult.Success) + return WebCallResult.CreateErrorResult(response.StatusCode, response.ResponseHeaders, new ServerError(data)); + var error = await TryParseError(parseResult.Data); + if(error != null) + return WebCallResult.CreateErrorResult(response.StatusCode, response.ResponseHeaders, error); + + var deserializeResult = Deserialize(parseResult.Data); + return new WebCallResult(response.StatusCode, response.ResponseHeaders, deserializeResult.Data, deserializeResult.Error); + } + else + { + var desResult = await Deserialize(responseStream).ConfigureAwait(false); + responseStream.Close(); + response.Close(); + + return new WebCallResult(statusCode, headers, desResult.Data, desResult.Error); + } } else { @@ -244,6 +276,17 @@ namespace CryptoExchange.Net } } + /// + /// Can be used to parse an error even though response status indicates success. Some apis always return 200 OK, even though there is an error. + /// This can be used together with ManualParseError to check if it is an error before deserializing to an object + /// + /// Received data + /// Null if not an error, Error otherwise + protected virtual Task TryParseError(JToken data) + { + return Task.FromResult(null); + } + /// /// Creates a request object /// @@ -280,7 +323,7 @@ namespace CryptoExchange.Net if(parameters?.Any() == true) WriteParamBody(request, parameters, contentType); else - request.SetContent("{}", contentType); + request.SetContent(requestBodyEmptyContent, contentType); } return request; diff --git a/README.md b/README.md index 614612c..2726beb 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,10 @@ The order book will automatically reconnect when the connection is lost and resy To stop synchronizing an order book use the `Stop` method. ## Release notes +* Version 3.0.8 - 02 Jun 2020 + * Added requestBodyEmptyContent setting for rest client + * Added TryParseError for rest implementations to check for error with success status code + * Version 3.0.7 - 20 May 2020 * Added error debug output * Fix for unsubscribe causing possible deadlock From c60131d46407371eee2bf8963fd131ab81060fbc Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Thu, 4 Jun 2020 22:01:42 +0200 Subject: [PATCH 13/22] wip orderbook rework --- CryptoExchange.Net/CryptoExchange.Net.xml | 9 +- .../OrderBook/ProcessQueueItem.cs | 13 ++ .../OrderBook/SymbolOrderBook.cs | 176 +++++++----------- 3 files changed, 86 insertions(+), 112 deletions(-) create mode 100644 CryptoExchange.Net/OrderBook/ProcessQueueItem.cs diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index aa6e171..b225bc8 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -2122,6 +2122,11 @@ How to serialize array parameters + + + What request body should be when no data is send + + Timeout for requests @@ -2939,7 +2944,9 @@ - + + +System.Diagnostics.CodeAnalysis.AllowNullAttribute"> Specifies that is allowed as an input even if the corresponding type disallows it. diff --git a/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs b/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs new file mode 100644 index 0000000..996e6d4 --- /dev/null +++ b/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs @@ -0,0 +1,13 @@ +using CryptoExchange.Net.Interfaces; +using System.Collections.Generic; + +namespace CryptoExchange.Net.OrderBook +{ + internal class ProcessQueueItem + { + public long StartUpdateId { get; set; } + public long EndUpdateId { get; set; } + public IEnumerable Bids { get; set; } = new List(); + public IEnumerable Asks { get; set; } = new List(); + } +} diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index ea35512..24adc47 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using CryptoExchange.Net.Interfaces; using CryptoExchange.Net.Logging; @@ -19,7 +21,7 @@ namespace CryptoExchange.Net.OrderBook /// /// The process buffer, used while syncing /// - protected readonly List processBuffer; + protected readonly List processBuffer; private readonly object bookLock = new object(); /// /// The ask list @@ -34,6 +36,10 @@ namespace CryptoExchange.Net.OrderBook private UpdateSubscription? subscription; private readonly bool sequencesAreConsecutive; + private Task _processTask; + private AutoResetEvent _queueEvent; + private ConcurrentQueue _processQueue; + /// /// Order book implementation id /// @@ -183,7 +189,10 @@ namespace CryptoExchange.Net.OrderBook throw new ArgumentNullException(nameof(options)); Id = options.OrderBookName; - processBuffer = new List(); + processBuffer = new List(); + _processQueue = new ConcurrentQueue(); + _queueEvent = new AutoResetEvent(false); + sequencesAreConsecutive = options.SequenceNumbersAreConsecutive; Symbol = symbol; Status = OrderBookStatus.Disconnected; @@ -209,6 +218,8 @@ namespace CryptoExchange.Net.OrderBook public async Task> StartAsync() { Status = OrderBookStatus.Connecting; + _processTask = Task.Run(ProcessQueue); + var startResult = await DoStart().ConfigureAwait(false); if (!startResult) return new CallResult(false, startResult.Error); @@ -224,6 +235,7 @@ namespace CryptoExchange.Net.OrderBook { log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} connection lost"); Status = OrderBookStatus.Connecting; + _queueEvent.Set(); processBuffer.Clear(); bookSet = false; DoReset(); @@ -259,6 +271,8 @@ namespace CryptoExchange.Net.OrderBook public async Task StopAsync() { Status = OrderBookStatus.Disconnected; + _queueEvent.Set(); + _processTask.Wait(); if(subscription != null) await subscription.Close().ConfigureAwait(false); } @@ -280,6 +294,46 @@ namespace CryptoExchange.Net.OrderBook /// protected abstract Task> DoResync(); + private void ProcessQueue() + { + while(Status != OrderBookStatus.Disconnected) + { + _queueEvent.WaitOne(); + + while (_processQueue.TryDequeue(out var item)) + ProcessQueueItem(item); + } + } + + private void ProcessQueueItem(ProcessQueueItem item) + { + lock (bookLock) + { + if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected) + return; + + if (!bookSet) + { + processBuffer.Add(new ProcessBufferRangeSequenceEntry() + { + Asks = item.Asks, + Bids = item.Bids, + FirstUpdateId = item.StartUpdateId, + LastUpdateId = item.EndUpdateId, + }); + log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update buffered #{item.StartUpdateId}-#{item.EndUpdateId} [{Asks.Count()} asks, {Bids.Count()} bids]"); + } + else + { + CheckProcessBuffer(); + var (prevBestBid, prevBestAsk) = BestOffers; + ProcessRangeUpdates(item.StartUpdateId, item.EndUpdateId, item.Bids, item.Asks); + OnOrderBookUpdate?.Invoke(item.Bids, item.Asks); + CheckBestOffersChanged(prevBestBid, prevBestAsk); + } + } + } + /// /// Set the initial data for the order book /// @@ -330,30 +384,8 @@ namespace CryptoExchange.Net.OrderBook /// protected void UpdateOrderBook(long rangeUpdateId, IEnumerable bids, IEnumerable asks) { - lock (bookLock) - { - if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected) - return; - - if (!bookSet) - { - processBuffer.Add(new ProcessBufferSingleSequenceEntry() - { - UpdateId = rangeUpdateId, - Asks = asks, - Bids = bids - }); - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update buffered #{rangeUpdateId}"); - } - else - { - CheckProcessBuffer(); - var (prevBestBid, prevBestAsk) = BestOffers; - ProcessSingleSequenceUpdates(rangeUpdateId, bids, asks); - OnOrderBookUpdate?.Invoke(bids, asks); - CheckBestOffersChanged(prevBestBid, prevBestAsk); - } - } + _processQueue.Enqueue(new ProcessQueueItem { StartUpdateId = rangeUpdateId, EndUpdateId = rangeUpdateId, Asks = asks, Bids = bids }); + _queueEvent.Set(); } /// @@ -365,31 +397,8 @@ namespace CryptoExchange.Net.OrderBook /// protected void UpdateOrderBook(long firstUpdateId, long lastUpdateId, IEnumerable bids, IEnumerable asks) { - lock (bookLock) - { - if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected) - return; - - if (!bookSet) - { - processBuffer.Add(new ProcessBufferRangeSequenceEntry() - { - Asks = asks, - Bids = bids, - FirstUpdateId = firstUpdateId, - LastUpdateId = lastUpdateId - }); - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update buffered #{firstUpdateId}-{lastUpdateId}"); - } - else - { - CheckProcessBuffer(); - var (prevBestBid, prevBestAsk) = BestOffers; - ProcessRangeUpdates(firstUpdateId, lastUpdateId, bids, asks); - OnOrderBookUpdate?.Invoke(bids, asks); - CheckBestOffersChanged(prevBestBid, prevBestAsk); - } - } + _processQueue.Enqueue(new ProcessQueueItem { StartUpdateId = firstUpdateId, EndUpdateId = lastUpdateId, Asks = asks, Bids = bids }); + _queueEvent.Set(); } /// @@ -399,42 +408,11 @@ namespace CryptoExchange.Net.OrderBook /// List of asks protected void UpdateOrderBook(IEnumerable bids, IEnumerable asks) { - lock (bookLock) - { - if (!bookSet) - { - processBuffer.Add(new ProcessBufferEntry - { - Asks = asks, - Bids = bids - }); - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update buffered #{Math.Min(bids.Min(b => b.Sequence), asks.Min(a => a.Sequence))}-{Math.Max(bids.Max(b => b.Sequence), asks.Max(a => a.Sequence))}"); - } - else - { - CheckProcessBuffer(); - var (prevBestBid, prevBestAsk) = BestOffers; - ProcessUpdates(bids, asks); - OnOrderBookUpdate?.Invoke(bids, asks); - CheckBestOffersChanged(prevBestBid, prevBestAsk); - } - } - } + var highest = Math.Max(bids.Any() ? bids.Max(b => b.Sequence) : 0, asks.Any() ? asks.Max(a => a.Sequence) : 0); + var lowest = Math.Min(bids.Any() ? bids.Min(b => b.Sequence) : long.MaxValue, asks.Any() ? asks.Min(a => a.Sequence) : long.MaxValue); - private void ProcessUpdates(IEnumerable bids, IEnumerable asks) - { - var entries = new Dictionary(); - foreach (var entry in asks.OrderBy(a => a.Sequence)) - entries.Add(entry, OrderBookEntryType.Ask); - foreach (var entry in bids.OrderBy(a => a.Sequence)) - entries.Add(entry, OrderBookEntryType.Bid); - - foreach (var entry in entries.OrderBy(e => e.Key.Sequence)) - { - if(ProcessUpdate(entry.Key.Sequence, entry.Value, entry.Key)) - LastSequenceNumber = entry.Key.Sequence; - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update #{LastSequenceNumber}"); - } + _processQueue.Enqueue(new ProcessQueueItem { StartUpdateId = lowest, EndUpdateId = highest , Asks = asks, Bids = bids }); + _queueEvent.Set(); } private void ProcessRangeUpdates(long firstUpdateId, long lastUpdateId, IEnumerable bids, IEnumerable asks) @@ -455,24 +433,6 @@ namespace CryptoExchange.Net.OrderBook log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update processed #{firstUpdateId}-{lastUpdateId}"); } - private void ProcessSingleSequenceUpdates(long updateId, IEnumerable bids, IEnumerable asks) - { - foreach (var entry in bids) - { - if (!ProcessUpdate(updateId, OrderBookEntryType.Bid, entry)) - return; - } - - foreach (var entry in asks) - { - if (!ProcessUpdate(updateId, OrderBookEntryType.Ask, entry)) - return; - } - - LastSequenceNumber = updateId; - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update processed #{LastSequenceNumber}"); - } - /// /// Check and empty the process buffer; see what entries to update the book with /// @@ -484,13 +444,7 @@ namespace CryptoExchange.Net.OrderBook foreach (var bufferEntry in pbList) { - if (bufferEntry is ProcessBufferEntry pbe) - ProcessUpdates(pbe.Bids, pbe.Asks); - else if(bufferEntry is ProcessBufferRangeSequenceEntry pbrse) - ProcessRangeUpdates(pbrse.FirstUpdateId, pbrse.LastUpdateId, pbrse.Bids, pbrse.Asks); - else if (bufferEntry is ProcessBufferSingleSequenceEntry pbsse) - ProcessSingleSequenceUpdates(pbsse.UpdateId, pbsse.Bids, pbsse.Asks); - + ProcessRangeUpdates(bufferEntry.FirstUpdateId, bufferEntry.LastUpdateId, bufferEntry.Bids, bufferEntry.Asks); processBuffer.Remove(bufferEntry); } } From 5a33bd554efd6ac8c4624b430fec0480fd806e42 Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Fri, 5 Jun 2020 20:50:54 +0200 Subject: [PATCH 14/22] Cleaned some now unused classes --- CryptoExchange.Net/CryptoExchange.Net.xml | 39 +------------------ .../OrderBook/ProcessBufferEntry.cs | 34 ---------------- .../OrderBook/SymbolOrderBook.cs | 5 ++- 3 files changed, 4 insertions(+), 74 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index b225bc8..b6b143d 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -1662,41 +1662,6 @@ - - - Buffer entry for order book - - - - - List of asks - - - - - List of bids - - - - - Buffer entry with a single update id per update - - - - - First update id - - - - - List of asks - - - - - List of bids - - Buffer entry with a first and last update id @@ -2944,9 +2909,7 @@ - - -System.Diagnostics.CodeAnalysis.AllowNullAttribute"> + Specifies that is allowed as an input even if the corresponding type disallows it. diff --git a/CryptoExchange.Net/OrderBook/ProcessBufferEntry.cs b/CryptoExchange.Net/OrderBook/ProcessBufferEntry.cs index 3cf64bc..1e7c919 100644 --- a/CryptoExchange.Net/OrderBook/ProcessBufferEntry.cs +++ b/CryptoExchange.Net/OrderBook/ProcessBufferEntry.cs @@ -3,40 +3,6 @@ using System.Collections.Generic; namespace CryptoExchange.Net.OrderBook { - /// - /// Buffer entry for order book - /// - public class ProcessBufferEntry - { - /// - /// List of asks - /// - public IEnumerable Asks { get; set; } = new List(); - /// - /// List of bids - /// - public IEnumerable Bids { get; set; } = new List(); - } - - /// - /// Buffer entry with a single update id per update - /// - public class ProcessBufferSingleSequenceEntry - { - /// - /// First update id - /// - public long UpdateId { get; set; } - /// - /// List of asks - /// - public IEnumerable Asks { get; set; } = new List(); - /// - /// List of bids - /// - public IEnumerable Bids { get; set; } = new List(); - } - /// /// Buffer entry with a first and last update id /// diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index 24adc47..5b973fe 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -22,7 +22,6 @@ namespace CryptoExchange.Net.OrderBook /// The process buffer, used while syncing /// protected readonly List processBuffer; - private readonly object bookLock = new object(); /// /// The ask list /// @@ -30,8 +29,10 @@ namespace CryptoExchange.Net.OrderBook /// /// The bid list /// - protected SortedList bids; + + private readonly object bookLock = new object(); + private OrderBookStatus status; private UpdateSubscription? subscription; private readonly bool sequencesAreConsecutive; From 128cab8e1ec3ecfa2401c1d55cc7eb9a4546cb48 Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Sun, 7 Jun 2020 12:40:26 +0200 Subject: [PATCH 15/22] Added postParameterPosition/arraySeralization settings to authentication provider interface, fixed array serialization for request body --- .../Authentication/AuthenticationProvider.cs | 13 +- CryptoExchange.Net/CryptoExchange.Net.xml | 161 ++---------------- CryptoExchange.Net/ExtensionMethods.cs | 7 +- .../OrderBook/SymbolOrderBook.cs | 2 + CryptoExchange.Net/RestClient.cs | 31 +++- 5 files changed, 51 insertions(+), 163 deletions(-) diff --git a/CryptoExchange.Net/Authentication/AuthenticationProvider.cs b/CryptoExchange.Net/Authentication/AuthenticationProvider.cs index 8739fed..21857b1 100644 --- a/CryptoExchange.Net/Authentication/AuthenticationProvider.cs +++ b/CryptoExchange.Net/Authentication/AuthenticationProvider.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using CryptoExchange.Net.Objects; +using System.Collections.Generic; using System.Net.Http; namespace CryptoExchange.Net.Authentication @@ -29,8 +30,11 @@ namespace CryptoExchange.Net.Authentication /// /// /// + /// + /// /// - public virtual Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed) + public virtual Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed, + PostParameters postParameterPosition, ArrayParametersSerialization arraySerialization) { return parameters; } @@ -42,8 +46,11 @@ namespace CryptoExchange.Net.Authentication /// /// /// + /// + /// /// - public virtual Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed) + public virtual Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed, + PostParameters postParameterPosition, ArrayParametersSerialization arraySerialization) { return new Dictionary(); } diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index b6b143d..fae41b2 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -97,7 +97,7 @@ - + Add authentication to the parameter list @@ -105,9 +105,11 @@ + + - + Add authentication to the header dictionary @@ -115,6 +117,8 @@ + + @@ -2142,7 +2146,7 @@ The roundtrip time of the ping request - + Execute a request @@ -2152,7 +2156,9 @@ Cancellation token The parameters of the request Whether or not the request should be authenticated - Whether or not the resulting object should be checked for missing properties in the mapping (only outputs if log verbosity is Debug) + Whether or not the resulting object should be checked for missing properties in the mapping (only outputs if log verbosity is Debug) + Where the post parameters should be placed + How array paramters should be serialized @@ -2171,7 +2177,7 @@ Received data Null if not an error, Error otherwise - + Creates a request object @@ -2179,6 +2185,8 @@ The method of the request The parameters of the request Whether or not the request should be authenticated + Where the post parameters should be placed + How array paramters should be serialized @@ -2909,148 +2917,5 @@ - - - Specifies that is allowed as an input even if the - corresponding type disallows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that is disallowed as an input even if the - corresponding type allows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that a method that will never return under any circumstance. - - - - - Initializes a new instance of the class. - - - - - Specifies that the method will not return if the associated - parameter is passed the specified value. - - - - - Gets the condition parameter value. - Code after the method is considered unreachable by diagnostics if the argument - to the associated parameter matches this value. - - - - - Initializes a new instance of the - class with the specified parameter value. - - - The condition parameter value. - Code after the method is considered unreachable by diagnostics if the argument - to the associated parameter matches this value. - - - - - Specifies that an output may be even if the - corresponding type disallows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that when a method returns , - the parameter may be even if the corresponding type disallows it. - - - - - Gets the return value condition. - If the method returns this value, the associated parameter may be . - - - - - Initializes the attribute with the specified return value condition. - - - The return value condition. - If the method returns this value, the associated parameter may be . - - - - - Specifies that an output is not even if the - corresponding type allows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that the output will be non- if the - named parameter is non-. - - - - - Gets the associated parameter name. - The output will be non- if the argument to the - parameter specified is non-. - - - - - Initializes the attribute with the associated parameter name. - - - The associated parameter name. - The output will be non- if the argument to the - parameter specified is non-. - - - - - Specifies that when a method returns , - the parameter will not be even if the corresponding type allows it. - - - - - Gets the return value condition. - If the method returns this value, the associated parameter will not be . - - - - - Initializes the attribute with the specified return value condition. - - - The return value condition. - If the method returns this value, the associated parameter will not be . - - diff --git a/CryptoExchange.Net/ExtensionMethods.cs b/CryptoExchange.Net/ExtensionMethods.cs index 1090cf9..1ac235b 100644 --- a/CryptoExchange.Net/ExtensionMethods.cs +++ b/CryptoExchange.Net/ExtensionMethods.cs @@ -7,6 +7,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Threading; using System.Threading.Tasks; +using System.Web; using CryptoExchange.Net.Logging; using CryptoExchange.Net.Objects; using Newtonsoft.Json; @@ -79,16 +80,16 @@ namespace CryptoExchange.Net foreach (var arrayEntry in arraysParameters) { if(serializationType == ArrayParametersSerialization.Array) - uriString += $"{string.Join("&", ((object[])(urlEncodeValues ? WebUtility.UrlEncode(arrayEntry.Value.ToString()) : arrayEntry.Value)).Select(v => $"{arrayEntry.Key}[]={v}"))}&"; + uriString += $"{string.Join("&", ((object[])(urlEncodeValues ? Uri.EscapeDataString(arrayEntry.Value.ToString()) : arrayEntry.Value)).Select(v => $"{arrayEntry.Key}[]={v}"))}&"; else { var array = (Array)arrayEntry.Value; - uriString += string.Join("&", array.OfType().Select(a => $"{arrayEntry.Key}={WebUtility.UrlEncode(a.ToString())}")); + uriString += string.Join("&", array.OfType().Select(a => $"{arrayEntry.Key}={Uri.EscapeDataString(a.ToString())}")); uriString += "&"; } } - uriString += $"{string.Join("&", parameters.Where(p => !p.Value.GetType().IsArray).Select(s => $"{s.Key}={(urlEncodeValues ? WebUtility.UrlEncode(s.Value.ToString()) : s.Value)}"))}"; + uriString += $"{string.Join("&", parameters.Where(p => !p.Value.GetType().IsArray).Select(s => $"{s.Key}={(urlEncodeValues ? Uri.EscapeDataString(s.Value.ToString()) : s.Value)}"))}"; uriString = uriString.TrimEnd('&'); return uriString; } diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index 5b973fe..c27e470 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -218,6 +218,7 @@ namespace CryptoExchange.Net.OrderBook /// public async Task> StartAsync() { + log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} starting"); Status = OrderBookStatus.Connecting; _processTask = Task.Run(ProcessQueue); @@ -271,6 +272,7 @@ namespace CryptoExchange.Net.OrderBook /// public async Task StopAsync() { + log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} stopping"); Status = OrderBookStatus.Disconnected; _queueEvent.Set(); _processTask.Wait(); diff --git a/CryptoExchange.Net/RestClient.cs b/CryptoExchange.Net/RestClient.cs index 42043c4..d3a0f55 100644 --- a/CryptoExchange.Net/RestClient.cs +++ b/CryptoExchange.Net/RestClient.cs @@ -164,11 +164,13 @@ namespace CryptoExchange.Net /// Cancellation token /// The parameters of the request /// Whether or not the request should be authenticated - /// Whether or not the resulting object should be checked for missing properties in the mapping (only outputs if log verbosity is Debug) + /// Whether or not the resulting object should be checked for missing properties in the mapping (only outputs if log verbosity is Debug) + /// Where the post parameters should be placed + /// How array paramters should be serialized /// [return: NotNull] protected virtual async Task> SendRequest(Uri uri, HttpMethod method, CancellationToken cancellationToken, - Dictionary? parameters = null, bool signed = false, bool checkResult = true) where T : class + Dictionary? parameters = null, bool signed = false, bool checkResult = true, PostParameters? postPosition = null, ArrayParametersSerialization? arraySerialization = null) where T : class { log.Write(LogVerbosity.Debug, "Creating request for " + uri); if (signed && authProvider == null) @@ -177,7 +179,7 @@ namespace CryptoExchange.Net return new WebCallResult(null, null, null, new NoApiCredentialsError()); } - var request = ConstructRequest(uri, method, parameters, signed); + var request = ConstructRequest(uri, method, parameters, signed, postPosition ?? postParametersPosition, arraySerialization ?? this.arraySerialization); foreach (var limiter in RateLimiters) { var limitResult = limiter.LimitRequest(this, uri.AbsolutePath, RateLimitBehaviour); @@ -294,17 +296,19 @@ namespace CryptoExchange.Net /// The method of the request /// The parameters of the request /// Whether or not the request should be authenticated + /// Where the post parameters should be placed + /// How array paramters should be serialized /// - protected virtual IRequest ConstructRequest(Uri uri, HttpMethod method, Dictionary? parameters, bool signed) + protected virtual IRequest ConstructRequest(Uri uri, HttpMethod method, Dictionary? parameters, bool signed, PostParameters postPosition, ArrayParametersSerialization arraySerialization) { if (parameters == null) parameters = new Dictionary(); var uriString = uri.ToString(); if(authProvider != null) - parameters = authProvider.AddAuthenticationToParameters(uriString, method, parameters, signed); + parameters = authProvider.AddAuthenticationToParameters(uriString, method, parameters, signed, postPosition, arraySerialization); - if((method == HttpMethod.Get || method == HttpMethod.Delete || postParametersPosition == PostParameters.InUri) && parameters?.Any() == true) + if((method == HttpMethod.Get || method == HttpMethod.Delete || postPosition == PostParameters.InUri) && parameters?.Any() == true) uriString += "?" + parameters.CreateParamString(true, arraySerialization); var contentType = requestBodyFormat == RequestBodyFormat.Json ? Constants.JsonContentHeader : Constants.FormContentHeader; @@ -313,12 +317,12 @@ namespace CryptoExchange.Net var headers = new Dictionary(); if (authProvider != null) - headers = authProvider.AddAuthenticationToHeaders(uriString, method, parameters!, signed); + headers = authProvider.AddAuthenticationToHeaders(uriString, method, parameters!, signed, postPosition, arraySerialization); foreach (var header in headers) request.AddHeader(header.Key, header.Value); - if ((method == HttpMethod.Post || method == HttpMethod.Put) && postParametersPosition != PostParameters.InUri) + if ((method == HttpMethod.Post || method == HttpMethod.Put) && postPosition != PostParameters.InUri) { if(parameters?.Any() == true) WriteParamBody(request, parameters, contentType); @@ -346,7 +350,16 @@ namespace CryptoExchange.Net { var formData = HttpUtility.ParseQueryString(string.Empty); foreach (var kvp in parameters.OrderBy(p => p.Key)) - formData.Add(kvp.Key, kvp.Value.ToString()); + { + if (kvp.Value.GetType().IsArray) + { + var array = (Array)kvp.Value; + foreach(var value in array) + formData.Add(kvp.Key, value.ToString()); + } + else + formData.Add(kvp.Key, kvp.Value.ToString()); + } var stringData = formData.ToString(); request.SetContent(stringData, contentType); } From ec362d7ab9385a6e0552685142bafed1870cac01 Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Sun, 7 Jun 2020 12:42:18 +0200 Subject: [PATCH 16/22] Updated version --- CryptoExchange.Net/CryptoExchange.Net.csproj | 4 ++-- README.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.csproj b/CryptoExchange.Net/CryptoExchange.Net.csproj index 279d669..b061108 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.csproj +++ b/CryptoExchange.Net/CryptoExchange.Net.csproj @@ -6,12 +6,12 @@ CryptoExchange.Net JKorf A base package for implementing cryptocurrency exchange API's - 3.0.8 + 3.0.9 false https://github.com/JKorf/CryptoExchange.Net en true - 3.0.8 - Added empty body content setting, added TryParseError virtual method + 3.0.9 - Added arraySerialization and postParameterPosition to AuthenticationProvider interface, fixed array serialization in request body enable 8.0 MIT diff --git a/README.md b/README.md index 2726beb..c3b538e 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,10 @@ The order book will automatically reconnect when the connection is lost and resy To stop synchronizing an order book use the `Stop` method. ## Release notes +* Version 3.0.9 - 07 Jun 2020 + * Added arraySerialization and postParameterPosition to AuthenticationProvider interface + * Fixed array serialization in request body + * Version 3.0.8 - 02 Jun 2020 * Added requestBodyEmptyContent setting for rest client * Added TryParseError for rest implementations to check for error with success status code From 07fbc0a3df474b065d4dcf19166b1c1652dfca26 Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Sun, 7 Jun 2020 12:48:15 +0200 Subject: [PATCH 17/22] Fixed tests --- .../TestImplementations/TestBaseClient.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CryptoExchange.Net.UnitTests/TestImplementations/TestBaseClient.cs b/CryptoExchange.Net.UnitTests/TestImplementations/TestBaseClient.cs index 9b36f42..d9e2e91 100644 --- a/CryptoExchange.Net.UnitTests/TestImplementations/TestBaseClient.cs +++ b/CryptoExchange.Net.UnitTests/TestImplementations/TestBaseClient.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Net.Http; using CryptoExchange.Net.Authentication; using CryptoExchange.Net.Logging; @@ -38,14 +39,14 @@ namespace CryptoExchange.Net.UnitTests { } - public override Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed) + public override Dictionary AddAuthenticationToHeaders(string uri, HttpMethod method, Dictionary parameters, bool signed, PostParameters postParameters, ArrayParametersSerialization arraySerialization) { - return base.AddAuthenticationToHeaders(uri, method, parameters, signed); + return base.AddAuthenticationToHeaders(uri, method, parameters, signed, postParameters, arraySerialization); } - public override Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed) + public override Dictionary AddAuthenticationToParameters(string uri, HttpMethod method, Dictionary parameters, bool signed, PostParameters postParameters, ArrayParametersSerialization arraySerialization) { - return base.AddAuthenticationToParameters(uri, method, parameters, signed); + return base.AddAuthenticationToParameters(uri, method, parameters, signed, postParameters, arraySerialization); } public override string Sign(string toSign) From 20bf8df4ef044b4fbe3fd77f118796f6856efc2e Mon Sep 17 00:00:00 2001 From: JKorf Date: Tue, 16 Jun 2020 16:18:57 +0200 Subject: [PATCH 18/22] Added strict levels to symbol order book --- CryptoExchange.Net/CryptoExchange.Net.xml | 161 ++---------------- CryptoExchange.Net/Objects/Options.cs | 14 +- .../OrderBook/SymbolOrderBook.cs | 32 ++++ 3 files changed, 59 insertions(+), 148 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index b225bc8..d674aa2 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -1541,11 +1541,20 @@ Whether each update should have a consecutive id number. Used to identify and reconnect when numbers are skipped. - + + + Whether or not a level should be removed from the book when it's pushed out of scope of the limit. For example with a book of limit 10, + when a new bid is added which makes the total amount of bids 11, should the last bid entry be removed + + + The name of the order book implementation Whether each update should have a consecutive id number. Used to identify and reconnect when numbers are skipped. + Whether or not a level should be removed from the book when it's pushed out of scope of the limit. For example with a book of limit 10, + when a new bid is added which makes the total amount of bids 11, should the last bid entry be removed + Amount of levels for this order book @@ -1757,6 +1766,11 @@ If order book is set + + + The amount of levels for this book + + The status of the order book. Order book is up to date when the status is `Synced` @@ -2946,148 +2960,3 @@ -System.Diagnostics.CodeAnalysis.AllowNullAttribute"> - - Specifies that is allowed as an input even if the - corresponding type disallows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that is disallowed as an input even if the - corresponding type allows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that a method that will never return under any circumstance. - - - - - Initializes a new instance of the class. - - - - - Specifies that the method will not return if the associated - parameter is passed the specified value. - - - - - Gets the condition parameter value. - Code after the method is considered unreachable by diagnostics if the argument - to the associated parameter matches this value. - - - - - Initializes a new instance of the - class with the specified parameter value. - - - The condition parameter value. - Code after the method is considered unreachable by diagnostics if the argument - to the associated parameter matches this value. - - - - - Specifies that an output may be even if the - corresponding type disallows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that when a method returns , - the parameter may be even if the corresponding type disallows it. - - - - - Gets the return value condition. - If the method returns this value, the associated parameter may be . - - - - - Initializes the attribute with the specified return value condition. - - - The return value condition. - If the method returns this value, the associated parameter may be . - - - - - Specifies that an output is not even if the - corresponding type allows it. - - - - - Initializes a new instance of the class. - - - - - Specifies that the output will be non- if the - named parameter is non-. - - - - - Gets the associated parameter name. - The output will be non- if the argument to the - parameter specified is non-. - - - - - Initializes the attribute with the associated parameter name. - - - The associated parameter name. - The output will be non- if the argument to the - parameter specified is non-. - - - - - Specifies that when a method returns , - the parameter will not be even if the corresponding type allows it. - - - - - Gets the return value condition. - If the method returns this value, the associated parameter will not be . - - - - - Initializes the attribute with the specified return value condition. - - - The return value condition. - If the method returns this value, the associated parameter will not be . - - - - diff --git a/CryptoExchange.Net/Objects/Options.cs b/CryptoExchange.Net/Objects/Options.cs index 18744ab..515494d 100644 --- a/CryptoExchange.Net/Objects/Options.cs +++ b/CryptoExchange.Net/Objects/Options.cs @@ -44,20 +44,30 @@ namespace CryptoExchange.Net.Objects /// public bool SequenceNumbersAreConsecutive { get; } + /// + /// Whether or not a level should be removed from the book when it's pushed out of scope of the limit. For example with a book of limit 10, + /// when a new bid is added which makes the total amount of bids 11, should the last bid entry be removed + /// + public bool StrictLevels { get; } + /// /// /// The name of the order book implementation /// Whether each update should have a consecutive id number. Used to identify and reconnect when numbers are skipped. - public OrderBookOptions(string name, bool sequencesAreConsecutive) + /// Whether or not a level should be removed from the book when it's pushed out of scope of the limit. For example with a book of limit 10, + /// when a new bid is added which makes the total amount of bids 11, should the last bid entry be removed + /// Amount of levels for this order book + public OrderBookOptions(string name, bool sequencesAreConsecutive, bool strictLevels) { OrderBookName = name; SequenceNumbersAreConsecutive = sequencesAreConsecutive; + StrictLevels = strictLevels; } /// public override string ToString() { - return $"{base.ToString()}, OrderBookName: {OrderBookName}, SequenceNumbersAreConsequtive: {SequenceNumbersAreConsecutive}"; + return $"{base.ToString()}, OrderBookName: {OrderBookName}, SequenceNumbersAreConsequtive: {SequenceNumbersAreConsecutive}, StrictLevels: {StrictLevels}"; } } diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index 24adc47..b8674f5 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -35,6 +35,7 @@ namespace CryptoExchange.Net.OrderBook private OrderBookStatus status; private UpdateSubscription? subscription; private readonly bool sequencesAreConsecutive; + private readonly bool strictLevels; private Task _processTask; private AutoResetEvent _queueEvent; @@ -54,6 +55,11 @@ namespace CryptoExchange.Net.OrderBook /// protected bool bookSet; + /// + /// The amount of levels for this book + /// + protected int? Levels { get; set; } = null; + /// /// The status of the order book. Order book is up to date when the status is `Synced` /// @@ -194,6 +200,7 @@ namespace CryptoExchange.Net.OrderBook _queueEvent = new AutoResetEvent(false); sequencesAreConsecutive = options.SequenceNumbersAreConsecutive; + strictLevels = options.StrictLevels; Symbol = symbol; Status = OrderBookStatus.Disconnected; @@ -236,6 +243,8 @@ namespace CryptoExchange.Net.OrderBook log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} connection lost"); Status = OrderBookStatus.Connecting; _queueEvent.Set(); + // Clear queue + while(_processQueue.TryDequeue(out _)) processBuffer.Clear(); bookSet = false; DoReset(); @@ -328,6 +337,14 @@ namespace CryptoExchange.Net.OrderBook CheckProcessBuffer(); var (prevBestBid, prevBestAsk) = BestOffers; ProcessRangeUpdates(item.StartUpdateId, item.EndUpdateId, item.Bids, item.Asks); + + if (asks.First().Key < bids.First().Key) + { + log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} detected out of sync order book. Resyncing"); + _ = subscription?.Reconnect(); + return; + } + OnOrderBookUpdate?.Invoke(item.Bids, item.Asks); CheckBestOffersChanged(prevBestBid, prevBestAsk); } @@ -429,6 +446,21 @@ namespace CryptoExchange.Net.OrderBook foreach (var entry in asks) ProcessUpdate(LastSequenceNumber + 1, OrderBookEntryType.Ask, entry); + if (Levels.HasValue && strictLevels) + { + while (this.bids.Count() > Levels.Value) + { + BidCount--; + this.bids.Remove(this.bids.Last().Key); + } + + while (this.asks.Count() > Levels.Value) + { + AskCount--; + this.asks.Remove(this.asks.Last().Key); + } + } + LastSequenceNumber = lastUpdateId; log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} update processed #{firstUpdateId}-{lastUpdateId}"); } From 5105e995e8606167d51d5c9539bfc966ff22c060 Mon Sep 17 00:00:00 2001 From: JKorf Date: Tue, 16 Jun 2020 16:29:15 +0200 Subject: [PATCH 19/22] Fixed test, updated version --- CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs | 2 +- CryptoExchange.Net/CryptoExchange.Net.csproj | 4 ++-- README.md | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs index 59917c6..beefa97 100644 --- a/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs +++ b/CryptoExchange.Net.UnitTests/SymbolOrderBookTests.cs @@ -14,7 +14,7 @@ namespace CryptoExchange.Net.UnitTests [TestFixture] public class SymbolOrderBookTests { - private static OrderBookOptions defaultOrderBookOptions = new OrderBookOptions("Test", true); + private static OrderBookOptions defaultOrderBookOptions = new OrderBookOptions("Test", true, false); private class TestableSymbolOrderBook : SymbolOrderBook { diff --git a/CryptoExchange.Net/CryptoExchange.Net.csproj b/CryptoExchange.Net/CryptoExchange.Net.csproj index b061108..62bb836 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.csproj +++ b/CryptoExchange.Net/CryptoExchange.Net.csproj @@ -6,12 +6,12 @@ CryptoExchange.Net JKorf A base package for implementing cryptocurrency exchange API's - 3.0.9 + 3.0.10 false https://github.com/JKorf/CryptoExchange.Net en true - 3.0.9 - Added arraySerialization and postParameterPosition to AuthenticationProvider interface, fixed array serialization in request body + 3.0.10 - Fix for order book synchronization enable 8.0 MIT diff --git a/README.md b/README.md index c3b538e..664643c 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,9 @@ The order book will automatically reconnect when the connection is lost and resy To stop synchronizing an order book use the `Stop` method. ## Release notes +* Version 3.0.10 - 16 Jun 2020 + * Fix for order book synchronization + * Version 3.0.9 - 07 Jun 2020 * Added arraySerialization and postParameterPosition to AuthenticationProvider interface * Fixed array serialization in request body From ddc4ebe638585e55605a1a27c4144769f746bc0a Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Sat, 20 Jun 2020 20:34:48 +0200 Subject: [PATCH 20/22] Rework order book; added support for checksum --- CryptoExchange.Net/CryptoExchange.Net.xml | 155 ++++++++++++++++++ CryptoExchange.Net/Objects/Enums.cs | 4 + .../OrderBook/ProcessQueueItem.cs | 13 ++ .../OrderBook/SymbolOrderBook.cs | 123 +++++++++----- 4 files changed, 257 insertions(+), 38 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index 4f03bd5..bf6762e 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -1333,6 +1333,11 @@ Connecting + + + Reconnecting + + Syncing data @@ -1858,6 +1863,13 @@ + + + Validate a checksum with the current order book + + + + Set the initial data for the order book @@ -2931,5 +2943,148 @@ + + + Specifies that is allowed as an input even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that is disallowed as an input even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that a method that will never return under any circumstance. + + + + + Initializes a new instance of the class. + + + + + Specifies that the method will not return if the associated + parameter is passed the specified value. + + + + + Gets the condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Initializes a new instance of the + class with the specified parameter value. + + + The condition parameter value. + Code after the method is considered unreachable by diagnostics if the argument + to the associated parameter matches this value. + + + + + Specifies that an output may be even if the + corresponding type disallows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that when a method returns , + the parameter may be even if the corresponding type disallows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter may be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter may be . + + + + + Specifies that an output is not even if the + corresponding type allows it. + + + + + Initializes a new instance of the class. + + + + + Specifies that the output will be non- if the + named parameter is non-. + + + + + Gets the associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Initializes the attribute with the associated parameter name. + + + The associated parameter name. + The output will be non- if the argument to the + parameter specified is non-. + + + + + Specifies that when a method returns , + the parameter will not be even if the corresponding type allows it. + + + + + Gets the return value condition. + If the method returns this value, the associated parameter will not be . + + + + + Initializes the attribute with the specified return value condition. + + + The return value condition. + If the method returns this value, the associated parameter will not be . + + diff --git a/CryptoExchange.Net/Objects/Enums.cs b/CryptoExchange.Net/Objects/Enums.cs index 3af3694..785b8ff 100644 --- a/CryptoExchange.Net/Objects/Enums.cs +++ b/CryptoExchange.Net/Objects/Enums.cs @@ -59,6 +59,10 @@ /// Connecting, /// + /// Reconnecting + /// + Reconnecting, + /// /// Syncing data /// Syncing, diff --git a/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs b/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs index 996e6d4..14832a0 100644 --- a/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs +++ b/CryptoExchange.Net/OrderBook/ProcessQueueItem.cs @@ -10,4 +10,17 @@ namespace CryptoExchange.Net.OrderBook public IEnumerable Bids { get; set; } = new List(); public IEnumerable Asks { get; set; } = new List(); } + + internal class InitialOrderBookItem + { + public long StartUpdateId { get; set; } + public long EndUpdateId { get; set; } + public IEnumerable Bids { get; set; } = new List(); + public IEnumerable Asks { get; set; } = new List(); + } + + internal class ChecksumItem + { + public int Checksum { get; set; } + } } diff --git a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs index 471dc45..8f4e32f 100644 --- a/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs +++ b/CryptoExchange.Net/OrderBook/SymbolOrderBook.cs @@ -40,7 +40,7 @@ namespace CryptoExchange.Net.OrderBook private Task _processTask; private AutoResetEvent _queueEvent; - private ConcurrentQueue _processQueue; + private ConcurrentQueue _processQueue; /// /// Order book implementation id @@ -197,7 +197,7 @@ namespace CryptoExchange.Net.OrderBook Id = options.OrderBookName; processBuffer = new List(); - _processQueue = new ConcurrentQueue(); + _processQueue = new ConcurrentQueue(); _queueEvent = new AutoResetEvent(false); sequencesAreConsecutive = options.SequenceNumbersAreConsecutive; @@ -243,7 +243,7 @@ namespace CryptoExchange.Net.OrderBook private void Reset() { log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} connection lost"); - Status = OrderBookStatus.Connecting; + Status = OrderBookStatus.Reconnecting; _queueEvent.Set(); // Clear queue while(_processQueue.TryDequeue(out _)) @@ -306,14 +306,58 @@ namespace CryptoExchange.Net.OrderBook /// protected abstract Task> DoResync(); + /// + /// Validate a checksum with the current order book + /// + /// + /// + protected virtual bool DoChecksum(int checksum) => true; + private void ProcessQueue() { while(Status != OrderBookStatus.Disconnected) { _queueEvent.WaitOne(); - + while (_processQueue.TryDequeue(out var item)) - ProcessQueueItem(item); + { + if (Status == OrderBookStatus.Disconnected) + break; + + if (item is InitialOrderBookItem iobi) + ProcessInitialOrderBookItem(iobi); + if (item is ProcessQueueItem pqi) + ProcessQueueItem(pqi); + else if (item is ChecksumItem ci) + ProcessChecksum(ci); + } + } + } + + private void ProcessInitialOrderBookItem(InitialOrderBookItem item) + { + lock (bookLock) + { + if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected) + return; + + asks.Clear(); + foreach (var ask in item.Asks) + asks.Add(ask.Price, ask); + bids.Clear(); + foreach (var bid in item.Bids) + bids.Add(bid.Price, bid); + + LastSequenceNumber = item.EndUpdateId; + + AskCount = asks.Count; + BidCount = bids.Count; + + LastOrderBookUpdate = DateTime.UtcNow; + log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} data set: {BidCount} bids, {AskCount} asks. #{item.EndUpdateId}"); + CheckProcessBuffer(); + OnOrderBookUpdate?.Invoke(item.Asks, item.Bids); + OnBestOffersChanged?.Invoke(BestBid, BestAsk); } } @@ -354,6 +398,20 @@ namespace CryptoExchange.Net.OrderBook } } + private void ProcessChecksum(ChecksumItem ci) + { + lock (bookLock) + { + var checksumResult = DoChecksum(ci.Checksum); + if(!checksumResult) + { + log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} out of sync. Resyncing"); + _ = subscription?.Reconnect(); + return; + } + } + } + /// /// Set the initial data for the order book /// @@ -362,38 +420,10 @@ namespace CryptoExchange.Net.OrderBook /// List of bids protected void SetInitialOrderBook(long orderBookSequenceNumber, IEnumerable bidList, IEnumerable askList) { - lock (bookLock) - { - if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected) - return; + bookSet = true; - asks.Clear(); - foreach (var ask in askList) - asks.Add(ask.Price, ask); - bids.Clear(); - foreach (var bid in bidList) - bids.Add(bid.Price, bid); - - LastSequenceNumber = orderBookSequenceNumber; - - AskCount = asks.Count; - BidCount = asks.Count; - - bookSet = true; - LastOrderBookUpdate = DateTime.UtcNow; - log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} data set: {BidCount} bids, {AskCount} asks. #{orderBookSequenceNumber}"); - CheckProcessBuffer(); - OnOrderBookUpdate?.Invoke(bidList, askList); - OnBestOffersChanged?.Invoke(BestBid, BestAsk); - } - } - - private void CheckBestOffersChanged(ISymbolOrderBookEntry prevBestBid, ISymbolOrderBookEntry prevBestAsk) - { - var (bestBid, bestAsk) = BestOffers; - if (bestBid.Price != prevBestBid.Price || bestBid.Quantity != prevBestBid.Quantity || - bestAsk.Price != prevBestAsk.Price || bestAsk.Quantity != prevBestAsk.Quantity) - OnBestOffersChanged?.Invoke(bestBid, bestAsk); + _processQueue.Enqueue(new InitialOrderBookItem { StartUpdateId = orderBookSequenceNumber, EndUpdateId = orderBookSequenceNumber, Asks = askList, Bids = bidList }); + _queueEvent.Set(); } /// @@ -408,6 +438,16 @@ namespace CryptoExchange.Net.OrderBook _queueEvent.Set(); } + /// + /// Add a checksum to the process queue + /// + /// + protected void AddChecksum(int checksum) + { + _processQueue.Enqueue(new ChecksumItem() { Checksum = checksum }); + _queueEvent.Set(); + } + /// /// Update the order book using a first/last update id /// @@ -505,7 +545,6 @@ namespace CryptoExchange.Net.OrderBook { // Out of sync log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} out of sync (expected { LastSequenceNumber + 1}, was {sequence}), reconnecting"); - Status = OrderBookStatus.Connecting; subscription?.Reconnect(); return false; } @@ -531,7 +570,7 @@ namespace CryptoExchange.Net.OrderBook } else { - listToChange[entry.Price].Quantity = entry.Quantity; + listToChange[entry.Price] = entry; } } @@ -557,6 +596,14 @@ namespace CryptoExchange.Net.OrderBook return new CallResult(true, null); } + private void CheckBestOffersChanged(ISymbolOrderBookEntry prevBestBid, ISymbolOrderBookEntry prevBestAsk) + { + var (bestBid, bestAsk) = BestOffers; + if (bestBid.Price != prevBestBid.Price || bestBid.Quantity != prevBestBid.Quantity || + bestAsk.Price != prevBestAsk.Price || bestAsk.Quantity != prevBestAsk.Quantity) + OnBestOffersChanged?.Invoke(bestBid, bestAsk); + } + /// /// Dispose the order book /// From ed5abb6f2266db7cb2ea2935722da11f806b10cf Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Sat, 20 Jun 2020 20:36:15 +0200 Subject: [PATCH 21/22] Updated version --- CryptoExchange.Net/CryptoExchange.Net.csproj | 4 ++-- README.md | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CryptoExchange.Net/CryptoExchange.Net.csproj b/CryptoExchange.Net/CryptoExchange.Net.csproj index 62bb836..dab827b 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.csproj +++ b/CryptoExchange.Net/CryptoExchange.Net.csproj @@ -6,12 +6,12 @@ CryptoExchange.Net JKorf A base package for implementing cryptocurrency exchange API's - 3.0.10 + 3.0.11 false https://github.com/JKorf/CryptoExchange.Net en true - 3.0.10 - Fix for order book synchronization + 3.0.11 - Added support for checksum in SymbolOrderBook enable 8.0 MIT diff --git a/README.md b/README.md index 664643c..3d85219 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,9 @@ The order book will automatically reconnect when the connection is lost and resy To stop synchronizing an order book use the `Stop` method. ## Release notes +* Version 3.0.11 - 20 Jun 2020 + * Added support for checksum in SymbolOrderBook + * Version 3.0.10 - 16 Jun 2020 * Fix for order book synchronization From 1b4b4007810d903e70840193599fbb1f32efd3a4 Mon Sep 17 00:00:00 2001 From: Jan Korf Date: Sun, 21 Jun 2020 11:48:01 +0200 Subject: [PATCH 22/22] Update CryptoExchange.Net.xml --- CryptoExchange.Net/CryptoExchange.Net.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CryptoExchange.Net/CryptoExchange.Net.xml b/CryptoExchange.Net/CryptoExchange.Net.xml index bf6762e..abf449e 100644 --- a/CryptoExchange.Net/CryptoExchange.Net.xml +++ b/CryptoExchange.Net/CryptoExchange.Net.xml @@ -1886,6 +1886,12 @@ + + + Add a checksum to the process queue + + + Update the order book using a first/last update id