Rework order book; added support for checksum

CryptoExchange.Net/CryptoExchange.Net.xml

@@ -1333,6 +1333,11 @@
             Connecting
             </summary>
         </member>
+        <member name="F:CryptoExchange.Net.Objects.OrderBookStatus.Reconnecting">
+            <summary>
+            Reconnecting
+            </summary>
+        </member>
         <member name="F:CryptoExchange.Net.Objects.OrderBookStatus.Syncing">
             <summary>
             Syncing data
@@ -1858,6 +1863,13 @@
             </summary>
             <returns></returns>
         </member>
+        <member name="M:CryptoExchange.Net.OrderBook.SymbolOrderBook.DoChecksum(System.Int32)">
+            <summary>
+            Validate a checksum with the current order book
+            </summary>
+            <param name="checksum"></param>
+            <returns></returns>
+        </member>
         <member name="M:CryptoExchange.Net.OrderBook.SymbolOrderBook.SetInitialOrderBook(System.Int64,System.Collections.Generic.IEnumerable{CryptoExchange.Net.Interfaces.ISymbolOrderBookEntry},System.Collections.Generic.IEnumerable{CryptoExchange.Net.Interfaces.ISymbolOrderBookEntry})">
             <summary>
             Set the initial data for the order book
@@ -2931,5 +2943,148 @@
         <member name="M:CryptoExchange.Net.Sockets.WebsocketFactory.CreateWebsocket(CryptoExchange.Net.Logging.Log,System.String,System.Collections.Generic.IDictionary{System.String,System.String},System.Collections.Generic.IDictionary{System.String,System.String})">
             <inheritdoc />
         </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.AllowNullAttribute">
+            <summary>
+                Specifies that <see langword="null"/> is allowed as an input even if the
+                corresponding type disallows it.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.AllowNullAttribute.#ctor">
+            <summary>
+                Initializes a new instance of the <see cref="T:System.Diagnostics.CodeAnalysis.AllowNullAttribute"/> class.
+            </summary>
+        </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.DisallowNullAttribute">
+            <summary>
+                Specifies that <see langword="null"/> is disallowed as an input even if the
+                corresponding type allows it.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.DisallowNullAttribute.#ctor">
+            <summary>
+                Initializes a new instance of the <see cref="T:System.Diagnostics.CodeAnalysis.DisallowNullAttribute"/> class.
+            </summary>
+        </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.DoesNotReturnAttribute">
+            <summary>
+                Specifies that a method that will never return under any circumstance.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.DoesNotReturnAttribute.#ctor">
+            <summary>
+                Initializes a new instance of the <see cref="T:System.Diagnostics.CodeAnalysis.DoesNotReturnAttribute"/> class.
+            </summary>
+        </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.DoesNotReturnIfAttribute">
+            <summary>
+                Specifies that the method will not return if the associated <see cref="T:System.Boolean"/>
+                parameter is passed the specified value.
+            </summary>
+        </member>
+        <member name="P:System.Diagnostics.CodeAnalysis.DoesNotReturnIfAttribute.ParameterValue">
+            <summary>
+                Gets the condition parameter value.
+                Code after the method is considered unreachable by diagnostics if the argument
+                to the associated parameter matches this value.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.DoesNotReturnIfAttribute.#ctor(System.Boolean)">
+            <summary>
+                Initializes a new instance of the <see cref="T:System.Diagnostics.CodeAnalysis.DoesNotReturnIfAttribute"/>
+                class with the specified parameter value.
+            </summary>
+            <param name="parameterValue">
+                The condition parameter value.
+                Code after the method is considered unreachable by diagnostics if the argument
+                to the associated parameter matches this value.
+            </param>
+        </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.MaybeNullAttribute">
+            <summary>
+                Specifies that an output may be <see langword="null"/> even if the
+                corresponding type disallows it.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.MaybeNullAttribute.#ctor">
+            <summary>
+                Initializes a new instance of the <see cref="T:System.Diagnostics.CodeAnalysis.MaybeNullAttribute"/> class.
+            </summary>
+        </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute">
+            <summary>
+                Specifies that when a method returns <see cref="P:System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute.ReturnValue"/>, 
+                the parameter may be <see langword="null"/> even if the corresponding type disallows it.
+            </summary>
+        </member>
+        <member name="P:System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute.ReturnValue">
+            <summary>
+                Gets the return value condition.
+                If the method returns this value, the associated parameter may be <see langword="null"/>.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.MaybeNullWhenAttribute.#ctor(System.Boolean)">
+            <summary>
+                 Initializes the attribute with the specified return value condition.
+            </summary>
+            <param name="returnValue">
+                The return value condition.
+                If the method returns this value, the associated parameter may be <see langword="null"/>.
+            </param>
+        </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.NotNullAttribute">
+            <summary>
+                Specifies that an output is not <see langword="null"/> even if the
+                corresponding type allows it.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.NotNullAttribute.#ctor">
+            <summary>
+                Initializes a new instance of the <see cref="T:System.Diagnostics.CodeAnalysis.NotNullAttribute"/> class.
+            </summary>
+        </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.NotNullIfNotNullAttribute">
+            <summary>
+                Specifies that the output will be non-<see langword="null"/> if the
+                named parameter is non-<see langword="null"/>.
+            </summary>
+        </member>
+        <member name="P:System.Diagnostics.CodeAnalysis.NotNullIfNotNullAttribute.ParameterName">
+            <summary>
+                Gets the associated parameter name.
+                The output will be non-<see langword="null"/> if the argument to the
+                parameter specified is non-<see langword="null"/>.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.NotNullIfNotNullAttribute.#ctor(System.String)">
+            <summary>
+                Initializes the attribute with the associated parameter name.
+            </summary>
+            <param name="parameterName">
+                The associated parameter name.
+                The output will be non-<see langword="null"/> if the argument to the
+                parameter specified is non-<see langword="null"/>.
+            </param>
+        </member>
+        <member name="T:System.Diagnostics.CodeAnalysis.NotNullWhenAttribute">
+            <summary>
+                Specifies that when a method returns <see cref="P:System.Diagnostics.CodeAnalysis.NotNullWhenAttribute.ReturnValue"/>,
+                the parameter will not be <see langword="null"/> even if the corresponding type allows it.
+            </summary>
+        </member>
+        <member name="P:System.Diagnostics.CodeAnalysis.NotNullWhenAttribute.ReturnValue">
+            <summary>
+                Gets the return value condition.
+                If the method returns this value, the associated parameter will not be <see langword="null"/>.
+            </summary>
+        </member>
+        <member name="M:System.Diagnostics.CodeAnalysis.NotNullWhenAttribute.#ctor(System.Boolean)">
+            <summary>
+                Initializes the attribute with the specified return value condition.
+            </summary>
+            <param name="returnValue">
+                The return value condition.
+                If the method returns this value, the associated parameter will not be <see langword="null"/>.
+            </param>
+        </member>
     </members>
 </doc>

CryptoExchange.Net/Objects/Enums.cs

@@ -59,6 +59,10 @@
         /// </summary>
         Connecting,
         /// <summary>
+        /// Reconnecting
+        /// </summary>
+        Reconnecting,
+        /// <summary>
         /// Syncing data
         /// </summary>
         Syncing,

CryptoExchange.Net/OrderBook/ProcessQueueItem.cs

@@ -10,4 +10,17 @@ namespace CryptoExchange.Net.OrderBook
         public IEnumerable<ISymbolOrderBookEntry> Bids { get; set; } = new List<ISymbolOrderBookEntry>();
         public IEnumerable<ISymbolOrderBookEntry> Asks { get; set; } = new List<ISymbolOrderBookEntry>();
     }
+
+    internal class InitialOrderBookItem
+    {
+        public long StartUpdateId { get; set; }
+        public long EndUpdateId { get; set; }
+        public IEnumerable<ISymbolOrderBookEntry> Bids { get; set; } = new List<ISymbolOrderBookEntry>();
+        public IEnumerable<ISymbolOrderBookEntry> Asks { get; set; } = new List<ISymbolOrderBookEntry>();
+    }
+
+    internal class ChecksumItem
+    {
+        public int Checksum { get; set; }
+    }
 }

CryptoExchange.Net/OrderBook/SymbolOrderBook.cs

@@ -40,7 +40,7 @@ namespace CryptoExchange.Net.OrderBook
 
         private Task _processTask;
         private AutoResetEvent _queueEvent;
-        private ConcurrentQueue<ProcessQueueItem> _processQueue;
+        private ConcurrentQueue<object> _processQueue;
 
         /// <summary>
         /// Order book implementation id
@@ -197,7 +197,7 @@ namespace CryptoExchange.Net.OrderBook
 
             Id = options.OrderBookName;
             processBuffer = new List<ProcessBufferRangeSequenceEntry>();
-            _processQueue = new ConcurrentQueue<ProcessQueueItem>();
+            _processQueue = new ConcurrentQueue<object>();
             _queueEvent = new AutoResetEvent(false);
 
             sequencesAreConsecutive = options.SequenceNumbersAreConsecutive;
@@ -243,7 +243,7 @@ namespace CryptoExchange.Net.OrderBook
         private void Reset()
         {
             log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} connection lost");
-            Status = OrderBookStatus.Connecting;
+            Status = OrderBookStatus.Reconnecting;
             _queueEvent.Set();
             // Clear queue
             while(_processQueue.TryDequeue(out _))
@@ -306,6 +306,13 @@ namespace CryptoExchange.Net.OrderBook
         /// <returns></returns>
         protected abstract Task<CallResult<bool>> DoResync();
 
+        /// <summary>
+        /// Validate a checksum with the current order book
+        /// </summary>
+        /// <param name="checksum"></param>
+        /// <returns></returns>
+        protected virtual bool DoChecksum(int checksum) => true;
+
         private void ProcessQueue()
         {
             while(Status != OrderBookStatus.Disconnected)
@@ -313,7 +320,44 @@ namespace CryptoExchange.Net.OrderBook
                 _queueEvent.WaitOne();
 
                 while (_processQueue.TryDequeue(out var item))
-                    ProcessQueueItem(item);
+                {
+                    if (Status == OrderBookStatus.Disconnected)
+                        break;
+
+                    if (item is InitialOrderBookItem iobi)
+                        ProcessInitialOrderBookItem(iobi);
+                    if (item is ProcessQueueItem pqi)
+                        ProcessQueueItem(pqi);
+                    else if (item is ChecksumItem ci)
+                        ProcessChecksum(ci);
+                }
+            }
+        }
+
+        private void ProcessInitialOrderBookItem(InitialOrderBookItem item)
+        {
+            lock (bookLock)
+            {
+                if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected)
+                    return;
+
+                asks.Clear();
+                foreach (var ask in item.Asks)
+                    asks.Add(ask.Price, ask);
+                bids.Clear();
+                foreach (var bid in item.Bids)
+                    bids.Add(bid.Price, bid);
+
+                LastSequenceNumber = item.EndUpdateId;
+
+                AskCount = asks.Count;
+                BidCount = bids.Count;
+
+                LastOrderBookUpdate = DateTime.UtcNow;
+                log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} data set: {BidCount} bids, {AskCount} asks. #{item.EndUpdateId}");
+                CheckProcessBuffer();
+                OnOrderBookUpdate?.Invoke(item.Asks, item.Bids);
+                OnBestOffersChanged?.Invoke(BestBid, BestAsk);
             }
         }
 
@@ -354,6 +398,20 @@ namespace CryptoExchange.Net.OrderBook
             }
         }
 
+        private void ProcessChecksum(ChecksumItem ci)
+        {
+            lock (bookLock)
+            {
+                var checksumResult = DoChecksum(ci.Checksum);
+                if(!checksumResult)
+                {
+                    log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} out of sync. Resyncing");
+                    _ = subscription?.Reconnect();
+                    return;
+                }
+            }
+        }
+
         /// <summary>
         /// Set the initial data for the order book
         /// </summary>
@@ -362,38 +420,10 @@ namespace CryptoExchange.Net.OrderBook
         /// <param name="bidList">List of bids</param>
         protected void SetInitialOrderBook(long orderBookSequenceNumber, IEnumerable<ISymbolOrderBookEntry> bidList, IEnumerable<ISymbolOrderBookEntry> askList)
         {
-            lock (bookLock)
+            bookSet = true;
-            {
-                if (Status == OrderBookStatus.Connecting || Status == OrderBookStatus.Disconnected)
-                    return;
 
-                asks.Clear();
+            _processQueue.Enqueue(new InitialOrderBookItem { StartUpdateId = orderBookSequenceNumber, EndUpdateId = orderBookSequenceNumber, Asks = askList, Bids = bidList });
-                foreach (var ask in askList)
+            _queueEvent.Set();
-                    asks.Add(ask.Price, ask);
-                bids.Clear();
-                foreach (var bid in bidList)
-                    bids.Add(bid.Price, bid);
-
-                LastSequenceNumber = orderBookSequenceNumber;
-
-                AskCount = asks.Count;
-                BidCount = asks.Count;
-
-                bookSet = true;
-                LastOrderBookUpdate = DateTime.UtcNow;
-                log.Write(LogVerbosity.Debug, $"{Id} order book {Symbol} data set: {BidCount} bids, {AskCount} asks. #{orderBookSequenceNumber}");
-                CheckProcessBuffer();
-                OnOrderBookUpdate?.Invoke(bidList, askList);
-                OnBestOffersChanged?.Invoke(BestBid, BestAsk);
-            }
-        }
-
-        private void CheckBestOffersChanged(ISymbolOrderBookEntry prevBestBid, ISymbolOrderBookEntry prevBestAsk)
-        {
-            var (bestBid, bestAsk) = BestOffers;
-            if (bestBid.Price != prevBestBid.Price || bestBid.Quantity != prevBestBid.Quantity ||
-                   bestAsk.Price != prevBestAsk.Price || bestAsk.Quantity != prevBestAsk.Quantity)
-                OnBestOffersChanged?.Invoke(bestBid, bestAsk);
         }
 
         /// <summary>
@@ -408,6 +438,16 @@ namespace CryptoExchange.Net.OrderBook
             _queueEvent.Set();
         }
 
+        /// <summary>
+        /// Add a checksum to the process queue
+        /// </summary>
+        /// <param name="checksum"></param>
+        protected void AddChecksum(int checksum)
+        {
+            _processQueue.Enqueue(new ChecksumItem() { Checksum = checksum });
+            _queueEvent.Set();
+        }
+
         /// <summary>
         /// Update the order book using a first/last update id
         /// </summary>
@@ -505,7 +545,6 @@ namespace CryptoExchange.Net.OrderBook
             {
                 // Out of sync
                 log.Write(LogVerbosity.Warning, $"{Id} order book {Symbol} out of sync (expected { LastSequenceNumber + 1}, was {sequence}), reconnecting");
-                Status = OrderBookStatus.Connecting;
                 subscription?.Reconnect();
                 return false;
             }
@@ -531,7 +570,7 @@ namespace CryptoExchange.Net.OrderBook
                 }
                 else
                 {
-                    listToChange[entry.Price].Quantity = entry.Quantity;
+                    listToChange[entry.Price] = entry;
                 }
             }
 
@@ -557,6 +596,14 @@ namespace CryptoExchange.Net.OrderBook
             return new CallResult<bool>(true, null);
         }
 
+        private void CheckBestOffersChanged(ISymbolOrderBookEntry prevBestBid, ISymbolOrderBookEntry prevBestAsk)
+        {
+            var (bestBid, bestAsk) = BestOffers;
+            if (bestBid.Price != prevBestBid.Price || bestBid.Quantity != prevBestBid.Quantity ||
+                   bestAsk.Price != prevBestAsk.Price || bestAsk.Quantity != prevBestAsk.Quantity)
+                OnBestOffersChanged?.Invoke(bestBid, bestAsk);
+        }
+
         /// <summary>
         /// Dispose the order book
         /// </summary>