Skip to main content

nautilus_coinapi/book/
sync.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Adapter-local order book synchronization state for CoinAPI.
17//!
18//! [`BookSyncTracker`] applies the CoinAPI sequence rule to the shared per-book [`BookSync`]
19//! lifecycle and converts accepted messages into deltas. A snapshot lists the whole book, so it is
20//! accepted at any sequence; an update is accepted only at the sequence after the last accepted
21//! message. A book's position is that last sequence.
22//!
23//! The reader captures each book frame's subscription-write identity before queuing it. The
24//! consumer waits for that write to commit, and the tracker rejects identities replaced or removed
25//! since the frame arrived. `BookWrites` opens snapshot gates at commit and aborts pending writes
26//! on drop, so cancellation wakes waiting consumers without admitting their frames.
27//!
28//! CoinAPI restarts a pair's sequence at one whenever it restarts the pair or the connection, and
29//! a managed `OrderBook` keeps its sequence as a high-water mark across snapshots. Emitted deltas
30//! therefore carry the venue sequence plus a per-book offset, set at each accepted snapshot so the
31//! emitted sequence keeps rising across restarts, unsubscribes, and reconnects. Gap detection uses
32//! the venue sequence.
33//!
34//! Conversion happens under the tracker's lock, so an L3 book's `OrderIndex` changes only with
35//! its accepted position. A message that fails conversion leaves the book out of sync, as a gap
36//! does. This module performs state transitions; `super::recovery` runs the asynchronous
37//! scope changes and retries.
38
39use std::sync::{
40    Arc,
41    atomic::{AtomicBool, Ordering},
42};
43
44use ahash::AHashMap;
45use nautilus_common::live::dst::time::Instant;
46use nautilus_core::AtomicMap;
47use nautilus_live::book::{
48    BookSequenceOutcome, BookSyncSignal, BookSyncSignalKind,
49    snapshot::SnapshotGate,
50    sync::{BookPhase, BookSync},
51};
52use nautilus_model::{
53    data::OrderBookDeltas, identifiers::InstrumentId, instruments::InstrumentAny,
54};
55use parking_lot::Mutex;
56use tokio_util::sync::CancellationToken;
57
58use super::{CoinApiBookError, orders::OrderIndex, recovery::BookRecovery};
59use crate::websocket::{
60    messages::{Book, BookL3, DataType, Subscription},
61    parse::parse_book_deltas,
62};
63
64/// A `book` or `book_l3` message routed to its book.
65#[derive(Debug, Clone, Copy)]
66pub enum BookFrame<'a> {
67    Levels(&'a Book),
68    Orders(&'a BookL3),
69}
70
71impl BookFrame<'_> {
72    fn sequence(self) -> u64 {
73        match self {
74            Self::Levels(book) => book.sequence,
75            Self::Orders(book) => book.sequence,
76        }
77    }
78
79    fn is_snapshot(self) -> bool {
80        match self {
81            Self::Levels(book) => book.is_snapshot == Some(true),
82            Self::Orders(book) => book.is_snapshot,
83        }
84    }
85}
86
87#[derive(Debug, Clone)]
88pub(crate) struct BookSubscription {
89    pub(crate) data_type: DataType,
90    pub(crate) depth: Option<usize>,
91    pub(crate) active: Arc<AtomicBool>,
92}
93
94// Registration and retirement share one writer lock; market-data reads use immutable snapshots
95#[derive(Debug, Default)]
96pub(crate) struct BookSubscriptions {
97    pub(crate) entries: AtomicMap<InstrumentId, BookSubscription>,
98    pub(crate) writes: Mutex<()>,
99    pub(crate) sync: BookSyncTracker,
100}
101
102impl BookSubscriptions {
103    pub(crate) fn begin_writes(
104        &self,
105        snapshots: &[(Subscription, Option<Arc<AtomicBool>>, SnapshotGate)],
106    ) -> BookWrites {
107        let _write = self.writes.lock();
108        let entries = self.entries.load();
109        let mut books = self.sync.books.lock();
110        let mut receipts = (**self.sync.receipts.load()).clone();
111        let mut pending = Vec::new();
112
113        for (subscription, registration, gate) in snapshots {
114            let Some((instrument_id, state)) = books
115                .iter_mut()
116                .find(|(_, state)| state.subscription == *subscription)
117            else {
118                continue;
119            };
120
121            let Some(current) = entries.get(instrument_id) else {
122                continue;
123            };
124
125            if registration
126                .as_ref()
127                .is_some_and(|registration| !Arc::ptr_eq(registration, &current.active))
128            {
129                continue;
130            }
131
132            let write = Arc::new(BookWrite {
133                gate: gate.clone(),
134                outcome: tokio::sync::watch::channel(None).0,
135            });
136
137            if let Some(previous) = state.write.replace(Arc::clone(&write)) {
138                previous.abort();
139            }
140
141            receipts.insert(*subscription, Arc::clone(&write));
142            pending.push((*instrument_id, write));
143        }
144
145        self.sync.receipts.store(receipts);
146
147        BookWrites {
148            tracker: self.sync.clone(),
149            pending,
150        }
151    }
152
153    pub(crate) fn clear(&self) {
154        let _write = self.writes.lock();
155        self.sync.clear();
156        self.entries.store(AHashMap::new());
157    }
158
159    pub(crate) fn retire(&self) {
160        let _write = self.writes.lock();
161        let mut entries = (**self.entries.load()).clone();
162        entries.retain(|instrument_id, registration| {
163            if registration.active.load(Ordering::Relaxed) {
164                self.sync.remove(*instrument_id);
165                false
166            } else {
167                true
168            }
169        });
170
171        self.entries.store(entries);
172    }
173}
174
175/// Emitted sequences use a fixed offset from venue sequences within each snapshot episode.
176#[derive(Debug)]
177struct BookState {
178    subscription: Subscription,
179    sync: BookSync<CoinApiBookError, u64>,
180    write: Option<Arc<BookWrite>>,
181    orders: OrderIndex,
182    offset: u64,
183}
184
185/// Sequence, snapshot, and recovery state of each subscribed book.
186///
187/// A book accepts messages only after [`Self::record_subscription`] registers it; [`Self::apply`]
188/// suppresses messages for any other book.
189///
190/// Emitted sequences persist across subscriptions and connections. Depth streams track both the
191/// last venue sequence and the last emitted sequence.
192#[derive(Debug, Clone, Default)]
193pub struct BookSyncTracker {
194    books: Arc<Mutex<AHashMap<InstrumentId, BookState>>>,
195    receipts: Arc<AtomicMap<Subscription, Arc<BookWrite>>>,
196    emitted: Arc<Mutex<AHashMap<InstrumentId, u64>>>,
197    depths: Arc<Mutex<AHashMap<InstrumentId, (u64, u64)>>>,
198}
199
200impl BookSyncTracker {
201    /// Replaces the book with one waiting for its first snapshot behind `gate`.
202    pub fn record_subscription(
203        &self,
204        instrument_id: InstrumentId,
205        subscription: Subscription,
206        now: Instant,
207        gate: SnapshotGate,
208    ) -> CancellationToken {
209        let mut sync = BookSync::new(now);
210        let cancel = sync.expect_snapshot(gate);
211
212        let state = BookState {
213            subscription,
214            sync,
215            write: None,
216            orders: OrderIndex::default(),
217            offset: 0,
218        };
219
220        let mut books = self.books.lock();
221        if let Some(previous) = books.insert(instrument_id, state) {
222            self.receipts.remove(&previous.subscription);
223            if let Some(write) = previous.write {
224                write.abort();
225            }
226        }
227
228        cancel
229    }
230
231    pub(crate) fn remove(&self, instrument_id: InstrumentId) {
232        let mut books = self.books.lock();
233        if let Some(state) = books.remove(&instrument_id) {
234            self.receipts.remove(&state.subscription);
235            if let Some(write) = state.write {
236                write.abort();
237            }
238        }
239    }
240
241    pub(crate) fn clear(&self) {
242        let mut books = self.books.lock();
243        for state in books.values() {
244            if let Some(write) = &state.write {
245                write.abort();
246            }
247        }
248
249        books.clear();
250        self.receipts.store(AHashMap::new());
251    }
252
253    pub(crate) fn receipts(&self) -> BookReceipts {
254        Arc::clone(&self.receipts.load())
255    }
256
257    pub(crate) fn apply_received(
258        &self,
259        instrument_id: InstrumentId,
260        frame: BookFrame<'_>,
261        instrument: &InstrumentAny,
262        now: Instant,
263        write: &Arc<BookWrite>,
264    ) -> (BookSequenceOutcome, Option<OrderBookDeltas>) {
265        self.apply_frame(instrument_id, frame, instrument, now, Some(write))
266    }
267
268    /// Validates a message and converts it once accepted.
269    ///
270    /// Returns [`BookSequenceOutcome::Recover`] when the caller must start a recovery, and deltas
271    /// only for an accepted message that changes the book.
272    pub fn apply(
273        &self,
274        instrument_id: InstrumentId,
275        frame: BookFrame<'_>,
276        instrument: &InstrumentAny,
277        now: Instant,
278    ) -> (BookSequenceOutcome, Option<OrderBookDeltas>) {
279        self.apply_frame(instrument_id, frame, instrument, now, None)
280    }
281
282    fn apply_frame(
283        &self,
284        instrument_id: InstrumentId,
285        frame: BookFrame<'_>,
286        instrument: &InstrumentAny,
287        now: Instant,
288        write: Option<&Arc<BookWrite>>,
289    ) -> (BookSequenceOutcome, Option<OrderBookDeltas>) {
290        let mut books = self.books.lock();
291
292        let Some(state) = books.get_mut(&instrument_id) else {
293            return (BookSequenceOutcome::Suppress, None);
294        };
295
296        if let Some(write) = write
297            && !state
298                .write
299                .as_ref()
300                .is_some_and(|current| Arc::ptr_eq(current, write))
301        {
302            return (BookSequenceOutcome::Suppress, None);
303        }
304
305        // Frames before the subscribing write belong to a scope that did not include the book
306        if state.sync.is_send_pending() {
307            return (BookSequenceOutcome::Suppress, None);
308        }
309
310        let sequence = frame.sequence();
311        if frame.is_snapshot() {
312            let converted = match frame {
313                BookFrame::Levels(book) => parse_book_deltas(book, instrument).map(|d| (None, d)),
314                BookFrame::Orders(book) => OrderIndex::snapshot(book, instrument)
315                    .map(|(orders, deltas)| (Some(orders), Some(deltas))),
316            };
317
318            let (orders, deltas) = match converted {
319                Ok(converted) => converted,
320                Err(e) => return (invalid(state, instrument_id, sequence, &e), None),
321            };
322
323            if !state.sync.accept_snapshot(sequence, now) {
324                return (BookSequenceOutcome::Suppress, None);
325            }
326
327            if let Some(orders) = orders {
328                state.orders = orders;
329            }
330
331            // The snapshot's emitted sequence follows the book's last one, even after a restart
332            let mut emitted = self.emitted.lock();
333            let last = emitted.get(&instrument_id).copied().unwrap_or(0);
334            state.offset = (last + 1).saturating_sub(sequence);
335            emitted.insert(instrument_id, state.offset + sequence);
336            return (
337                BookSequenceOutcome::Accept,
338                deltas.map(|deltas| offset(deltas, state.offset)),
339            );
340        }
341
342        // An out-of-sync book suppresses updates while an owner restores it, and asks for one
343        // otherwise
344        match *state.sync.phase() {
345            BookPhase::Synced(position) if position.checked_add(1) == Some(sequence) => {}
346            BookPhase::Synced(_) | BookPhase::Waiting | BookPhase::Recovering => {
347                return (gap(state, instrument_id, sequence), None);
348            }
349        }
350
351        let converted = match frame {
352            BookFrame::Levels(book) => parse_book_deltas(book, instrument),
353            BookFrame::Orders(book) => state.orders.update(book, instrument),
354        };
355
356        match converted {
357            Ok(deltas) => {
358                state.sync.advance(sequence, now);
359                self.emitted
360                    .lock()
361                    .insert(instrument_id, state.offset + sequence);
362                (
363                    BookSequenceOutcome::Accept,
364                    deltas.map(|deltas| offset(deltas, state.offset)),
365                )
366            }
367            Err(e) => (invalid(state, instrument_id, sequence, &e), None),
368        }
369    }
370
371    /// Returns the emitted sequence of a depth snapshot at venue `sequence`.
372    ///
373    /// Each depth message lists complete top levels, so the stream advances by the venue step and
374    /// continues from its last emitted sequence when CoinAPI restarts the venue sequence.
375    pub fn depth_sequence(&self, instrument_id: InstrumentId, sequence: u64) -> u64 {
376        let mut depths = self.depths.lock();
377
378        let emitted = match depths.get(&instrument_id) {
379            Some(&(venue, emitted)) if sequence > venue => emitted + (sequence - venue),
380            Some(&(_, emitted)) => emitted + 1,
381            None => sequence,
382        };
383
384        depths.insert(instrument_id, (sequence, emitted));
385        emitted
386    }
387
388    /// Restarts every book after its connection reconnects, arming a snapshot `deadline` when
389    /// given, and returns how many deadlines were armed.
390    pub fn reset_on_reconnect(&self, deadline: Option<Instant>) -> usize {
391        let mut armed = 0;
392
393        for state in self.books.lock().values_mut() {
394            state.sync.reset_on_reconnect();
395
396            if deadline.is_some_and(|deadline| state.sync.arm_deadline(deadline)) {
397                armed += 1;
398            }
399        }
400
401        armed
402    }
403
404    pub(crate) fn take_expired_snapshots(&self, now: Instant) -> Vec<BookSyncSignal> {
405        let mut expired = self
406            .books
407            .lock()
408            .iter_mut()
409            .filter_map(|(instrument_id, state)| {
410                state.sync.take_expired(now).then_some(instrument_id)
411            })
412            .map(|instrument_id| BookSyncSignal {
413                instrument_id: *instrument_id,
414                kind: BookSyncSignalKind::SnapshotMissing,
415            })
416            .collect::<Vec<_>>();
417
418        // Sort by instrument; the book map iterates in per-process hash order
419        expired.sort_by_key(|signal| signal.instrument_id);
420        expired
421    }
422
423    /// Claims a recovery for a subscription whose initial snapshot did not arrive, unless the
424    /// subscription was replaced or removed.
425    pub(crate) fn claim_subscription_recovery(
426        &self,
427        instrument_id: InstrumentId,
428        cancel: &CancellationToken,
429    ) -> Option<(Arc<BookRecovery>, Subscription)> {
430        let mut books = self.books.lock();
431
432        if cancel.is_cancelled() {
433            return None;
434        }
435
436        let state = books.get_mut(&instrument_id)?;
437        let recovery = state.sync.claim()?;
438        Some((recovery, state.subscription))
439    }
440
441    pub(crate) fn claim_recovery(
442        &self,
443        instrument_id: InstrumentId,
444    ) -> Option<(Arc<BookRecovery>, Subscription)> {
445        let mut books = self.books.lock();
446        let state = books
447            .get_mut(&instrument_id)
448            .filter(|state| !state.sync.is_send_pending())?;
449        let recovery = state.sync.claim()?;
450        Some((recovery, state.subscription))
451    }
452}
453
454pub(crate) type BookReceipts = Arc<AHashMap<Subscription, Arc<BookWrite>>>;
455
456#[derive(Debug)]
457pub(crate) struct BookWrite {
458    gate: SnapshotGate,
459    outcome: tokio::sync::watch::Sender<Option<bool>>,
460}
461
462impl BookWrite {
463    pub(crate) async fn committed(&self) -> bool {
464        if let Some(committed) = *self.outcome.borrow() {
465            return committed;
466        }
467
468        let mut outcome = self.outcome.subscribe();
469        outcome.wait_for(|outcome| outcome.is_some()).await.is_ok()
470            && *outcome.borrow() == Some(true)
471    }
472
473    fn abort(&self) {
474        self.outcome.send_if_modified(|outcome| {
475            if outcome.is_none() {
476                *outcome = Some(false);
477                true
478            } else {
479                false
480            }
481        });
482    }
483}
484
485pub(crate) struct BookWrites {
486    tracker: BookSyncTracker,
487    pending: Vec<(InstrumentId, Arc<BookWrite>)>,
488}
489
490impl BookWrites {
491    pub(crate) fn commit(&self) {
492        let books = self.tracker.books.lock();
493        for (instrument_id, write) in &self.pending {
494            let current = books.get(instrument_id).is_some_and(|state| {
495                state
496                    .write
497                    .as_ref()
498                    .is_some_and(|current| Arc::ptr_eq(current, write))
499            });
500
501            if current {
502                write.gate.open();
503            }
504
505            write.outcome.send_replace(Some(current));
506        }
507    }
508}
509
510impl Drop for BookWrites {
511    fn drop(&mut self) {
512        for (_, write) in &self.pending {
513            write.abort();
514        }
515    }
516}
517
518fn offset(mut deltas: OrderBookDeltas, offset: u64) -> OrderBookDeltas {
519    for delta in &mut deltas.deltas {
520        delta.sequence += offset;
521    }
522
523    deltas.sequence += offset;
524    deltas
525}
526
527fn gap(state: &mut BookState, instrument_id: InstrumentId, sequence: u64) -> BookSequenceOutcome {
528    let position = state.sync.position().copied();
529    let outcome = state.sync.gap();
530
531    if outcome == BookSequenceOutcome::Recover {
532        log::warn!(
533            "Book sequence gap for {instrument_id}: last sequence {position:?}, received \
534             {sequence}; requesting a fresh snapshot"
535        );
536    }
537
538    outcome
539}
540
541fn invalid(
542    state: &mut BookState,
543    instrument_id: InstrumentId,
544    sequence: u64,
545    error: &anyhow::Error,
546) -> BookSequenceOutcome {
547    log::warn!("Invalid book message for {instrument_id} at sequence {sequence}: {error:#}");
548    state.sync.gap()
549}
550
551#[cfg(test)]
552mod tests {
553    use nautilus_common::live::dst::time::Duration;
554    use nautilus_core::UnixNanos;
555    use nautilus_model::{
556        enums::BookAction,
557        identifiers::InstrumentId,
558        instruments::CurrencyPair,
559        types::{Currency, Price, Quantity},
560    };
561    use rstest::rstest;
562    use ustr::Ustr;
563
564    use super::*;
565    use crate::websocket::messages::{Channel, DataType};
566
567    fn instrument_id() -> InstrumentId {
568        InstrumentId::from("BTC-USD.COINBASE")
569    }
570
571    fn instrument() -> InstrumentAny {
572        InstrumentAny::CurrencyPair(
573            CurrencyPair::builder()
574                .instrument_id(instrument_id())
575                .raw_symbol("BTC-USD".into())
576                .base_currency(Currency::BTC())
577                .quote_currency(Currency::USD())
578                .price_precision(2)
579                .size_precision(8)
580                .price_increment(Price::from("0.01"))
581                .size_increment(Quantity::from("0.00000001"))
582                .ts_event(UnixNanos::default())
583                .ts_init(UnixNanos::default())
584                .build()
585                .unwrap(),
586        )
587    }
588
589    fn subscription(data_type: DataType) -> Subscription {
590        Subscription {
591            channel: Channel {
592                data_type,
593                period_id: None,
594            },
595            symbol_id: Ustr::from("COINBASE_SPOT_BTC_USD"),
596        }
597    }
598
599    fn levels(sequence: u64, is_snapshot: bool) -> Book {
600        let mut book: Book =
601            serde_json::from_str(include_str!("../../test_data/ws_book.json")).unwrap();
602        book.sequence = sequence;
603        book.is_snapshot = Some(is_snapshot);
604        book
605    }
606
607    fn orders(sequence: u64, is_snapshot: bool) -> BookL3 {
608        let mut book: BookL3 =
609            serde_json::from_str(include_str!("../../test_data/ws_book_l3.json")).unwrap();
610        book.sequence = sequence;
611        book.is_snapshot = is_snapshot;
612        book
613    }
614
615    // A tracker whose book's subscribing write has completed
616    fn subscribed(data_type: DataType, now: Instant) -> BookSyncTracker {
617        let tracker = BookSyncTracker::default();
618        tracker.record_subscription(
619            instrument_id(),
620            subscription(data_type),
621            now,
622            SnapshotGate::default(),
623        );
624        tracker
625    }
626
627    fn apply(
628        tracker: &BookSyncTracker,
629        book: &Book,
630        now: Instant,
631    ) -> (BookSequenceOutcome, Option<Vec<BookAction>>) {
632        let (outcome, deltas) =
633            tracker.apply(instrument_id(), BookFrame::Levels(book), &instrument(), now);
634        let actions = deltas.map(|deltas| deltas.deltas.iter().map(|d| d.action).collect());
635        (outcome, actions)
636    }
637
638    fn registered() -> BookSubscriptions {
639        let books = BookSubscriptions::default();
640        let gate = SnapshotGate::default();
641        books.sync.record_subscription(
642            instrument_id(),
643            subscription(DataType::Book),
644            Instant::now(),
645            gate,
646        );
647        books.entries.insert(
648            instrument_id(),
649            BookSubscription {
650                data_type: DataType::Book,
651                depth: None,
652                active: Arc::new(AtomicBool::new(false)),
653            },
654        );
655
656        books
657    }
658
659    #[tokio::test]
660    async fn scope_change_keeps_unaffected_book_receipt() {
661        let books = registered();
662        let initial =
663            books.begin_writes(&[(subscription(DataType::Book), None, SnapshotGate::default())]);
664        initial.commit();
665        let receipt = Arc::clone(&books.sync.receipts()[&subscription(DataType::Book)]);
666        let other = books.begin_writes(&[]);
667        other.commit();
668        let result = books.sync.apply_received(
669            instrument_id(),
670            BookFrame::Levels(&levels(1, true)),
671            &instrument(),
672            Instant::now(),
673            &receipt,
674        );
675        assert!(Arc::ptr_eq(
676            &books.sync.receipts()[&subscription(DataType::Book)],
677            &receipt
678        ));
679        assert_eq!(result.0, BookSequenceOutcome::Accept);
680    }
681
682    #[tokio::test]
683    async fn old_registration_cannot_admit_replacement_frames() {
684        let books = registered();
685        let registration = books.entries.get_cloned(&instrument_id()).unwrap().active;
686        let first = books.begin_writes(&[(
687            subscription(DataType::Book),
688            Some(registration.clone()),
689            SnapshotGate::default(),
690        )]);
691        first.commit();
692        let previous = Arc::clone(&books.sync.receipts()[&subscription(DataType::Book)]);
693        let gate = SnapshotGate::default();
694        gate.lock().close();
695        books.sync.record_subscription(
696            instrument_id(),
697            subscription(DataType::Book),
698            Instant::now(),
699            gate.clone(),
700        );
701        let replacement = Arc::new(AtomicBool::new(false));
702        books.entries.insert(
703            instrument_id(),
704            BookSubscription {
705                data_type: DataType::Book,
706                depth: None,
707                active: replacement.clone(),
708            },
709        );
710
711        let stale = books.begin_writes(&[(
712            subscription(DataType::Book),
713            Some(registration),
714            gate.clone(),
715        )]);
716        stale.commit();
717        assert!(books.sync.receipts().is_empty());
718        assert!(gate.lock().is_closed());
719        let current =
720            books.begin_writes(&[(subscription(DataType::Book), Some(replacement), gate)]);
721        current.commit();
722        let frame = levels(1, true);
723        let now = Instant::now();
724        let stale = books.sync.apply_received(
725            instrument_id(),
726            BookFrame::Levels(&frame),
727            &instrument(),
728            now,
729            &previous,
730        );
731        let accepted = books.sync.apply_received(
732            instrument_id(),
733            BookFrame::Levels(&frame),
734            &instrument(),
735            now,
736            &books.sync.receipts()[&subscription(DataType::Book)],
737        );
738        assert_eq!(stale, (BookSequenceOutcome::Suppress, None));
739        assert_eq!(accepted.0, BookSequenceOutcome::Accept);
740        assert_eq!(accepted.1.unwrap().sequence, 1);
741    }
742
743    #[rstest]
744    #[case::initial(false)]
745    #[case::recovery(true)]
746    #[tokio::test]
747    async fn received_snapshot_keeps_its_write_identity(#[case] recovery: bool) {
748        let books = registered();
749        let now = Instant::now();
750        let gate = SnapshotGate::default();
751        gate.lock().close();
752        let initial = books.begin_writes(&[(subscription(DataType::Book), None, gate)]);
753        let old = Arc::clone(&books.sync.receipts()[&subscription(DataType::Book)]);
754        initial.commit();
755        let first = books.sync.apply_received(
756            instrument_id(),
757            BookFrame::Levels(&levels(8, true)),
758            &instrument(),
759            now,
760            &old,
761        );
762        assert_eq!(first.0, BookSequenceOutcome::Accept);
763
764        let recovery_owner = recovery.then(|| {
765            assert_eq!(
766                apply(&books.sync, &levels(10, false), now).0,
767                BookSequenceOutcome::Recover
768            );
769            let (owner, _) = books.sync.claim_recovery(instrument_id()).unwrap();
770            assert!(owner.begin_replacement());
771            owner
772        });
773
774        let gate = recovery_owner
775            .as_ref()
776            .map_or_else(SnapshotGate::default, |owner| owner.gate.clone());
777        gate.lock().close();
778        let replacement = books.begin_writes(&[(subscription(DataType::Book), None, gate)]);
779        let current = Arc::clone(&books.sync.receipts()[&subscription(DataType::Book)]);
780        let waiting = current.committed();
781        tokio::pin!(waiting);
782        assert!(futures_util::poll!(&mut waiting).is_pending());
783        replacement.commit();
784        assert!(waiting.await);
785        let stale = books.sync.apply_received(
786            instrument_id(),
787            BookFrame::Levels(&levels(1, true)),
788            &instrument(),
789            now,
790            &old,
791        );
792        let accepted = books.sync.apply_received(
793            instrument_id(),
794            BookFrame::Levels(&levels(1, true)),
795            &instrument(),
796            now,
797            &current,
798        );
799        assert_eq!(stale, (BookSequenceOutcome::Suppress, None));
800        assert_eq!(accepted.0, BookSequenceOutcome::Accept);
801        assert_eq!(accepted.1.unwrap().sequence, 9);
802
803        if let Some(owner) = recovery_owner {
804            assert!(owner.is_accepted());
805        }
806    }
807
808    #[rstest]
809    #[case::failed(false)]
810    #[case::unsubscribed(true)]
811    #[tokio::test]
812    async fn abandoned_write_wakes_received_snapshot(#[case] unsubscribe: bool) {
813        let books = registered();
814        let gate = SnapshotGate::default();
815        gate.lock().close();
816        let pending = books.begin_writes(&[(subscription(DataType::Book), None, gate.clone())]);
817        let write = Arc::clone(&books.sync.receipts()[&subscription(DataType::Book)]);
818        let waiting = write.committed();
819        tokio::pin!(waiting);
820        assert!(futures_util::poll!(&mut waiting).is_pending());
821
822        if unsubscribe {
823            books.sync.remove(instrument_id());
824        } else {
825            drop(pending);
826        }
827
828        assert!(
829            !tokio::time::timeout(std::time::Duration::from_secs(1), waiting)
830                .await
831                .unwrap()
832        );
833        assert!(gate.lock().is_closed());
834    }
835
836    #[tokio::test]
837    async fn old_completion_cannot_open_replacement_gate() {
838        let books = registered();
839        let gate = SnapshotGate::default();
840        gate.lock().close();
841        let old = books.begin_writes(&[(subscription(DataType::Book), None, gate.clone())]);
842        let old_write = Arc::clone(&books.sync.receipts()[&subscription(DataType::Book)]);
843        let current = books.begin_writes(&[(subscription(DataType::Book), None, gate.clone())]);
844        old.commit();
845        assert!(!old_write.committed().await);
846        assert!(gate.lock().is_closed());
847        current.commit();
848        assert!(!gate.lock().is_closed());
849    }
850
851    #[rstest]
852    #[case(true)]
853    #[case(false)]
854    fn retirement_removes_active_and_preserves_pending(#[case] active: bool) {
855        let subscriptions = BookSubscriptions::default();
856        subscriptions.entries.insert(
857            instrument_id(),
858            BookSubscription {
859                data_type: DataType::Book,
860                depth: None,
861                active: Arc::new(AtomicBool::new(active)),
862            },
863        );
864
865        let cancel = subscriptions.sync.record_subscription(
866            instrument_id(),
867            subscription(DataType::Book),
868            Instant::now(),
869            SnapshotGate::default(),
870        );
871        subscriptions.retire();
872
873        assert_eq!(
874            subscriptions.entries.contains_key(&instrument_id()),
875            !active
876        );
877        assert_eq!(cancel.is_cancelled(), active);
878    }
879
880    #[rstest]
881    fn retirement_keeps_replacement_registration_when_old_command_commits() {
882        let subscriptions = BookSubscriptions::default();
883        let old = Arc::new(AtomicBool::new(false));
884        let replacement = Arc::new(AtomicBool::new(false));
885        subscriptions.entries.insert(
886            instrument_id(),
887            BookSubscription {
888                data_type: DataType::Book20,
889                depth: Some(10),
890                active: Arc::clone(&replacement),
891            },
892        );
893
894        old.store(true, Ordering::Relaxed);
895        subscriptions.retire();
896
897        let registration = subscriptions.entries.get_cloned(&instrument_id()).unwrap();
898        assert_eq!(registration.data_type, DataType::Book20);
899        assert_eq!(registration.depth, Some(10));
900        assert!(!registration.active.load(Ordering::Relaxed));
901        assert!(Arc::ptr_eq(&registration.active, &replacement));
902    }
903
904    #[rstest]
905    fn snapshot_then_linked_updates_are_accepted() {
906        let now = Instant::now();
907        let tracker = subscribed(DataType::Book, now);
908
909        let snapshot = apply(&tracker, &levels(1, true), now);
910        let update = apply(&tracker, &levels(2, false), now);
911
912        assert_eq!(
913            snapshot,
914            (
915                BookSequenceOutcome::Accept,
916                Some(vec![
917                    BookAction::Clear,
918                    BookAction::Add,
919                    BookAction::Add,
920                    BookAction::Add,
921                    BookAction::Add,
922                ])
923            )
924        );
925        assert_eq!(
926            update,
927            (
928                BookSequenceOutcome::Accept,
929                Some(vec![BookAction::Update; 4])
930            )
931        );
932    }
933
934    // A snapshot lists the whole book, so it resynchronizes at any sequence, including after a skip
935    #[rstest]
936    fn gap_recovers_once_claimed_and_snapshot_completes_recovery() {
937        let now = Instant::now();
938        let tracker = subscribed(DataType::Book, now);
939        apply(&tracker, &levels(1, true), now);
940
941        let gap = apply(&tracker, &levels(3, false), now);
942        let unowned = apply(&tracker, &levels(4, false), now);
943        let (recovery, claimed) = tracker.claim_recovery(instrument_id()).unwrap();
944        assert!(recovery.begin_replacement());
945        let owned = apply(&tracker, &levels(5, false), now);
946        let during_write = apply(&tracker, &levels(1, true), now);
947        recovery.gate.open();
948        let restarted = apply(&tracker, &levels(1, true), now);
949        let linked = apply(&tracker, &levels(2, false), now);
950
951        assert_eq!(gap, (BookSequenceOutcome::Recover, None));
952        assert_eq!(unowned, (BookSequenceOutcome::Recover, None));
953        assert_eq!(claimed, subscription(DataType::Book));
954        assert_eq!(owned, (BookSequenceOutcome::Suppress, None));
955        assert_eq!(during_write, (BookSequenceOutcome::Suppress, None));
956        assert_eq!(restarted.0, BookSequenceOutcome::Accept);
957        assert!(recovery.is_accepted());
958        assert_eq!(linked.0, BookSequenceOutcome::Accept);
959    }
960
961    // CoinAPI restarts sequences at one, so emitted sequences continue from the last emitted one,
962    // including after the book is subscribed again
963    #[rstest]
964    fn emitted_sequences_keep_rising_across_restarts_and_resubscribes() {
965        let now = Instant::now();
966        let tracker = subscribed(DataType::Book, now);
967        let emitted = |frame: &Book| {
968            tracker
969                .apply(
970                    instrument_id(),
971                    BookFrame::Levels(frame),
972                    &instrument(),
973                    now,
974                )
975                .1
976                .map(|deltas| {
977                    let sequences: Vec<u64> = deltas.deltas.iter().map(|d| d.sequence).collect();
978                    assert!(
979                        sequences
980                            .iter()
981                            .all(|sequence| *sequence == deltas.sequence)
982                    );
983                    deltas.sequence
984                })
985        };
986
987        let first = emitted(&levels(1, true));
988        let second = emitted(&levels(2, false));
989        let doubled = emitted(&levels(2, true));
990        let restarted = emitted(&levels(1, true));
991        let linked = emitted(&levels(2, false));
992        tracker.record_subscription(
993            instrument_id(),
994            subscription(DataType::Book),
995            now,
996            SnapshotGate::default(),
997        );
998        let resubscribed = emitted(&levels(1, true));
999
1000        assert_eq!(
1001            [first, second, doubled, restarted, linked, resubscribed],
1002            [Some(1), Some(2), Some(3), Some(4), Some(5), Some(6)]
1003        );
1004    }
1005
1006    #[rstest]
1007    fn depth_sequences_keep_rising_when_the_venue_restarts() {
1008        let tracker = BookSyncTracker::default();
1009        let emitted: Vec<u64> = [7, 9, 1, 2, 5]
1010            .into_iter()
1011            .map(|sequence| tracker.depth_sequence(instrument_id(), sequence))
1012            .collect();
1013
1014        assert_eq!(emitted, [7, 9, 10, 11, 14]);
1015    }
1016
1017    #[rstest]
1018    fn update_before_snapshot_requests_recovery() {
1019        let now = Instant::now();
1020        let tracker = subscribed(DataType::Book, now);
1021
1022        let result = apply(&tracker, &levels(7, false), now);
1023
1024        assert_eq!(result, (BookSequenceOutcome::Recover, None));
1025    }
1026
1027    #[rstest]
1028    fn frames_wait_for_subscribing_write() {
1029        let now = Instant::now();
1030        let tracker = BookSyncTracker::default();
1031        let gate = SnapshotGate::default();
1032        gate.lock().close();
1033        let cancel = tracker.record_subscription(
1034            instrument_id(),
1035            subscription(DataType::Book),
1036            now,
1037            gate.clone(),
1038        );
1039
1040        let update = apply(&tracker, &levels(7, false), now);
1041        let snapshot = apply(&tracker, &levels(1, true), now);
1042        let refused = tracker.claim_recovery(instrument_id());
1043        gate.open();
1044        let accepted = apply(&tracker, &levels(1, true), now);
1045
1046        assert_eq!(update, (BookSequenceOutcome::Suppress, None));
1047        assert_eq!(snapshot, (BookSequenceOutcome::Suppress, None));
1048        assert!(refused.is_none());
1049        assert_eq!(accepted.0, BookSequenceOutcome::Accept);
1050        assert!(cancel.is_cancelled());
1051    }
1052
1053    #[rstest]
1054    fn invalid_messages_request_recovery_without_output() {
1055        let now = Instant::now();
1056        let tracker = subscribed(DataType::BookL3, now);
1057        let mut aggregated = orders(1, true);
1058        aggregated.bids[1].update_type = None;
1059        aggregated
1060            .asks
1061            .iter_mut()
1062            .for_each(|entry| entry.update_type = None);
1063
1064        let (snapshot, snapshot_deltas) = tracker.apply(
1065            instrument_id(),
1066            BookFrame::Orders(&aggregated),
1067            &instrument(),
1068            now,
1069        );
1070
1071        assert_eq!(snapshot, BookSequenceOutcome::Recover);
1072        assert_eq!(snapshot_deltas, None);
1073    }
1074
1075    // The index follows the accepted snapshot, so a later ADD for a listed order is inconsistent
1076    #[rstest]
1077    fn l3_updates_apply_to_orders_from_snapshot() {
1078        let now = Instant::now();
1079        let tracker = subscribed(DataType::BookL3, now);
1080        let mut snapshot = orders(1, true);
1081        snapshot.bids.truncate(1);
1082        let mut delete = orders(2, false);
1083        delete.asks.remove(0);
1084        delete.bids.clear();
1085        let mut add_listed = orders(3, false);
1086        add_listed.asks.truncate(1);
1087        add_listed.bids.clear();
1088
1089        let apply = |book: &BookL3| {
1090            let (outcome, deltas) =
1091                tracker.apply(instrument_id(), BookFrame::Orders(book), &instrument(), now);
1092            let actions: Option<Vec<BookAction>> =
1093                deltas.map(|deltas| deltas.deltas.iter().map(|d| d.action).collect());
1094            (outcome, actions)
1095        };
1096
1097        let accepted = apply(&snapshot);
1098        let deleted = apply(&delete);
1099        let inconsistent = apply(&add_listed);
1100
1101        assert_eq!(
1102            accepted,
1103            (
1104                BookSequenceOutcome::Accept,
1105                Some(vec![
1106                    BookAction::Clear,
1107                    BookAction::Add,
1108                    BookAction::Add,
1109                    BookAction::Add
1110                ])
1111            )
1112        );
1113        assert_eq!(
1114            deleted,
1115            (BookSequenceOutcome::Accept, Some(vec![BookAction::Delete]))
1116        );
1117        assert_eq!(inconsistent, (BookSequenceOutcome::Recover, None));
1118    }
1119
1120    #[rstest]
1121    fn reconnect_waits_for_replayed_snapshot_until_deadline() {
1122        let now = Instant::now();
1123        let tracker = subscribed(DataType::Book, now);
1124        apply(&tracker, &levels(1, true), now);
1125        apply(&tracker, &levels(2, false), now);
1126
1127        let armed = tracker.reset_on_reconnect(Some(now + Duration::from_secs(10)));
1128        let stale = apply(&tracker, &levels(3, false), now);
1129        let early = tracker.take_expired_snapshots(now + Duration::from_secs(9));
1130        let expired = tracker.take_expired_snapshots(now + Duration::from_secs(10));
1131        let unowned = apply(&tracker, &levels(3, false), now);
1132
1133        assert_eq!(armed, 1);
1134        assert_eq!(stale, (BookSequenceOutcome::Suppress, None));
1135        assert!(early.is_empty());
1136        assert_eq!(
1137            expired,
1138            vec![BookSyncSignal {
1139                instrument_id: instrument_id(),
1140                kind: BookSyncSignalKind::SnapshotMissing,
1141            }]
1142        );
1143        assert_eq!(unowned, (BookSequenceOutcome::Recover, None));
1144    }
1145
1146    #[rstest]
1147    fn removed_book_suppresses_frames_and_cancels_recovery() {
1148        let now = Instant::now();
1149        let tracker = subscribed(DataType::Book, now);
1150        let (recovery, _) = tracker.claim_recovery(instrument_id()).unwrap();
1151
1152        tracker.remove(instrument_id());
1153        let result = apply(&tracker, &levels(1, true), now);
1154
1155        assert!(recovery.cancellation.is_cancelled());
1156        assert_eq!(result, (BookSequenceOutcome::Suppress, None));
1157    }
1158}