Skip to main content

nautilus_kalshi/book/
sync.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    collections::{BTreeMap, HashMap},
18    sync::{Arc, Mutex},
19    time::Duration,
20};
21
22use anyhow::Context;
23use nautilus_common::{
24    live::{dst::time, sender::EventSender},
25    messages::DataEvent,
26};
27use nautilus_core::UnixNanos;
28use nautilus_live::{
29    book::{
30        recovery::BookRecovery,
31        snapshot::SnapshotGate,
32        sync::{BookPhase, BookSync},
33    },
34    task::TaskSpawner,
35};
36use nautilus_model::{
37    data::{BookOrder, Data, OrderBookDelta, OrderBookDeltas},
38    enums::{BookAction, OrderSide, RecordFlag},
39    identifiers::InstrumentId,
40    instruments::BinaryOption,
41    types::Quantity,
42};
43use rust_decimal::Decimal;
44use serde::Deserialize;
45use tokio_util::sync::CancellationToken;
46
47use crate::{
48    data::parse,
49    http::models::{FixedPoint, Orderbook, OutcomeSide},
50    websocket::{
51        client::{KalshiWebSocketClient, KalshiWsEvent},
52        error::KalshiWsError,
53        messages::{KalshiChannel, KalshiDataMessage, KalshiSubscription, KalshiWsMessage},
54    },
55};
56
57/// Applies order-book protocol events for the markets a data client has subscribed to.
58#[derive(Debug, Clone)]
59pub struct BookSyncTracker {
60    state: Arc<Mutex<State>>,
61    subscription: KalshiSubscription,
62    snapshot_timeout: Duration,
63}
64
65#[allow(
66    clippy::missing_panics_doc,
67    reason = "book state mutex poisoning is not expected"
68)]
69impl BookSyncTracker {
70    /// Creates a tracker for the selected markets.
71    ///
72    /// `snapshot_timeout` bounds how long an initial book may wait before recovery. Zero starts that
73    /// wait immediately.
74    #[must_use]
75    pub fn new(instruments: &[InstrumentId], snapshot_timeout: Duration) -> Self {
76        let mut market_tickers: Vec<_> =
77            instruments.iter().map(|id| id.symbol.to_string()).collect();
78        market_tickers.sort();
79
80        Self {
81            state: Arc::default(),
82            snapshot_timeout,
83            subscription: KalshiSubscription {
84                channel: KalshiChannel::OrderbookDelta,
85                market_tickers,
86            },
87        }
88    }
89
90    /// Adds one local subscription reference.
91    ///
92    /// The first reference records WebSocket intent when `ws_client` is present. Later references
93    /// only increase demand.
94    ///
95    /// # Errors
96    ///
97    /// Returns an error if the WebSocket client rejects the subscription or the reference count
98    /// overflows.
99    pub fn subscribe(
100        &self,
101        id: InstrumentId,
102        ws_client: Option<&KalshiWebSocketClient>,
103    ) -> anyhow::Result<()> {
104        let mut state = self.state.lock().expect("book state lock is not poisoned");
105        let first = state.books.is_empty();
106        if first && let Some(ws_client) = ws_client {
107            ws_client.subscribe(self.subscription.clone())?;
108        }
109
110        let initial = state.sid.is_none();
111
112        let book = state.books.entry(id).or_insert_with(|| {
113            let mut book = Book::new(time::Instant::now());
114            book.initial = initial;
115            book
116        });
117
118        book.demand = book
119            .demand
120            .checked_add(1)
121            .context("book subscription reference count overflow")?;
122        Ok(())
123    }
124
125    pub(crate) fn unsubscribe(
126        &self,
127        id: InstrumentId,
128        ws_client: Option<&KalshiWebSocketClient>,
129    ) -> anyhow::Result<()> {
130        let mut state = self.state.lock().expect("book state lock is not poisoned");
131        if let Some(book) = state.books.get_mut(&id) {
132            book.demand -= 1;
133            if book.demand == 0 {
134                state.books.remove(&id);
135            }
136        }
137
138        if state.books.is_empty()
139            && let Some(ws_client) = ws_client
140        {
141            ws_client.unsubscribe(&self.subscription)?;
142        }
143
144        Ok(())
145    }
146
147    /// Restores book subscription intent and marks the session running.
148    ///
149    /// # Errors
150    ///
151    /// Returns an error if the WebSocket client rejects the retained subscription.
152    pub fn replay(&self, ws_client: &KalshiWebSocketClient) -> anyhow::Result<()> {
153        let mut state = self.state.lock().expect("book state lock is not poisoned");
154        if !state.books.is_empty() {
155            ws_client.subscribe(self.subscription.clone())?;
156        }
157
158        state.running = true;
159        Ok(())
160    }
161
162    pub(crate) fn stop(&self, reset: bool) {
163        let mut state = self.state.lock().expect("book state lock is not poisoned");
164        state.running = false;
165        state.generation = None;
166        state.sid = None;
167        if reset {
168            state.books.clear();
169        }
170
171        for book in state.books.values_mut() {
172            let demand = book.demand;
173            *book = Book::new(time::Instant::now());
174            book.demand = demand;
175        }
176    }
177
178    /// Applies one book protocol event and sends any resulting deltas.
179    ///
180    /// Ticker and trade frames are ignored. Events are ignored until [`Self::replay`] marks the
181    /// tracker running, and an older generation is ignored. A newer generation resets stream state
182    /// before the event is applied. Invalid snapshots and deltas gap the affected book and do not
183    /// emit a batch.
184    ///
185    /// # Errors
186    ///
187    /// Returns an error if the data receiver is closed.
188    pub fn event(
189        &self,
190        event: &KalshiWsEvent,
191        instruments: &HashMap<InstrumentId, BinaryOption>,
192        sender: &EventSender<DataEvent>,
193        now: UnixNanos,
194    ) -> anyhow::Result<()> {
195        if matches!(
196            event,
197            KalshiWsEvent::Message {
198                message: KalshiWsMessage::Ticker(_) | KalshiWsMessage::Trade(_),
199                ..
200            }
201        ) {
202            return Ok(());
203        }
204
205        let generation = event.generation();
206        let mut state = self.state.lock().expect("book state lock is not poisoned");
207        if !state.advance_generation(generation) {
208            return Ok(());
209        }
210
211        match event {
212            KalshiWsEvent::Message {
213                message: KalshiWsMessage::Subscribed { msg, .. },
214                ..
215            } if msg.channel == KalshiChannel::OrderbookDelta => {
216                state.subscribed(msg.sid, self.snapshot_timeout);
217            }
218            KalshiWsEvent::Message {
219                message: KalshiWsMessage::Unsubscribed { sid, .. },
220                ..
221            } if state.sid == Some(*sid) => {
222                state.reset_stream();
223            }
224            KalshiWsEvent::BookSequenceGap { sid, .. } => {
225                state.gap(*sid);
226            }
227            KalshiWsEvent::BookSnapshotRequested {
228                sid, id, attempt, ..
229            } => {
230                for book in state.books.values_mut() {
231                    if let Some(current) = &mut book.attempt
232                        && current.token == *attempt
233                        && !current.cancel.is_cancelled()
234                    {
235                        current.correlation = Some((generation, *sid, *id));
236                        current.gate.open();
237                    }
238                }
239            }
240            KalshiWsEvent::Message {
241                message: KalshiWsMessage::OrderbookSnapshot(data),
242                ..
243            } => {
244                self.snapshot(&mut state, data, instruments, sender, generation, now)?;
245            }
246            KalshiWsEvent::Message {
247                message: KalshiWsMessage::OrderbookDelta(data),
248                ..
249            } => {
250                self.delta(&mut state, data, instruments, sender, now)?;
251            }
252            KalshiWsEvent::Message {
253                message:
254                    KalshiWsMessage::Error {
255                        id: Some(id),
256                        sid,
257                        msg,
258                    },
259                ..
260            } => {
261                state.reject_snapshot(generation, *id, *sid, msg.code);
262            }
263            KalshiWsEvent::Error { message, .. } => {
264                for book in state.books.values_mut() {
265                    book.sync.reject(KalshiWsError::Client(message.clone()));
266                    book.gap();
267                    book.initial = false;
268                }
269            }
270            _ => {}
271        }
272
273        Ok(())
274    }
275
276    fn snapshot(
277        &self,
278        state: &mut State,
279        data: &KalshiDataMessage,
280        instruments: &HashMap<InstrumentId, BinaryOption>,
281        sender: &EventSender<DataEvent>,
282        generation: u64,
283        now: UnixNanos,
284    ) -> anyhow::Result<()> {
285        if state.sid != Some(data.sid) {
286            return Ok(());
287        }
288
289        let Some((id, book)) = selected(state, data) else {
290            return Ok(());
291        };
292
293        let initial = book.initial && data.id.is_none();
294
295        let replacement = book.attempt.as_ref().is_some_and(|attempt| {
296            !attempt.cancel.is_cancelled()
297                && data
298                    .id
299                    .is_some_and(|id| attempt.correlation == Some((generation, data.sid, id)))
300        });
301
302        if !initial && !replacement {
303            return Ok(());
304        }
305
306        let Some(instrument) = instruments.get(&id) else {
307            return Ok(());
308        };
309
310        let parsed = (|| {
311            let seq = data.seq.context("book snapshot sequence missing")?;
312            let payload: Snapshot = serde_json::from_value(data.msg.clone())?;
313            let levels = Levels::from_snapshot(
314                &payload.yes_dollars_fp,
315                &payload.no_dollars_fp,
316                false,
317                instrument,
318            )?;
319            let event_time = event_time(data, now)?;
320            let batch = levels.snapshot(instrument, seq, event_time, now)?;
321            anyhow::Ok((seq, levels, batch))
322        })();
323
324        match parsed {
325            Ok((seq, levels, batch)) if book.sync.accept_snapshot(seq, time::Instant::now()) => {
326                sender
327                    .send(DataEvent::Data(Data::BookDeltas(batch.into())))
328                    .map_err(|_| anyhow::anyhow!("book event receiver closed"))?;
329                book.levels = levels;
330                book.initial = false;
331                book.gapped = false;
332                book.attempt = None;
333            }
334            Ok(_) => {}
335            Err(e) => {
336                log::warn!("Invalid Kalshi book snapshot: {e}");
337                book.gap();
338                book.initial = false;
339            }
340        }
341
342        Ok(())
343    }
344
345    fn delta(
346        &self,
347        state: &mut State,
348        data: &KalshiDataMessage,
349        instruments: &HashMap<InstrumentId, BinaryOption>,
350        sender: &EventSender<DataEvent>,
351        now: UnixNanos,
352    ) -> anyhow::Result<()> {
353        if state.sid != Some(data.sid) {
354            return Ok(());
355        }
356
357        let Some((id, book)) = selected(state, data) else {
358            return Ok(());
359        };
360
361        if !matches!(book.sync.phase(), BookPhase::Synced(_)) {
362            book.gap();
363            return Ok(());
364        }
365
366        let Some(instrument) = instruments.get(&id) else {
367            return Ok(());
368        };
369
370        let parsed = (|| {
371            let seq = data.seq.context("book delta sequence missing")?;
372            let payload: Delta = serde_json::from_value(data.msg.clone())?;
373
374            let event_time = match payload.ts_ms {
375                Some(value) => UnixNanos::from(
376                    value
377                        .checked_mul(1_000_000)
378                        .context("book timestamp overflow")?,
379                ),
380                None => event_time(data, now)?,
381            };
382
383            let batch = book
384                .levels
385                .delta(&payload, instrument, seq, event_time, now)?;
386            anyhow::Ok((seq, batch))
387        })();
388
389        match parsed {
390            Ok((seq, batch)) => {
391                book.sync.advance(seq, time::Instant::now());
392                sender
393                    .send(DataEvent::Data(Data::BookDeltas(batch.into())))
394                    .map_err(|_| anyhow::anyhow!("book event receiver closed"))?;
395            }
396            Err(e) => {
397                log::warn!("Invalid Kalshi book delta: {e}");
398                book.gap();
399                book.initial = false;
400            }
401        }
402
403        Ok(())
404    }
405
406    pub(crate) fn monitor(&self, ws_client: &KalshiWebSocketClient, tasks: &TaskSpawner) {
407        let mut state = self.state.lock().expect("book state lock is not poisoned");
408        if !state.running {
409            return;
410        }
411
412        let confirmed = state.sid.is_some();
413        for (id, book) in &mut state.books {
414            if !confirmed && book.initial {
415                continue;
416            }
417
418            let expired = book.sync.take_expired(time::Instant::now());
419            if !expired
420                && (matches!(book.sync.phase(), BookPhase::Synced(_))
421                    || book.waits_for_snapshot(self.snapshot_timeout))
422            {
423                continue;
424            }
425
426            let Some(recovery) = book.sync.claim() else {
427                continue;
428            };
429
430            if expired {
431                log::warn!("Kalshi book snapshot deadline expired for {id}");
432            }
433
434            book.initial = false;
435            let tracker = self.clone();
436            let ws_client = ws_client.clone();
437            let id = *id;
438            let running = Arc::clone(&recovery);
439            let shutdown = tasks.cancellation_token();
440            let snapshot_timeout = self.snapshot_timeout;
441
442            if let Err(e) = tasks.spawn(async move {
443                let run = running.run(
444                    id,
445                    snapshot_timeout,
446                    |cancel, gate| {
447                        let tracker = tracker.clone();
448                        let ws_client = ws_client.clone();
449                        let owner = Arc::clone(&running);
450                        async move {
451                            let token = tracker.attempt(id, cancel.clone(), gate, &owner)?;
452                            ws_client
453                                .request_book_snapshot(
454                                    &tracker.subscription,
455                                    id.symbol.to_string(),
456                                    token,
457                                    cancel,
458                                )
459                                .await
460                        }
461                    },
462                    |_| true,
463                    KalshiWsError::Client,
464                    || KalshiWsError::Client("book snapshot timed out".into()),
465                );
466
467                tokio::select! {
468                    biased;
469                    () = shutdown.cancelled() => {}
470                    () = run => {}
471                }
472                running.cancellation.cancel();
473            }) {
474                recovery.cancellation.cancel();
475                log::warn!("Kalshi book recovery task refused: {e}");
476            }
477        }
478    }
479
480    fn attempt(
481        &self,
482        id: InstrumentId,
483        cancel: CancellationToken,
484        gate: SnapshotGate,
485        owner: &Arc<BookRecovery<KalshiWsError>>,
486    ) -> Result<u64, KalshiWsError> {
487        let mut state = self.state.lock().expect("book state lock is not poisoned");
488        state.next_attempt = state
489            .next_attempt
490            .checked_add(1)
491            .ok_or_else(|| KalshiWsError::Client("book attempt IDs exhausted".into()))?;
492        let token = state.next_attempt;
493        if !state.running {
494            return Err(KalshiWsError::Client("book session stopped".into()));
495        }
496
497        let book = state
498            .books
499            .get_mut(&id)
500            .ok_or_else(|| KalshiWsError::Client("book subscription retired".into()))?;
501
502        if cancel.is_cancelled() || !book.sync.is_current(owner) {
503            return Err(KalshiWsError::Client("book recovery owner retired".into()));
504        }
505
506        book.attempt = Some(Attempt {
507            token,
508            cancel,
509            gate,
510            correlation: None,
511        });
512
513        Ok(token)
514    }
515}
516
517#[derive(Debug, Default)]
518struct State {
519    running: bool,
520    generation: Option<u64>,
521    sid: Option<u64>,
522    next_attempt: u64,
523    books: BTreeMap<InstrumentId, Book>,
524}
525
526impl State {
527    fn advance_generation(&mut self, generation: u64) -> bool {
528        if !self.running || self.generation.is_some_and(|current| generation < current) {
529            return false;
530        }
531
532        if self.generation.is_some_and(|current| generation > current) {
533            self.reset_stream();
534        }
535
536        self.generation = Some(generation);
537        true
538    }
539
540    fn subscribed(&mut self, sid: u64, snapshot_timeout: Duration) {
541        self.sid = Some(sid);
542
543        for book in self.books.values_mut() {
544            book.initial = true;
545            book.gapped = false;
546
547            if snapshot_timeout.is_zero() {
548                let gate = SnapshotGate::default();
549                gate.open();
550                book.sync.expect_snapshot(gate);
551            } else {
552                book.sync
553                    .arm_deadline(time::Instant::now() + snapshot_timeout);
554            }
555        }
556    }
557
558    fn reset_stream(&mut self) {
559        self.sid = None;
560
561        for book in self.books.values_mut() {
562            book.sync.reset_on_reconnect();
563            book.attempt = None;
564            book.initial = true;
565            book.gapped = false;
566            book.levels = Levels::default();
567        }
568    }
569
570    fn reject_snapshot(&mut self, generation: u64, id: u64, sid: Option<u64>, code: u32) {
571        for book in self.books.values_mut() {
572            if book.attempt.as_ref().is_some_and(|attempt| {
573                !attempt.cancel.is_cancelled()
574                    && attempt
575                        .correlation
576                        .is_some_and(|(current, subscription, request)| {
577                            current == generation
578                                && request == id
579                                && sid.is_none_or(|sid| sid == subscription)
580                        })
581            }) {
582                log::warn!("Kalshi book snapshot request {id} rejected with code {code}");
583                book.sync.reject(KalshiWsError::Client(format!(
584                    "book snapshot rejected with code {code}"
585                )));
586                book.gap();
587                book.initial = false;
588            }
589        }
590    }
591
592    fn gap(&mut self, sid: u64) {
593        if self.sid != Some(sid) {
594            return;
595        }
596
597        for book in self.books.values_mut() {
598            book.gap();
599            book.initial = false;
600        }
601    }
602}
603
604#[derive(Debug)]
605struct Book {
606    demand: usize,
607    sync: BookSync<KalshiWsError, u64>,
608    levels: Levels,
609    initial: bool,
610    gapped: bool,
611    attempt: Option<Attempt>,
612}
613
614impl Book {
615    fn new(now: time::Instant) -> Self {
616        Self {
617            demand: 0,
618            sync: BookSync::new(now),
619            levels: Levels::default(),
620            initial: true,
621            gapped: false,
622            attempt: None,
623        }
624    }
625
626    fn gap(&mut self) {
627        self.gapped = true;
628        self.sync.gap();
629    }
630
631    fn waits_for_snapshot(&self, snapshot_timeout: Duration) -> bool {
632        self.sync.has_pending_snapshot() && (!snapshot_timeout.is_zero() || !self.gapped)
633    }
634}
635
636#[derive(Debug)]
637struct Attempt {
638    token: u64,
639    cancel: CancellationToken,
640    gate: SnapshotGate,
641    correlation: Option<(u64, u64, u64)>,
642}
643
644fn selected<'a>(
645    state: &'a mut State,
646    data: &KalshiDataMessage,
647) -> Option<(InstrumentId, &'a mut Book)> {
648    let Some(ticker) = data
649        .msg
650        .get("market_ticker")
651        .and_then(serde_json::Value::as_str)
652    else {
653        log::warn!("Kalshi book frame has no market identity");
654        state.gap(data.sid);
655        return None;
656    };
657
658    state
659        .books
660        .iter_mut()
661        .find(|(id, _)| id.symbol.as_str() == ticker)
662        .map(|(id, book)| (*id, book))
663}
664
665fn event_time(data: &KalshiDataMessage, now: UnixNanos) -> anyhow::Result<UnixNanos> {
666    data.sending_ts_ms.map_or(Ok(now), |value| {
667        value
668            .checked_mul(1_000_000)
669            .map(UnixNanos::from)
670            .context("book timestamp overflow")
671    })
672}
673
674#[derive(Debug, Default)]
675pub(crate) struct Levels {
676    bids: BTreeMap<Decimal, Decimal>,
677    asks: BTreeMap<Decimal, Decimal>,
678}
679
680impl Levels {
681    pub(crate) fn rest(
682        book: &Orderbook,
683        instrument: &BinaryOption,
684        now: UnixNanos,
685    ) -> anyhow::Result<OrderBookDeltas> {
686        let levels = Self::from_snapshot(&book.yes_dollars, &book.no_dollars, true, instrument)?;
687        levels.snapshot(instrument, 0, now, now)
688    }
689
690    fn from_snapshot(
691        yes: &[[FixedPoint; 2]],
692        no: &[[FixedPoint; 2]],
693        complement_no: bool,
694        instrument: &BinaryOption,
695    ) -> anyhow::Result<Self> {
696        let mut levels = Self::default();
697
698        for (side, values) in [(OrderSide::Buy, yes), (OrderSide::Sell, no)] {
699            let target = if side == OrderSide::Buy {
700                &mut levels.bids
701            } else {
702                &mut levels.asks
703            };
704
705            for [price, qty] in values {
706                let price = if side == OrderSide::Sell && complement_no {
707                    Decimal::ONE
708                        .checked_sub(price.0)
709                        .context("book price overflow")?
710                } else {
711                    price.0
712                };
713
714                parse::price(FixedPoint(price), instrument.price_precision)?;
715                parse::size(*qty, instrument.size_precision)?;
716                anyhow::ensure!(
717                    target.insert(price, qty.0).is_none(),
718                    "duplicate book price level"
719                );
720            }
721        }
722
723        levels.integrity()?;
724        Ok(levels)
725    }
726
727    fn snapshot(
728        &self,
729        instrument: &BinaryOption,
730        seq: u64,
731        event_time: UnixNanos,
732        now: UnixNanos,
733    ) -> anyhow::Result<OrderBookDeltas> {
734        let mut deltas = vec![OrderBookDelta::clear(instrument.id, seq, event_time, now)];
735
736        for (side, levels) in [(OrderSide::Buy, &self.bids), (OrderSide::Sell, &self.asks)] {
737            for (price, qty) in levels {
738                let price = parse::price(FixedPoint(*price), instrument.price_precision)?;
739                let qty = parse::size(FixedPoint(*qty), instrument.size_precision)?;
740                let order = BookOrder::new(side, price, qty, 0);
741                deltas.push(OrderBookDelta::new_checked(
742                    instrument.id,
743                    BookAction::Add,
744                    order,
745                    RecordFlag::F_SNAPSHOT as u8,
746                    seq,
747                    event_time,
748                    now,
749                )?);
750            }
751        }
752
753        deltas.last_mut().expect("clear is present").flags |= RecordFlag::F_LAST as u8;
754        Ok(OrderBookDeltas::new(instrument.id, deltas))
755    }
756
757    fn delta(
758        &mut self,
759        delta: &Delta,
760        instrument: &BinaryOption,
761        seq: u64,
762        event_time: UnixNanos,
763        now: UnixNanos,
764    ) -> anyhow::Result<OrderBookDeltas> {
765        let side = match delta.side {
766            OutcomeSide::Yes => OrderSide::Buy,
767            OutcomeSide::No => OrderSide::Sell,
768        };
769
770        let price = parse::price(delta.price_dollars, instrument.price_precision)?;
771
772        let levels = if side == OrderSide::Buy {
773            &mut self.bids
774        } else {
775            &mut self.asks
776        };
777
778        let previous = levels
779            .get(&delta.price_dollars.0)
780            .copied()
781            .unwrap_or_default();
782        let qty = previous
783            .checked_add(delta.delta_fp.0)
784            .context("book size overflow")?;
785        anyhow::ensure!(qty >= Decimal::ZERO, "book delta produces negative size");
786        anyhow::ensure!(
787            !delta.delta_fp.0.is_zero(),
788            "book delta must change a level"
789        );
790
791        let size = if qty.is_zero() {
792            Quantity::zero(instrument.size_precision)
793        } else {
794            parse::size(FixedPoint(qty), instrument.size_precision)?
795        };
796
797        let action = if qty.is_zero() {
798            BookAction::Delete
799        } else if previous.is_zero() {
800            BookAction::Add
801        } else {
802            BookAction::Update
803        };
804
805        let order = BookOrder::new(side, price, size, 0);
806        let delta = OrderBookDelta::new_checked(
807            instrument.id,
808            action,
809            order,
810            RecordFlag::F_LAST as u8,
811            seq,
812            event_time,
813            now,
814        )?;
815
816        if qty.is_zero() {
817            levels.remove(&price.as_decimal());
818        } else {
819            levels.insert(price.as_decimal(), qty);
820        }
821
822        self.integrity()?;
823        Ok(OrderBookDeltas::new(instrument.id, vec![delta]))
824    }
825
826    fn integrity(&self) -> anyhow::Result<()> {
827        if let (Some((bid, _)), Some((ask, _))) =
828            (self.bids.last_key_value(), self.asks.first_key_value())
829        {
830            anyhow::ensure!(bid < ask, "crossed or locked book");
831        }
832
833        Ok(())
834    }
835}
836
837#[derive(Deserialize)]
838struct Snapshot {
839    #[serde(default)]
840    yes_dollars_fp: Vec<[FixedPoint; 2]>,
841    #[serde(default)]
842    no_dollars_fp: Vec<[FixedPoint; 2]>,
843}
844
845#[derive(Deserialize)]
846struct Delta {
847    price_dollars: FixedPoint,
848    delta_fp: FixedPoint,
849    side: OutcomeSide,
850    ts_ms: Option<u64>,
851}
852
853#[cfg(test)]
854mod tests {
855    use nautilus_live::book::conformance::BookStreamChecker;
856    #[cfg(not(all(feature = "simulation", madsim)))]
857    use nautilus_live::task::TaskGroup;
858    use nautilus_model::{
859        enums::BookType,
860        types::{Price, Quantity},
861    };
862    use rstest::rstest;
863    use serde_json::Value;
864
865    use super::*;
866    #[cfg(not(all(feature = "simulation", madsim)))]
867    use crate::config::KalshiDataClientConfig;
868    use crate::{
869        http::{
870            models::{Event, Market},
871            parse::parse_instrument,
872        },
873        websocket::messages::KalshiSubscribed,
874    };
875    #[cfg(not(all(feature = "simulation", madsim)))]
876    #[allow(dead_code, reason = "shared key fixtures serve multiple suites")]
877    mod keys {
878        include!(concat!(
879            env!("CARGO_MANIFEST_DIR"),
880            "/tests/integration/keys.rs"
881        ));
882    }
883    #[cfg(not(all(feature = "simulation", madsim)))]
884    mod http_request {
885        include!(concat!(
886            env!("CARGO_MANIFEST_DIR"),
887            "/tests/common/http_request.rs"
888        ));
889    }
890    #[cfg(not(all(feature = "simulation", madsim)))]
891    #[allow(
892        dead_code,
893        reason = "peer fields also serve other data lifecycle tests"
894    )]
895    mod peer {
896        include!(concat!(
897            env!("CARGO_MANIFEST_DIR"),
898            "/tests/common/data_peer.rs"
899        ));
900    }
901
902    fn fixtures() -> Value {
903        serde_json::from_str(include_str!("../../test_data/books.json")).unwrap()
904    }
905
906    fn instrument() -> BinaryOption {
907        let fixtures: Value =
908            serde_json::from_str(include_str!("../../test_data/instruments.json")).unwrap();
909        let mut market: Market =
910            serde_json::from_value(fixtures["definitions"]["live"]["market"].clone()).unwrap();
911        market.ticker = "MARKET-A".into();
912        let event: Event =
913            serde_json::from_value(fixtures["definitions"]["live"]["event"].clone()).unwrap();
914        parse_instrument(&market, &event, UnixNanos::from(5)).unwrap()
915    }
916
917    fn decimal(value: &str) -> Decimal {
918        value.parse().unwrap()
919    }
920
921    fn message(payload: Value, seq: u64, id: Option<u64>) -> KalshiDataMessage {
922        KalshiDataMessage {
923            id,
924            sid: 7,
925            seq: Some(seq),
926            sending_ts_ms: Some(1700000000001),
927            msg: payload,
928        }
929    }
930
931    fn subscribed(generation: u64, sid: u64) -> KalshiWsEvent {
932        KalshiWsEvent::Message {
933            generation,
934            message: KalshiWsMessage::Subscribed {
935                id: 1,
936                msg: KalshiSubscribed {
937                    channel: KalshiChannel::OrderbookDelta,
938                    sid,
939                },
940            },
941        }
942    }
943
944    fn snapshot(generation: u64, seq: u64, id: Option<u64>) -> KalshiWsEvent {
945        KalshiWsEvent::Message {
946            generation,
947            message: KalshiWsMessage::OrderbookSnapshot(message(
948                fixtures()["snapshot"].clone(),
949                seq,
950                id,
951            )),
952        }
953    }
954
955    fn snapshot_ticker(generation: u64, seq: u64, ticker: &str) -> KalshiWsEvent {
956        let mut payload = fixtures()["snapshot"].clone();
957        payload["market_ticker"] = ticker.into();
958
959        KalshiWsEvent::Message {
960            generation,
961            message: KalshiWsMessage::OrderbookSnapshot(message(payload, seq, None)),
962        }
963    }
964
965    fn snapshot_sid(generation: u64, seq: u64, sid: u64, ticker: &str) -> KalshiWsEvent {
966        let mut event = snapshot_ticker(generation, seq, ticker);
967
968        let KalshiWsEvent::Message {
969            message: KalshiWsMessage::OrderbookSnapshot(data),
970            ..
971        } = &mut event
972        else {
973            unreachable!("snapshot event");
974        };
975
976        data.sid = sid;
977        event
978    }
979
980    fn delta_ticker(
981        generation: u64,
982        seq: u64,
983        sid: u64,
984        ticker: &str,
985        mut payload: Value,
986    ) -> KalshiWsEvent {
987        payload["market_ticker"] = ticker.into();
988        let mut data = message(payload, seq, None);
989        data.sid = sid;
990
991        KalshiWsEvent::Message {
992            generation,
993            message: KalshiWsMessage::OrderbookDelta(data),
994        }
995    }
996
997    fn oracle_levels() -> (BTreeMap<Decimal, Decimal>, BTreeMap<Decimal, Decimal>) {
998        let snapshot = fixtures()["snapshot"].clone();
999        let mut bids = BTreeMap::new();
1000        let mut asks = BTreeMap::new();
1001
1002        for row in snapshot["yes_dollars_fp"].as_array().unwrap() {
1003            bids.insert(
1004                decimal(row[0].as_str().unwrap()),
1005                decimal(row[1].as_str().unwrap()),
1006            );
1007        }
1008
1009        for row in snapshot["no_dollars_fp"].as_array().unwrap() {
1010            asks.insert(
1011                decimal(row[0].as_str().unwrap()),
1012                decimal(row[1].as_str().unwrap()),
1013            );
1014        }
1015
1016        (bids, asks)
1017    }
1018
1019    fn batch(receiver: &mut tokio::sync::mpsc::UnboundedReceiver<DataEvent>) -> OrderBookDeltas {
1020        let DataEvent::Data(Data::BookDeltas(batch)) = receiver.try_recv().unwrap() else {
1021            panic!("expected book batch");
1022        };
1023
1024        *batch
1025    }
1026
1027    #[rstest]
1028    fn exact_levels_actions_flags_and_empty_replacement() {
1029        let instrument = instrument();
1030        let fixture = fixtures();
1031        let snapshot: Snapshot = serde_json::from_value(fixture["snapshot"].clone()).unwrap();
1032        let mut levels = Levels::from_snapshot(
1033            &snapshot.yes_dollars_fp,
1034            &snapshot.no_dollars_fp,
1035            false,
1036            &instrument,
1037        )
1038        .unwrap();
1039        let now = UnixNanos::from(123);
1040        let event_time = UnixNanos::from(91);
1041        let output = levels.snapshot(&instrument, 17, event_time, now).unwrap();
1042        let mut expected = vec![OrderBookDelta::clear(instrument.id, 17, event_time, now)];
1043
1044        for (side, price, qty, flags) in [
1045            (OrderSide::Buy, "0.123456", "7.25", 32),
1046            (OrderSide::Buy, "0.213456", "4.50", 32),
1047            (OrderSide::Sell, "0.713456", "9.75", 32),
1048            (OrderSide::Sell, "0.813456", "6.25", 160),
1049        ] {
1050            let price = Price::from_decimal_dp(decimal(price), instrument.price_precision).unwrap();
1051            let qty = Quantity::from_decimal_dp(decimal(qty), instrument.size_precision).unwrap();
1052            let order = BookOrder::new(side, price, qty, 0);
1053            expected.push(OrderBookDelta::new(
1054                instrument.id,
1055                BookAction::Add,
1056                order,
1057                flags,
1058                17,
1059                event_time,
1060                now,
1061            ));
1062        }
1063
1064        assert_eq!(output, OrderBookDeltas::new(instrument.id, expected));
1065        let mut checker = BookStreamChecker::new(BookType::L2_MBP, true);
1066        checker.open(instrument.id);
1067        checker.apply(&output).unwrap();
1068
1069        let cases = [
1070            (BookAction::Update, OrderSide::Buy, "0.123456", "9.75"),
1071            (BookAction::Update, OrderSide::Sell, "0.713456", "6.50"),
1072            (BookAction::Delete, OrderSide::Buy, "0.213456", "0"),
1073            (BookAction::Add, OrderSide::Sell, "0.613456", "8.25"),
1074        ];
1075
1076        let mut bids = BTreeMap::from([
1077            (decimal("0.123456"), decimal("7.25")),
1078            (decimal("0.213456"), decimal("4.50")),
1079        ]);
1080        let mut asks = BTreeMap::from([
1081            (decimal("0.713456"), decimal("9.75")),
1082            (decimal("0.813456"), decimal("6.25")),
1083        ]);
1084
1085        for (index, (action, side, price, qty)) in cases.into_iter().enumerate() {
1086            let payload: Delta = serde_json::from_value(fixture["steps"][index].clone()).unwrap();
1087            let seq = 18 + index as u64;
1088            let output = levels
1089                .delta(&payload, &instrument, seq, event_time, now)
1090                .unwrap();
1091            let price = Price::from_decimal_dp(decimal(price), instrument.price_precision).unwrap();
1092            let qty = Quantity::from_decimal_dp(decimal(qty), instrument.size_precision).unwrap();
1093            let order = BookOrder::new(side, price, qty, 0);
1094            assert_eq!(
1095                output,
1096                OrderBookDeltas::new(
1097                    instrument.id,
1098                    vec![OrderBookDelta::new(
1099                        instrument.id,
1100                        action,
1101                        order,
1102                        128,
1103                        seq,
1104                        event_time,
1105                        now
1106                    )]
1107                )
1108            );
1109
1110            let truth = if side == OrderSide::Buy {
1111                &mut bids
1112            } else {
1113                &mut asks
1114            };
1115
1116            if qty.is_zero() {
1117                truth.remove(&price.as_decimal());
1118            } else {
1119                truth.insert(price.as_decimal(), qty.as_decimal());
1120            }
1121
1122            checker.apply(&output).unwrap();
1123            checker.verify(instrument.id, 100, &bids, &asks).unwrap();
1124        }
1125
1126        let empty = Levels::default()
1127            .snapshot(&instrument, 30, event_time, now)
1128            .unwrap();
1129        let mut clear = OrderBookDelta::clear(instrument.id, 30, event_time, now);
1130        clear.flags = 160;
1131        assert_eq!(empty, OrderBookDeltas::new(instrument.id, vec![clear]));
1132        checker.apply(&empty).unwrap();
1133        checker
1134            .verify(instrument.id, 100, &BTreeMap::new(), &BTreeMap::new())
1135            .unwrap();
1136    }
1137
1138    #[rstest]
1139    fn rest_no_bids_complement_once_and_carry_no_watermark() {
1140        let instrument = instrument();
1141        let rest: Orderbook = serde_json::from_value(fixtures()["rest"].clone()).unwrap();
1142        let output = Levels::rest(&rest, &instrument, UnixNanos::from(13)).unwrap();
1143        assert_eq!(
1144            output.deltas[2].order.price.as_decimal(),
1145            decimal("0.713456")
1146        );
1147        assert_eq!(output.deltas[2].order.size.as_decimal(), decimal("9.75"));
1148        assert!(output.deltas.iter().all(|delta| delta.sequence == 0));
1149    }
1150
1151    #[rstest]
1152    #[case("delta_fp", "-100.00")]
1153    #[case("delta_fp", "0.0000001")]
1154    #[case("price_dollars", "0.1234567")]
1155    #[case("side", "invalid")]
1156    #[case("price_dollars", "1.5")]
1157    fn invalid_delta_cannot_publish(#[case] field: &str, #[case] value: &str) {
1158        let instrument = instrument();
1159        let snapshot: Snapshot = serde_json::from_value(fixtures()["snapshot"].clone()).unwrap();
1160        let mut levels = Levels::from_snapshot(
1161            &snapshot.yes_dollars_fp,
1162            &snapshot.no_dollars_fp,
1163            false,
1164            &instrument,
1165        )
1166        .unwrap();
1167        let mut payload = fixtures()["steps"][0].clone();
1168        payload[field] = value.into();
1169
1170        let result = serde_json::from_value::<Delta>(payload)
1171            .map_err(anyhow::Error::from)
1172            .and_then(|payload| {
1173                levels.delta(
1174                    &payload,
1175                    &instrument,
1176                    2,
1177                    UnixNanos::from(1),
1178                    UnixNanos::from(2),
1179                )
1180            });
1181
1182        assert!(result.is_err());
1183    }
1184
1185    #[rstest]
1186    fn canceled_confirmation_cannot_open_new_attempt_or_accept_late_snapshot() {
1187        let instrument = instrument();
1188        let id = instrument.id;
1189        let instruments = std::collections::HashMap::from([(id, instrument)]);
1190        let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1191        books.subscribe(id, None).unwrap();
1192        books.state.lock().unwrap().running = true;
1193        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1194        let sender = sender.into();
1195        let now = UnixNanos::from(9);
1196        books
1197            .event(&subscribed(0, 7), &instruments, &sender, now)
1198            .unwrap();
1199        books
1200            .event(&snapshot(0, 1, None), &instruments, &sender, now)
1201            .unwrap();
1202        let _ = batch(&mut output);
1203        let recovery = books
1204            .state
1205            .lock()
1206            .unwrap()
1207            .books
1208            .get_mut(&id)
1209            .unwrap()
1210            .sync
1211            .claim()
1212            .unwrap();
1213        assert!(recovery.begin_replacement());
1214        let cancel = recovery.cancellation.child_token();
1215        let token = books
1216            .attempt(id, cancel.clone(), recovery.gate.clone(), &recovery)
1217            .unwrap();
1218        cancel.cancel();
1219        assert!(recovery.begin_replacement());
1220        books
1221            .event(
1222                &KalshiWsEvent::BookSnapshotRequested {
1223                    generation: 0,
1224                    sid: 7,
1225                    id: 20,
1226                    attempt: token,
1227                },
1228                &instruments,
1229                &sender,
1230                now,
1231            )
1232            .unwrap();
1233
1234        assert!(recovery.gate.lock().is_closed());
1235        books
1236            .event(&snapshot(0, 2, Some(20)), &instruments, &sender, now)
1237            .unwrap();
1238        assert!(output.try_recv().is_err());
1239        let token = books
1240            .attempt(
1241                id,
1242                recovery.cancellation.child_token(),
1243                recovery.gate.clone(),
1244                &recovery,
1245            )
1246            .unwrap();
1247        books
1248            .event(
1249                &KalshiWsEvent::BookSnapshotRequested {
1250                    generation: 0,
1251                    sid: 7,
1252                    id: 21,
1253                    attempt: token,
1254                },
1255                &instruments,
1256                &sender,
1257                now,
1258            )
1259            .unwrap();
1260
1261        books
1262            .event(&snapshot(0, 3, Some(20)), &instruments, &sender, now)
1263            .unwrap();
1264        assert!(output.try_recv().is_err());
1265        books
1266            .event(&snapshot(0, 4, Some(21)), &instruments, &sender, now)
1267            .unwrap();
1268        assert_eq!(batch(&mut output).sequence, 4);
1269        assert!(recovery.is_accepted());
1270        books.unsubscribe(id, None).unwrap();
1271        assert!(recovery.cancellation.is_cancelled());
1272        books
1273            .event(&snapshot(0, 5, Some(21)), &instruments, &sender, now)
1274            .unwrap();
1275        assert!(output.try_recv().is_err());
1276    }
1277
1278    #[rstest]
1279    fn reconnect_preserves_owner_and_rejects_stale_generation() {
1280        let instrument = instrument();
1281        let id = instrument.id;
1282        let instruments = std::collections::HashMap::from([(id, instrument)]);
1283        let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1284        books.subscribe(id, None).unwrap();
1285        books.state.lock().unwrap().running = true;
1286        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1287        let sender = sender.into();
1288        let now = UnixNanos::from(9);
1289        books
1290            .event(&subscribed(0, 7), &instruments, &sender, now)
1291            .unwrap();
1292        let recovery = books
1293            .state
1294            .lock()
1295            .unwrap()
1296            .books
1297            .get_mut(&id)
1298            .unwrap()
1299            .sync
1300            .claim()
1301            .unwrap();
1302        assert!(recovery.begin_replacement());
1303        books
1304            .event(&subscribed(1, 7), &instruments, &sender, now)
1305            .unwrap();
1306        assert!(
1307            books.state.lock().unwrap().books[&id]
1308                .sync
1309                .is_current(&recovery)
1310        );
1311        assert!(
1312            books
1313                .state
1314                .lock()
1315                .unwrap()
1316                .books
1317                .get_mut(&id)
1318                .unwrap()
1319                .sync
1320                .claim()
1321                .is_none()
1322        );
1323        books
1324            .event(&snapshot(0, 1, None), &instruments, &sender, now)
1325            .unwrap();
1326        assert!(output.try_recv().is_err());
1327        let token = books
1328            .attempt(
1329                id,
1330                recovery.cancellation.child_token(),
1331                recovery.gate.clone(),
1332                &recovery,
1333            )
1334            .unwrap();
1335        books
1336            .event(
1337                &KalshiWsEvent::BookSnapshotRequested {
1338                    generation: 1,
1339                    sid: 7,
1340                    id: 21,
1341                    attempt: token,
1342                },
1343                &instruments,
1344                &sender,
1345                now,
1346            )
1347            .unwrap();
1348
1349        books
1350            .event(&snapshot(1, 2, Some(21)), &instruments, &sender, now)
1351            .unwrap();
1352        assert_eq!(batch(&mut output).sequence, 2);
1353        assert!(recovery.is_accepted());
1354        books.stop(false);
1355        books
1356            .event(&snapshot(1, 3, Some(21)), &instruments, &sender, now)
1357            .unwrap();
1358        assert!(output.try_recv().is_err());
1359    }
1360
1361    #[rstest]
1362    fn schedules_match_independent_publication_model() {
1363        let instrument = instrument();
1364        let id = instrument.id;
1365        let instruments = std::collections::HashMap::from([(id, instrument)]);
1366        let fixture = fixtures();
1367        let now = UnixNanos::from(9);
1368
1369        for schedule in 0_u32..6_u32.pow(5) {
1370            let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1371            books.state.lock().unwrap().running = true;
1372            let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1373            let sender = sender.into();
1374            let mut checker = BookStreamChecker::new(BookType::L2_MBP, true);
1375            let mut model = PublicationModel::default();
1376            let mut generation = 0;
1377            let mut seq = 0;
1378            let mut choices = schedule;
1379            for _ in 0..5 {
1380                let choice = choices % 6;
1381                choices /= 6;
1382                seq += 1;
1383
1384                match choice {
1385                    0 if !model.active => {
1386                        books.subscribe(id, None).unwrap();
1387                        books
1388                            .event(&subscribed(generation, 7), &instruments, &sender, now)
1389                            .unwrap();
1390                        checker.open(id);
1391                    }
1392                    1 => {
1393                        books
1394                            .event(&snapshot(generation, seq, None), &instruments, &sender, now)
1395                            .unwrap();
1396                    }
1397                    2 => {
1398                        let data = message(fixture["steps"][0].clone(), seq, None);
1399                        books
1400                            .event(
1401                                &KalshiWsEvent::Message {
1402                                    generation,
1403                                    message: KalshiWsMessage::OrderbookDelta(data),
1404                                },
1405                                &instruments,
1406                                &sender,
1407                                now,
1408                            )
1409                            .unwrap();
1410                    }
1411                    3 => {
1412                        books
1413                            .event(
1414                                &KalshiWsEvent::BookSequenceGap { generation, sid: 7 },
1415                                &instruments,
1416                                &sender,
1417                                now,
1418                            )
1419                            .unwrap();
1420                    }
1421                    4 => {
1422                        generation += 1;
1423                        books
1424                            .event(&subscribed(generation, 7), &instruments, &sender, now)
1425                            .unwrap();
1426                    }
1427                    5 if model.active => {
1428                        books.unsubscribe(id, None).unwrap();
1429                        checker.close(id);
1430                    }
1431                    _ => {}
1432                }
1433
1434                if model.step(choice) {
1435                    let batch = batch(&mut output);
1436                    assert_eq!(batch.sequence, seq, "schedule {schedule}, choice {choice}");
1437                    checker.apply(&batch).unwrap();
1438                    let bids = BTreeMap::from([
1439                        (decimal("0.123456"), model.bid_qty),
1440                        (decimal("0.213456"), decimal("4.50")),
1441                    ]);
1442                    let asks = BTreeMap::from([
1443                        (decimal("0.713456"), decimal("9.75")),
1444                        (decimal("0.813456"), decimal("6.25")),
1445                    ]);
1446                    checker.verify(id, 100, &bids, &asks).unwrap();
1447                }
1448
1449                assert!(
1450                    output.try_recv().is_err(),
1451                    "unexpected output: schedule {schedule}, choice {choice}"
1452                );
1453                assert_eq!(
1454                    books
1455                        .state
1456                        .lock()
1457                        .unwrap()
1458                        .books
1459                        .get(&id)
1460                        .is_some_and(|book| matches!(book.sync.phase(), BookPhase::Synced(_))),
1461                    model.synced,
1462                    "schedule {schedule}, choice {choice}"
1463                );
1464            }
1465        }
1466    }
1467
1468    #[rstest]
1469    fn multi_market_connection_schedule_matches_independent_oracle() {
1470        let first = instrument();
1471        let first_id = first.id;
1472        let mut second = first.clone();
1473        second.id = InstrumentId::from("MARKET-B.KALSHI");
1474        let second_id = second.id;
1475        let instruments = HashMap::from([(first_id, first), (second_id, second)]);
1476        let books = BookSyncTracker::new(&[first_id, second_id], Duration::from_secs(10));
1477        books.state.lock().unwrap().running = true;
1478        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1479        let sender = sender.into();
1480        let mut checker = BookStreamChecker::new(BookType::L2_MBP, true);
1481        checker.open(first_id);
1482        checker.open(second_id);
1483        let now = UnixNanos::from(9);
1484        let (mut bids, asks) = oracle_levels();
1485
1486        books.subscribe(first_id, None).unwrap();
1487        books.subscribe(second_id, None).unwrap();
1488        books
1489            .event(&subscribed(0, 7), &instruments, &sender, now)
1490            .unwrap();
1491        books
1492            .event(
1493                &snapshot_sid(0, 1, 99, "MARKET-A"),
1494                &instruments,
1495                &sender,
1496                now,
1497            )
1498            .unwrap();
1499        assert!(output.try_recv().is_err());
1500
1501        for (seq, ticker) in [(1, "MARKET-A"), (2, "MARKET-B")] {
1502            books
1503                .event(&snapshot_ticker(0, seq, ticker), &instruments, &sender, now)
1504                .unwrap();
1505            let emitted = batch(&mut output);
1506            checker.apply(&emitted).unwrap();
1507            checker
1508                .verify(emitted.instrument_id, 100, &bids, &asks)
1509                .unwrap();
1510        }
1511
1512        books
1513            .event(
1514                &delta_ticker(0, 3, 7, "MARKET-A", fixtures()["steps"][0].clone()),
1515                &instruments,
1516                &sender,
1517                now,
1518            )
1519            .unwrap();
1520        let emitted = batch(&mut output);
1521        assert_eq!(emitted.instrument_id, first_id);
1522        checker.apply(&emitted).unwrap();
1523        *bids.get_mut(&decimal("0.123456")).unwrap() += decimal("2.50");
1524        checker.verify(first_id, 100, &bids, &asks).unwrap();
1525        assert!(output.try_recv().is_err());
1526
1527        books
1528            .event(
1529                &KalshiWsEvent::BookSequenceGap {
1530                    generation: 0,
1531                    sid: 7,
1532                },
1533                &instruments,
1534                &sender,
1535                now,
1536            )
1537            .unwrap();
1538
1539        books
1540            .event(
1541                &delta_ticker(0, 4, 7, "MARKET-B", fixtures()["steps"][0].clone()),
1542                &instruments,
1543                &sender,
1544                now,
1545            )
1546            .unwrap();
1547        assert!(output.try_recv().is_err());
1548
1549        books
1550            .event(&subscribed(1, 8), &instruments, &sender, now)
1551            .unwrap();
1552        books
1553            .event(
1554                &snapshot_ticker(0, 5, "MARKET-A"),
1555                &instruments,
1556                &sender,
1557                now,
1558            )
1559            .unwrap();
1560        assert!(output.try_recv().is_err());
1561
1562        let (bids, asks) = oracle_levels();
1563
1564        for (seq, ticker, id) in [(1, "MARKET-A", first_id), (2, "MARKET-B", second_id)] {
1565            books
1566                .event(&snapshot_sid(1, seq, 8, ticker), &instruments, &sender, now)
1567                .unwrap();
1568            let emitted = batch(&mut output);
1569            assert_eq!(emitted.instrument_id, id);
1570            checker.apply(&emitted).unwrap();
1571            checker.verify(id, 100, &bids, &asks).unwrap();
1572        }
1573
1574        books.unsubscribe(first_id, None).unwrap();
1575        checker.close(first_id);
1576        books
1577            .event(
1578                &delta_ticker(1, 3, 8, "MARKET-B", fixtures()["steps"][1].clone()),
1579                &instruments,
1580                &sender,
1581                now,
1582            )
1583            .unwrap();
1584        let emitted = batch(&mut output);
1585        assert_eq!(emitted.instrument_id, second_id);
1586        checker.apply(&emitted).unwrap();
1587        let mut reduced_asks = asks.clone();
1588        *reduced_asks.get_mut(&decimal("0.713456")).unwrap() -= decimal("3.25");
1589        checker
1590            .verify(second_id, 100, &bids, &reduced_asks)
1591            .unwrap();
1592        assert!(output.try_recv().is_err());
1593
1594        books
1595            .event(
1596                &delta_ticker(1, 4, 8, "MARKET-A", fixtures()["steps"][0].clone()),
1597                &instruments,
1598                &sender,
1599                now,
1600            )
1601            .unwrap();
1602        assert!(output.try_recv().is_err());
1603        books.unsubscribe(second_id, None).unwrap();
1604        checker.close(second_id);
1605        books
1606            .event(
1607                &snapshot_sid(1, 5, 8, "MARKET-B"),
1608                &instruments,
1609                &sender,
1610                now,
1611            )
1612            .unwrap();
1613        assert!(output.try_recv().is_err());
1614    }
1615
1616    #[cfg(not(all(feature = "simulation", madsim)))]
1617    #[tokio::test]
1618    async fn expired_initial_wait_and_refused_task_are_retried_without_another_delta() {
1619        time::timeout(Duration::from_secs(8), async {
1620            let mut peer = peer::Peer::new().await;
1621            let instrument = instrument();
1622            let id = instrument.id;
1623            let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1624            books.subscribe(id, None).unwrap();
1625            let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
1626            books.replay(&ws_client).unwrap();
1627            ws_client.connect().await.unwrap();
1628            let _ = peer.command().await;
1629            let instruments = std::collections::HashMap::from([(id, instrument)]);
1630            let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1631            let sender = sender.into();
1632            let now = UnixNanos::from(9);
1633            let initial_tasks = TaskGroup::new();
1634            books.monitor(&ws_client, &initial_tasks.spawner().unwrap());
1635            assert_eq!(
1636                *books.state.lock().unwrap().books[&id].sync.phase(),
1637                BookPhase::Waiting
1638            );
1639            initial_tasks.begin_shutdown();
1640            initial_tasks
1641                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
1642                .await
1643                .unwrap();
1644
1645            loop {
1646                let event = input.recv().await.unwrap();
1647
1648                if matches!(
1649                    event,
1650                    KalshiWsEvent::Message {
1651                        message: KalshiWsMessage::OrderbookSnapshot(_),
1652                        ..
1653                    }
1654                ) {
1655                    break;
1656                }
1657
1658                books.event(&event, &instruments, &sender, now).unwrap();
1659            }
1660
1661            books
1662                .state
1663                .lock()
1664                .unwrap()
1665                .books
1666                .get_mut(&id)
1667                .unwrap()
1668                .sync
1669                .arm_deadline(time::Instant::now() - Duration::from_secs(1));
1670            let tasks = TaskGroup::new();
1671            let refused = tasks.spawner().unwrap();
1672            tasks.begin_shutdown();
1673            books.monitor(&ws_client, &refused);
1674            {
1675                let mut state = books.state.lock().unwrap();
1676                let recovery = state
1677                    .books
1678                    .get_mut(&id)
1679                    .unwrap()
1680                    .sync
1681                    .claim()
1682                    .expect("refused task must release ownership");
1683                recovery.cancellation.cancel();
1684            }
1685
1686            tasks
1687                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
1688                .await
1689                .unwrap();
1690            tasks.start_generation().unwrap();
1691            books.monitor(&ws_client, &tasks.spawner().unwrap());
1692            let command = peer.command().await;
1693            assert_eq!(command["params"]["action"], "get_snapshot");
1694
1695            loop {
1696                books
1697                    .event(&input.recv().await.unwrap(), &instruments, &sender, now)
1698                    .unwrap();
1699
1700                if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
1701                    assert_eq!(batch.sequence, 2);
1702                    break;
1703                }
1704            }
1705
1706            books.unsubscribe(id, None).unwrap();
1707            tasks.begin_shutdown();
1708            tasks
1709                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
1710                .await
1711                .unwrap();
1712            ws_client.dispose().await.unwrap();
1713        })
1714        .await
1715        .unwrap();
1716    }
1717
1718    #[rstest]
1719    #[case::missing_identity("market_ticker", None)]
1720    #[case::timestamp_overflow("ts_ms", Some(Value::from(u64::MAX)))]
1721    #[case::crossed("price_dollars", Some(Value::from("0.900000")))]
1722    #[case::negative_size("delta_fp", Some(Value::from("-100.00")))]
1723    #[case::invalid_side("side", Some(Value::from("invalid")))]
1724    fn invalid_frame_stops_publication(#[case] field: &str, #[case] value: Option<Value>) {
1725        let instrument = instrument();
1726        let id = instrument.id;
1727        let instruments = std::collections::HashMap::from([(id, instrument)]);
1728        let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1729        books.subscribe(id, None).unwrap();
1730        books.state.lock().unwrap().running = true;
1731        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1732        let sender = sender.into();
1733        let now = UnixNanos::from(9);
1734        books
1735            .event(&subscribed(0, 7), &instruments, &sender, now)
1736            .unwrap();
1737        books
1738            .event(&snapshot(0, 1, None), &instruments, &sender, now)
1739            .unwrap();
1740        let _ = batch(&mut output);
1741        let mut payload = fixtures()["steps"][0].clone();
1742        if let Some(value) = value {
1743            payload[field] = value;
1744        } else {
1745            payload.as_object_mut().unwrap().remove(field);
1746        }
1747
1748        books
1749            .event(
1750                &KalshiWsEvent::Message {
1751                    generation: 0,
1752                    message: KalshiWsMessage::OrderbookDelta(message(payload, 2, None)),
1753                },
1754                &instruments,
1755                &sender,
1756                now,
1757            )
1758            .unwrap();
1759
1760        books
1761            .event(
1762                &KalshiWsEvent::Message {
1763                    generation: 0,
1764                    message: KalshiWsMessage::OrderbookDelta(message(
1765                        fixtures()["steps"][0].clone(),
1766                        3,
1767                        None,
1768                    )),
1769                },
1770                &instruments,
1771                &sender,
1772                now,
1773            )
1774            .unwrap();
1775
1776        assert_eq!(
1777            *books.state.lock().unwrap().books[&id].sync.phase(),
1778            BookPhase::Recovering
1779        );
1780        assert!(output.try_recv().is_err());
1781    }
1782
1783    #[derive(Default)]
1784    struct PublicationModel {
1785        active: bool,
1786        synced: bool,
1787        initial: bool,
1788        bid_qty: Decimal,
1789    }
1790
1791    impl PublicationModel {
1792        fn step(&mut self, choice: u32) -> bool {
1793            match choice {
1794                0 if !self.active => {
1795                    self.active = true;
1796                    self.synced = false;
1797                    self.initial = true;
1798                }
1799                1 if self.active && self.initial => {
1800                    self.synced = true;
1801                    self.initial = false;
1802                    self.bid_qty = decimal("7.25");
1803                    return true;
1804                }
1805                2 if self.active && self.synced => {
1806                    self.bid_qty += decimal("2.50");
1807                    return true;
1808                }
1809                3 if self.active => {
1810                    self.synced = false;
1811                    self.initial = false;
1812                }
1813                4 if self.active => {
1814                    self.synced = false;
1815                    self.initial = true;
1816                }
1817                5 if self.active => {
1818                    self.active = false;
1819                    self.synced = false;
1820                    self.initial = false;
1821                }
1822                _ => {}
1823            }
1824
1825            false
1826        }
1827    }
1828
1829    #[rstest]
1830    #[case::unsubscribe(0)]
1831    #[case::stop(1)]
1832    #[case::reset(2)]
1833    #[case::resubscribe(3)]
1834    fn retirement_before_confirmation_cannot_open_gate(#[case] action: u8) {
1835        let instrument = instrument();
1836        let id = instrument.id;
1837        let instruments = std::collections::HashMap::from([(id, instrument)]);
1838        let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1839        books.subscribe(id, None).unwrap();
1840        books.state.lock().unwrap().running = true;
1841        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1842        let sender = sender.into();
1843        let now = UnixNanos::from(9);
1844        books
1845            .event(&subscribed(0, 7), &instruments, &sender, now)
1846            .unwrap();
1847        let owner = books
1848            .state
1849            .lock()
1850            .unwrap()
1851            .books
1852            .get_mut(&id)
1853            .unwrap()
1854            .sync
1855            .claim()
1856            .unwrap();
1857        assert!(owner.begin_replacement());
1858        let token = books
1859            .attempt(
1860                id,
1861                owner.cancellation.child_token(),
1862                owner.gate.clone(),
1863                &owner,
1864            )
1865            .unwrap();
1866
1867        match action {
1868            0 => books.unsubscribe(id, None).unwrap(),
1869            1 => books.stop(false),
1870            2 => books.stop(true),
1871            3 => {
1872                books.unsubscribe(id, None).unwrap();
1873                books.subscribe(id, None).unwrap();
1874            }
1875            _ => unreachable!(),
1876        }
1877
1878        books
1879            .event(
1880                &KalshiWsEvent::BookSnapshotRequested {
1881                    generation: 0,
1882                    sid: 7,
1883                    id: 21,
1884                    attempt: token,
1885                },
1886                &instruments,
1887                &sender,
1888                now,
1889            )
1890            .unwrap();
1891
1892        books
1893            .event(&snapshot(0, 2, Some(21)), &instruments, &sender, now)
1894            .unwrap();
1895        assert!(owner.cancellation.is_cancelled());
1896        assert!(owner.gate.lock().is_closed());
1897        assert!(output.try_recv().is_err());
1898
1899        if action == 3 {
1900            let result = books.attempt(id, CancellationToken::new(), owner.gate.clone(), &owner);
1901            assert_eq!(
1902                result.unwrap_err().to_string(),
1903                "client error: book recovery owner retired"
1904            );
1905        }
1906    }
1907
1908    #[cfg(not(all(feature = "simulation", madsim)))]
1909    #[tokio::test]
1910    async fn native_reconnect_retains_pending_recovery_owner() {
1911        time::timeout(Duration::from_secs(10), async {
1912            let mut peer = peer::Peer::new().await;
1913            let instrument = instrument();
1914            let id = instrument.id;
1915            let instruments = std::collections::HashMap::from([(id, instrument)]);
1916            let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1917            books.subscribe(id, None).unwrap();
1918            let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
1919            books.replay(&ws_client).unwrap();
1920            ws_client.connect().await.unwrap();
1921            let _ = peer.command().await;
1922            let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1923            let sender = sender.into();
1924            let now = UnixNanos::from(9);
1925
1926            loop {
1927                books
1928                    .event(&input.recv().await.unwrap(), &instruments, &sender, now)
1929                    .unwrap();
1930
1931                if output.try_recv().is_ok() {
1932                    break;
1933                }
1934            }
1935
1936            let owner = books
1937                .state
1938                .lock()
1939                .unwrap()
1940                .books
1941                .get_mut(&id)
1942                .unwrap()
1943                .sync
1944                .claim()
1945                .unwrap();
1946            assert!(owner.begin_replacement());
1947            let cancel = owner.cancellation.child_token();
1948            let token = books
1949                .attempt(id, cancel.clone(), owner.gate.clone(), &owner)
1950                .unwrap();
1951            peer.confirm
1952                .store(false, std::sync::atomic::Ordering::SeqCst);
1953            ws_client
1954                .request_book_snapshot(&books.subscription, id.symbol.to_string(), token, cancel)
1955                .await
1956                .unwrap();
1957            let request = peer.command().await;
1958            assert_eq!(request["params"]["action"], "get_snapshot");
1959
1960            loop {
1961                let event = input.recv().await.unwrap();
1962                books.event(&event, &instruments, &sender, now).unwrap();
1963
1964                if matches!(event, KalshiWsEvent::BookSnapshotRequested { .. }) {
1965                    break;
1966                }
1967            }
1968
1969            peer.confirm
1970                .store(true, std::sync::atomic::Ordering::SeqCst);
1971            ws_client.stop().await.unwrap();
1972            ws_client.connect().await.unwrap();
1973            let tasks = TaskGroup::new();
1974
1975            loop {
1976                let event = input.recv().await.unwrap();
1977                books.event(&event, &instruments, &sender, now).unwrap();
1978                books.monitor(&ws_client, &tasks.spawner().unwrap());
1979
1980                if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
1981                    assert_eq!(batch.sequence, 1);
1982                    assert_eq!(
1983                        batch.flags,
1984                        RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8
1985                    );
1986                    break;
1987                }
1988
1989                let mut state = books.state.lock().unwrap();
1990                assert!(state.books[&id].sync.is_current(&owner));
1991                assert!(state.books.get_mut(&id).unwrap().sync.claim().is_none());
1992            }
1993
1994            assert!(owner.is_accepted());
1995            assert!(!owner.cancellation.is_cancelled());
1996            books
1997                .event(
1998                    &snapshot(0, 999, Some(request["id"].as_u64().unwrap())),
1999                    &instruments,
2000                    &sender,
2001                    now,
2002                )
2003                .unwrap();
2004            assert!(output.try_recv().is_err());
2005            tasks.begin_shutdown();
2006            tasks
2007                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
2008                .await
2009                .unwrap();
2010            books.stop(false);
2011            ws_client.dispose().await.unwrap();
2012        })
2013        .await
2014        .unwrap();
2015    }
2016
2017    #[rstest]
2018    fn book_demand_survives_until_the_last_reference_retires() {
2019        let instrument = instrument();
2020        let id = instrument.id;
2021        let instruments = std::collections::HashMap::from([(id, instrument)]);
2022        let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2023        books.subscribe(id, None).unwrap();
2024        books.subscribe(id, None).unwrap();
2025        books.state.lock().unwrap().running = true;
2026        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2027        let sender = sender.into();
2028        let now = UnixNanos::from(9);
2029        books
2030            .event(&subscribed(0, 7), &instruments, &sender, now)
2031            .unwrap();
2032        books
2033            .event(&snapshot(0, 1, None), &instruments, &sender, now)
2034            .unwrap();
2035        let _ = batch(&mut output);
2036        books.unsubscribe(id, None).unwrap();
2037
2038        let delta = KalshiWsEvent::Message {
2039            generation: 0,
2040            message: KalshiWsMessage::OrderbookDelta(message(
2041                fixtures()["steps"][0].clone(),
2042                2,
2043                None,
2044            )),
2045        };
2046
2047        books.event(&delta, &instruments, &sender, now).unwrap();
2048        let output_delta = batch(&mut output);
2049        assert_eq!(output_delta.sequence, 2);
2050        assert_eq!(
2051            output_delta.deltas[0].order.size.as_decimal(),
2052            decimal("9.75")
2053        );
2054        books.unsubscribe(id, None).unwrap();
2055        books.event(&delta, &instruments, &sender, now).unwrap();
2056        assert!(output.try_recv().is_err());
2057    }
2058
2059    #[cfg(not(all(feature = "simulation", madsim)))]
2060    #[tokio::test]
2061    async fn last_book_readd_after_rejected_unsubscribe_recovers() {
2062        time::timeout(Duration::from_secs(8), async {
2063            let mut peer = peer::Peer::new().await;
2064            let instrument = instrument();
2065            let id = instrument.id;
2066            let instruments = std::collections::HashMap::from([(id, instrument)]);
2067            let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2068            books.subscribe(id, None).unwrap();
2069            let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
2070            books.replay(&ws_client).unwrap();
2071            ws_client.connect().await.unwrap();
2072            let _ = peer.command().await;
2073            let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2074            let sender = sender.into();
2075            let now = UnixNanos::from(9);
2076
2077            loop {
2078                books
2079                    .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2080                    .unwrap();
2081
2082                if output.try_recv().is_ok() {
2083                    break;
2084                }
2085            }
2086
2087            peer.confirm
2088                .store(false, std::sync::atomic::Ordering::SeqCst);
2089            books.unsubscribe(id, Some(&ws_client)).unwrap();
2090            let unsubscribe = peer.command().await;
2091            assert_eq!(unsubscribe["cmd"], "unsubscribe");
2092            books.subscribe(id, Some(&ws_client)).unwrap();
2093            peer.confirm
2094                .store(true, std::sync::atomic::Ordering::SeqCst);
2095            let mut frame = serde_json::from_str::<Value>(include_str!(
2096                "../../test_data/websocket_messages.json"
2097            ))
2098            .unwrap()["unsubscribe_rate_limit"]
2099                .clone();
2100            frame["id"] = serde_json::json!(unsubscribe["id"]);
2101            peer.actions.send(("raw".into(), frame)).unwrap();
2102
2103            loop {
2104                let event = input.recv().await.unwrap();
2105                books.event(&event, &instruments, &sender, now).unwrap();
2106
2107                if matches!(event, KalshiWsEvent::Ready { .. }) {
2108                    break;
2109                }
2110            }
2111
2112            assert!(ws_client.is_active());
2113            let tasks = TaskGroup::new();
2114            books.monitor(&ws_client, &tasks.spawner().unwrap());
2115            let extra_owner = books
2116                .state
2117                .lock()
2118                .unwrap()
2119                .books
2120                .get_mut(&id)
2121                .unwrap()
2122                .sync
2123                .claim();
2124
2125            if let Some(owner) = extra_owner {
2126                owner.cancellation.cancel();
2127                panic!("ready retained SID left the re-added book without a recovery owner");
2128            }
2129
2130            let request = peer.command().await;
2131            assert_eq!(request["params"]["action"], "get_snapshot");
2132
2133            loop {
2134                books
2135                    .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2136                    .unwrap();
2137
2138                if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
2139                    assert_eq!(batch.sequence, 2);
2140                    break;
2141                }
2142            }
2143
2144            books.stop(false);
2145            tasks.begin_shutdown();
2146            tasks
2147                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
2148                .await
2149                .unwrap();
2150            ws_client.dispose().await.unwrap();
2151        })
2152        .await
2153        .unwrap();
2154    }
2155
2156    #[rstest]
2157    fn unsubscribe_ack_retires_sid_before_new_subscription() {
2158        let instrument = instrument();
2159        let id = instrument.id;
2160        let instruments = HashMap::from([(id, instrument)]);
2161        let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2162        books.subscribe(id, None).unwrap();
2163        books.state.lock().unwrap().running = true;
2164        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2165        let sender = sender.into();
2166        let now = UnixNanos::from(9);
2167        books
2168            .event(&subscribed(0, 7), &instruments, &sender, now)
2169            .unwrap();
2170        books
2171            .event(&snapshot(0, 1, None), &instruments, &sender, now)
2172            .unwrap();
2173        let _ = batch(&mut output);
2174        books.unsubscribe(id, None).unwrap();
2175        books
2176            .event(
2177                &KalshiWsEvent::Message {
2178                    generation: 0,
2179                    message: KalshiWsMessage::Unsubscribed {
2180                        id: Some(2),
2181                        sid: 7,
2182                        seq: 2,
2183                    },
2184                },
2185                &instruments,
2186                &sender,
2187                now,
2188            )
2189            .unwrap();
2190
2191        assert_eq!(books.state.lock().unwrap().sid, None);
2192        books.subscribe(id, None).unwrap();
2193        books
2194            .event(&snapshot(0, 3, None), &instruments, &sender, now)
2195            .unwrap();
2196        assert!(output.try_recv().is_err());
2197        books
2198            .event(&subscribed(0, 9), &instruments, &sender, now)
2199            .unwrap();
2200        books
2201            .event(&snapshot(0, 4, None), &instruments, &sender, now)
2202            .unwrap();
2203        assert!(output.try_recv().is_err());
2204
2205        let KalshiWsEvent::Message {
2206            message: KalshiWsMessage::OrderbookSnapshot(mut data),
2207            ..
2208        } = snapshot(0, 1, None)
2209        else {
2210            unreachable!()
2211        };
2212
2213        data.sid = 9;
2214        books
2215            .event(
2216                &KalshiWsEvent::Message {
2217                    generation: 0,
2218                    message: KalshiWsMessage::OrderbookSnapshot(data),
2219                },
2220                &instruments,
2221                &sender,
2222                now,
2223            )
2224            .unwrap();
2225
2226        assert_eq!(batch(&mut output).sequence, 1);
2227        assert_eq!(books.state.lock().unwrap().sid, Some(9));
2228    }
2229
2230    #[rstest]
2231    fn last_book_readd_without_unsubscribe_ack_requires_correlated_snapshot() {
2232        let instrument = instrument();
2233        let id = instrument.id;
2234        let instruments = HashMap::from([(id, instrument)]);
2235        let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2236        books.subscribe(id, None).unwrap();
2237        books.state.lock().unwrap().running = true;
2238        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2239        let sender = sender.into();
2240        let now = UnixNanos::from(9);
2241        books
2242            .event(&subscribed(0, 7), &instruments, &sender, now)
2243            .unwrap();
2244        books
2245            .event(&snapshot(0, 1, None), &instruments, &sender, now)
2246            .unwrap();
2247        let _ = batch(&mut output);
2248        books.unsubscribe(id, None).unwrap();
2249        books.subscribe(id, None).unwrap();
2250        books
2251            .event(&snapshot(0, 2, None), &instruments, &sender, now)
2252            .unwrap();
2253        assert!(output.try_recv().is_err());
2254        assert_eq!(books.state.lock().unwrap().sid, Some(7));
2255        let owner = books
2256            .state
2257            .lock()
2258            .unwrap()
2259            .books
2260            .get_mut(&id)
2261            .unwrap()
2262            .sync
2263            .claim()
2264            .unwrap();
2265        assert!(owner.begin_replacement());
2266        let token = books
2267            .attempt(
2268                id,
2269                owner.cancellation.child_token(),
2270                owner.gate.clone(),
2271                &owner,
2272            )
2273            .unwrap();
2274        books
2275            .event(
2276                &KalshiWsEvent::BookSnapshotRequested {
2277                    generation: 0,
2278                    sid: 7,
2279                    id: 21,
2280                    attempt: token,
2281                },
2282                &instruments,
2283                &sender,
2284                now,
2285            )
2286            .unwrap();
2287
2288        books
2289            .event(&snapshot(0, 3, Some(21)), &instruments, &sender, now)
2290            .unwrap();
2291        assert_eq!(batch(&mut output).sequence, 3);
2292        assert!(owner.is_accepted());
2293    }
2294
2295    #[cfg(not(all(feature = "simulation", madsim)))]
2296    #[tokio::test]
2297    async fn non_book_rejection_does_not_block_book_recovery() {
2298        time::timeout(Duration::from_secs(8), async {
2299            let mut peer = peer::Peer::new().await;
2300            let instrument = instrument();
2301            let id = instrument.id;
2302            let instruments = HashMap::from([(id, instrument)]);
2303            let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2304            books.subscribe(id, None).unwrap();
2305            let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
2306            books.replay(&ws_client).unwrap();
2307            ws_client.connect().await.unwrap();
2308            let _ = peer.command().await;
2309            let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2310            let sender = sender.into();
2311            let now = UnixNanos::from(9);
2312
2313            loop {
2314                books
2315                    .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2316                    .unwrap();
2317
2318                if output.try_recv().is_ok() {
2319                    break;
2320                }
2321            }
2322
2323            peer.confirm
2324                .store(false, std::sync::atomic::Ordering::SeqCst);
2325            ws_client
2326                .subscribe(KalshiSubscription {
2327                    channel: KalshiChannel::Ticker,
2328                    market_tickers: vec!["MARKET-A".into()],
2329                })
2330                .unwrap();
2331
2332            let rejected = peer.command().await;
2333            assert_eq!(rejected["params"]["channels"][0], "ticker");
2334            let mut frame = serde_json::from_str::<Value>(include_str!(
2335                "../../test_data/websocket_messages.json"
2336            ))
2337            .unwrap()["command_rate_limit"]
2338                .clone();
2339            frame["id"] = serde_json::json!(rejected["id"]);
2340            peer.actions.send(("raw".into(), frame)).unwrap();
2341
2342            loop {
2343                let event = input.recv().await.unwrap();
2344                books.event(&event, &instruments, &sender, now).unwrap();
2345
2346                if matches!(event, KalshiWsEvent::Error { .. }) {
2347                    break;
2348                }
2349            }
2350
2351            assert!(!ws_client.is_active());
2352            peer.confirm
2353                .store(true, std::sync::atomic::Ordering::SeqCst);
2354            let tasks = TaskGroup::new();
2355            books.monitor(&ws_client, &tasks.spawner().unwrap());
2356            let extra = books
2357                .state
2358                .lock()
2359                .unwrap()
2360                .books
2361                .get_mut(&id)
2362                .unwrap()
2363                .sync
2364                .claim();
2365            assert!(
2366                extra.is_none(),
2367                "unrelated command rejection prevented the book recovery owner"
2368            );
2369            let request = peer.command().await;
2370            assert_eq!(request["params"]["action"], "get_snapshot");
2371
2372            loop {
2373                books
2374                    .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2375                    .unwrap();
2376
2377                if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
2378                    assert_eq!(batch.sequence, 2);
2379                    break;
2380                }
2381            }
2382
2383            assert!(!ws_client.is_active());
2384            books.stop(false);
2385            tasks.begin_shutdown();
2386            tasks
2387                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
2388                .await
2389                .unwrap();
2390            ws_client.dispose().await.unwrap();
2391        })
2392        .await
2393        .unwrap();
2394    }
2395
2396    #[rstest]
2397    #[case::empty_arrays(0)]
2398    #[case::both_omitted(1)]
2399    #[case::yes_omitted(2)]
2400    #[case::no_omitted(3)]
2401    fn empty_and_one_sided_replacements_clear_the_published_book(#[case] shape: u8) {
2402        let instrument = instrument();
2403        let id = instrument.id;
2404        let instruments = HashMap::from([(id, instrument)]);
2405        let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2406        books.subscribe(id, None).unwrap();
2407        books.state.lock().unwrap().running = true;
2408        let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2409        let sender = sender.into();
2410        let now = UnixNanos::from(9);
2411        let mut checker = BookStreamChecker::new(BookType::L2_MBP, true);
2412        checker.open(id);
2413        books
2414            .event(&subscribed(0, 7), &instruments, &sender, now)
2415            .unwrap();
2416        books
2417            .event(&snapshot(0, 1, None), &instruments, &sender, now)
2418            .unwrap();
2419        checker.apply(&batch(&mut output)).unwrap();
2420        books
2421            .event(
2422                &KalshiWsEvent::BookSequenceGap {
2423                    generation: 0,
2424                    sid: 7,
2425                },
2426                &instruments,
2427                &sender,
2428                now,
2429            )
2430            .unwrap();
2431
2432        let owner = books
2433            .state
2434            .lock()
2435            .unwrap()
2436            .books
2437            .get_mut(&id)
2438            .unwrap()
2439            .sync
2440            .claim()
2441            .unwrap();
2442        assert!(owner.begin_replacement());
2443        let token = books
2444            .attempt(
2445                id,
2446                owner.cancellation.child_token(),
2447                owner.gate.clone(),
2448                &owner,
2449            )
2450            .unwrap();
2451        books
2452            .event(
2453                &KalshiWsEvent::BookSnapshotRequested {
2454                    generation: 0,
2455                    sid: 7,
2456                    id: 21,
2457                    attempt: token,
2458                },
2459                &instruments,
2460                &sender,
2461                now,
2462            )
2463            .unwrap();
2464
2465        let mut payload = fixtures()[if shape <= 1 { "empty" } else { "snapshot" }].clone();
2466
2467        for (side, remove) in [
2468            ("yes_dollars_fp", shape == 1 || shape == 2),
2469            ("no_dollars_fp", shape == 1 || shape == 3),
2470        ] {
2471            if remove {
2472                payload.as_object_mut().unwrap().remove(side);
2473            }
2474        }
2475
2476        books
2477            .event(
2478                &KalshiWsEvent::Message {
2479                    generation: 0,
2480                    message: KalshiWsMessage::OrderbookSnapshot(message(payload, 2, Some(21))),
2481                },
2482                &instruments,
2483                &sender,
2484                now,
2485            )
2486            .unwrap();
2487
2488        let replacement = batch(&mut output);
2489        assert_eq!(replacement.sequence, 2);
2490        let mut clear = OrderBookDelta::clear(id, 2, UnixNanos::from(1700000000001000000), now);
2491        if shape <= 1 {
2492            clear.flags |= RecordFlag::F_LAST as u8;
2493        }
2494
2495        assert_eq!(replacement.deltas[0], clear);
2496        assert_eq!(
2497            replacement.flags,
2498            RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8
2499        );
2500        checker.apply(&replacement).unwrap();
2501
2502        let bids = if shape == 3 {
2503            BTreeMap::from([
2504                (decimal("0.123456"), decimal("7.25")),
2505                (decimal("0.213456"), decimal("4.50")),
2506            ])
2507        } else {
2508            BTreeMap::new()
2509        };
2510
2511        let asks = if shape == 2 {
2512            BTreeMap::from([
2513                (decimal("0.713456"), decimal("9.75")),
2514                (decimal("0.813456"), decimal("6.25")),
2515            ])
2516        } else {
2517            BTreeMap::new()
2518        };
2519
2520        checker.verify(id, 100, &bids, &asks).unwrap();
2521        assert_eq!(replacement.deltas.len(), 1 + bids.len() + asks.len());
2522        assert!(owner.is_accepted());
2523        assert!(output.try_recv().is_err());
2524    }
2525
2526    #[rstest]
2527    fn snapshot_rejection_reaches_only_its_correlated_recovery() {
2528        use nautilus_live::book::recovery::BookRecoveryOutcome;
2529
2530        use crate::websocket::messages::KalshiServerError;
2531        let instrument = instrument();
2532        let id = instrument.id;
2533        let mut other = instrument.clone();
2534        other.id = InstrumentId::from("MARKET-B.KALSHI");
2535        let other_id = other.id;
2536        let instruments = HashMap::from([(id, instrument), (other_id, other)]);
2537        let books = BookSyncTracker::new(&[id, other_id], Duration::from_secs(10));
2538        let (sender, _output) = tokio::sync::mpsc::unbounded_channel();
2539        let sender = sender.into();
2540        books.state.lock().unwrap().running = true;
2541        let now = UnixNanos::from(9);
2542        books.subscribe(id, None).unwrap();
2543        books.subscribe(other_id, None).unwrap();
2544        books
2545            .event(&subscribed(0, 7), &instruments, &sender, now)
2546            .unwrap();
2547        let mut owners = Vec::new();
2548
2549        for (id, request) in [(id, 21), (other_id, 22)] {
2550            let owner = books
2551                .state
2552                .lock()
2553                .unwrap()
2554                .books
2555                .get_mut(&id)
2556                .unwrap()
2557                .sync
2558                .claim()
2559                .unwrap();
2560            assert!(owner.begin_replacement());
2561            let token = books
2562                .attempt(
2563                    id,
2564                    owner.cancellation.child_token(),
2565                    owner.gate.clone(),
2566                    &owner,
2567                )
2568                .unwrap();
2569            books
2570                .event(
2571                    &KalshiWsEvent::BookSnapshotRequested {
2572                        generation: 0,
2573                        sid: 7,
2574                        id: request,
2575                        attempt: token,
2576                    },
2577                    &instruments,
2578                    &sender,
2579                    now,
2580                )
2581                .unwrap();
2582
2583            owners.push(owner);
2584        }
2585
2586        books
2587            .event(
2588                &KalshiWsEvent::Message {
2589                    generation: 0,
2590                    message: KalshiWsMessage::Error {
2591                        id: Some(21),
2592                        sid: Some(7),
2593                        msg: KalshiServerError {
2594                            code: 27,
2595                            msg: "rate limit exceeded".into(),
2596                        },
2597                    },
2598                },
2599                &instruments,
2600                &sender,
2601                now,
2602            )
2603            .unwrap();
2604
2605        assert!(matches!(
2606            &*owners[0].outcome.borrow(),
2607            BookRecoveryOutcome::Rejected(_)
2608        ));
2609        assert!(matches!(
2610            &*owners[1].outcome.borrow(),
2611            BookRecoveryOutcome::Pending
2612        ));
2613        assert!(!owners[1].gate.lock().is_closed());
2614        assert!(!owners[1].cancellation.is_cancelled());
2615    }
2616
2617    #[rstest]
2618    #[case::disabled(0)]
2619    #[case::short(1)]
2620    #[case::nondefault(31)]
2621    fn snapshot_deadline_uses_the_configured_duration(#[case] seconds: u64) {
2622        let instrument = instrument();
2623        let id = instrument.id;
2624        let instruments = HashMap::from([(id, instrument)]);
2625        let books = BookSyncTracker::new(&[id], Duration::from_secs(seconds));
2626        books.subscribe(id, None).unwrap();
2627        books.state.lock().unwrap().running = true;
2628        let (sender, _output) = tokio::sync::mpsc::unbounded_channel();
2629        let sender = sender.into();
2630        let start = time::Instant::now();
2631        books
2632            .event(&subscribed(0, 7), &instruments, &sender, UnixNanos::from(9))
2633            .unwrap();
2634        let end = time::Instant::now();
2635        let mut state = books.state.lock().unwrap();
2636        let book = state.books.get_mut(&id).unwrap();
2637        assert!(book.waits_for_snapshot(Duration::from_secs(seconds)));
2638        book.gap();
2639        assert_eq!(
2640            book.waits_for_snapshot(Duration::from_secs(seconds)),
2641            seconds != 0
2642        );
2643
2644        if seconds == 0 {
2645            assert!(book.sync.has_pending_snapshot());
2646            assert!(!book.sync.is_send_pending());
2647            assert!(!book.sync.take_expired(end + Duration::from_secs(1000)));
2648        } else {
2649            assert!(
2650                !book
2651                    .sync
2652                    .take_expired(start + Duration::from_millis(seconds * 1000 - 1))
2653            );
2654            assert!(book.sync.take_expired(end + Duration::from_secs(seconds)));
2655        }
2656    }
2657
2658    #[cfg(not(all(feature = "simulation", madsim)))]
2659    #[tokio::test]
2660    async fn rejected_replay_keeps_a_recovery_owner() {
2661        time::timeout(Duration::from_secs(10), async {
2662            let mut peer = peer::Peer::new().await;
2663            let instrument = instrument();
2664            let id = instrument.id;
2665            let instruments = HashMap::from([(id, instrument)]);
2666            let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2667            books.subscribe(id, None).unwrap();
2668            let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
2669            books.replay(&ws_client).unwrap();
2670            ws_client.connect().await.unwrap();
2671            let _ = peer.command().await;
2672            let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2673            let sender = sender.into();
2674            let now = UnixNanos::from(9);
2675
2676            loop {
2677                books
2678                    .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2679                    .unwrap();
2680
2681                if output.try_recv().is_ok() {
2682                    break;
2683                }
2684            }
2685
2686            peer.confirm
2687                .store(false, std::sync::atomic::Ordering::SeqCst);
2688            ws_client.stop().await.unwrap();
2689            let connect = ws_client.connect();
2690            tokio::pin!(connect);
2691            let rejected = tokio::select! {
2692                result = &mut connect => panic!("replayed connection finished before rejection: {result:?}"),
2693                command = peer.command() => command,
2694            };
2695            assert_eq!(rejected["cmd"], "subscribe");
2696            let mut frame = serde_json::from_str::<Value>(include_str!(
2697                "../../test_data/websocket_messages.json"
2698            ))
2699            .unwrap()["command_rate_limit"]
2700                .clone();
2701            frame["id"] = serde_json::json!(rejected["id"]);
2702            peer.actions.send(("raw".into(), frame)).unwrap();
2703
2704            loop {
2705                let event = input.recv().await.unwrap();
2706                books.event(&event, &instruments, &sender, now).unwrap();
2707
2708                if matches!(event, KalshiWsEvent::Error { generation: 1, .. }) {
2709                    break;
2710                }
2711            }
2712
2713            assert_eq!(
2714                connect.await.unwrap_err().to_string(),
2715                "Kalshi WebSocket subscription restoration timed out"
2716            );
2717            peer.confirm
2718                .store(true, std::sync::atomic::Ordering::SeqCst);
2719            let tasks = TaskGroup::new();
2720            books.monitor(&ws_client, &tasks.spawner().unwrap());
2721            let extra = books
2722                .state
2723                .lock()
2724                .unwrap()
2725                .books
2726                .get_mut(&id)
2727                .unwrap()
2728                .sync
2729                .claim();
2730
2731            if let Some(owner) = extra {
2732                owner.cancellation.cancel();
2733                panic!("rejected replay left the book unowned");
2734            }
2735
2736            ws_client.stop().await.unwrap();
2737            ws_client.connect().await.unwrap();
2738            let replay = peer.command().await;
2739            assert_eq!(replay["cmd"], "subscribe");
2740
2741            loop {
2742                let event = input.recv().await.unwrap();
2743                books.event(&event, &instruments, &sender, now).unwrap();
2744
2745                if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
2746                    assert_eq!(batch.sequence, 2);
2747                    assert_eq!(
2748                        batch.flags,
2749                        RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8
2750                    );
2751                    break;
2752                }
2753
2754                let extra = books
2755                    .state
2756                    .lock()
2757                    .unwrap()
2758                    .books
2759                    .get_mut(&id)
2760                    .unwrap()
2761                    .sync
2762                    .claim();
2763
2764                if let Some(owner) = extra {
2765                    owner.cancellation.cancel();
2766                    panic!("reconnect replaced the running owner");
2767                }
2768            }
2769
2770            books.stop(false);
2771            tasks.begin_shutdown();
2772            tasks
2773                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
2774                .await
2775                .unwrap();
2776            ws_client.dispose().await.unwrap();
2777        })
2778        .await
2779        .unwrap();
2780    }
2781
2782    #[rstest]
2783    #[case::initial(0)]
2784    #[case::resubscribe(1)]
2785    #[case::reconnect(2)]
2786    fn zero_timeout_waits_for_initial_snapshot_after_subscription(#[case] transition: u8) {
2787        let instrument = instrument();
2788        let id = instrument.id;
2789        let instruments = HashMap::from([(id, instrument)]);
2790        let books = BookSyncTracker::new(&[id], Duration::ZERO);
2791        books.subscribe(id, None).unwrap();
2792        books.state.lock().unwrap().running = true;
2793        let (sender, _output) = tokio::sync::mpsc::unbounded_channel();
2794        let sender = sender.into();
2795        let now = UnixNanos::from(9);
2796        books
2797            .event(&subscribed(0, 7), &instruments, &sender, now)
2798            .unwrap();
2799
2800        if transition != 0 {
2801            books
2802                .event(
2803                    &KalshiWsEvent::Message {
2804                        generation: 0,
2805                        message: KalshiWsMessage::OrderbookSnapshot(message(
2806                            fixtures()["snapshot"].clone(),
2807                            1,
2808                            None,
2809                        )),
2810                    },
2811                    &instruments,
2812                    &sender,
2813                    now,
2814                )
2815                .unwrap();
2816
2817            assert_eq!(
2818                books.state.lock().unwrap().books[&id].sync.position(),
2819                Some(&1)
2820            );
2821
2822            let event = if transition == 1 {
2823                KalshiWsEvent::Message {
2824                    generation: 0,
2825                    message: KalshiWsMessage::Unsubscribed {
2826                        id: Some(2),
2827                        sid: 7,
2828                        seq: 1,
2829                    },
2830                }
2831            } else {
2832                KalshiWsEvent::Ready { generation: 1 }
2833            };
2834
2835            books.event(&event, &instruments, &sender, now).unwrap();
2836            books
2837                .event(
2838                    &subscribed(u64::from(transition == 2), 7),
2839                    &instruments,
2840                    &sender,
2841                    now,
2842                )
2843                .unwrap();
2844        }
2845
2846        {
2847            let mut state = books.state.lock().unwrap();
2848            let book = state.books.get_mut(&id).unwrap();
2849            assert!(
2850                book.waits_for_snapshot(Duration::ZERO),
2851                "confirmed initial wait was treated as a gap"
2852            );
2853            assert!(
2854                !book
2855                    .sync
2856                    .take_expired(time::Instant::now() + Duration::from_secs(1000))
2857            );
2858        }
2859
2860        books
2861            .event(
2862                &KalshiWsEvent::BookSequenceGap {
2863                    generation: u64::from(transition == 2),
2864                    sid: 7,
2865                },
2866                &instruments,
2867                &sender,
2868                now,
2869            )
2870            .unwrap();
2871
2872        let state = books.state.lock().unwrap();
2873        assert!(
2874            !state.books[&id].waits_for_snapshot(Duration::ZERO),
2875            "a real gap failed to end the initial wait"
2876        );
2877    }
2878}