1use std::{
17 collections::{BTreeMap, HashMap},
18 sync::{Arc, Mutex},
19 time::Duration,
20};
21
22use anyhow::Context;
23use nautilus_common::{
24 live::{dst::time, sender::EventSender},
25 messages::DataEvent,
26};
27use nautilus_core::UnixNanos;
28use nautilus_live::{
29 book::{
30 recovery::BookRecovery,
31 snapshot::SnapshotGate,
32 sync::{BookPhase, BookSync},
33 },
34 task::TaskSpawner,
35};
36use nautilus_model::{
37 data::{BookOrder, Data, OrderBookDelta, OrderBookDeltas},
38 enums::{BookAction, OrderSide, RecordFlag},
39 identifiers::InstrumentId,
40 instruments::BinaryOption,
41 types::Quantity,
42};
43use rust_decimal::Decimal;
44use serde::Deserialize;
45use tokio_util::sync::CancellationToken;
46
47use crate::{
48 data::parse,
49 http::models::{FixedPoint, Orderbook, OutcomeSide},
50 websocket::{
51 client::{KalshiWebSocketClient, KalshiWsEvent},
52 error::KalshiWsError,
53 messages::{KalshiChannel, KalshiDataMessage, KalshiSubscription, KalshiWsMessage},
54 },
55};
56
57#[derive(Debug, Clone)]
59pub struct BookSyncTracker {
60 state: Arc<Mutex<State>>,
61 subscription: KalshiSubscription,
62 snapshot_timeout: Duration,
63}
64
65#[allow(
66 clippy::missing_panics_doc,
67 reason = "book state mutex poisoning is not expected"
68)]
69impl BookSyncTracker {
70 #[must_use]
75 pub fn new(instruments: &[InstrumentId], snapshot_timeout: Duration) -> Self {
76 let mut market_tickers: Vec<_> =
77 instruments.iter().map(|id| id.symbol.to_string()).collect();
78 market_tickers.sort();
79
80 Self {
81 state: Arc::default(),
82 snapshot_timeout,
83 subscription: KalshiSubscription {
84 channel: KalshiChannel::OrderbookDelta,
85 market_tickers,
86 },
87 }
88 }
89
90 pub fn subscribe(
100 &self,
101 id: InstrumentId,
102 ws_client: Option<&KalshiWebSocketClient>,
103 ) -> anyhow::Result<()> {
104 let mut state = self.state.lock().expect("book state lock is not poisoned");
105 let first = state.books.is_empty();
106 if first && let Some(ws_client) = ws_client {
107 ws_client.subscribe(self.subscription.clone())?;
108 }
109
110 let initial = state.sid.is_none();
111
112 let book = state.books.entry(id).or_insert_with(|| {
113 let mut book = Book::new(time::Instant::now());
114 book.initial = initial;
115 book
116 });
117
118 book.demand = book
119 .demand
120 .checked_add(1)
121 .context("book subscription reference count overflow")?;
122 Ok(())
123 }
124
125 pub(crate) fn unsubscribe(
126 &self,
127 id: InstrumentId,
128 ws_client: Option<&KalshiWebSocketClient>,
129 ) -> anyhow::Result<()> {
130 let mut state = self.state.lock().expect("book state lock is not poisoned");
131 if let Some(book) = state.books.get_mut(&id) {
132 book.demand -= 1;
133 if book.demand == 0 {
134 state.books.remove(&id);
135 }
136 }
137
138 if state.books.is_empty()
139 && let Some(ws_client) = ws_client
140 {
141 ws_client.unsubscribe(&self.subscription)?;
142 }
143
144 Ok(())
145 }
146
147 pub fn replay(&self, ws_client: &KalshiWebSocketClient) -> anyhow::Result<()> {
153 let mut state = self.state.lock().expect("book state lock is not poisoned");
154 if !state.books.is_empty() {
155 ws_client.subscribe(self.subscription.clone())?;
156 }
157
158 state.running = true;
159 Ok(())
160 }
161
162 pub(crate) fn stop(&self, reset: bool) {
163 let mut state = self.state.lock().expect("book state lock is not poisoned");
164 state.running = false;
165 state.generation = None;
166 state.sid = None;
167 if reset {
168 state.books.clear();
169 }
170
171 for book in state.books.values_mut() {
172 let demand = book.demand;
173 *book = Book::new(time::Instant::now());
174 book.demand = demand;
175 }
176 }
177
178 pub fn event(
189 &self,
190 event: &KalshiWsEvent,
191 instruments: &HashMap<InstrumentId, BinaryOption>,
192 sender: &EventSender<DataEvent>,
193 now: UnixNanos,
194 ) -> anyhow::Result<()> {
195 if matches!(
196 event,
197 KalshiWsEvent::Message {
198 message: KalshiWsMessage::Ticker(_) | KalshiWsMessage::Trade(_),
199 ..
200 }
201 ) {
202 return Ok(());
203 }
204
205 let generation = event.generation();
206 let mut state = self.state.lock().expect("book state lock is not poisoned");
207 if !state.advance_generation(generation) {
208 return Ok(());
209 }
210
211 match event {
212 KalshiWsEvent::Message {
213 message: KalshiWsMessage::Subscribed { msg, .. },
214 ..
215 } if msg.channel == KalshiChannel::OrderbookDelta => {
216 state.subscribed(msg.sid, self.snapshot_timeout);
217 }
218 KalshiWsEvent::Message {
219 message: KalshiWsMessage::Unsubscribed { sid, .. },
220 ..
221 } if state.sid == Some(*sid) => {
222 state.reset_stream();
223 }
224 KalshiWsEvent::BookSequenceGap { sid, .. } => {
225 state.gap(*sid);
226 }
227 KalshiWsEvent::BookSnapshotRequested {
228 sid, id, attempt, ..
229 } => {
230 for book in state.books.values_mut() {
231 if let Some(current) = &mut book.attempt
232 && current.token == *attempt
233 && !current.cancel.is_cancelled()
234 {
235 current.correlation = Some((generation, *sid, *id));
236 current.gate.open();
237 }
238 }
239 }
240 KalshiWsEvent::Message {
241 message: KalshiWsMessage::OrderbookSnapshot(data),
242 ..
243 } => {
244 self.snapshot(&mut state, data, instruments, sender, generation, now)?;
245 }
246 KalshiWsEvent::Message {
247 message: KalshiWsMessage::OrderbookDelta(data),
248 ..
249 } => {
250 self.delta(&mut state, data, instruments, sender, now)?;
251 }
252 KalshiWsEvent::Message {
253 message:
254 KalshiWsMessage::Error {
255 id: Some(id),
256 sid,
257 msg,
258 },
259 ..
260 } => {
261 state.reject_snapshot(generation, *id, *sid, msg.code);
262 }
263 KalshiWsEvent::Error { message, .. } => {
264 for book in state.books.values_mut() {
265 book.sync.reject(KalshiWsError::Client(message.clone()));
266 book.gap();
267 book.initial = false;
268 }
269 }
270 _ => {}
271 }
272
273 Ok(())
274 }
275
276 fn snapshot(
277 &self,
278 state: &mut State,
279 data: &KalshiDataMessage,
280 instruments: &HashMap<InstrumentId, BinaryOption>,
281 sender: &EventSender<DataEvent>,
282 generation: u64,
283 now: UnixNanos,
284 ) -> anyhow::Result<()> {
285 if state.sid != Some(data.sid) {
286 return Ok(());
287 }
288
289 let Some((id, book)) = selected(state, data) else {
290 return Ok(());
291 };
292
293 let initial = book.initial && data.id.is_none();
294
295 let replacement = book.attempt.as_ref().is_some_and(|attempt| {
296 !attempt.cancel.is_cancelled()
297 && data
298 .id
299 .is_some_and(|id| attempt.correlation == Some((generation, data.sid, id)))
300 });
301
302 if !initial && !replacement {
303 return Ok(());
304 }
305
306 let Some(instrument) = instruments.get(&id) else {
307 return Ok(());
308 };
309
310 let parsed = (|| {
311 let seq = data.seq.context("book snapshot sequence missing")?;
312 let payload: Snapshot = serde_json::from_value(data.msg.clone())?;
313 let levels = Levels::from_snapshot(
314 &payload.yes_dollars_fp,
315 &payload.no_dollars_fp,
316 false,
317 instrument,
318 )?;
319 let event_time = event_time(data, now)?;
320 let batch = levels.snapshot(instrument, seq, event_time, now)?;
321 anyhow::Ok((seq, levels, batch))
322 })();
323
324 match parsed {
325 Ok((seq, levels, batch)) if book.sync.accept_snapshot(seq, time::Instant::now()) => {
326 sender
327 .send(DataEvent::Data(Data::BookDeltas(batch.into())))
328 .map_err(|_| anyhow::anyhow!("book event receiver closed"))?;
329 book.levels = levels;
330 book.initial = false;
331 book.gapped = false;
332 book.attempt = None;
333 }
334 Ok(_) => {}
335 Err(e) => {
336 log::warn!("Invalid Kalshi book snapshot: {e}");
337 book.gap();
338 book.initial = false;
339 }
340 }
341
342 Ok(())
343 }
344
345 fn delta(
346 &self,
347 state: &mut State,
348 data: &KalshiDataMessage,
349 instruments: &HashMap<InstrumentId, BinaryOption>,
350 sender: &EventSender<DataEvent>,
351 now: UnixNanos,
352 ) -> anyhow::Result<()> {
353 if state.sid != Some(data.sid) {
354 return Ok(());
355 }
356
357 let Some((id, book)) = selected(state, data) else {
358 return Ok(());
359 };
360
361 if !matches!(book.sync.phase(), BookPhase::Synced(_)) {
362 book.gap();
363 return Ok(());
364 }
365
366 let Some(instrument) = instruments.get(&id) else {
367 return Ok(());
368 };
369
370 let parsed = (|| {
371 let seq = data.seq.context("book delta sequence missing")?;
372 let payload: Delta = serde_json::from_value(data.msg.clone())?;
373
374 let event_time = match payload.ts_ms {
375 Some(value) => UnixNanos::from(
376 value
377 .checked_mul(1_000_000)
378 .context("book timestamp overflow")?,
379 ),
380 None => event_time(data, now)?,
381 };
382
383 let batch = book
384 .levels
385 .delta(&payload, instrument, seq, event_time, now)?;
386 anyhow::Ok((seq, batch))
387 })();
388
389 match parsed {
390 Ok((seq, batch)) => {
391 book.sync.advance(seq, time::Instant::now());
392 sender
393 .send(DataEvent::Data(Data::BookDeltas(batch.into())))
394 .map_err(|_| anyhow::anyhow!("book event receiver closed"))?;
395 }
396 Err(e) => {
397 log::warn!("Invalid Kalshi book delta: {e}");
398 book.gap();
399 book.initial = false;
400 }
401 }
402
403 Ok(())
404 }
405
406 pub(crate) fn monitor(&self, ws_client: &KalshiWebSocketClient, tasks: &TaskSpawner) {
407 let mut state = self.state.lock().expect("book state lock is not poisoned");
408 if !state.running {
409 return;
410 }
411
412 let confirmed = state.sid.is_some();
413 for (id, book) in &mut state.books {
414 if !confirmed && book.initial {
415 continue;
416 }
417
418 let expired = book.sync.take_expired(time::Instant::now());
419 if !expired
420 && (matches!(book.sync.phase(), BookPhase::Synced(_))
421 || book.waits_for_snapshot(self.snapshot_timeout))
422 {
423 continue;
424 }
425
426 let Some(recovery) = book.sync.claim() else {
427 continue;
428 };
429
430 if expired {
431 log::warn!("Kalshi book snapshot deadline expired for {id}");
432 }
433
434 book.initial = false;
435 let tracker = self.clone();
436 let ws_client = ws_client.clone();
437 let id = *id;
438 let running = Arc::clone(&recovery);
439 let shutdown = tasks.cancellation_token();
440 let snapshot_timeout = self.snapshot_timeout;
441
442 if let Err(e) = tasks.spawn(async move {
443 let run = running.run(
444 id,
445 snapshot_timeout,
446 |cancel, gate| {
447 let tracker = tracker.clone();
448 let ws_client = ws_client.clone();
449 let owner = Arc::clone(&running);
450 async move {
451 let token = tracker.attempt(id, cancel.clone(), gate, &owner)?;
452 ws_client
453 .request_book_snapshot(
454 &tracker.subscription,
455 id.symbol.to_string(),
456 token,
457 cancel,
458 )
459 .await
460 }
461 },
462 |_| true,
463 KalshiWsError::Client,
464 || KalshiWsError::Client("book snapshot timed out".into()),
465 );
466
467 tokio::select! {
468 biased;
469 () = shutdown.cancelled() => {}
470 () = run => {}
471 }
472 running.cancellation.cancel();
473 }) {
474 recovery.cancellation.cancel();
475 log::warn!("Kalshi book recovery task refused: {e}");
476 }
477 }
478 }
479
480 fn attempt(
481 &self,
482 id: InstrumentId,
483 cancel: CancellationToken,
484 gate: SnapshotGate,
485 owner: &Arc<BookRecovery<KalshiWsError>>,
486 ) -> Result<u64, KalshiWsError> {
487 let mut state = self.state.lock().expect("book state lock is not poisoned");
488 state.next_attempt = state
489 .next_attempt
490 .checked_add(1)
491 .ok_or_else(|| KalshiWsError::Client("book attempt IDs exhausted".into()))?;
492 let token = state.next_attempt;
493 if !state.running {
494 return Err(KalshiWsError::Client("book session stopped".into()));
495 }
496
497 let book = state
498 .books
499 .get_mut(&id)
500 .ok_or_else(|| KalshiWsError::Client("book subscription retired".into()))?;
501
502 if cancel.is_cancelled() || !book.sync.is_current(owner) {
503 return Err(KalshiWsError::Client("book recovery owner retired".into()));
504 }
505
506 book.attempt = Some(Attempt {
507 token,
508 cancel,
509 gate,
510 correlation: None,
511 });
512
513 Ok(token)
514 }
515}
516
517#[derive(Debug, Default)]
518struct State {
519 running: bool,
520 generation: Option<u64>,
521 sid: Option<u64>,
522 next_attempt: u64,
523 books: BTreeMap<InstrumentId, Book>,
524}
525
526impl State {
527 fn advance_generation(&mut self, generation: u64) -> bool {
528 if !self.running || self.generation.is_some_and(|current| generation < current) {
529 return false;
530 }
531
532 if self.generation.is_some_and(|current| generation > current) {
533 self.reset_stream();
534 }
535
536 self.generation = Some(generation);
537 true
538 }
539
540 fn subscribed(&mut self, sid: u64, snapshot_timeout: Duration) {
541 self.sid = Some(sid);
542
543 for book in self.books.values_mut() {
544 book.initial = true;
545 book.gapped = false;
546
547 if snapshot_timeout.is_zero() {
548 let gate = SnapshotGate::default();
549 gate.open();
550 book.sync.expect_snapshot(gate);
551 } else {
552 book.sync
553 .arm_deadline(time::Instant::now() + snapshot_timeout);
554 }
555 }
556 }
557
558 fn reset_stream(&mut self) {
559 self.sid = None;
560
561 for book in self.books.values_mut() {
562 book.sync.reset_on_reconnect();
563 book.attempt = None;
564 book.initial = true;
565 book.gapped = false;
566 book.levels = Levels::default();
567 }
568 }
569
570 fn reject_snapshot(&mut self, generation: u64, id: u64, sid: Option<u64>, code: u32) {
571 for book in self.books.values_mut() {
572 if book.attempt.as_ref().is_some_and(|attempt| {
573 !attempt.cancel.is_cancelled()
574 && attempt
575 .correlation
576 .is_some_and(|(current, subscription, request)| {
577 current == generation
578 && request == id
579 && sid.is_none_or(|sid| sid == subscription)
580 })
581 }) {
582 log::warn!("Kalshi book snapshot request {id} rejected with code {code}");
583 book.sync.reject(KalshiWsError::Client(format!(
584 "book snapshot rejected with code {code}"
585 )));
586 book.gap();
587 book.initial = false;
588 }
589 }
590 }
591
592 fn gap(&mut self, sid: u64) {
593 if self.sid != Some(sid) {
594 return;
595 }
596
597 for book in self.books.values_mut() {
598 book.gap();
599 book.initial = false;
600 }
601 }
602}
603
604#[derive(Debug)]
605struct Book {
606 demand: usize,
607 sync: BookSync<KalshiWsError, u64>,
608 levels: Levels,
609 initial: bool,
610 gapped: bool,
611 attempt: Option<Attempt>,
612}
613
614impl Book {
615 fn new(now: time::Instant) -> Self {
616 Self {
617 demand: 0,
618 sync: BookSync::new(now),
619 levels: Levels::default(),
620 initial: true,
621 gapped: false,
622 attempt: None,
623 }
624 }
625
626 fn gap(&mut self) {
627 self.gapped = true;
628 self.sync.gap();
629 }
630
631 fn waits_for_snapshot(&self, snapshot_timeout: Duration) -> bool {
632 self.sync.has_pending_snapshot() && (!snapshot_timeout.is_zero() || !self.gapped)
633 }
634}
635
636#[derive(Debug)]
637struct Attempt {
638 token: u64,
639 cancel: CancellationToken,
640 gate: SnapshotGate,
641 correlation: Option<(u64, u64, u64)>,
642}
643
644fn selected<'a>(
645 state: &'a mut State,
646 data: &KalshiDataMessage,
647) -> Option<(InstrumentId, &'a mut Book)> {
648 let Some(ticker) = data
649 .msg
650 .get("market_ticker")
651 .and_then(serde_json::Value::as_str)
652 else {
653 log::warn!("Kalshi book frame has no market identity");
654 state.gap(data.sid);
655 return None;
656 };
657
658 state
659 .books
660 .iter_mut()
661 .find(|(id, _)| id.symbol.as_str() == ticker)
662 .map(|(id, book)| (*id, book))
663}
664
665fn event_time(data: &KalshiDataMessage, now: UnixNanos) -> anyhow::Result<UnixNanos> {
666 data.sending_ts_ms.map_or(Ok(now), |value| {
667 value
668 .checked_mul(1_000_000)
669 .map(UnixNanos::from)
670 .context("book timestamp overflow")
671 })
672}
673
674#[derive(Debug, Default)]
675pub(crate) struct Levels {
676 bids: BTreeMap<Decimal, Decimal>,
677 asks: BTreeMap<Decimal, Decimal>,
678}
679
680impl Levels {
681 pub(crate) fn rest(
682 book: &Orderbook,
683 instrument: &BinaryOption,
684 now: UnixNanos,
685 ) -> anyhow::Result<OrderBookDeltas> {
686 let levels = Self::from_snapshot(&book.yes_dollars, &book.no_dollars, true, instrument)?;
687 levels.snapshot(instrument, 0, now, now)
688 }
689
690 fn from_snapshot(
691 yes: &[[FixedPoint; 2]],
692 no: &[[FixedPoint; 2]],
693 complement_no: bool,
694 instrument: &BinaryOption,
695 ) -> anyhow::Result<Self> {
696 let mut levels = Self::default();
697
698 for (side, values) in [(OrderSide::Buy, yes), (OrderSide::Sell, no)] {
699 let target = if side == OrderSide::Buy {
700 &mut levels.bids
701 } else {
702 &mut levels.asks
703 };
704
705 for [price, qty] in values {
706 let price = if side == OrderSide::Sell && complement_no {
707 Decimal::ONE
708 .checked_sub(price.0)
709 .context("book price overflow")?
710 } else {
711 price.0
712 };
713
714 parse::price(FixedPoint(price), instrument.price_precision)?;
715 parse::size(*qty, instrument.size_precision)?;
716 anyhow::ensure!(
717 target.insert(price, qty.0).is_none(),
718 "duplicate book price level"
719 );
720 }
721 }
722
723 levels.integrity()?;
724 Ok(levels)
725 }
726
727 fn snapshot(
728 &self,
729 instrument: &BinaryOption,
730 seq: u64,
731 event_time: UnixNanos,
732 now: UnixNanos,
733 ) -> anyhow::Result<OrderBookDeltas> {
734 let mut deltas = vec![OrderBookDelta::clear(instrument.id, seq, event_time, now)];
735
736 for (side, levels) in [(OrderSide::Buy, &self.bids), (OrderSide::Sell, &self.asks)] {
737 for (price, qty) in levels {
738 let price = parse::price(FixedPoint(*price), instrument.price_precision)?;
739 let qty = parse::size(FixedPoint(*qty), instrument.size_precision)?;
740 let order = BookOrder::new(side, price, qty, 0);
741 deltas.push(OrderBookDelta::new_checked(
742 instrument.id,
743 BookAction::Add,
744 order,
745 RecordFlag::F_SNAPSHOT as u8,
746 seq,
747 event_time,
748 now,
749 )?);
750 }
751 }
752
753 deltas.last_mut().expect("clear is present").flags |= RecordFlag::F_LAST as u8;
754 Ok(OrderBookDeltas::new(instrument.id, deltas))
755 }
756
757 fn delta(
758 &mut self,
759 delta: &Delta,
760 instrument: &BinaryOption,
761 seq: u64,
762 event_time: UnixNanos,
763 now: UnixNanos,
764 ) -> anyhow::Result<OrderBookDeltas> {
765 let side = match delta.side {
766 OutcomeSide::Yes => OrderSide::Buy,
767 OutcomeSide::No => OrderSide::Sell,
768 };
769
770 let price = parse::price(delta.price_dollars, instrument.price_precision)?;
771
772 let levels = if side == OrderSide::Buy {
773 &mut self.bids
774 } else {
775 &mut self.asks
776 };
777
778 let previous = levels
779 .get(&delta.price_dollars.0)
780 .copied()
781 .unwrap_or_default();
782 let qty = previous
783 .checked_add(delta.delta_fp.0)
784 .context("book size overflow")?;
785 anyhow::ensure!(qty >= Decimal::ZERO, "book delta produces negative size");
786 anyhow::ensure!(
787 !delta.delta_fp.0.is_zero(),
788 "book delta must change a level"
789 );
790
791 let size = if qty.is_zero() {
792 Quantity::zero(instrument.size_precision)
793 } else {
794 parse::size(FixedPoint(qty), instrument.size_precision)?
795 };
796
797 let action = if qty.is_zero() {
798 BookAction::Delete
799 } else if previous.is_zero() {
800 BookAction::Add
801 } else {
802 BookAction::Update
803 };
804
805 let order = BookOrder::new(side, price, size, 0);
806 let delta = OrderBookDelta::new_checked(
807 instrument.id,
808 action,
809 order,
810 RecordFlag::F_LAST as u8,
811 seq,
812 event_time,
813 now,
814 )?;
815
816 if qty.is_zero() {
817 levels.remove(&price.as_decimal());
818 } else {
819 levels.insert(price.as_decimal(), qty);
820 }
821
822 self.integrity()?;
823 Ok(OrderBookDeltas::new(instrument.id, vec![delta]))
824 }
825
826 fn integrity(&self) -> anyhow::Result<()> {
827 if let (Some((bid, _)), Some((ask, _))) =
828 (self.bids.last_key_value(), self.asks.first_key_value())
829 {
830 anyhow::ensure!(bid < ask, "crossed or locked book");
831 }
832
833 Ok(())
834 }
835}
836
837#[derive(Deserialize)]
838struct Snapshot {
839 #[serde(default)]
840 yes_dollars_fp: Vec<[FixedPoint; 2]>,
841 #[serde(default)]
842 no_dollars_fp: Vec<[FixedPoint; 2]>,
843}
844
845#[derive(Deserialize)]
846struct Delta {
847 price_dollars: FixedPoint,
848 delta_fp: FixedPoint,
849 side: OutcomeSide,
850 ts_ms: Option<u64>,
851}
852
853#[cfg(test)]
854mod tests {
855 use nautilus_live::book::conformance::BookStreamChecker;
856 #[cfg(not(all(feature = "simulation", madsim)))]
857 use nautilus_live::task::TaskGroup;
858 use nautilus_model::{
859 enums::BookType,
860 types::{Price, Quantity},
861 };
862 use rstest::rstest;
863 use serde_json::Value;
864
865 use super::*;
866 #[cfg(not(all(feature = "simulation", madsim)))]
867 use crate::config::KalshiDataClientConfig;
868 use crate::{
869 http::{
870 models::{Event, Market},
871 parse::parse_instrument,
872 },
873 websocket::messages::KalshiSubscribed,
874 };
875 #[cfg(not(all(feature = "simulation", madsim)))]
876 #[allow(dead_code, reason = "shared key fixtures serve multiple suites")]
877 mod keys {
878 include!(concat!(
879 env!("CARGO_MANIFEST_DIR"),
880 "/tests/integration/keys.rs"
881 ));
882 }
883 #[cfg(not(all(feature = "simulation", madsim)))]
884 mod http_request {
885 include!(concat!(
886 env!("CARGO_MANIFEST_DIR"),
887 "/tests/common/http_request.rs"
888 ));
889 }
890 #[cfg(not(all(feature = "simulation", madsim)))]
891 #[allow(
892 dead_code,
893 reason = "peer fields also serve other data lifecycle tests"
894 )]
895 mod peer {
896 include!(concat!(
897 env!("CARGO_MANIFEST_DIR"),
898 "/tests/common/data_peer.rs"
899 ));
900 }
901
902 fn fixtures() -> Value {
903 serde_json::from_str(include_str!("../../test_data/books.json")).unwrap()
904 }
905
906 fn instrument() -> BinaryOption {
907 let fixtures: Value =
908 serde_json::from_str(include_str!("../../test_data/instruments.json")).unwrap();
909 let mut market: Market =
910 serde_json::from_value(fixtures["definitions"]["live"]["market"].clone()).unwrap();
911 market.ticker = "MARKET-A".into();
912 let event: Event =
913 serde_json::from_value(fixtures["definitions"]["live"]["event"].clone()).unwrap();
914 parse_instrument(&market, &event, UnixNanos::from(5)).unwrap()
915 }
916
917 fn decimal(value: &str) -> Decimal {
918 value.parse().unwrap()
919 }
920
921 fn message(payload: Value, seq: u64, id: Option<u64>) -> KalshiDataMessage {
922 KalshiDataMessage {
923 id,
924 sid: 7,
925 seq: Some(seq),
926 sending_ts_ms: Some(1700000000001),
927 msg: payload,
928 }
929 }
930
931 fn subscribed(generation: u64, sid: u64) -> KalshiWsEvent {
932 KalshiWsEvent::Message {
933 generation,
934 message: KalshiWsMessage::Subscribed {
935 id: 1,
936 msg: KalshiSubscribed {
937 channel: KalshiChannel::OrderbookDelta,
938 sid,
939 },
940 },
941 }
942 }
943
944 fn snapshot(generation: u64, seq: u64, id: Option<u64>) -> KalshiWsEvent {
945 KalshiWsEvent::Message {
946 generation,
947 message: KalshiWsMessage::OrderbookSnapshot(message(
948 fixtures()["snapshot"].clone(),
949 seq,
950 id,
951 )),
952 }
953 }
954
955 fn snapshot_ticker(generation: u64, seq: u64, ticker: &str) -> KalshiWsEvent {
956 let mut payload = fixtures()["snapshot"].clone();
957 payload["market_ticker"] = ticker.into();
958
959 KalshiWsEvent::Message {
960 generation,
961 message: KalshiWsMessage::OrderbookSnapshot(message(payload, seq, None)),
962 }
963 }
964
965 fn snapshot_sid(generation: u64, seq: u64, sid: u64, ticker: &str) -> KalshiWsEvent {
966 let mut event = snapshot_ticker(generation, seq, ticker);
967
968 let KalshiWsEvent::Message {
969 message: KalshiWsMessage::OrderbookSnapshot(data),
970 ..
971 } = &mut event
972 else {
973 unreachable!("snapshot event");
974 };
975
976 data.sid = sid;
977 event
978 }
979
980 fn delta_ticker(
981 generation: u64,
982 seq: u64,
983 sid: u64,
984 ticker: &str,
985 mut payload: Value,
986 ) -> KalshiWsEvent {
987 payload["market_ticker"] = ticker.into();
988 let mut data = message(payload, seq, None);
989 data.sid = sid;
990
991 KalshiWsEvent::Message {
992 generation,
993 message: KalshiWsMessage::OrderbookDelta(data),
994 }
995 }
996
997 fn oracle_levels() -> (BTreeMap<Decimal, Decimal>, BTreeMap<Decimal, Decimal>) {
998 let snapshot = fixtures()["snapshot"].clone();
999 let mut bids = BTreeMap::new();
1000 let mut asks = BTreeMap::new();
1001
1002 for row in snapshot["yes_dollars_fp"].as_array().unwrap() {
1003 bids.insert(
1004 decimal(row[0].as_str().unwrap()),
1005 decimal(row[1].as_str().unwrap()),
1006 );
1007 }
1008
1009 for row in snapshot["no_dollars_fp"].as_array().unwrap() {
1010 asks.insert(
1011 decimal(row[0].as_str().unwrap()),
1012 decimal(row[1].as_str().unwrap()),
1013 );
1014 }
1015
1016 (bids, asks)
1017 }
1018
1019 fn batch(receiver: &mut tokio::sync::mpsc::UnboundedReceiver<DataEvent>) -> OrderBookDeltas {
1020 let DataEvent::Data(Data::BookDeltas(batch)) = receiver.try_recv().unwrap() else {
1021 panic!("expected book batch");
1022 };
1023
1024 *batch
1025 }
1026
1027 #[rstest]
1028 fn exact_levels_actions_flags_and_empty_replacement() {
1029 let instrument = instrument();
1030 let fixture = fixtures();
1031 let snapshot: Snapshot = serde_json::from_value(fixture["snapshot"].clone()).unwrap();
1032 let mut levels = Levels::from_snapshot(
1033 &snapshot.yes_dollars_fp,
1034 &snapshot.no_dollars_fp,
1035 false,
1036 &instrument,
1037 )
1038 .unwrap();
1039 let now = UnixNanos::from(123);
1040 let event_time = UnixNanos::from(91);
1041 let output = levels.snapshot(&instrument, 17, event_time, now).unwrap();
1042 let mut expected = vec![OrderBookDelta::clear(instrument.id, 17, event_time, now)];
1043
1044 for (side, price, qty, flags) in [
1045 (OrderSide::Buy, "0.123456", "7.25", 32),
1046 (OrderSide::Buy, "0.213456", "4.50", 32),
1047 (OrderSide::Sell, "0.713456", "9.75", 32),
1048 (OrderSide::Sell, "0.813456", "6.25", 160),
1049 ] {
1050 let price = Price::from_decimal_dp(decimal(price), instrument.price_precision).unwrap();
1051 let qty = Quantity::from_decimal_dp(decimal(qty), instrument.size_precision).unwrap();
1052 let order = BookOrder::new(side, price, qty, 0);
1053 expected.push(OrderBookDelta::new(
1054 instrument.id,
1055 BookAction::Add,
1056 order,
1057 flags,
1058 17,
1059 event_time,
1060 now,
1061 ));
1062 }
1063
1064 assert_eq!(output, OrderBookDeltas::new(instrument.id, expected));
1065 let mut checker = BookStreamChecker::new(BookType::L2_MBP, true);
1066 checker.open(instrument.id);
1067 checker.apply(&output).unwrap();
1068
1069 let cases = [
1070 (BookAction::Update, OrderSide::Buy, "0.123456", "9.75"),
1071 (BookAction::Update, OrderSide::Sell, "0.713456", "6.50"),
1072 (BookAction::Delete, OrderSide::Buy, "0.213456", "0"),
1073 (BookAction::Add, OrderSide::Sell, "0.613456", "8.25"),
1074 ];
1075
1076 let mut bids = BTreeMap::from([
1077 (decimal("0.123456"), decimal("7.25")),
1078 (decimal("0.213456"), decimal("4.50")),
1079 ]);
1080 let mut asks = BTreeMap::from([
1081 (decimal("0.713456"), decimal("9.75")),
1082 (decimal("0.813456"), decimal("6.25")),
1083 ]);
1084
1085 for (index, (action, side, price, qty)) in cases.into_iter().enumerate() {
1086 let payload: Delta = serde_json::from_value(fixture["steps"][index].clone()).unwrap();
1087 let seq = 18 + index as u64;
1088 let output = levels
1089 .delta(&payload, &instrument, seq, event_time, now)
1090 .unwrap();
1091 let price = Price::from_decimal_dp(decimal(price), instrument.price_precision).unwrap();
1092 let qty = Quantity::from_decimal_dp(decimal(qty), instrument.size_precision).unwrap();
1093 let order = BookOrder::new(side, price, qty, 0);
1094 assert_eq!(
1095 output,
1096 OrderBookDeltas::new(
1097 instrument.id,
1098 vec![OrderBookDelta::new(
1099 instrument.id,
1100 action,
1101 order,
1102 128,
1103 seq,
1104 event_time,
1105 now
1106 )]
1107 )
1108 );
1109
1110 let truth = if side == OrderSide::Buy {
1111 &mut bids
1112 } else {
1113 &mut asks
1114 };
1115
1116 if qty.is_zero() {
1117 truth.remove(&price.as_decimal());
1118 } else {
1119 truth.insert(price.as_decimal(), qty.as_decimal());
1120 }
1121
1122 checker.apply(&output).unwrap();
1123 checker.verify(instrument.id, 100, &bids, &asks).unwrap();
1124 }
1125
1126 let empty = Levels::default()
1127 .snapshot(&instrument, 30, event_time, now)
1128 .unwrap();
1129 let mut clear = OrderBookDelta::clear(instrument.id, 30, event_time, now);
1130 clear.flags = 160;
1131 assert_eq!(empty, OrderBookDeltas::new(instrument.id, vec![clear]));
1132 checker.apply(&empty).unwrap();
1133 checker
1134 .verify(instrument.id, 100, &BTreeMap::new(), &BTreeMap::new())
1135 .unwrap();
1136 }
1137
1138 #[rstest]
1139 fn rest_no_bids_complement_once_and_carry_no_watermark() {
1140 let instrument = instrument();
1141 let rest: Orderbook = serde_json::from_value(fixtures()["rest"].clone()).unwrap();
1142 let output = Levels::rest(&rest, &instrument, UnixNanos::from(13)).unwrap();
1143 assert_eq!(
1144 output.deltas[2].order.price.as_decimal(),
1145 decimal("0.713456")
1146 );
1147 assert_eq!(output.deltas[2].order.size.as_decimal(), decimal("9.75"));
1148 assert!(output.deltas.iter().all(|delta| delta.sequence == 0));
1149 }
1150
1151 #[rstest]
1152 #[case("delta_fp", "-100.00")]
1153 #[case("delta_fp", "0.0000001")]
1154 #[case("price_dollars", "0.1234567")]
1155 #[case("side", "invalid")]
1156 #[case("price_dollars", "1.5")]
1157 fn invalid_delta_cannot_publish(#[case] field: &str, #[case] value: &str) {
1158 let instrument = instrument();
1159 let snapshot: Snapshot = serde_json::from_value(fixtures()["snapshot"].clone()).unwrap();
1160 let mut levels = Levels::from_snapshot(
1161 &snapshot.yes_dollars_fp,
1162 &snapshot.no_dollars_fp,
1163 false,
1164 &instrument,
1165 )
1166 .unwrap();
1167 let mut payload = fixtures()["steps"][0].clone();
1168 payload[field] = value.into();
1169
1170 let result = serde_json::from_value::<Delta>(payload)
1171 .map_err(anyhow::Error::from)
1172 .and_then(|payload| {
1173 levels.delta(
1174 &payload,
1175 &instrument,
1176 2,
1177 UnixNanos::from(1),
1178 UnixNanos::from(2),
1179 )
1180 });
1181
1182 assert!(result.is_err());
1183 }
1184
1185 #[rstest]
1186 fn canceled_confirmation_cannot_open_new_attempt_or_accept_late_snapshot() {
1187 let instrument = instrument();
1188 let id = instrument.id;
1189 let instruments = std::collections::HashMap::from([(id, instrument)]);
1190 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1191 books.subscribe(id, None).unwrap();
1192 books.state.lock().unwrap().running = true;
1193 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1194 let sender = sender.into();
1195 let now = UnixNanos::from(9);
1196 books
1197 .event(&subscribed(0, 7), &instruments, &sender, now)
1198 .unwrap();
1199 books
1200 .event(&snapshot(0, 1, None), &instruments, &sender, now)
1201 .unwrap();
1202 let _ = batch(&mut output);
1203 let recovery = books
1204 .state
1205 .lock()
1206 .unwrap()
1207 .books
1208 .get_mut(&id)
1209 .unwrap()
1210 .sync
1211 .claim()
1212 .unwrap();
1213 assert!(recovery.begin_replacement());
1214 let cancel = recovery.cancellation.child_token();
1215 let token = books
1216 .attempt(id, cancel.clone(), recovery.gate.clone(), &recovery)
1217 .unwrap();
1218 cancel.cancel();
1219 assert!(recovery.begin_replacement());
1220 books
1221 .event(
1222 &KalshiWsEvent::BookSnapshotRequested {
1223 generation: 0,
1224 sid: 7,
1225 id: 20,
1226 attempt: token,
1227 },
1228 &instruments,
1229 &sender,
1230 now,
1231 )
1232 .unwrap();
1233
1234 assert!(recovery.gate.lock().is_closed());
1235 books
1236 .event(&snapshot(0, 2, Some(20)), &instruments, &sender, now)
1237 .unwrap();
1238 assert!(output.try_recv().is_err());
1239 let token = books
1240 .attempt(
1241 id,
1242 recovery.cancellation.child_token(),
1243 recovery.gate.clone(),
1244 &recovery,
1245 )
1246 .unwrap();
1247 books
1248 .event(
1249 &KalshiWsEvent::BookSnapshotRequested {
1250 generation: 0,
1251 sid: 7,
1252 id: 21,
1253 attempt: token,
1254 },
1255 &instruments,
1256 &sender,
1257 now,
1258 )
1259 .unwrap();
1260
1261 books
1262 .event(&snapshot(0, 3, Some(20)), &instruments, &sender, now)
1263 .unwrap();
1264 assert!(output.try_recv().is_err());
1265 books
1266 .event(&snapshot(0, 4, Some(21)), &instruments, &sender, now)
1267 .unwrap();
1268 assert_eq!(batch(&mut output).sequence, 4);
1269 assert!(recovery.is_accepted());
1270 books.unsubscribe(id, None).unwrap();
1271 assert!(recovery.cancellation.is_cancelled());
1272 books
1273 .event(&snapshot(0, 5, Some(21)), &instruments, &sender, now)
1274 .unwrap();
1275 assert!(output.try_recv().is_err());
1276 }
1277
1278 #[rstest]
1279 fn reconnect_preserves_owner_and_rejects_stale_generation() {
1280 let instrument = instrument();
1281 let id = instrument.id;
1282 let instruments = std::collections::HashMap::from([(id, instrument)]);
1283 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1284 books.subscribe(id, None).unwrap();
1285 books.state.lock().unwrap().running = true;
1286 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1287 let sender = sender.into();
1288 let now = UnixNanos::from(9);
1289 books
1290 .event(&subscribed(0, 7), &instruments, &sender, now)
1291 .unwrap();
1292 let recovery = books
1293 .state
1294 .lock()
1295 .unwrap()
1296 .books
1297 .get_mut(&id)
1298 .unwrap()
1299 .sync
1300 .claim()
1301 .unwrap();
1302 assert!(recovery.begin_replacement());
1303 books
1304 .event(&subscribed(1, 7), &instruments, &sender, now)
1305 .unwrap();
1306 assert!(
1307 books.state.lock().unwrap().books[&id]
1308 .sync
1309 .is_current(&recovery)
1310 );
1311 assert!(
1312 books
1313 .state
1314 .lock()
1315 .unwrap()
1316 .books
1317 .get_mut(&id)
1318 .unwrap()
1319 .sync
1320 .claim()
1321 .is_none()
1322 );
1323 books
1324 .event(&snapshot(0, 1, None), &instruments, &sender, now)
1325 .unwrap();
1326 assert!(output.try_recv().is_err());
1327 let token = books
1328 .attempt(
1329 id,
1330 recovery.cancellation.child_token(),
1331 recovery.gate.clone(),
1332 &recovery,
1333 )
1334 .unwrap();
1335 books
1336 .event(
1337 &KalshiWsEvent::BookSnapshotRequested {
1338 generation: 1,
1339 sid: 7,
1340 id: 21,
1341 attempt: token,
1342 },
1343 &instruments,
1344 &sender,
1345 now,
1346 )
1347 .unwrap();
1348
1349 books
1350 .event(&snapshot(1, 2, Some(21)), &instruments, &sender, now)
1351 .unwrap();
1352 assert_eq!(batch(&mut output).sequence, 2);
1353 assert!(recovery.is_accepted());
1354 books.stop(false);
1355 books
1356 .event(&snapshot(1, 3, Some(21)), &instruments, &sender, now)
1357 .unwrap();
1358 assert!(output.try_recv().is_err());
1359 }
1360
1361 #[rstest]
1362 fn schedules_match_independent_publication_model() {
1363 let instrument = instrument();
1364 let id = instrument.id;
1365 let instruments = std::collections::HashMap::from([(id, instrument)]);
1366 let fixture = fixtures();
1367 let now = UnixNanos::from(9);
1368
1369 for schedule in 0_u32..6_u32.pow(5) {
1370 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1371 books.state.lock().unwrap().running = true;
1372 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1373 let sender = sender.into();
1374 let mut checker = BookStreamChecker::new(BookType::L2_MBP, true);
1375 let mut model = PublicationModel::default();
1376 let mut generation = 0;
1377 let mut seq = 0;
1378 let mut choices = schedule;
1379 for _ in 0..5 {
1380 let choice = choices % 6;
1381 choices /= 6;
1382 seq += 1;
1383
1384 match choice {
1385 0 if !model.active => {
1386 books.subscribe(id, None).unwrap();
1387 books
1388 .event(&subscribed(generation, 7), &instruments, &sender, now)
1389 .unwrap();
1390 checker.open(id);
1391 }
1392 1 => {
1393 books
1394 .event(&snapshot(generation, seq, None), &instruments, &sender, now)
1395 .unwrap();
1396 }
1397 2 => {
1398 let data = message(fixture["steps"][0].clone(), seq, None);
1399 books
1400 .event(
1401 &KalshiWsEvent::Message {
1402 generation,
1403 message: KalshiWsMessage::OrderbookDelta(data),
1404 },
1405 &instruments,
1406 &sender,
1407 now,
1408 )
1409 .unwrap();
1410 }
1411 3 => {
1412 books
1413 .event(
1414 &KalshiWsEvent::BookSequenceGap { generation, sid: 7 },
1415 &instruments,
1416 &sender,
1417 now,
1418 )
1419 .unwrap();
1420 }
1421 4 => {
1422 generation += 1;
1423 books
1424 .event(&subscribed(generation, 7), &instruments, &sender, now)
1425 .unwrap();
1426 }
1427 5 if model.active => {
1428 books.unsubscribe(id, None).unwrap();
1429 checker.close(id);
1430 }
1431 _ => {}
1432 }
1433
1434 if model.step(choice) {
1435 let batch = batch(&mut output);
1436 assert_eq!(batch.sequence, seq, "schedule {schedule}, choice {choice}");
1437 checker.apply(&batch).unwrap();
1438 let bids = BTreeMap::from([
1439 (decimal("0.123456"), model.bid_qty),
1440 (decimal("0.213456"), decimal("4.50")),
1441 ]);
1442 let asks = BTreeMap::from([
1443 (decimal("0.713456"), decimal("9.75")),
1444 (decimal("0.813456"), decimal("6.25")),
1445 ]);
1446 checker.verify(id, 100, &bids, &asks).unwrap();
1447 }
1448
1449 assert!(
1450 output.try_recv().is_err(),
1451 "unexpected output: schedule {schedule}, choice {choice}"
1452 );
1453 assert_eq!(
1454 books
1455 .state
1456 .lock()
1457 .unwrap()
1458 .books
1459 .get(&id)
1460 .is_some_and(|book| matches!(book.sync.phase(), BookPhase::Synced(_))),
1461 model.synced,
1462 "schedule {schedule}, choice {choice}"
1463 );
1464 }
1465 }
1466 }
1467
1468 #[rstest]
1469 fn multi_market_connection_schedule_matches_independent_oracle() {
1470 let first = instrument();
1471 let first_id = first.id;
1472 let mut second = first.clone();
1473 second.id = InstrumentId::from("MARKET-B.KALSHI");
1474 let second_id = second.id;
1475 let instruments = HashMap::from([(first_id, first), (second_id, second)]);
1476 let books = BookSyncTracker::new(&[first_id, second_id], Duration::from_secs(10));
1477 books.state.lock().unwrap().running = true;
1478 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1479 let sender = sender.into();
1480 let mut checker = BookStreamChecker::new(BookType::L2_MBP, true);
1481 checker.open(first_id);
1482 checker.open(second_id);
1483 let now = UnixNanos::from(9);
1484 let (mut bids, asks) = oracle_levels();
1485
1486 books.subscribe(first_id, None).unwrap();
1487 books.subscribe(second_id, None).unwrap();
1488 books
1489 .event(&subscribed(0, 7), &instruments, &sender, now)
1490 .unwrap();
1491 books
1492 .event(
1493 &snapshot_sid(0, 1, 99, "MARKET-A"),
1494 &instruments,
1495 &sender,
1496 now,
1497 )
1498 .unwrap();
1499 assert!(output.try_recv().is_err());
1500
1501 for (seq, ticker) in [(1, "MARKET-A"), (2, "MARKET-B")] {
1502 books
1503 .event(&snapshot_ticker(0, seq, ticker), &instruments, &sender, now)
1504 .unwrap();
1505 let emitted = batch(&mut output);
1506 checker.apply(&emitted).unwrap();
1507 checker
1508 .verify(emitted.instrument_id, 100, &bids, &asks)
1509 .unwrap();
1510 }
1511
1512 books
1513 .event(
1514 &delta_ticker(0, 3, 7, "MARKET-A", fixtures()["steps"][0].clone()),
1515 &instruments,
1516 &sender,
1517 now,
1518 )
1519 .unwrap();
1520 let emitted = batch(&mut output);
1521 assert_eq!(emitted.instrument_id, first_id);
1522 checker.apply(&emitted).unwrap();
1523 *bids.get_mut(&decimal("0.123456")).unwrap() += decimal("2.50");
1524 checker.verify(first_id, 100, &bids, &asks).unwrap();
1525 assert!(output.try_recv().is_err());
1526
1527 books
1528 .event(
1529 &KalshiWsEvent::BookSequenceGap {
1530 generation: 0,
1531 sid: 7,
1532 },
1533 &instruments,
1534 &sender,
1535 now,
1536 )
1537 .unwrap();
1538
1539 books
1540 .event(
1541 &delta_ticker(0, 4, 7, "MARKET-B", fixtures()["steps"][0].clone()),
1542 &instruments,
1543 &sender,
1544 now,
1545 )
1546 .unwrap();
1547 assert!(output.try_recv().is_err());
1548
1549 books
1550 .event(&subscribed(1, 8), &instruments, &sender, now)
1551 .unwrap();
1552 books
1553 .event(
1554 &snapshot_ticker(0, 5, "MARKET-A"),
1555 &instruments,
1556 &sender,
1557 now,
1558 )
1559 .unwrap();
1560 assert!(output.try_recv().is_err());
1561
1562 let (bids, asks) = oracle_levels();
1563
1564 for (seq, ticker, id) in [(1, "MARKET-A", first_id), (2, "MARKET-B", second_id)] {
1565 books
1566 .event(&snapshot_sid(1, seq, 8, ticker), &instruments, &sender, now)
1567 .unwrap();
1568 let emitted = batch(&mut output);
1569 assert_eq!(emitted.instrument_id, id);
1570 checker.apply(&emitted).unwrap();
1571 checker.verify(id, 100, &bids, &asks).unwrap();
1572 }
1573
1574 books.unsubscribe(first_id, None).unwrap();
1575 checker.close(first_id);
1576 books
1577 .event(
1578 &delta_ticker(1, 3, 8, "MARKET-B", fixtures()["steps"][1].clone()),
1579 &instruments,
1580 &sender,
1581 now,
1582 )
1583 .unwrap();
1584 let emitted = batch(&mut output);
1585 assert_eq!(emitted.instrument_id, second_id);
1586 checker.apply(&emitted).unwrap();
1587 let mut reduced_asks = asks.clone();
1588 *reduced_asks.get_mut(&decimal("0.713456")).unwrap() -= decimal("3.25");
1589 checker
1590 .verify(second_id, 100, &bids, &reduced_asks)
1591 .unwrap();
1592 assert!(output.try_recv().is_err());
1593
1594 books
1595 .event(
1596 &delta_ticker(1, 4, 8, "MARKET-A", fixtures()["steps"][0].clone()),
1597 &instruments,
1598 &sender,
1599 now,
1600 )
1601 .unwrap();
1602 assert!(output.try_recv().is_err());
1603 books.unsubscribe(second_id, None).unwrap();
1604 checker.close(second_id);
1605 books
1606 .event(
1607 &snapshot_sid(1, 5, 8, "MARKET-B"),
1608 &instruments,
1609 &sender,
1610 now,
1611 )
1612 .unwrap();
1613 assert!(output.try_recv().is_err());
1614 }
1615
1616 #[cfg(not(all(feature = "simulation", madsim)))]
1617 #[tokio::test]
1618 async fn expired_initial_wait_and_refused_task_are_retried_without_another_delta() {
1619 time::timeout(Duration::from_secs(8), async {
1620 let mut peer = peer::Peer::new().await;
1621 let instrument = instrument();
1622 let id = instrument.id;
1623 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1624 books.subscribe(id, None).unwrap();
1625 let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
1626 books.replay(&ws_client).unwrap();
1627 ws_client.connect().await.unwrap();
1628 let _ = peer.command().await;
1629 let instruments = std::collections::HashMap::from([(id, instrument)]);
1630 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1631 let sender = sender.into();
1632 let now = UnixNanos::from(9);
1633 let initial_tasks = TaskGroup::new();
1634 books.monitor(&ws_client, &initial_tasks.spawner().unwrap());
1635 assert_eq!(
1636 *books.state.lock().unwrap().books[&id].sync.phase(),
1637 BookPhase::Waiting
1638 );
1639 initial_tasks.begin_shutdown();
1640 initial_tasks
1641 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
1642 .await
1643 .unwrap();
1644
1645 loop {
1646 let event = input.recv().await.unwrap();
1647
1648 if matches!(
1649 event,
1650 KalshiWsEvent::Message {
1651 message: KalshiWsMessage::OrderbookSnapshot(_),
1652 ..
1653 }
1654 ) {
1655 break;
1656 }
1657
1658 books.event(&event, &instruments, &sender, now).unwrap();
1659 }
1660
1661 books
1662 .state
1663 .lock()
1664 .unwrap()
1665 .books
1666 .get_mut(&id)
1667 .unwrap()
1668 .sync
1669 .arm_deadline(time::Instant::now() - Duration::from_secs(1));
1670 let tasks = TaskGroup::new();
1671 let refused = tasks.spawner().unwrap();
1672 tasks.begin_shutdown();
1673 books.monitor(&ws_client, &refused);
1674 {
1675 let mut state = books.state.lock().unwrap();
1676 let recovery = state
1677 .books
1678 .get_mut(&id)
1679 .unwrap()
1680 .sync
1681 .claim()
1682 .expect("refused task must release ownership");
1683 recovery.cancellation.cancel();
1684 }
1685
1686 tasks
1687 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
1688 .await
1689 .unwrap();
1690 tasks.start_generation().unwrap();
1691 books.monitor(&ws_client, &tasks.spawner().unwrap());
1692 let command = peer.command().await;
1693 assert_eq!(command["params"]["action"], "get_snapshot");
1694
1695 loop {
1696 books
1697 .event(&input.recv().await.unwrap(), &instruments, &sender, now)
1698 .unwrap();
1699
1700 if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
1701 assert_eq!(batch.sequence, 2);
1702 break;
1703 }
1704 }
1705
1706 books.unsubscribe(id, None).unwrap();
1707 tasks.begin_shutdown();
1708 tasks
1709 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
1710 .await
1711 .unwrap();
1712 ws_client.dispose().await.unwrap();
1713 })
1714 .await
1715 .unwrap();
1716 }
1717
1718 #[rstest]
1719 #[case::missing_identity("market_ticker", None)]
1720 #[case::timestamp_overflow("ts_ms", Some(Value::from(u64::MAX)))]
1721 #[case::crossed("price_dollars", Some(Value::from("0.900000")))]
1722 #[case::negative_size("delta_fp", Some(Value::from("-100.00")))]
1723 #[case::invalid_side("side", Some(Value::from("invalid")))]
1724 fn invalid_frame_stops_publication(#[case] field: &str, #[case] value: Option<Value>) {
1725 let instrument = instrument();
1726 let id = instrument.id;
1727 let instruments = std::collections::HashMap::from([(id, instrument)]);
1728 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1729 books.subscribe(id, None).unwrap();
1730 books.state.lock().unwrap().running = true;
1731 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1732 let sender = sender.into();
1733 let now = UnixNanos::from(9);
1734 books
1735 .event(&subscribed(0, 7), &instruments, &sender, now)
1736 .unwrap();
1737 books
1738 .event(&snapshot(0, 1, None), &instruments, &sender, now)
1739 .unwrap();
1740 let _ = batch(&mut output);
1741 let mut payload = fixtures()["steps"][0].clone();
1742 if let Some(value) = value {
1743 payload[field] = value;
1744 } else {
1745 payload.as_object_mut().unwrap().remove(field);
1746 }
1747
1748 books
1749 .event(
1750 &KalshiWsEvent::Message {
1751 generation: 0,
1752 message: KalshiWsMessage::OrderbookDelta(message(payload, 2, None)),
1753 },
1754 &instruments,
1755 &sender,
1756 now,
1757 )
1758 .unwrap();
1759
1760 books
1761 .event(
1762 &KalshiWsEvent::Message {
1763 generation: 0,
1764 message: KalshiWsMessage::OrderbookDelta(message(
1765 fixtures()["steps"][0].clone(),
1766 3,
1767 None,
1768 )),
1769 },
1770 &instruments,
1771 &sender,
1772 now,
1773 )
1774 .unwrap();
1775
1776 assert_eq!(
1777 *books.state.lock().unwrap().books[&id].sync.phase(),
1778 BookPhase::Recovering
1779 );
1780 assert!(output.try_recv().is_err());
1781 }
1782
1783 #[derive(Default)]
1784 struct PublicationModel {
1785 active: bool,
1786 synced: bool,
1787 initial: bool,
1788 bid_qty: Decimal,
1789 }
1790
1791 impl PublicationModel {
1792 fn step(&mut self, choice: u32) -> bool {
1793 match choice {
1794 0 if !self.active => {
1795 self.active = true;
1796 self.synced = false;
1797 self.initial = true;
1798 }
1799 1 if self.active && self.initial => {
1800 self.synced = true;
1801 self.initial = false;
1802 self.bid_qty = decimal("7.25");
1803 return true;
1804 }
1805 2 if self.active && self.synced => {
1806 self.bid_qty += decimal("2.50");
1807 return true;
1808 }
1809 3 if self.active => {
1810 self.synced = false;
1811 self.initial = false;
1812 }
1813 4 if self.active => {
1814 self.synced = false;
1815 self.initial = true;
1816 }
1817 5 if self.active => {
1818 self.active = false;
1819 self.synced = false;
1820 self.initial = false;
1821 }
1822 _ => {}
1823 }
1824
1825 false
1826 }
1827 }
1828
1829 #[rstest]
1830 #[case::unsubscribe(0)]
1831 #[case::stop(1)]
1832 #[case::reset(2)]
1833 #[case::resubscribe(3)]
1834 fn retirement_before_confirmation_cannot_open_gate(#[case] action: u8) {
1835 let instrument = instrument();
1836 let id = instrument.id;
1837 let instruments = std::collections::HashMap::from([(id, instrument)]);
1838 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1839 books.subscribe(id, None).unwrap();
1840 books.state.lock().unwrap().running = true;
1841 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1842 let sender = sender.into();
1843 let now = UnixNanos::from(9);
1844 books
1845 .event(&subscribed(0, 7), &instruments, &sender, now)
1846 .unwrap();
1847 let owner = books
1848 .state
1849 .lock()
1850 .unwrap()
1851 .books
1852 .get_mut(&id)
1853 .unwrap()
1854 .sync
1855 .claim()
1856 .unwrap();
1857 assert!(owner.begin_replacement());
1858 let token = books
1859 .attempt(
1860 id,
1861 owner.cancellation.child_token(),
1862 owner.gate.clone(),
1863 &owner,
1864 )
1865 .unwrap();
1866
1867 match action {
1868 0 => books.unsubscribe(id, None).unwrap(),
1869 1 => books.stop(false),
1870 2 => books.stop(true),
1871 3 => {
1872 books.unsubscribe(id, None).unwrap();
1873 books.subscribe(id, None).unwrap();
1874 }
1875 _ => unreachable!(),
1876 }
1877
1878 books
1879 .event(
1880 &KalshiWsEvent::BookSnapshotRequested {
1881 generation: 0,
1882 sid: 7,
1883 id: 21,
1884 attempt: token,
1885 },
1886 &instruments,
1887 &sender,
1888 now,
1889 )
1890 .unwrap();
1891
1892 books
1893 .event(&snapshot(0, 2, Some(21)), &instruments, &sender, now)
1894 .unwrap();
1895 assert!(owner.cancellation.is_cancelled());
1896 assert!(owner.gate.lock().is_closed());
1897 assert!(output.try_recv().is_err());
1898
1899 if action == 3 {
1900 let result = books.attempt(id, CancellationToken::new(), owner.gate.clone(), &owner);
1901 assert_eq!(
1902 result.unwrap_err().to_string(),
1903 "client error: book recovery owner retired"
1904 );
1905 }
1906 }
1907
1908 #[cfg(not(all(feature = "simulation", madsim)))]
1909 #[tokio::test]
1910 async fn native_reconnect_retains_pending_recovery_owner() {
1911 time::timeout(Duration::from_secs(10), async {
1912 let mut peer = peer::Peer::new().await;
1913 let instrument = instrument();
1914 let id = instrument.id;
1915 let instruments = std::collections::HashMap::from([(id, instrument)]);
1916 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
1917 books.subscribe(id, None).unwrap();
1918 let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
1919 books.replay(&ws_client).unwrap();
1920 ws_client.connect().await.unwrap();
1921 let _ = peer.command().await;
1922 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
1923 let sender = sender.into();
1924 let now = UnixNanos::from(9);
1925
1926 loop {
1927 books
1928 .event(&input.recv().await.unwrap(), &instruments, &sender, now)
1929 .unwrap();
1930
1931 if output.try_recv().is_ok() {
1932 break;
1933 }
1934 }
1935
1936 let owner = books
1937 .state
1938 .lock()
1939 .unwrap()
1940 .books
1941 .get_mut(&id)
1942 .unwrap()
1943 .sync
1944 .claim()
1945 .unwrap();
1946 assert!(owner.begin_replacement());
1947 let cancel = owner.cancellation.child_token();
1948 let token = books
1949 .attempt(id, cancel.clone(), owner.gate.clone(), &owner)
1950 .unwrap();
1951 peer.confirm
1952 .store(false, std::sync::atomic::Ordering::SeqCst);
1953 ws_client
1954 .request_book_snapshot(&books.subscription, id.symbol.to_string(), token, cancel)
1955 .await
1956 .unwrap();
1957 let request = peer.command().await;
1958 assert_eq!(request["params"]["action"], "get_snapshot");
1959
1960 loop {
1961 let event = input.recv().await.unwrap();
1962 books.event(&event, &instruments, &sender, now).unwrap();
1963
1964 if matches!(event, KalshiWsEvent::BookSnapshotRequested { .. }) {
1965 break;
1966 }
1967 }
1968
1969 peer.confirm
1970 .store(true, std::sync::atomic::Ordering::SeqCst);
1971 peer.actions.send(("close".into(), Value::Null)).unwrap();
1972 ws_client.stop().await.unwrap();
1973 ws_client.connect().await.unwrap();
1974 let tasks = TaskGroup::new();
1975
1976 loop {
1977 let event = input.recv().await.unwrap();
1978 books.event(&event, &instruments, &sender, now).unwrap();
1979 books.monitor(&ws_client, &tasks.spawner().unwrap());
1980
1981 if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
1982 assert_eq!(batch.sequence, 1);
1983 assert_eq!(
1984 batch.flags,
1985 RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8
1986 );
1987 break;
1988 }
1989
1990 let mut state = books.state.lock().unwrap();
1991 assert!(state.books[&id].sync.is_current(&owner));
1992 assert!(state.books.get_mut(&id).unwrap().sync.claim().is_none());
1993 }
1994
1995 assert!(owner.is_accepted());
1996 assert!(!owner.cancellation.is_cancelled());
1997 books
1998 .event(
1999 &snapshot(0, 999, Some(request["id"].as_u64().unwrap())),
2000 &instruments,
2001 &sender,
2002 now,
2003 )
2004 .unwrap();
2005 assert!(output.try_recv().is_err());
2006 tasks.begin_shutdown();
2007 tasks
2008 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
2009 .await
2010 .unwrap();
2011 books.stop(false);
2012 ws_client.dispose().await.unwrap();
2013 })
2014 .await
2015 .unwrap();
2016 }
2017
2018 #[rstest]
2019 fn book_demand_survives_until_the_last_reference_retires() {
2020 let instrument = instrument();
2021 let id = instrument.id;
2022 let instruments = std::collections::HashMap::from([(id, instrument)]);
2023 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2024 books.subscribe(id, None).unwrap();
2025 books.subscribe(id, None).unwrap();
2026 books.state.lock().unwrap().running = true;
2027 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2028 let sender = sender.into();
2029 let now = UnixNanos::from(9);
2030 books
2031 .event(&subscribed(0, 7), &instruments, &sender, now)
2032 .unwrap();
2033 books
2034 .event(&snapshot(0, 1, None), &instruments, &sender, now)
2035 .unwrap();
2036 let _ = batch(&mut output);
2037 books.unsubscribe(id, None).unwrap();
2038
2039 let delta = KalshiWsEvent::Message {
2040 generation: 0,
2041 message: KalshiWsMessage::OrderbookDelta(message(
2042 fixtures()["steps"][0].clone(),
2043 2,
2044 None,
2045 )),
2046 };
2047
2048 books.event(&delta, &instruments, &sender, now).unwrap();
2049 let output_delta = batch(&mut output);
2050 assert_eq!(output_delta.sequence, 2);
2051 assert_eq!(
2052 output_delta.deltas[0].order.size.as_decimal(),
2053 decimal("9.75")
2054 );
2055 books.unsubscribe(id, None).unwrap();
2056 books.event(&delta, &instruments, &sender, now).unwrap();
2057 assert!(output.try_recv().is_err());
2058 }
2059
2060 #[cfg(not(all(feature = "simulation", madsim)))]
2061 #[tokio::test]
2062 async fn last_book_readd_after_rejected_unsubscribe_recovers() {
2063 time::timeout(Duration::from_secs(8), async {
2064 let mut peer = peer::Peer::new().await;
2065 let instrument = instrument();
2066 let id = instrument.id;
2067 let instruments = std::collections::HashMap::from([(id, instrument)]);
2068 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2069 books.subscribe(id, None).unwrap();
2070 let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
2071 books.replay(&ws_client).unwrap();
2072 ws_client.connect().await.unwrap();
2073 let _ = peer.command().await;
2074 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2075 let sender = sender.into();
2076 let now = UnixNanos::from(9);
2077
2078 loop {
2079 books
2080 .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2081 .unwrap();
2082
2083 if output.try_recv().is_ok() {
2084 break;
2085 }
2086 }
2087
2088 peer.confirm
2089 .store(false, std::sync::atomic::Ordering::SeqCst);
2090 books.unsubscribe(id, Some(&ws_client)).unwrap();
2091 let unsubscribe = peer.command().await;
2092 assert_eq!(unsubscribe["cmd"], "unsubscribe");
2093 books.subscribe(id, Some(&ws_client)).unwrap();
2094 peer.confirm
2095 .store(true, std::sync::atomic::Ordering::SeqCst);
2096 let mut frame = serde_json::from_str::<Value>(include_str!(
2097 "../../test_data/websocket_messages.json"
2098 ))
2099 .unwrap()["unsubscribe_rate_limit"]
2100 .clone();
2101 frame["id"] = serde_json::json!(unsubscribe["id"]);
2102 peer.actions.send(("raw".into(), frame)).unwrap();
2103
2104 loop {
2105 let event = input.recv().await.unwrap();
2106 books.event(&event, &instruments, &sender, now).unwrap();
2107
2108 if matches!(event, KalshiWsEvent::Ready { .. }) {
2109 break;
2110 }
2111 }
2112
2113 assert!(ws_client.is_active());
2114 let tasks = TaskGroup::new();
2115 books.monitor(&ws_client, &tasks.spawner().unwrap());
2116 let extra_owner = books
2117 .state
2118 .lock()
2119 .unwrap()
2120 .books
2121 .get_mut(&id)
2122 .unwrap()
2123 .sync
2124 .claim();
2125
2126 if let Some(owner) = extra_owner {
2127 owner.cancellation.cancel();
2128 panic!("ready retained SID left the re-added book without a recovery owner");
2129 }
2130
2131 let request = peer.command().await;
2132 assert_eq!(request["params"]["action"], "get_snapshot");
2133
2134 loop {
2135 books
2136 .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2137 .unwrap();
2138
2139 if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
2140 assert_eq!(batch.sequence, 2);
2141 break;
2142 }
2143 }
2144
2145 books.stop(false);
2146 tasks.begin_shutdown();
2147 tasks
2148 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
2149 .await
2150 .unwrap();
2151 ws_client.dispose().await.unwrap();
2152 })
2153 .await
2154 .unwrap();
2155 }
2156
2157 #[rstest]
2158 fn unsubscribe_ack_retires_sid_before_new_subscription() {
2159 let instrument = instrument();
2160 let id = instrument.id;
2161 let instruments = HashMap::from([(id, instrument)]);
2162 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2163 books.subscribe(id, None).unwrap();
2164 books.state.lock().unwrap().running = true;
2165 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2166 let sender = sender.into();
2167 let now = UnixNanos::from(9);
2168 books
2169 .event(&subscribed(0, 7), &instruments, &sender, now)
2170 .unwrap();
2171 books
2172 .event(&snapshot(0, 1, None), &instruments, &sender, now)
2173 .unwrap();
2174 let _ = batch(&mut output);
2175 books.unsubscribe(id, None).unwrap();
2176 books
2177 .event(
2178 &KalshiWsEvent::Message {
2179 generation: 0,
2180 message: KalshiWsMessage::Unsubscribed {
2181 id: Some(2),
2182 sid: 7,
2183 seq: 2,
2184 },
2185 },
2186 &instruments,
2187 &sender,
2188 now,
2189 )
2190 .unwrap();
2191
2192 assert_eq!(books.state.lock().unwrap().sid, None);
2193 books.subscribe(id, None).unwrap();
2194 books
2195 .event(&snapshot(0, 3, None), &instruments, &sender, now)
2196 .unwrap();
2197 assert!(output.try_recv().is_err());
2198 books
2199 .event(&subscribed(0, 9), &instruments, &sender, now)
2200 .unwrap();
2201 books
2202 .event(&snapshot(0, 4, None), &instruments, &sender, now)
2203 .unwrap();
2204 assert!(output.try_recv().is_err());
2205
2206 let KalshiWsEvent::Message {
2207 message: KalshiWsMessage::OrderbookSnapshot(mut data),
2208 ..
2209 } = snapshot(0, 1, None)
2210 else {
2211 unreachable!()
2212 };
2213
2214 data.sid = 9;
2215 books
2216 .event(
2217 &KalshiWsEvent::Message {
2218 generation: 0,
2219 message: KalshiWsMessage::OrderbookSnapshot(data),
2220 },
2221 &instruments,
2222 &sender,
2223 now,
2224 )
2225 .unwrap();
2226
2227 assert_eq!(batch(&mut output).sequence, 1);
2228 assert_eq!(books.state.lock().unwrap().sid, Some(9));
2229 }
2230
2231 #[rstest]
2232 fn last_book_readd_without_unsubscribe_ack_requires_correlated_snapshot() {
2233 let instrument = instrument();
2234 let id = instrument.id;
2235 let instruments = HashMap::from([(id, instrument)]);
2236 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2237 books.subscribe(id, None).unwrap();
2238 books.state.lock().unwrap().running = true;
2239 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2240 let sender = sender.into();
2241 let now = UnixNanos::from(9);
2242 books
2243 .event(&subscribed(0, 7), &instruments, &sender, now)
2244 .unwrap();
2245 books
2246 .event(&snapshot(0, 1, None), &instruments, &sender, now)
2247 .unwrap();
2248 let _ = batch(&mut output);
2249 books.unsubscribe(id, None).unwrap();
2250 books.subscribe(id, None).unwrap();
2251 books
2252 .event(&snapshot(0, 2, None), &instruments, &sender, now)
2253 .unwrap();
2254 assert!(output.try_recv().is_err());
2255 assert_eq!(books.state.lock().unwrap().sid, Some(7));
2256 let owner = books
2257 .state
2258 .lock()
2259 .unwrap()
2260 .books
2261 .get_mut(&id)
2262 .unwrap()
2263 .sync
2264 .claim()
2265 .unwrap();
2266 assert!(owner.begin_replacement());
2267 let token = books
2268 .attempt(
2269 id,
2270 owner.cancellation.child_token(),
2271 owner.gate.clone(),
2272 &owner,
2273 )
2274 .unwrap();
2275 books
2276 .event(
2277 &KalshiWsEvent::BookSnapshotRequested {
2278 generation: 0,
2279 sid: 7,
2280 id: 21,
2281 attempt: token,
2282 },
2283 &instruments,
2284 &sender,
2285 now,
2286 )
2287 .unwrap();
2288
2289 books
2290 .event(&snapshot(0, 3, Some(21)), &instruments, &sender, now)
2291 .unwrap();
2292 assert_eq!(batch(&mut output).sequence, 3);
2293 assert!(owner.is_accepted());
2294 }
2295
2296 #[cfg(not(all(feature = "simulation", madsim)))]
2297 #[tokio::test]
2298 async fn non_book_rejection_does_not_block_book_recovery() {
2299 time::timeout(Duration::from_secs(8), async {
2300 let mut peer = peer::Peer::new().await;
2301 let instrument = instrument();
2302 let id = instrument.id;
2303 let instruments = HashMap::from([(id, instrument)]);
2304 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2305 books.subscribe(id, None).unwrap();
2306 let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
2307 books.replay(&ws_client).unwrap();
2308 ws_client.connect().await.unwrap();
2309 let _ = peer.command().await;
2310 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2311 let sender = sender.into();
2312 let now = UnixNanos::from(9);
2313
2314 loop {
2315 books
2316 .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2317 .unwrap();
2318
2319 if output.try_recv().is_ok() {
2320 break;
2321 }
2322 }
2323
2324 peer.confirm
2325 .store(false, std::sync::atomic::Ordering::SeqCst);
2326 ws_client
2327 .subscribe(KalshiSubscription {
2328 channel: KalshiChannel::Ticker,
2329 market_tickers: vec!["MARKET-A".into()],
2330 })
2331 .unwrap();
2332
2333 let rejected = peer.command().await;
2334 assert_eq!(rejected["params"]["channels"][0], "ticker");
2335 let mut frame = serde_json::from_str::<Value>(include_str!(
2336 "../../test_data/websocket_messages.json"
2337 ))
2338 .unwrap()["command_rate_limit"]
2339 .clone();
2340 frame["id"] = serde_json::json!(rejected["id"]);
2341 peer.actions.send(("raw".into(), frame)).unwrap();
2342
2343 loop {
2344 let event = input.recv().await.unwrap();
2345 books.event(&event, &instruments, &sender, now).unwrap();
2346
2347 if matches!(event, KalshiWsEvent::Error { .. }) {
2348 break;
2349 }
2350 }
2351
2352 assert!(!ws_client.is_active());
2353 peer.confirm
2354 .store(true, std::sync::atomic::Ordering::SeqCst);
2355 let tasks = TaskGroup::new();
2356 books.monitor(&ws_client, &tasks.spawner().unwrap());
2357 let extra = books
2358 .state
2359 .lock()
2360 .unwrap()
2361 .books
2362 .get_mut(&id)
2363 .unwrap()
2364 .sync
2365 .claim();
2366 assert!(
2367 extra.is_none(),
2368 "unrelated command rejection prevented the book recovery owner"
2369 );
2370 let request = peer.command().await;
2371 assert_eq!(request["params"]["action"], "get_snapshot");
2372
2373 loop {
2374 books
2375 .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2376 .unwrap();
2377
2378 if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
2379 assert_eq!(batch.sequence, 2);
2380 break;
2381 }
2382 }
2383
2384 assert!(!ws_client.is_active());
2385 books.stop(false);
2386 tasks.begin_shutdown();
2387 tasks
2388 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
2389 .await
2390 .unwrap();
2391 ws_client.dispose().await.unwrap();
2392 })
2393 .await
2394 .unwrap();
2395 }
2396
2397 #[rstest]
2398 #[case::empty_arrays(0)]
2399 #[case::both_omitted(1)]
2400 #[case::yes_omitted(2)]
2401 #[case::no_omitted(3)]
2402 fn empty_and_one_sided_replacements_clear_the_published_book(#[case] shape: u8) {
2403 let instrument = instrument();
2404 let id = instrument.id;
2405 let instruments = HashMap::from([(id, instrument)]);
2406 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2407 books.subscribe(id, None).unwrap();
2408 books.state.lock().unwrap().running = true;
2409 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2410 let sender = sender.into();
2411 let now = UnixNanos::from(9);
2412 let mut checker = BookStreamChecker::new(BookType::L2_MBP, true);
2413 checker.open(id);
2414 books
2415 .event(&subscribed(0, 7), &instruments, &sender, now)
2416 .unwrap();
2417 books
2418 .event(&snapshot(0, 1, None), &instruments, &sender, now)
2419 .unwrap();
2420 checker.apply(&batch(&mut output)).unwrap();
2421 books
2422 .event(
2423 &KalshiWsEvent::BookSequenceGap {
2424 generation: 0,
2425 sid: 7,
2426 },
2427 &instruments,
2428 &sender,
2429 now,
2430 )
2431 .unwrap();
2432
2433 let owner = books
2434 .state
2435 .lock()
2436 .unwrap()
2437 .books
2438 .get_mut(&id)
2439 .unwrap()
2440 .sync
2441 .claim()
2442 .unwrap();
2443 assert!(owner.begin_replacement());
2444 let token = books
2445 .attempt(
2446 id,
2447 owner.cancellation.child_token(),
2448 owner.gate.clone(),
2449 &owner,
2450 )
2451 .unwrap();
2452 books
2453 .event(
2454 &KalshiWsEvent::BookSnapshotRequested {
2455 generation: 0,
2456 sid: 7,
2457 id: 21,
2458 attempt: token,
2459 },
2460 &instruments,
2461 &sender,
2462 now,
2463 )
2464 .unwrap();
2465
2466 let mut payload = fixtures()[if shape <= 1 { "empty" } else { "snapshot" }].clone();
2467
2468 for (side, remove) in [
2469 ("yes_dollars_fp", shape == 1 || shape == 2),
2470 ("no_dollars_fp", shape == 1 || shape == 3),
2471 ] {
2472 if remove {
2473 payload.as_object_mut().unwrap().remove(side);
2474 }
2475 }
2476
2477 books
2478 .event(
2479 &KalshiWsEvent::Message {
2480 generation: 0,
2481 message: KalshiWsMessage::OrderbookSnapshot(message(payload, 2, Some(21))),
2482 },
2483 &instruments,
2484 &sender,
2485 now,
2486 )
2487 .unwrap();
2488
2489 let replacement = batch(&mut output);
2490 assert_eq!(replacement.sequence, 2);
2491 let mut clear = OrderBookDelta::clear(id, 2, UnixNanos::from(1700000000001000000), now);
2492 if shape <= 1 {
2493 clear.flags |= RecordFlag::F_LAST as u8;
2494 }
2495
2496 assert_eq!(replacement.deltas[0], clear);
2497 assert_eq!(
2498 replacement.flags,
2499 RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8
2500 );
2501 checker.apply(&replacement).unwrap();
2502
2503 let bids = if shape == 3 {
2504 BTreeMap::from([
2505 (decimal("0.123456"), decimal("7.25")),
2506 (decimal("0.213456"), decimal("4.50")),
2507 ])
2508 } else {
2509 BTreeMap::new()
2510 };
2511
2512 let asks = if shape == 2 {
2513 BTreeMap::from([
2514 (decimal("0.713456"), decimal("9.75")),
2515 (decimal("0.813456"), decimal("6.25")),
2516 ])
2517 } else {
2518 BTreeMap::new()
2519 };
2520
2521 checker.verify(id, 100, &bids, &asks).unwrap();
2522 assert_eq!(replacement.deltas.len(), 1 + bids.len() + asks.len());
2523 assert!(owner.is_accepted());
2524 assert!(output.try_recv().is_err());
2525 }
2526
2527 #[rstest]
2528 fn snapshot_rejection_reaches_only_its_correlated_recovery() {
2529 use nautilus_live::book::recovery::BookRecoveryOutcome;
2530
2531 use crate::websocket::messages::KalshiServerError;
2532 let instrument = instrument();
2533 let id = instrument.id;
2534 let mut other = instrument.clone();
2535 other.id = InstrumentId::from("MARKET-B.KALSHI");
2536 let other_id = other.id;
2537 let instruments = HashMap::from([(id, instrument), (other_id, other)]);
2538 let books = BookSyncTracker::new(&[id, other_id], Duration::from_secs(10));
2539 let (sender, _output) = tokio::sync::mpsc::unbounded_channel();
2540 let sender = sender.into();
2541 books.state.lock().unwrap().running = true;
2542 let now = UnixNanos::from(9);
2543 books.subscribe(id, None).unwrap();
2544 books.subscribe(other_id, None).unwrap();
2545 books
2546 .event(&subscribed(0, 7), &instruments, &sender, now)
2547 .unwrap();
2548 let mut owners = Vec::new();
2549
2550 for (id, request) in [(id, 21), (other_id, 22)] {
2551 let owner = books
2552 .state
2553 .lock()
2554 .unwrap()
2555 .books
2556 .get_mut(&id)
2557 .unwrap()
2558 .sync
2559 .claim()
2560 .unwrap();
2561 assert!(owner.begin_replacement());
2562 let token = books
2563 .attempt(
2564 id,
2565 owner.cancellation.child_token(),
2566 owner.gate.clone(),
2567 &owner,
2568 )
2569 .unwrap();
2570 books
2571 .event(
2572 &KalshiWsEvent::BookSnapshotRequested {
2573 generation: 0,
2574 sid: 7,
2575 id: request,
2576 attempt: token,
2577 },
2578 &instruments,
2579 &sender,
2580 now,
2581 )
2582 .unwrap();
2583
2584 owners.push(owner);
2585 }
2586
2587 books
2588 .event(
2589 &KalshiWsEvent::Message {
2590 generation: 0,
2591 message: KalshiWsMessage::Error {
2592 id: Some(21),
2593 sid: Some(7),
2594 msg: KalshiServerError {
2595 code: 27,
2596 msg: "rate limit exceeded".into(),
2597 },
2598 },
2599 },
2600 &instruments,
2601 &sender,
2602 now,
2603 )
2604 .unwrap();
2605
2606 assert!(matches!(
2607 &*owners[0].outcome.borrow(),
2608 BookRecoveryOutcome::Rejected(_)
2609 ));
2610 assert!(matches!(
2611 &*owners[1].outcome.borrow(),
2612 BookRecoveryOutcome::Pending
2613 ));
2614 assert!(!owners[1].gate.lock().is_closed());
2615 assert!(!owners[1].cancellation.is_cancelled());
2616 }
2617
2618 #[rstest]
2619 #[case::disabled(0)]
2620 #[case::short(1)]
2621 #[case::nondefault(31)]
2622 fn snapshot_deadline_uses_the_configured_duration(#[case] seconds: u64) {
2623 let instrument = instrument();
2624 let id = instrument.id;
2625 let instruments = HashMap::from([(id, instrument)]);
2626 let books = BookSyncTracker::new(&[id], Duration::from_secs(seconds));
2627 books.subscribe(id, None).unwrap();
2628 books.state.lock().unwrap().running = true;
2629 let (sender, _output) = tokio::sync::mpsc::unbounded_channel();
2630 let sender = sender.into();
2631 let start = time::Instant::now();
2632 books
2633 .event(&subscribed(0, 7), &instruments, &sender, UnixNanos::from(9))
2634 .unwrap();
2635 let end = time::Instant::now();
2636 let mut state = books.state.lock().unwrap();
2637 let book = state.books.get_mut(&id).unwrap();
2638 assert!(book.waits_for_snapshot(Duration::from_secs(seconds)));
2639 book.gap();
2640 assert_eq!(
2641 book.waits_for_snapshot(Duration::from_secs(seconds)),
2642 seconds != 0
2643 );
2644
2645 if seconds == 0 {
2646 assert!(book.sync.has_pending_snapshot());
2647 assert!(!book.sync.is_send_pending());
2648 assert!(!book.sync.take_expired(end + Duration::from_secs(1000)));
2649 } else {
2650 assert!(
2651 !book
2652 .sync
2653 .take_expired(start + Duration::from_millis(seconds * 1000 - 1))
2654 );
2655 assert!(book.sync.take_expired(end + Duration::from_secs(seconds)));
2656 }
2657 }
2658
2659 #[cfg(not(all(feature = "simulation", madsim)))]
2660 #[tokio::test]
2661 async fn rejected_replay_keeps_a_recovery_owner() {
2662 time::timeout(Duration::from_secs(10), async {
2663 let mut peer = peer::Peer::new().await;
2664 let instrument = instrument();
2665 let id = instrument.id;
2666 let instruments = HashMap::from([(id, instrument)]);
2667 let books = BookSyncTracker::new(&[id], Duration::from_secs(10));
2668 books.subscribe(id, None).unwrap();
2669 let (ws_client, mut input) = KalshiWebSocketClient::new(peer.config.clone()).unwrap();
2670 books.replay(&ws_client).unwrap();
2671 ws_client.connect().await.unwrap();
2672 let _ = peer.command().await;
2673 let (sender, mut output) = tokio::sync::mpsc::unbounded_channel();
2674 let sender = sender.into();
2675 let now = UnixNanos::from(9);
2676
2677 loop {
2678 books
2679 .event(&input.recv().await.unwrap(), &instruments, &sender, now)
2680 .unwrap();
2681
2682 if output.try_recv().is_ok() {
2683 break;
2684 }
2685 }
2686
2687 peer.confirm
2688 .store(false, std::sync::atomic::Ordering::SeqCst);
2689 peer.actions.send(("close".into(), Value::Null)).unwrap();
2690 ws_client.stop().await.unwrap();
2691 let connect = ws_client.connect();
2692 tokio::pin!(connect);
2693 let rejected = tokio::select! {
2694 result = &mut connect => panic!("replayed connection finished before rejection: {result:?}"),
2695 command = peer.command() => command,
2696 };
2697 assert_eq!(rejected["cmd"], "subscribe");
2698 let mut frame = serde_json::from_str::<Value>(include_str!(
2699 "../../test_data/websocket_messages.json"
2700 ))
2701 .unwrap()["command_rate_limit"]
2702 .clone();
2703 frame["id"] = serde_json::json!(rejected["id"]);
2704 peer.actions.send(("raw".into(), frame)).unwrap();
2705
2706 loop {
2707 let event = input.recv().await.unwrap();
2708 books.event(&event, &instruments, &sender, now).unwrap();
2709
2710 if matches!(event, KalshiWsEvent::Error { generation: 1, .. }) {
2711 break;
2712 }
2713 }
2714
2715 assert_eq!(
2716 connect.await.unwrap_err().to_string(),
2717 "Kalshi WebSocket subscription restoration timed out"
2718 );
2719 peer.confirm
2720 .store(true, std::sync::atomic::Ordering::SeqCst);
2721 let tasks = TaskGroup::new();
2722 books.monitor(&ws_client, &tasks.spawner().unwrap());
2723 let extra = books
2724 .state
2725 .lock()
2726 .unwrap()
2727 .books
2728 .get_mut(&id)
2729 .unwrap()
2730 .sync
2731 .claim();
2732
2733 if let Some(owner) = extra {
2734 owner.cancellation.cancel();
2735 panic!("rejected replay left the book unowned");
2736 }
2737
2738 ws_client.stop().await.unwrap();
2739 ws_client.connect().await.unwrap();
2740 let replay = peer.command().await;
2741 assert_eq!(replay["cmd"], "subscribe");
2742
2743 loop {
2744 let event = input.recv().await.unwrap();
2745 books.event(&event, &instruments, &sender, now).unwrap();
2746
2747 if let Ok(DataEvent::Data(Data::BookDeltas(batch))) = output.try_recv() {
2748 assert_eq!(batch.sequence, 2);
2749 assert_eq!(
2750 batch.flags,
2751 RecordFlag::F_SNAPSHOT as u8 | RecordFlag::F_LAST as u8
2752 );
2753 break;
2754 }
2755
2756 let extra = books
2757 .state
2758 .lock()
2759 .unwrap()
2760 .books
2761 .get_mut(&id)
2762 .unwrap()
2763 .sync
2764 .claim();
2765
2766 if let Some(owner) = extra {
2767 owner.cancellation.cancel();
2768 panic!("reconnect replaced the running owner");
2769 }
2770 }
2771
2772 books.stop(false);
2773 tasks.begin_shutdown();
2774 tasks
2775 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
2776 .await
2777 .unwrap();
2778 ws_client.dispose().await.unwrap();
2779 })
2780 .await
2781 .unwrap();
2782 }
2783
2784 #[rstest]
2785 #[case::initial(0)]
2786 #[case::resubscribe(1)]
2787 #[case::reconnect(2)]
2788 fn zero_timeout_waits_for_initial_snapshot_after_subscription(#[case] transition: u8) {
2789 let instrument = instrument();
2790 let id = instrument.id;
2791 let instruments = HashMap::from([(id, instrument)]);
2792 let books = BookSyncTracker::new(&[id], Duration::ZERO);
2793 books.subscribe(id, None).unwrap();
2794 books.state.lock().unwrap().running = true;
2795 let (sender, _output) = tokio::sync::mpsc::unbounded_channel();
2796 let sender = sender.into();
2797 let now = UnixNanos::from(9);
2798 books
2799 .event(&subscribed(0, 7), &instruments, &sender, now)
2800 .unwrap();
2801
2802 if transition != 0 {
2803 books
2804 .event(
2805 &KalshiWsEvent::Message {
2806 generation: 0,
2807 message: KalshiWsMessage::OrderbookSnapshot(message(
2808 fixtures()["snapshot"].clone(),
2809 1,
2810 None,
2811 )),
2812 },
2813 &instruments,
2814 &sender,
2815 now,
2816 )
2817 .unwrap();
2818
2819 assert_eq!(
2820 books.state.lock().unwrap().books[&id].sync.position(),
2821 Some(&1)
2822 );
2823
2824 let event = if transition == 1 {
2825 KalshiWsEvent::Message {
2826 generation: 0,
2827 message: KalshiWsMessage::Unsubscribed {
2828 id: Some(2),
2829 sid: 7,
2830 seq: 1,
2831 },
2832 }
2833 } else {
2834 KalshiWsEvent::Ready { generation: 1 }
2835 };
2836
2837 books.event(&event, &instruments, &sender, now).unwrap();
2838 books
2839 .event(
2840 &subscribed(u64::from(transition == 2), 7),
2841 &instruments,
2842 &sender,
2843 now,
2844 )
2845 .unwrap();
2846 }
2847
2848 {
2849 let mut state = books.state.lock().unwrap();
2850 let book = state.books.get_mut(&id).unwrap();
2851 assert!(
2852 book.waits_for_snapshot(Duration::ZERO),
2853 "confirmed initial wait was treated as a gap"
2854 );
2855 assert!(
2856 !book
2857 .sync
2858 .take_expired(time::Instant::now() + Duration::from_secs(1000))
2859 );
2860 }
2861
2862 books
2863 .event(
2864 &KalshiWsEvent::BookSequenceGap {
2865 generation: u64::from(transition == 2),
2866 sid: 7,
2867 },
2868 &instruments,
2869 &sender,
2870 now,
2871 )
2872 .unwrap();
2873
2874 let state = books.state.lock().unwrap();
2875 assert!(
2876 !state.books[&id].waits_for_snapshot(Duration::ZERO),
2877 "a real gap failed to end the initial wait"
2878 );
2879 }
2880}