Skip to main content

nautilus_kalshi/websocket/
handler.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Command correlation and subscription restoration for one multiplexed connection.
17
18use std::{
19    collections::{BTreeMap, BTreeSet},
20    sync::{Arc, atomic::Ordering},
21    time::Duration,
22};
23
24use futures_util::{StreamExt, stream::FuturesUnordered};
25use nautilus_common::live::dst::time;
26use nautilus_network::{RECONNECTED, transport::Message, websocket::WebSocketClient};
27use tokio_util::sync::CancellationToken;
28
29use super::{
30    client::{KalshiWsEvent, Shared},
31    error::KalshiWsError,
32    messages::{
33        KalshiCommand, KalshiOperation, KalshiSubscription, KalshiUpdateAction, KalshiWsMessage,
34    },
35    parse::decode_frame,
36};
37use crate::common::enums::KalshiWsErrorCode;
38
39pub(super) struct KalshiWsHandler {
40    shared: Arc<Shared>,
41    ws_client: Arc<WebSocketClient>,
42    receiver: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
43    cancellation: CancellationToken,
44    epoch: u64,
45    generation: u64,
46    next_id: u64,
47    active: BTreeMap<String, ActiveSubscription>,
48    pending: BTreeMap<u64, PendingCommand>,
49    blocked: BTreeMap<String, Option<KalshiSubscription>>,
50    sequences: BTreeMap<u64, u64>,
51}
52
53impl KalshiWsHandler {
54    pub(super) fn new(
55        shared: Arc<Shared>,
56        ws_client: Arc<WebSocketClient>,
57        receiver: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
58        cancellation: CancellationToken,
59    ) -> Self {
60        let generation = shared.generation.fetch_add(1, Ordering::AcqRel);
61
62        Self {
63            shared,
64            ws_client,
65            receiver,
66            cancellation,
67            epoch: 0,
68            generation,
69            next_id: 1,
70            active: BTreeMap::new(),
71            pending: BTreeMap::new(),
72            blocked: BTreeMap::new(),
73            sequences: BTreeMap::new(),
74        }
75    }
76
77    pub(super) async fn run(mut self) {
78        loop {
79            if self.cancellation.is_cancelled() || self.shared.events.is_closed() {
80                break;
81            }
82
83            let pending = self.pending.len();
84            self.pending.retain(|_, command| {
85                command
86                    .book_cancel
87                    .as_ref()
88                    .is_none_or(|cancel| !cancel.is_cancelled())
89            });
90
91            if self.pending.len() != pending {
92                self.reconcile().await;
93            }
94
95            let timeout = Duration::from_secs(self.shared.config.ws_timeout_secs);
96            let deadline = self
97                .pending
98                .values()
99                .filter(|pending| pending.book_cancel.is_none())
100                .map(|pending| pending.deadline)
101                .min()
102                .unwrap_or_else(|| time::Instant::now() + timeout);
103            let mut snapshots: FuturesUnordered<_> = self
104                .pending
105                .values()
106                .filter_map(|pending| pending.book_cancel.clone())
107                .map(CancellationToken::cancelled_owned)
108                .collect();
109            tokio::select! {
110                biased;
111                () = self.cancellation.cancelled() => break,
112                () = self.shared.events.closed() => break,
113                Some(()) = snapshots.next() => {},
114                () = time::sleep_until(deadline) => {
115                    self.expire().await;
116
117                    if self.ws_client.is_closed() { break; }
118                }
119                command = self.receiver.recv() => {
120                    let Some(command) = command else { break; };
121                    self.command(command).await;
122                }
123            }
124        }
125
126        self.shared.ready.store(false, Ordering::Release);
127        self.shared.auth.fail("Kalshi WebSocket handler stopped");
128        self.ws_client.disconnect().await;
129    }
130
131    async fn expire(&mut self) {
132        let queued = self.receiver.len();
133        for _ in 0..queued {
134            if self.cancellation.is_cancelled() {
135                return;
136            }
137
138            let Ok(command) = self.receiver.try_recv() else {
139                break;
140            };
141
142            self.command(command).await;
143        }
144
145        self.expire_pending();
146    }
147
148    fn expire_pending(&mut self) {
149        if self.pending.values().any(|pending| {
150            pending.book_cancel.is_none() && pending.deadline <= time::Instant::now()
151        }) {
152            self.error("Kalshi WebSocket command acknowledgement timed out");
153            self.recover();
154        }
155    }
156
157    async fn command(&mut self, command: HandlerCommand) {
158        if let HandlerCommand::Frame { received, .. } = &command
159            && self
160                .pending
161                .values()
162                .any(|pending| pending.book_cancel.is_none() && pending.deadline < *received)
163        {
164            self.expire_pending();
165        }
166
167        match command {
168            HandlerCommand::Reconcile => self.reconcile().await,
169            HandlerCommand::Snapshot { topic, markets } => self.snapshot(topic, markets).await,
170            HandlerCommand::BookSnapshot {
171                topic,
172                markets,
173                attempt,
174                cancel,
175                completed,
176            } => {
177                let result = self.book_snapshot(topic, markets, attempt, cancel).await;
178                let _ = completed.send(result);
179            }
180            HandlerCommand::Frame { epoch, message, .. } => self.frame(epoch, message).await,
181        }
182    }
183
184    async fn reconcile(&mut self) {
185        if !self.ws_client.is_active()
186            || !self.shared.auth.is_authenticated()
187            || !self.owns_connection()
188        {
189            return;
190        }
191
192        let desired = self
193            .shared
194            .state
195            .lock()
196            .expect("client state lock is not poisoned")
197            .desired
198            .clone();
199        self.blocked
200            .retain(|topic, failed| desired.get(topic) == failed.as_ref());
201        let mut topics: BTreeSet<String> = desired.keys().cloned().collect();
202        topics.extend(self.active.keys().cloned());
203        for topic in topics {
204            if self.blocked.contains_key(&topic)
205                || self.pending.values().any(|pending| pending.topic == topic)
206            {
207                continue;
208            }
209
210            let wanted = self
211                .shared
212                .state
213                .lock()
214                .expect("client state lock is not poisoned")
215                .desired
216                .get(&topic)
217                .cloned();
218            let wanted = wanted.as_ref();
219            let active = self.active.get(&topic);
220
221            let operation = match (wanted, active) {
222                (Some(args), None) => KalshiOperation::Subscribe {
223                    channels: [args.channel],
224                    market_tickers: args.market_tickers.clone(),
225                    use_yes_price: (args.channel == super::messages::KalshiChannel::OrderbookDelta)
226                        .then_some(true),
227                    send_initial_snapshot: (args.channel == super::messages::KalshiChannel::Ticker)
228                        .then_some(true),
229                },
230                (None, Some(active)) => KalshiOperation::Unsubscribe { sids: [active.sid] },
231                (Some(args), Some(active)) if args != &active.args => {
232                    let additions: Vec<_> = args
233                        .market_tickers
234                        .iter()
235                        .filter(|market| !active.args.market_tickers.contains(market))
236                        .cloned()
237                        .collect();
238
239                    let (action, markets) = if additions.is_empty() {
240                        (
241                            KalshiUpdateAction::DeleteMarkets,
242                            active
243                                .args
244                                .market_tickers
245                                .iter()
246                                .filter(|market| !args.market_tickers.contains(market))
247                                .cloned()
248                                .collect(),
249                        )
250                    } else {
251                        (KalshiUpdateAction::AddMarkets, additions)
252                    };
253
254                    KalshiOperation::UpdateSubscription {
255                        sid: active.sid,
256                        market_tickers: markets,
257                        action,
258                        send_initial_snapshot: (args.channel
259                            == super::messages::KalshiChannel::Ticker
260                            && action == KalshiUpdateAction::AddMarkets)
261                            .then_some(true),
262                    }
263                }
264                (Some(_), Some(_)) => {
265                    self.shared.subscriptions.confirm_subscribe(&topic);
266                    continue;
267                }
268                _ => continue,
269            };
270
271            if !self.send(topic, operation, wanted.cloned()).await {
272                break;
273            }
274        }
275
276        self.publish_ready();
277    }
278
279    async fn send(
280        &mut self,
281        topic: String,
282        operation: KalshiOperation,
283        desired: Option<KalshiSubscription>,
284    ) -> bool {
285        let id = self.next_id;
286        self.next_id += 1;
287        let command = KalshiCommand { id, operation };
288        let wire = serde_json::to_string(&command).expect("command serialization is infallible");
289        let timeout = Duration::from_secs(self.shared.config.ws_timeout_secs);
290        self.pending.insert(
291            id,
292            PendingCommand {
293                topic,
294                operation: command.operation,
295                desired,
296                deadline: time::Instant::now() + timeout,
297                book_cancel: None,
298            },
299        );
300
301        if !matches!(
302            time::timeout(
303                timeout,
304                self.ws_client
305                    .send_text_on_connection(wire, None, self.epoch)
306            )
307            .await,
308            Ok(Ok(()))
309        ) {
310            if self.owns_connection() {
311                self.error("Kalshi WebSocket command send failed");
312            }
313
314            self.recover();
315            return false;
316        }
317
318        true
319    }
320
321    async fn snapshot(&mut self, topic: String, markets: Vec<String>) {
322        if !self.ws_client.is_active()
323            || self.pending.values().any(|pending| pending.topic == topic)
324        {
325            self.error("Kalshi WebSocket snapshot requires a confirmed subscription");
326            return;
327        }
328
329        let Some(active) = self.active.get(&topic) else {
330            return;
331        };
332
333        let operation = KalshiOperation::UpdateSubscription {
334            sid: active.sid,
335            market_tickers: markets,
336            action: KalshiUpdateAction::GetSnapshot,
337            send_initial_snapshot: None,
338        };
339
340        let desired = Some(active.args.clone());
341        self.send(topic, operation, desired).await;
342    }
343
344    async fn frame(&mut self, epoch: u64, message: Message) {
345        if let Message::Text(text) = &message
346            && text.as_ref() == RECONNECTED.as_bytes()
347        {
348            if epoch <= self.epoch
349                || epoch
350                    != self
351                        .ws_client
352                        .connection_epoch_atomic()
353                        .load(Ordering::Acquire)
354            {
355                return;
356            }
357
358            self.epoch = epoch;
359            self.generation = self.shared.generation.fetch_add(1, Ordering::AcqRel);
360            self.active.clear();
361            self.pending.clear();
362            self.blocked.clear();
363            self.sequences.clear();
364            self.shared.ready.store(false, Ordering::Release);
365            self.shared.subscriptions.reset_after_reconnect();
366            self.shared.auth.succeed();
367            self.reconcile().await;
368            return;
369        }
370
371        if epoch != self.epoch
372            || !self.ws_client.is_active()
373            || !self.shared.auth.is_authenticated()
374        {
375            return;
376        }
377
378        let bytes = match &message {
379            Message::Text(text) => text.as_ref(),
380            Message::Binary(bytes) => bytes.as_ref(),
381            _ => return,
382        };
383
384        let decoded = match decode_frame(bytes) {
385            Ok(decoded) => decoded,
386            Err(_) => {
387                self.error("Malformed Kalshi WebSocket frame");
388                return;
389            }
390        };
391
392        if !self.sequence(&decoded) {
393            return;
394        }
395
396        if decoded.data().is_some() {
397            self.dispatch_data(decoded).await;
398            return;
399        }
400
401        if self.acknowledge(&decoded) {
402            let _ = self.shared.events.send(KalshiWsEvent::Message {
403                generation: self.generation,
404                message: decoded,
405            });
406
407            self.reconcile().await;
408        }
409    }
410
411    async fn dispatch_data(&mut self, decoded: KalshiWsMessage) {
412        let snapshot_completed = match &decoded {
413            KalshiWsMessage::OrderbookSnapshot(data) if data.id.is_some() => {
414                let Some(completed) = self.acknowledge_snapshot(data) else {
415                    return;
416                };
417
418                completed
419            }
420            _ => false,
421        };
422
423        if let Some((channel, data)) = decoded.data() {
424            let topic = self.active.iter().find_map(|(topic, active)| {
425                (active.sid == data.sid && active.args.channel == channel).then_some(topic)
426            });
427
428            if let Some(topic) = topic
429                && self
430                    .shared
431                    .state
432                    .lock()
433                    .expect("client state lock is not poisoned")
434                    .desired
435                    .contains_key(topic)
436            {
437                let _ = self.shared.events.send(KalshiWsEvent::Message {
438                    generation: self.generation,
439                    message: decoded,
440                });
441            }
442
443            if snapshot_completed {
444                self.reconcile().await;
445            }
446        }
447    }
448
449    async fn book_snapshot(
450        &mut self,
451        topic: String,
452        markets: Vec<String>,
453        attempt: u64,
454        cancel: CancellationToken,
455    ) -> Result<(), KalshiWsError> {
456        if cancel.is_cancelled()
457            || !self.ws_client.is_active()
458            || !self.shared.auth.is_authenticated()
459            || !self.owns_connection()
460        {
461            return Err(KalshiWsError::Client(
462                "snapshot connection is unavailable".into(),
463            ));
464        }
465
466        let Some(active) = self.active.get(&topic) else {
467            if self.blocked.contains_key(&topic) {
468                self.recover();
469            }
470
471            return Err(KalshiWsError::Client(
472                "subscription is not confirmed".into(),
473            ));
474        };
475
476        if self.pending.values().any(|pending| {
477            pending.topic == topic
478                && !matches!(
479                    pending.operation,
480                    KalshiOperation::UpdateSubscription {
481                        action: KalshiUpdateAction::GetSnapshot,
482                        ..
483                    }
484                )
485        }) {
486            return Err(KalshiWsError::Client(
487                "subscription has a pending command".into(),
488            ));
489        }
490
491        let sid = active.sid;
492        let desired = Some(active.args.clone());
493        let id = self.next_id;
494
495        let operation = KalshiOperation::UpdateSubscription {
496            sid,
497            market_tickers: markets,
498            action: KalshiUpdateAction::GetSnapshot,
499            send_initial_snapshot: None,
500        };
501
502        let sent = tokio::select! {
503            biased;
504            () = cancel.cancelled() => false,
505            sent = self.send(topic, operation, desired) => sent,
506        };
507
508        if !sent || cancel.is_cancelled() || !self.owns_connection() {
509            self.pending.remove(&id);
510            return Err(KalshiWsError::Client(
511                "snapshot write did not complete on the intended connection".into(),
512            ));
513        }
514
515        self.pending
516            .get_mut(&id)
517            .expect("snapshot write is pending")
518            .book_cancel = Some(cancel);
519
520        self.shared
521            .events
522            .send(KalshiWsEvent::BookSnapshotRequested {
523                generation: self.generation,
524                sid,
525                id,
526                attempt,
527            })
528            .map_err(|_| KalshiWsError::Client("book event receiver closed".into()))?;
529
530        Ok(())
531    }
532
533    fn sequence(&mut self, message: &KalshiWsMessage) -> bool {
534        let position = match message {
535            KalshiWsMessage::Ok {
536                sid: Some(sid),
537                seq: Some(seq),
538                ..
539            }
540            | KalshiWsMessage::Unsubscribed { sid, seq, .. } => Some((*sid, *seq)),
541            _ => message
542                .data()
543                .and_then(|(_, data)| data.seq.map(|seq| (data.sid, seq))),
544        };
545
546        let Some((sid, seq)) = position else {
547            return true;
548        };
549
550        let Some(previous) = self.sequences.get_mut(&sid) else {
551            return true;
552        };
553
554        let Some(gap) = advance_sid_sequence(*previous, seq) else {
555            return false;
556        };
557
558        if gap {
559            log::warn!("Kalshi book sequence gap on SID {sid}: after {previous}, received {seq}");
560
561            let _ = self.shared.events.send(KalshiWsEvent::BookSequenceGap {
562                generation: self.generation,
563                sid,
564            });
565        }
566
567        *previous = seq;
568        true
569    }
570
571    fn acknowledge_snapshot(&mut self, data: &super::messages::KalshiDataMessage) -> Option<bool> {
572        let id = data.id?;
573        let pending = self.pending.get_mut(&id)?;
574
575        let KalshiOperation::UpdateSubscription {
576            sid,
577            market_tickers,
578            action: KalshiUpdateAction::GetSnapshot,
579            ..
580        } = &mut pending.operation
581        else {
582            return None;
583        };
584
585        if *sid != data.sid {
586            return None;
587        }
588
589        let market = data
590            .msg
591            .get("market_ticker")
592            .and_then(serde_json::Value::as_str)?;
593
594        if !market_tickers.iter().any(|ticker| ticker == market) {
595            return None;
596        }
597
598        market_tickers.retain(|ticker| ticker != market);
599        let completed = market_tickers.is_empty();
600        if completed {
601            self.pending.remove(&id);
602        }
603
604        Some(completed)
605    }
606
607    fn acknowledge(&mut self, message: &KalshiWsMessage) -> bool {
608        match message {
609            KalshiWsMessage::Subscribed { id, msg } => self.acknowledge_subscribe(id, msg),
610            KalshiWsMessage::Unsubscribed { id, sid, .. } => self.acknowledge_unsubscribe(id, sid),
611            KalshiWsMessage::Ok {
612                id: Some(id),
613                sid: Some(sid),
614                msg,
615                ..
616            } => self.acknowledge_update(id, sid, msg),
617            KalshiWsMessage::Error { id, sid, msg } => self.acknowledge_error(id, sid, msg),
618            _ => false,
619        }
620    }
621
622    fn acknowledge_subscribe(&mut self, id: &u64, msg: &super::messages::KalshiSubscribed) -> bool {
623        let Some(pending) = self.pending.get(id) else {
624            return false;
625        };
626
627        let KalshiOperation::Subscribe { channels, .. } = &pending.operation else {
628            return false;
629        };
630
631        if channels[0] != msg.channel
632            || msg.sid == 0
633            || self.active.values().any(|active| active.sid == msg.sid)
634        {
635            return false;
636        }
637
638        let pending = self.pending.remove(id).expect("pending command exists");
639        let args = pending.desired.expect("subscribe retains arguments");
640        if self
641            .shared
642            .state
643            .lock()
644            .expect("client state lock is not poisoned")
645            .desired
646            .get(&pending.topic)
647            == Some(&args)
648        {
649            self.shared.subscriptions.confirm_subscribe(&pending.topic);
650        }
651
652        self.active
653            .insert(pending.topic, ActiveSubscription { sid: msg.sid, args });
654        if msg.channel == super::messages::KalshiChannel::OrderbookDelta {
655            self.sequences.insert(msg.sid, 0);
656        }
657
658        true
659    }
660
661    fn acknowledge_unsubscribe(&mut self, id: &Option<u64>, sid: &u64) -> bool {
662        let id = id.or_else(|| self.pending.iter().find_map(|(id, pending)| matches!(pending.operation, KalshiOperation::Unsubscribe { sids } if sids[0] == *sid).then_some(*id)));
663
664        let Some(id) = id else {
665            return false;
666        };
667
668        let Some(pending) = self.pending.get(&id) else {
669            return false;
670        };
671
672        if !matches!(pending.operation, KalshiOperation::Unsubscribe { sids } if sids[0] == *sid) {
673            return false;
674        }
675
676        let pending = self.pending.remove(&id).expect("pending command exists");
677        self.active.remove(&pending.topic);
678        self.sequences.remove(sid);
679        self.shared
680            .subscriptions
681            .confirm_unsubscribe(&pending.topic);
682        true
683    }
684
685    fn acknowledge_update(
686        &mut self,
687        id: &u64,
688        sid: &u64,
689        msg: &Option<super::messages::KalshiUpdated>,
690    ) -> bool {
691        let Some(pending) = self.pending.get(id) else {
692            return false;
693        };
694
695        let KalshiOperation::UpdateSubscription {
696            sid: expected,
697            market_tickers,
698            action,
699            ..
700        } = &pending.operation
701        else {
702            return false;
703        };
704
705        if sid != expected || *action == KalshiUpdateAction::GetSnapshot {
706            return false;
707        }
708
709        let Some(active) = self.active.get_mut(&pending.topic) else {
710            return false;
711        };
712
713        if *action != KalshiUpdateAction::GetSnapshot {
714            let mut updated = active.args.market_tickers.clone();
715            match action {
716                KalshiUpdateAction::AddMarkets => updated.extend(market_tickers.iter().cloned()),
717                KalshiUpdateAction::DeleteMarkets => {
718                    updated.retain(|market| !market_tickers.contains(market));
719                }
720                KalshiUpdateAction::GetSnapshot => unreachable!(),
721            }
722
723            updated.sort();
724            updated.dedup();
725
726            let Some(msg) = msg else {
727                return false;
728            };
729
730            let mut confirmed = msg.market_tickers.clone();
731            confirmed.sort();
732            if confirmed != updated {
733                return false;
734            }
735
736            active.args.market_tickers = updated;
737        }
738
739        let pending = self.pending.remove(id).expect("pending command exists");
740        if self
741            .shared
742            .state
743            .lock()
744            .expect("client state lock is not poisoned")
745            .desired
746            .get(&pending.topic)
747            == Some(&active.args)
748        {
749            self.shared.subscriptions.confirm_subscribe(&pending.topic);
750        }
751
752        true
753    }
754
755    fn acknowledge_error(
756        &mut self,
757        id: &Option<u64>,
758        sid: &Option<u64>,
759        msg: &super::messages::KalshiServerError,
760    ) -> bool {
761        let pending = id.and_then(|id| self.pending.get(&id));
762        if id.is_some() && pending.is_none() {
763            return false;
764        }
765
766        let expected_sid = pending.and_then(|pending| match pending.operation {
767            KalshiOperation::Subscribe { .. } => None,
768            KalshiOperation::Unsubscribe { sids } => Some(sids[0]),
769            KalshiOperation::UpdateSubscription { sid, .. } => Some(sid),
770        });
771
772        if let Some(sid) = sid {
773            if id.is_some() {
774                if expected_sid != Some(*sid) {
775                    return false;
776                }
777            } else if !self.active.values().any(|active| active.sid == *sid) {
778                return false;
779            }
780        }
781
782        let terminal = matches!(
783            KalshiWsErrorCode::from_repr(msg.code),
784            Some(
785                KalshiWsErrorCode::AuthenticationRequired
786                    | KalshiWsErrorCode::ChannelError
787                    | KalshiWsErrorCode::SubscriptionBufferOverflow
788            )
789        );
790
791        if !terminal && pending.is_some_and(|pending| pending.book_cancel.is_some()) {
792            self.pending
793                .remove(&id.expect("snapshot command has an ID"));
794            return true;
795        }
796
797        self.error(&format!("Kalshi WebSocket protocol error {}", msg.code));
798
799        if terminal {
800            self.recover();
801            return true;
802        }
803
804        let Some(id) = id else {
805            return true;
806        };
807
808        let pending = self.pending.remove(id).expect("pending command exists");
809        self.shared.subscriptions.mark_failure(&pending.topic);
810        self.blocked.insert(pending.topic, pending.desired);
811        true
812    }
813
814    fn recover(&mut self) {
815        self.shared.ready.store(false, Ordering::Release);
816        self.shared.auth.invalidate();
817        self.pending.clear();
818
819        if self.owns_connection() {
820            let _ = self.ws_client.request_reconnect();
821        }
822    }
823
824    fn owns_connection(&self) -> bool {
825        self.epoch
826            == self
827                .ws_client
828                .connection_epoch_atomic()
829                .load(Ordering::Acquire)
830    }
831
832    fn publish_ready(&self) {
833        let state = self
834            .shared
835            .state
836            .lock()
837            .expect("client state lock is not poisoned");
838
839        let ready = self.pending.is_empty()
840            && self.owns_connection()
841            && self.ws_client.is_active()
842            && self.shared.auth.is_authenticated()
843            && self.active.len() == state.desired.len()
844            && state.desired.iter().all(|(topic, args)| {
845                self.active
846                    .get(topic)
847                    .is_some_and(|active| &active.args == args)
848            });
849
850        if ready && !self.shared.ready.swap(true, Ordering::AcqRel) {
851            let _ = self.shared.events.send(KalshiWsEvent::Ready {
852                generation: self.generation,
853            });
854
855            self.shared.ready_notify.notify_waiters();
856        }
857    }
858
859    fn error(&self, message: &str) {
860        log::warn!("{message}");
861
862        let _ = self.shared.events.send(KalshiWsEvent::Error {
863            generation: self.generation,
864            message: message.to_owned(),
865        });
866    }
867}
868
869pub(super) enum HandlerCommand {
870    Reconcile,
871    Snapshot {
872        topic: String,
873        markets: Vec<String>,
874    },
875    BookSnapshot {
876        topic: String,
877        markets: Vec<String>,
878        attempt: u64,
879        cancel: CancellationToken,
880        completed: tokio::sync::oneshot::Sender<Result<(), KalshiWsError>>,
881    },
882    Frame {
883        epoch: u64,
884        received: time::Instant,
885        message: Message,
886    },
887}
888
889struct ActiveSubscription {
890    sid: u64,
891    args: KalshiSubscription,
892}
893
894struct PendingCommand {
895    topic: String,
896    operation: KalshiOperation,
897    desired: Option<KalshiSubscription>,
898    deadline: time::Instant,
899    book_cancel: Option<CancellationToken>,
900}
901
902/// Returns how `seq` advances from `previous`.
903///
904/// `None` means `seq` is not strictly newer and the frame must be ignored. `Some(false)` means
905/// `seq` is the next value. `Some(true)` means at least one sequence value was skipped.
906#[must_use]
907pub fn advance_sid_sequence(previous: u64, seq: u64) -> Option<bool> {
908    if seq <= previous {
909        return None;
910    }
911
912    Some(previous.checked_add(1) != Some(seq))
913}
914
915#[cfg(test)]
916mod sequence_tests {
917    use rstest::rstest;
918
919    use super::advance_sid_sequence;
920
921    #[rstest]
922    fn advance_sid_sequence_rejects_stale_values_and_reports_gaps() {
923        assert_eq!(advance_sid_sequence(4, 4), None);
924        assert_eq!(advance_sid_sequence(4, 3), None);
925        assert_eq!(advance_sid_sequence(4, 5), Some(false));
926        assert_eq!(advance_sid_sequence(4, 7), Some(true));
927        assert_eq!(advance_sid_sequence(u64::MAX, u64::MAX), None);
928    }
929}
930
931#[cfg(all(test, not(all(feature = "simulation", madsim))))]
932mod tests {
933    use std::{
934        sync::{Arc, atomic::Ordering},
935        time::Duration,
936    };
937
938    use futures_util::{SinkExt, StreamExt};
939    use nautilus_common::live::dst::{task, time};
940    use nautilus_network::{
941        RECONNECTED,
942        dst::net::TcpListener,
943        transport::Message as TransportMessage,
944        websocket::{WebSocketClient, auth::AuthState},
945    };
946    use rstest::rstest;
947    use tokio_tungstenite::{accept_async, tungstenite::Message};
948    use tokio_util::sync::CancellationToken;
949
950    use super::{HandlerCommand, KalshiWsHandler};
951    use crate::{
952        config::KalshiDataClientConfig,
953        websocket::{
954            client::{
955                KalshiWsEvent,
956                tests::{shared, transport_config},
957            },
958            messages::{
959                KalshiChannel, KalshiDataMessage, KalshiOperation, KalshiSubscription,
960                KalshiUpdated, KalshiWsMessage,
961            },
962        },
963    };
964
965    #[allow(
966        dead_code,
967        reason = "shared key fixtures also serve authentication tests"
968    )]
969    mod keys {
970        include!(concat!(
971            env!("CARGO_MANIFEST_DIR"),
972            "/tests/integration/keys.rs"
973        ));
974    }
975
976    #[tokio::test]
977    async fn stale_send_leaves_healthy_replacement_active() {
978        time::timeout(Duration::from_secs(15), async {
979            let (listener, config) = peer_config().await;
980            let peer = task::spawn(async move {
981                let (stream, _) = listener.accept().await.unwrap();
982                let mut ws_client = accept_async(stream).await.unwrap();
983                ws_client.close(None).await.unwrap();
984                let (stream, _) = listener.accept().await.unwrap();
985                let mut ws_client = accept_async(stream).await.unwrap();
986                assert!(matches!(
987                    ws_client.next().await,
988                    None | Some(Err(_) | Ok(Message::Close(_)))
989                ));
990            });
991
992            let (shared, mut events) = shared(config.clone());
993            let (ws_client, receiver) = reconnected_transport(&config, 1).await;
994            ws_client.set_auth_tracker(shared.auth.clone(), false);
995
996            let mut handler = KalshiWsHandler::new(
997                shared,
998                Arc::clone(&ws_client),
999                receiver,
1000                CancellationToken::new(),
1001            );
1002
1003            let args = KalshiSubscription {
1004                channel: KalshiChannel::OrderbookDelta,
1005                market_tickers: vec!["MARKET-A".into()],
1006            };
1007
1008            let operation = KalshiOperation::Subscribe {
1009                channels: [args.channel],
1010                market_tickers: args.market_tickers.clone(),
1011                use_yes_price: Some(true),
1012                send_initial_snapshot: None,
1013            };
1014
1015            let sent = handler.send(args.topic(), operation, Some(args)).await;
1016            assert!(!sent);
1017            assert!(ws_client.is_active());
1018            assert_eq!(
1019                ws_client.connection_epoch_atomic().load(Ordering::Acquire),
1020                1
1021            );
1022            assert_eq!(
1023                events.try_recv(),
1024                Err(tokio::sync::mpsc::error::TryRecvError::Empty)
1025            );
1026            ws_client.disconnect().await;
1027            peer.await.unwrap();
1028        })
1029        .await
1030        .unwrap();
1031    }
1032
1033    #[tokio::test]
1034    async fn obsolete_reconnect_notification_does_not_authenticate_current_connection() {
1035        time::timeout(Duration::from_secs(15), async {
1036            let (listener, config) = peer_config().await;
1037
1038            let peer = task::spawn(async move {
1039                for _ in 0..2 {
1040                    let (stream, _) = listener.accept().await.unwrap();
1041                    let mut ws_client = accept_async(stream).await.unwrap();
1042                    ws_client.close(None).await.unwrap();
1043                }
1044
1045                let (stream, _) = listener.accept().await.unwrap();
1046                let mut ws_client = accept_async(stream).await.unwrap();
1047                assert!(matches!(
1048                    ws_client.next().await,
1049                    None | Some(Err(_) | Ok(Message::Close(_)))
1050                ));
1051            });
1052
1053            let (shared, mut events) = shared(config.clone());
1054            let (ws_client, receiver) = reconnected_transport(&config, 2).await;
1055            ws_client.set_auth_tracker(shared.auth.clone(), false);
1056
1057            let mut handler = KalshiWsHandler::new(
1058                Arc::clone(&shared),
1059                Arc::clone(&ws_client),
1060                receiver,
1061                CancellationToken::new(),
1062            );
1063
1064            let Some(HandlerCommand::Frame { epoch, message, .. }) = handler.receiver.recv().await
1065            else {
1066                panic!("missing first reconnect notification");
1067            };
1068
1069            assert_eq!(epoch, 1);
1070            handler.frame(epoch, message).await;
1071            assert_eq!(shared.auth.auth_state(), AuthState::Unauthenticated);
1072            assert_eq!(
1073                events.try_recv(),
1074                Err(tokio::sync::mpsc::error::TryRecvError::Empty)
1075            );
1076
1077            let Some(HandlerCommand::Frame { epoch, message, .. }) = handler.receiver.recv().await
1078            else {
1079                panic!("missing current reconnect notification");
1080            };
1081
1082            assert_eq!(epoch, 2);
1083            handler.frame(epoch, message).await;
1084            assert_eq!(shared.auth.auth_state(), AuthState::Authenticated);
1085            assert_eq!(
1086                events.recv().await,
1087                Some(KalshiWsEvent::Ready { generation: 1 })
1088            );
1089            assert!(ws_client.is_active());
1090            assert_eq!(
1091                ws_client.connection_epoch_atomic().load(Ordering::Acquire),
1092                2
1093            );
1094            ws_client.disconnect().await;
1095            peer.await.unwrap();
1096        })
1097        .await
1098        .unwrap();
1099    }
1100
1101    #[tokio::test]
1102    async fn replay_rechecks_intent_after_each_awaited_send() {
1103        time::timeout(Duration::from_secs(15), async {
1104            let (listener, config) = peer_config().await;
1105            let peer = task::spawn(async move {
1106                let (stream, _) = listener.accept().await.unwrap();
1107                let mut ws_client = accept_async(stream).await.unwrap();
1108                let fixtures: serde_json::Value =
1109                    serde_json::from_str(include_str!("../../test_data/websocket.json")).unwrap();
1110                let expected = fixtures["book_subscribe"].as_str().unwrap();
1111                assert_eq!(
1112                    ws_client.next().await.unwrap().unwrap(),
1113                    Message::text(expected)
1114                );
1115                assert!(matches!(
1116                    ws_client.next().await,
1117                    None | Some(Err(_) | Ok(Message::Close(_)))
1118                ));
1119            });
1120
1121            let (shared, _events) = shared(config.clone());
1122
1123            let first = KalshiSubscription {
1124                channel: KalshiChannel::OrderbookDelta,
1125                market_tickers: vec!["MARKET-A".into()],
1126            };
1127
1128            let removed = KalshiSubscription {
1129                channel: KalshiChannel::OrderbookDelta,
1130                market_tickers: vec!["MARKET-B".into()],
1131            };
1132
1133            {
1134                let mut state = shared.state.lock().unwrap();
1135                state.desired.insert(first.topic(), first);
1136                state.desired.insert(removed.topic(), removed.clone());
1137            }
1138
1139            let (ws_client, receiver) = reconnected_transport(&config, 0).await;
1140            ws_client.set_auth_tracker(shared.auth.clone(), false);
1141            shared.auth.succeed();
1142
1143            let mut handler = KalshiWsHandler::new(
1144                Arc::clone(&shared),
1145                Arc::clone(&ws_client),
1146                receiver,
1147                CancellationToken::new(),
1148            );
1149
1150            let mut replay = std::pin::pin!(handler.reconcile());
1151            assert!(futures_util::poll!(replay.as_mut()).is_pending());
1152            shared
1153                .state
1154                .lock()
1155                .unwrap()
1156                .desired
1157                .remove(&removed.topic());
1158            replay.await;
1159            ws_client.disconnect().await;
1160            peer.await.unwrap();
1161        })
1162        .await
1163        .unwrap();
1164    }
1165
1166    async fn peer_config() -> (TcpListener, KalshiDataClientConfig) {
1167        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1168        let address = listener.local_addr().unwrap();
1169
1170        let config = KalshiDataClientConfig {
1171            api_key_id: Some("auth-test-id".into()),
1172            private_key: Some(keys::rsa_pkcs8().into()),
1173            base_url_ws: Some(format!("ws://{address}/custom/ws/v3").into()),
1174            ws_timeout_secs: 5,
1175            ..Default::default()
1176        };
1177
1178        (listener, config)
1179    }
1180
1181    async fn reconnected_transport(
1182        config: &KalshiDataClientConfig,
1183        expected_epoch: u64,
1184    ) -> (
1185        Arc<WebSocketClient>,
1186        tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
1187    ) {
1188        let mut config = transport_config(config);
1189        // Exercise shared epoch handling independently of Kalshi's disabled reconnect policy.
1190        config.reconnect_max_attempts = None;
1191        let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
1192        let changed = Arc::new(tokio::sync::Notify::new());
1193        let notification = Arc::clone(&changed);
1194
1195        let ws_client = WebSocketClient::epoch_builder()
1196            .config(config)
1197            .epoch_handler(Arc::new(move |epoch, message| {
1198                let reconnect = message.as_text() == Some(RECONNECTED);
1199
1200                let _ = sender.send(HandlerCommand::Frame {
1201                    epoch,
1202                    received: time::Instant::now(),
1203                    message,
1204                });
1205
1206                if reconnect {
1207                    notification.notify_one();
1208                }
1209            }))
1210            .connect()
1211            .await
1212            .unwrap();
1213
1214        while ws_client.connection_epoch_atomic().load(Ordering::Acquire) < expected_epoch {
1215            changed.notified().await;
1216        }
1217
1218        (Arc::new(ws_client), receiver)
1219    }
1220
1221    #[tokio::test]
1222    async fn sequence_is_shared_by_markets_and_control_replies_before_correlation() {
1223        let (listener, config) = peer_config().await;
1224        let peer = task::spawn(async move {
1225            let (stream, _) = listener.accept().await.unwrap();
1226            let mut ws_client = accept_async(stream).await.unwrap();
1227            let _ = ws_client.next().await;
1228        });
1229
1230        let (shared, mut events) = shared(config.clone());
1231        let (ws_client, receiver) = reconnected_transport(&config, 0).await;
1232
1233        let mut handler = KalshiWsHandler::new(
1234            shared,
1235            Arc::clone(&ws_client),
1236            receiver,
1237            CancellationToken::new(),
1238        );
1239
1240        handler.sequences.insert(7, 0);
1241        let fixture: serde_json::Value =
1242            serde_json::from_str(include_str!("../../test_data/books.json")).unwrap();
1243
1244        let data = |seq, market: &str| {
1245            let mut msg = fixture["snapshot"].clone();
1246            msg["market_ticker"] = market.into();
1247            KalshiWsMessage::OrderbookSnapshot(KalshiDataMessage {
1248                id: None,
1249                sid: 7,
1250                seq: Some(seq),
1251                sending_ts_ms: None,
1252                msg,
1253            })
1254        };
1255
1256        assert!(handler.sequence(&data(1, "MARKET-A")));
1257        assert!(handler.sequence(&data(2, "MARKET-B")));
1258        assert!(handler.sequence(&KalshiWsMessage::Ok {
1259            id: Some(999),
1260            sid: Some(7),
1261            seq: Some(3),
1262            sending_ts_ms: None,
1263            msg: Some(KalshiUpdated {
1264                market_tickers: vec!["MARKET-A".into(), "MARKET-B".into()]
1265            })
1266        }));
1267        assert!(handler.sequence(&data(4, "MARKET-A")));
1268        assert!(!handler.sequence(&data(4, "MARKET-A")));
1269        assert!(!handler.sequence(&data(2, "MARKET-B")));
1270        assert!(events.try_recv().is_err());
1271        assert!(handler.sequence(&data(6, "MARKET-B")));
1272        assert_eq!(
1273            events.try_recv().unwrap(),
1274            KalshiWsEvent::BookSequenceGap {
1275                generation: 0,
1276                sid: 7
1277            }
1278        );
1279        assert_eq!(handler.sequences.get(&7), Some(&6));
1280        ws_client.disconnect().await;
1281        peer.await.unwrap();
1282    }
1283
1284    #[rstest]
1285    #[case::timely_timer(true, true)]
1286    #[case::late_timer(false, true)]
1287    #[case::late_receiver(false, false)]
1288    #[tokio::test]
1289    async fn queued_acknowledgement_uses_arrival_time(#[case] timely: bool, #[case] timer: bool) {
1290        time::timeout(Duration::from_secs(5), async {
1291            let (listener, config) = peer_config().await;
1292            let peer = task::spawn(async move {
1293                let (stream, _) = listener.accept().await.unwrap();
1294                let mut ws_client = accept_async(stream).await.unwrap();
1295                let _ = ws_client.next().await.unwrap().unwrap();
1296                let fixture: serde_json::Value =
1297                    serde_json::from_str(include_str!("../../test_data/books.json")).unwrap();
1298                ws_client
1299                    .send(Message::text(fixture["subscribed"].to_string()))
1300                    .await
1301                    .unwrap();
1302                let _ = ws_client.next().await;
1303            });
1304
1305            let (shared, mut events) = shared(config.clone());
1306
1307            let args = KalshiSubscription {
1308                channel: KalshiChannel::OrderbookDelta,
1309                market_tickers: vec!["MARKET-A".into()],
1310            };
1311
1312            shared
1313                .state
1314                .lock()
1315                .unwrap()
1316                .desired
1317                .insert(args.topic(), args.clone());
1318            let (ws_client, receiver) = reconnected_transport(&config, 0).await;
1319            ws_client.set_auth_tracker(shared.auth.clone(), false);
1320            shared.auth.succeed();
1321            let cancellation = CancellationToken::new();
1322
1323            let mut handler = KalshiWsHandler::new(
1324                shared,
1325                Arc::clone(&ws_client),
1326                receiver,
1327                cancellation.clone(),
1328            );
1329
1330            let operation = KalshiOperation::Subscribe {
1331                channels: [args.channel],
1332                market_tickers: args.market_tickers.clone(),
1333                use_yes_price: Some(true),
1334                send_initial_snapshot: None,
1335            };
1336
1337            assert!(handler.send(args.topic(), operation, Some(args)).await);
1338            let frame = handler.receiver.recv().await.unwrap();
1339
1340            let HandlerCommand::Frame { received, .. } = &frame else {
1341                panic!("expected acknowledgement frame");
1342            };
1343
1344            let deadline = if timely {
1345                *received + Duration::from_nanos(1)
1346            } else {
1347                *received - Duration::from_nanos(1)
1348            };
1349
1350            assert!(deadline < time::Instant::now());
1351            handler.pending.get_mut(&1).unwrap().deadline = deadline;
1352
1353            if timer {
1354                let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
1355                sender.send(frame).unwrap();
1356                handler.receiver = receiver;
1357                handler.expire().await;
1358            } else {
1359                handler.command(frame).await;
1360            }
1361
1362            let first = events.try_recv().unwrap();
1363
1364            if timely {
1365                assert!(matches!(
1366                    first,
1367                    KalshiWsEvent::Message {
1368                        message: KalshiWsMessage::Subscribed {
1369                            id: 1,
1370                            msg: crate::websocket::messages::KalshiSubscribed { sid: 7, .. }
1371                        },
1372                        generation: 0
1373                    }
1374                ));
1375            } else {
1376                assert_eq!(
1377                    first,
1378                    KalshiWsEvent::Error {
1379                        generation: 0,
1380                        message: "Kalshi WebSocket command acknowledgement timed out".into()
1381                    }
1382                );
1383            }
1384
1385            ws_client.disconnect().await;
1386            peer.await.unwrap();
1387        })
1388        .await
1389        .unwrap();
1390    }
1391
1392    #[tokio::test]
1393    async fn frames_arriving_during_expiry_cannot_extend_the_drain() {
1394        time::timeout(Duration::from_secs(5), async {
1395            let (listener, config) = peer_config().await;
1396            let peer = task::spawn(async move {
1397                let (stream, _) = listener.accept().await.unwrap();
1398                let mut ws_client = accept_async(stream).await.unwrap();
1399
1400                for _ in 0..2 {
1401                    assert!(matches!(ws_client.next().await, Some(Ok(Message::Text(_)))));
1402                }
1403
1404                let _ = ws_client.next().await;
1405            });
1406
1407            let (shared, mut events) = shared(config.clone());
1408
1409            let first = KalshiSubscription {
1410                channel: KalshiChannel::OrderbookDelta,
1411                market_tickers: vec!["MARKET-A".into()],
1412            };
1413
1414            let second = KalshiSubscription {
1415                channel: KalshiChannel::OrderbookDelta,
1416                market_tickers: vec!["MARKET-B".into()],
1417            };
1418
1419            shared
1420                .state
1421                .lock()
1422                .unwrap()
1423                .desired
1424                .insert(second.topic(), second);
1425            let (ws_client, receiver) = reconnected_transport(&config, 0).await;
1426            ws_client.set_auth_tracker(shared.auth.clone(), false);
1427            shared.auth.succeed();
1428
1429            let mut handler = KalshiWsHandler::new(
1430                shared,
1431                Arc::clone(&ws_client),
1432                receiver,
1433                CancellationToken::new(),
1434            );
1435
1436            let operation = KalshiOperation::Subscribe {
1437                channels: [first.channel],
1438                market_tickers: first.market_tickers.clone(),
1439                use_yes_price: Some(true),
1440                send_initial_snapshot: None,
1441            };
1442
1443            assert!(handler.send(first.topic(), operation, Some(first)).await);
1444            let deadline = time::Instant::now() - Duration::from_secs(1);
1445            handler.pending.get_mut(&1).unwrap().deadline = deadline;
1446            let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
1447            sender.send(HandlerCommand::Reconcile).unwrap();
1448            handler.receiver = receiver;
1449            {
1450                let mut expiry = std::pin::pin!(handler.expire());
1451                assert!(futures_util::poll!(expiry.as_mut()).is_pending());
1452
1453                for _ in 0..256 {
1454                    sender
1455                        .send(HandlerCommand::Frame {
1456                            epoch: 0,
1457                            received: deadline,
1458                            message: TransportMessage::Ping(Vec::new().into()),
1459                        })
1460                        .unwrap();
1461                }
1462
1463                expiry.await;
1464            }
1465
1466            assert_eq!(handler.receiver.len(), 256);
1467            assert!(handler.pending.is_empty());
1468            assert_eq!(
1469                events.try_recv().unwrap(),
1470                KalshiWsEvent::Error {
1471                    generation: 0,
1472                    message: "Kalshi WebSocket command acknowledgement timed out".into()
1473                }
1474            );
1475            ws_client.disconnect().await;
1476            peer.await.unwrap();
1477        })
1478        .await
1479        .unwrap();
1480    }
1481
1482    #[tokio::test]
1483    async fn snapshot_writer_overflow_cannot_confirm_recovery() {
1484        time::timeout(Duration::from_secs(5), async {
1485            let (listener, config) = peer_config().await;
1486            let peer = task::spawn(async move {
1487                let (stream, _) = listener.accept().await.unwrap();
1488                let mut ws_client = accept_async(stream).await.unwrap();
1489                while let Some(Ok(message)) = ws_client.next().await {
1490                    assert!(!matches!(message, Message::Text(_)));
1491
1492                    if matches!(message, Message::Close(_)) {
1493                        break;
1494                    }
1495                }
1496            });
1497
1498            let (shared, mut events) = shared(config.clone());
1499            shared.auth.succeed();
1500            let mut transport = transport_config(&config);
1501            transport.writer_capacity = Some(1);
1502
1503            let ws_client = Arc::new(
1504                WebSocketClient::epoch_builder()
1505                    .config(transport)
1506                    .epoch_handler(Arc::new(|_, _| {}))
1507                    .connect()
1508                    .await
1509                    .unwrap(),
1510            );
1511
1512            let (_sender, receiver) = tokio::sync::mpsc::unbounded_channel();
1513
1514            let mut handler = KalshiWsHandler::new(
1515                shared,
1516                Arc::clone(&ws_client),
1517                receiver,
1518                CancellationToken::new(),
1519            );
1520
1521            let args = KalshiSubscription {
1522                channel: KalshiChannel::OrderbookDelta,
1523                market_tickers: vec!["MARKET-A".into()],
1524            };
1525
1526            let topic = args.topic();
1527            handler
1528                .active
1529                .insert(topic.clone(), super::ActiveSubscription { sid: 7, args });
1530            ws_client
1531                .send_pong_on_connection(Vec::new(), 0)
1532                .await
1533                .unwrap();
1534            let result = handler
1535                .book_snapshot(topic, vec!["MARKET-A".into()], 41, CancellationToken::new())
1536                .await;
1537            assert_eq!(
1538                result.unwrap_err().to_string(),
1539                "client error: snapshot write did not complete on the intended connection"
1540            );
1541            assert!(handler.pending.is_empty());
1542            assert_eq!(
1543                events.try_recv().unwrap(),
1544                KalshiWsEvent::Error {
1545                    generation: 0,
1546                    message: "Kalshi WebSocket command send failed".into()
1547                }
1548            );
1549            assert!(events.try_recv().is_err());
1550            ws_client.disconnect().await;
1551            peer.await.unwrap();
1552        })
1553        .await
1554        .unwrap();
1555    }
1556
1557    #[tokio::test]
1558    async fn concurrent_book_snapshots_keep_independent_pending_correlations() {
1559        time::timeout(Duration::from_secs(5), async {
1560            let (listener, config) = peer_config().await;
1561            let peer = task::spawn(async move {
1562                let (stream, _) = listener.accept().await.unwrap();
1563                let mut ws_client = accept_async(stream).await.unwrap();
1564                for (id, market) in [(1, "MARKET-A"), (2, "MARKET-B")] {
1565                    let Message::Text(wire) = ws_client.next().await.unwrap().unwrap() else {
1566                        panic!("expected snapshot command")
1567                    };
1568
1569                    let command: serde_json::Value = serde_json::from_str(&wire).unwrap();
1570                    assert_eq!(
1571                        command,
1572                        serde_json::json!({
1573                            "id": id,
1574                            "cmd": "update_subscription",
1575                            "params": {"sid": 7, "market_tickers": [market], "action": "get_snapshot"}
1576                        })
1577                    );
1578                }
1579
1580                let _ = ws_client.next().await;
1581            });
1582
1583            let (shared, mut events) = shared(config.clone());
1584            shared.auth.succeed();
1585            let (ws_client, receiver) = reconnected_transport(&config, 0).await;
1586
1587            let mut handler = KalshiWsHandler::new(
1588                shared,
1589                Arc::clone(&ws_client),
1590                receiver,
1591                CancellationToken::new(),
1592            );
1593
1594            let args = KalshiSubscription {
1595                channel: KalshiChannel::OrderbookDelta,
1596                market_tickers: vec!["MARKET-A".into(), "MARKET-B".into()],
1597            };
1598
1599            let topic = args.topic();
1600            handler
1601                .active
1602                .insert(topic.clone(), super::ActiveSubscription { sid: 7, args });
1603            handler
1604                .book_snapshot(
1605                    topic.clone(),
1606                    vec!["MARKET-A".into()],
1607                    41,
1608                    CancellationToken::new(),
1609                )
1610                .await
1611                .unwrap();
1612            handler
1613                .book_snapshot(topic, vec!["MARKET-B".into()], 42, CancellationToken::new())
1614                .await
1615                .unwrap();
1616            assert_eq!(handler.pending.len(), 2);
1617
1618            for (id, attempt) in [(1, 41), (2, 42)] {
1619                assert_eq!(
1620                    events.try_recv().unwrap(),
1621                    KalshiWsEvent::BookSnapshotRequested {
1622                        generation: 0,
1623                        sid: 7,
1624                        id,
1625                        attempt
1626                    }
1627                );
1628            }
1629
1630            let reply = |id, market: &str| KalshiDataMessage {
1631                id: Some(id),
1632                sid: 7,
1633                seq: Some(1),
1634                sending_ts_ms: None,
1635                msg: serde_json::json!({"market_ticker":market}),
1636            };
1637
1638            assert_eq!(handler.acknowledge_snapshot(&reply(2, "MARKET-A")), None);
1639            assert_eq!(handler.pending.len(), 2);
1640            assert_eq!(
1641                handler.acknowledge_snapshot(&reply(2, "MARKET-B")),
1642                Some(true)
1643            );
1644            assert!(handler.pending.contains_key(&1));
1645            assert_eq!(
1646                handler.acknowledge_snapshot(&reply(1, "MARKET-A")),
1647                Some(true)
1648            );
1649            assert!(handler.pending.is_empty());
1650            ws_client.disconnect().await;
1651            peer.await.unwrap();
1652        })
1653        .await
1654        .unwrap();
1655    }
1656
1657    #[rstest]
1658    #[case::managed_rejection(true, true)]
1659    #[case::managed_deadline(false, true)]
1660    #[case::public_rejection(true, false)]
1661    #[tokio::test]
1662    async fn snapshot_failures_preserve_their_ownership(
1663        #[case] rejection: bool,
1664        #[case] managed: bool,
1665    ) {
1666        time::timeout(Duration::from_secs(5), async {
1667            let (listener, config) = peer_config().await;
1668            let peer = task::spawn(async move {
1669                let (stream, _) = listener.accept().await.unwrap();
1670                let mut ws_client = accept_async(stream).await.unwrap();
1671                let _ = ws_client.next().await;
1672                let _ = ws_client.next().await;
1673            });
1674
1675            let (shared, mut events) = shared(config.clone());
1676            shared.auth.succeed();
1677            let (ws_client, receiver) = reconnected_transport(&config, 0).await;
1678
1679            let mut handler = KalshiWsHandler::new(
1680                shared,
1681                Arc::clone(&ws_client),
1682                receiver,
1683                CancellationToken::new(),
1684            );
1685
1686            let args = KalshiSubscription {
1687                channel: KalshiChannel::OrderbookDelta,
1688                market_tickers: vec!["MARKET-A".into()],
1689            };
1690
1691            let topic = args.topic();
1692            handler
1693                .shared
1694                .state
1695                .lock()
1696                .unwrap()
1697                .desired
1698                .insert(topic.clone(), args.clone());
1699            handler
1700                .active
1701                .insert(topic.clone(), super::ActiveSubscription { sid: 7, args });
1702
1703            if managed {
1704                handler
1705                    .book_snapshot(
1706                        topic.clone(),
1707                        vec!["MARKET-A".into()],
1708                        41,
1709                        CancellationToken::new(),
1710                    )
1711                    .await
1712                    .unwrap();
1713                assert!(matches!(
1714                    events.try_recv().unwrap(),
1715                    KalshiWsEvent::BookSnapshotRequested { .. }
1716                ));
1717            } else {
1718                handler
1719                    .snapshot(topic.clone(), vec!["MARKET-A".into()])
1720                    .await;
1721            }
1722
1723            if rejection {
1724                let error = serde_json::from_str::<serde_json::Value>(include_str!(
1725                    "../../test_data/websocket_messages.json"
1726                ))
1727                .unwrap()["snapshot_rate_limit"]
1728                    .clone();
1729                handler
1730                    .frame(
1731                        0,
1732                        TransportMessage::Text(error.to_string().into_bytes().into()),
1733                    )
1734                    .await;
1735
1736                if !managed {
1737                    assert!(matches!(
1738                        events.try_recv().unwrap(),
1739                        KalshiWsEvent::Error { .. }
1740                    ));
1741                }
1742
1743                assert!(matches!(
1744                    events.try_recv().unwrap(),
1745                    KalshiWsEvent::Message {
1746                        message: KalshiWsMessage::Error { .. },
1747                        ..
1748                    }
1749                ));
1750                assert_eq!(handler.blocked.contains_key(&topic), !managed);
1751            } else {
1752                handler.pending.get_mut(&1).unwrap().deadline =
1753                    time::Instant::now() - Duration::from_secs(1);
1754                handler.expire_pending();
1755                assert!(
1756                    handler.shared.auth.is_authenticated(),
1757                    "snapshot deadline forced a transport reconnect"
1758                );
1759            }
1760
1761            ws_client.disconnect().await;
1762            peer.await.unwrap();
1763        })
1764        .await
1765        .unwrap();
1766    }
1767
1768    #[tokio::test]
1769    async fn canceled_snapshot_retires_pending_command_before_unsubscribe() {
1770        time::timeout(Duration::from_secs(5), async {
1771            let (listener, config) = peer_config().await;
1772            let peer = task::spawn(async move {
1773                let (stream, _) = listener.accept().await.unwrap();
1774                let mut ws_client = accept_async(stream).await.unwrap();
1775                let snapshot = ws_client.next().await.unwrap().unwrap();
1776                let unsubscribe = ws_client.next().await.unwrap().unwrap();
1777                let snapshot: serde_json::Value =
1778                    serde_json::from_str(snapshot.to_text().unwrap()).unwrap();
1779                let unsubscribe: serde_json::Value =
1780                    serde_json::from_str(unsubscribe.to_text().unwrap()).unwrap();
1781                ws_client.close(None).await.unwrap();
1782                (snapshot, unsubscribe)
1783            });
1784
1785            let (shared, mut events) = shared(config.clone());
1786            shared.auth.succeed();
1787            let (ws_client, receiver) = reconnected_transport(&config, 0).await;
1788            let cancellation = CancellationToken::new();
1789
1790            let mut handler = KalshiWsHandler::new(
1791                shared,
1792                Arc::clone(&ws_client),
1793                receiver,
1794                cancellation.clone(),
1795            );
1796
1797            let args = KalshiSubscription {
1798                channel: KalshiChannel::OrderbookDelta,
1799                market_tickers: vec!["MARKET-A".into()],
1800            };
1801
1802            let topic = args.topic();
1803            handler
1804                .active
1805                .insert(topic.clone(), super::ActiveSubscription { sid: 7, args });
1806            let snapshot_cancel = CancellationToken::new();
1807            handler
1808                .book_snapshot(topic, vec!["MARKET-A".into()], 41, snapshot_cancel.clone())
1809                .await
1810                .unwrap();
1811            let event = events.try_recv().unwrap();
1812            snapshot_cancel.cancel();
1813
1814            let running = task::spawn(handler.run());
1815            let (snapshot, unsubscribe) = peer.await.unwrap();
1816            cancellation.cancel();
1817            running.await.unwrap();
1818            assert!(matches!(
1819                event,
1820                KalshiWsEvent::BookSnapshotRequested {
1821                    generation: 0,
1822                    sid: 7,
1823                    id: 1,
1824                    attempt: 41
1825                }
1826            ));
1827            let fixtures: serde_json::Value =
1828                serde_json::from_str(include_str!("../../test_data/websocket_messages.json"))
1829                    .unwrap();
1830            let expected_snapshot = fixtures["managed_snapshot"].clone();
1831            let expected_unsubscribe = fixtures["managed_unsubscribe"].clone();
1832            assert_eq!(snapshot, expected_snapshot);
1833            assert_eq!(unsubscribe, expected_unsubscribe);
1834        })
1835        .await
1836        .unwrap();
1837    }
1838
1839    #[tokio::test]
1840    async fn running_snapshot_cancellation_restores_readiness_without_a_frame() {
1841        time::timeout(Duration::from_secs(5), async {
1842            let (listener, mut config) = peer_config().await;
1843            config.ws_timeout_secs = 30;
1844
1845            let peer = task::spawn(async move {
1846                let (stream, _) = listener.accept().await.unwrap();
1847                let mut ws_client = accept_async(stream).await.unwrap();
1848                let _ = ws_client.next().await;
1849                let _ = ws_client.next().await;
1850            });
1851
1852            let (shared, mut events) = shared(config.clone());
1853            shared.auth.succeed();
1854            let (ws_client, _transport_receiver) = reconnected_transport(&config, 0).await;
1855            let (commands, receiver) = tokio::sync::mpsc::unbounded_channel();
1856            let cancellation = CancellationToken::new();
1857
1858            let mut handler = KalshiWsHandler::new(
1859                Arc::clone(&shared),
1860                ws_client,
1861                receiver,
1862                cancellation.clone(),
1863            );
1864
1865            let args = KalshiSubscription {
1866                channel: KalshiChannel::OrderbookDelta,
1867                market_tickers: vec!["MARKET-A".into()],
1868            };
1869
1870            let topic = args.topic();
1871            shared
1872                .state
1873                .lock()
1874                .unwrap()
1875                .desired
1876                .insert(topic.clone(), args.clone());
1877            handler
1878                .active
1879                .insert(topic.clone(), super::ActiveSubscription { sid: 7, args });
1880
1881            let running = task::spawn(handler.run());
1882            let snapshot_cancel = CancellationToken::new();
1883            let (completed, result) = tokio::sync::oneshot::channel();
1884            commands
1885                .send(HandlerCommand::BookSnapshot {
1886                    topic,
1887                    markets: vec!["MARKET-A".into()],
1888                    attempt: 41,
1889                    cancel: snapshot_cancel.clone(),
1890                    completed,
1891                })
1892                .unwrap();
1893
1894            result.await.unwrap().unwrap();
1895            let requested = events.recv().await.unwrap();
1896            snapshot_cancel.cancel();
1897            let ready = time::timeout(Duration::from_secs(1), events.recv()).await;
1898            let active = shared.ready.load(Ordering::Acquire);
1899            cancellation.cancel();
1900            running.await.unwrap();
1901            peer.await.unwrap();
1902            assert!(matches!(
1903                requested,
1904                KalshiWsEvent::BookSnapshotRequested {
1905                    generation: 0,
1906                    sid: 7,
1907                    id: 1,
1908                    attempt: 41
1909                }
1910            ));
1911            assert!(
1912                matches!(ready, Ok(Some(KalshiWsEvent::Ready { generation: 0 }))),
1913                "cancelled request waited for the native command timeout"
1914            );
1915            assert!(active);
1916        })
1917        .await
1918        .unwrap();
1919    }
1920}