Skip to main content

nautilus_lighter/websocket/
handler.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Inner WebSocket feed handler running on a dedicated tokio task.
17
18use std::{
19    collections::VecDeque,
20    fmt::Debug,
21    future::Future,
22    pin::Pin,
23    sync::{
24        Arc,
25        atomic::{AtomicBool, Ordering},
26    },
27    time::Duration,
28};
29
30use ahash::AHashMap;
31use futures_util::{StreamExt, stream::FuturesUnordered};
32use nautilus_core::{
33    AtomicTime,
34    nanos::UnixNanos,
35    string::secret::{REDACTED, SecretString},
36    time::get_atomic_clock_realtime,
37};
38#[cfg(test)]
39use nautilus_live::book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS;
40use nautilus_live::book::{
41    BookSequenceOutcome,
42    recovery::{BookRecoveryOutcome, BookRecoveryState},
43    snapshot::{PendingSnapshot, SnapshotGate},
44};
45use nautilus_model::{identifiers::AccountId, instruments::InstrumentAny, types::Currency};
46use nautilus_network::{
47    RECONNECTED,
48    error::SendError,
49    retry::{RetryManager, create_websocket_retry_manager},
50    websocket::{SubscriptionState, WebSocketClient},
51};
52use tokio_tungstenite::tungstenite::Message;
53use tokio_util::sync::CancellationToken;
54use ustr::Ustr;
55use zeroize::Zeroize;
56
57use super::{
58    account_state::LighterAccountStateReconciler,
59    error::LighterWsError,
60    messages::{
61        AccountStream, ExecutionReport, LighterAsset, LighterPosition, LighterUserStats,
62        LighterWsCandle, LighterWsChannel, LighterWsChannelKind, LighterWsFrame,
63        LighterWsOrderBook, LighterWsRequest, LighterWsTx, NautilusWsMessage,
64        SendTxRejectionSource, TX_BATCH_ID_PREFIX,
65    },
66    parse::{
67        parse_ws_bar, parse_ws_funding_rate_update, parse_ws_index_price_update,
68        parse_ws_mark_price_update, parse_ws_position_status_report, parse_ws_quote_tick,
69        parse_ws_spot_index_price_update, parse_ws_trade_tick,
70    },
71};
72use crate::{
73    book::{
74        recovery::{self, BookWorkResult, BookWrite},
75        sync::BookSyncTracker,
76    },
77    common::{
78        consts::{
79            LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED, LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED,
80            LIGHTER_ERROR_CODE_TX_RANGE, LIGHTER_ERROR_CODE_WS_RATE_LIMITED,
81            LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED, LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL,
82            SUBSCRIBE_INFLIGHT_MAX, SUBSCRIBE_RETRY_BASE_BACKOFF, SUBSCRIBE_RETRY_MAX,
83        },
84        enums::LighterCandleResolution,
85        rate_limit::LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY,
86    },
87    http::models::{LighterOrder, LighterTrade},
88};
89
90// Lighter control-frame `type` field values that fall outside the typed
91// `LighterWsFrame` variants. These are protocol-level frames the handler
92// inspects via the typed-frame parse fallback in `handle_control_value`.
93const CTRL_TYPE_CONNECTED: &str = "connected";
94const CTRL_TYPE_SUBSCRIBED: &str = "subscribed";
95const CTRL_TYPE_UNSUBSCRIBED: &str = "unsubscribed";
96const CTRL_TYPE_PING: &str = "ping";
97const CTRL_TYPE_PONG: &str = "pong";
98const CTRL_TYPE_ERROR: &str = "error";
99const CTRL_TYPE_SEND_TX: &str = "jsonapi/sendtx";
100
101#[derive(serde::Deserialize)]
102struct LighterWsFrameHeader<'a> {
103    #[serde(rename = "type", borrow)]
104    kind: &'a str,
105    #[serde(borrow)]
106    channel: Option<&'a str>,
107}
108
109/// Commands sent from the outer client to the inner feed handler.
110#[expect(
111    clippy::large_enum_variant,
112    reason = "commands are ephemeral and immediately consumed"
113)]
114pub enum HandlerCommand {
115    /// Hand the live `WebSocketClient` to the handler after the outer client
116    /// completes the network connect.
117    SetClient(WebSocketClient),
118    /// Drain the queue and shut the handler down.
119    Disconnect,
120    /// Subscribe to a channel. `auth` carries an optional auth token for the
121    /// account-scoped channels.
122    Subscribe {
123        channel: LighterWsChannel,
124        auth: Option<SecretString>,
125        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
126    },
127    /// Unsubscribe from a channel.
128    Unsubscribe { channel: LighterWsChannel },
129    /// Replace a book subscription for the current recovery owner.
130    RecoverBook {
131        market_index: i64,
132        cancel: CancellationToken,
133        gate: SnapshotGate,
134        completion: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
135    },
136    /// Replace the handler's instrument cache (used on initial connect).
137    InitializeInstruments(Vec<(i64, InstrumentAny)>),
138    /// Insert or replace a single instrument by `market_index`.
139    UpdateInstrument {
140        market_index: i64,
141        instrument: InstrumentAny,
142    },
143    /// Toggle whether `update/order_book` frames for `market_index` should
144    /// be emitted as [`NautilusWsMessage::Deltas`].
145    SetBookDeltasSub { market_index: i64, subscribed: bool },
146    /// Toggle whether `update/order_book` frames for `market_index` should
147    /// also be emitted as a [`NautilusWsMessage::Depth`] snapshot.
148    SetDepthSub { market_index: i64, subscribed: bool },
149    /// Provide the execution context (`AccountId` and venue `account_index`)
150    /// the handler stamps onto reports parsed from `account_*` frames.
151    /// Without this context the handler cannot construct typed reports and
152    /// account frames are forwarded as raw JSON instead.
153    SetExecutionContext {
154        account_id: AccountId,
155        account_index: i64,
156    },
157    /// Dispatch a signed L2 transaction over the WebSocket.
158    ///
159    /// The handler serializes a [`LighterWsRequest::SendTx`] frame; the venue
160    /// confirms acceptance via the `account_*` streams (and reports any
161    /// rejection via a control-plane `error` frame).
162    SendTx {
163        tx_type: u8,
164        tx_info: Box<serde_json::value::RawValue>,
165        connection_epoch: u64,
166        response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
167    },
168    SendTxBatch {
169        data: super::messages::LighterWsSendTxBatch,
170        connection_epoch: u64,
171        response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
172    },
173}
174
175impl Debug for HandlerCommand {
176    /// Custom `Debug` that redacts the `auth` field of `Subscribe`. The
177    /// command is routed in-process via mpsc and may be printed by error
178    /// paths or trace logging; deriving `Debug` would otherwise leak the
179    /// venue bearer token verbatim.
180    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181        match self {
182            Self::SetClient(_) => f.write_str("SetClient(<WebSocketClient>)"),
183            Self::Disconnect => f.write_str("Disconnect"),
184            Self::Subscribe { channel, auth, .. } => f
185                .debug_struct(stringify!(Subscribe))
186                .field("channel", channel)
187                .field("auth", auth)
188                .finish(),
189            Self::Unsubscribe { channel } => f
190                .debug_struct(stringify!(Unsubscribe))
191                .field("channel", channel)
192                .finish(),
193            Self::RecoverBook { market_index, .. } => f
194                .debug_struct(stringify!(RecoverBook))
195                .field("market_index", market_index)
196                .finish(),
197            Self::InitializeInstruments(instruments) => f
198                .debug_tuple(stringify!(InitializeInstruments))
199                .field(&instruments.len())
200                .finish(),
201            Self::UpdateInstrument { market_index, .. } => f
202                .debug_struct(stringify!(UpdateInstrument))
203                .field("market_index", market_index)
204                .finish(),
205            Self::SetBookDeltasSub {
206                market_index,
207                subscribed,
208            } => f
209                .debug_struct(stringify!(SetBookDeltasSub))
210                .field("market_index", market_index)
211                .field("subscribed", subscribed)
212                .finish(),
213            Self::SetDepthSub {
214                market_index,
215                subscribed,
216            } => f
217                .debug_struct(stringify!(SetDepthSub))
218                .field("market_index", market_index)
219                .field("subscribed", subscribed)
220                .finish(),
221            Self::SetExecutionContext {
222                account_id,
223                account_index,
224            } => f
225                .debug_struct(stringify!(SetExecutionContext))
226                .field("account_id", account_id)
227                .field("account_index", account_index)
228                .finish(),
229            Self::SendTxBatch { .. } => f.write_str("SendTxBatch(<redacted>)"),
230            Self::SendTx { tx_type, .. } => f
231                .debug_struct(stringify!(SendTx))
232                .field("tx_type", tx_type)
233                .field("tx_info", &REDACTED)
234                .finish(),
235        }
236    }
237}
238
239/// Inner feed handler. Owns the [`WebSocketClient`] exclusively and routes
240/// raw frames into the venue-message channel.
241///
242/// Subscribe dispatch is gated by a closed-loop inflight count: the venue caps
243/// unacknowledged client messages per connection, and acknowledgement latency
244/// is multi-second, so queued subscribes are held back until acks free a slot.
245pub(super) struct FeedHandler {
246    clock: &'static AtomicTime,
247    signal: Arc<AtomicBool>,
248    inner: Option<Arc<WebSocketClient>>,
249    cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
250    cmd_tx: Option<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>,
251    raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
252    out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
253    subscriptions: SubscriptionState,
254    retry_manager: RetryManager<LighterWsError>,
255    pending_messages: VecDeque<NautilusWsMessage>,
256    pending_subs: VecDeque<(Ustr, u64)>,
257    inflight_subs: AHashMap<Ustr, u64>,
258    subscription_attempts: AHashMap<Ustr, SubscriptionAttempt>,
259    subscription_retries: FuturesUnordered<SubscriptionRetry>,
260    ignored_completions: AHashMap<Ustr, CompletionKind>,
261    next_subscription_generation: u64,
262    instruments: AHashMap<i64, InstrumentAny>,
263    book: BookSyncTracker,
264    book_snapshot_timeout: Duration,
265    last_candles: AHashMap<(i64, LighterCandleResolution), LighterWsCandle>,
266    exec_account: Option<(AccountId, i64)>,
267    account_state_reconciler: LighterAccountStateReconciler,
268}
269
270type SubscriptionRetry = Pin<Box<dyn Future<Output = (Ustr, u64)> + Send + Sync + 'static>>;
271
272struct SubscriptionAttempt {
273    channel: LighterWsChannel,
274    auth: Option<SecretString>,
275    pending_auth: Option<SecretString>,
276    generation: u64,
277    retries: u8,
278    response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
279    pending_response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
280}
281
282impl SubscriptionAttempt {
283    fn fold_pending_auth(&mut self) {
284        if let Some(auth) = self.pending_auth.take() {
285            self.auth = Some(auth);
286            self.response_txs.append(&mut self.pending_response_txs);
287        } else {
288            debug_assert!(self.pending_response_txs.is_empty());
289        }
290    }
291}
292
293/// Which venue frame produced a subscription completion.
294///
295/// The venue answers a subscribe with a generic control ack followed by a
296/// typed `subscribed/<channel>` snapshot, and both complete by topic. The kind
297/// lets `complete_subscription` swallow the predecessor's trailing frame so a
298/// same-topic generation dispatched between the two cannot be completed by a
299/// response that belongs to the earlier request.
300#[derive(Clone, Copy, Debug, PartialEq, Eq)]
301enum CompletionKind {
302    ControlAck,
303    Typed,
304    AlreadySubscribed,
305}
306
307impl FeedHandler {
308    #[cfg(test)]
309    pub(super) fn new(
310        signal: Arc<AtomicBool>,
311        cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
312        raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
313        out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
314        subscriptions: SubscriptionState,
315    ) -> Self {
316        Self::new_with_settlement_currency(
317            signal,
318            cmd_rx,
319            raw_rx,
320            out_tx,
321            subscriptions,
322            Currency::get_or_create_crypto("USDC"),
323            Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
324        )
325    }
326
327    pub(super) fn new_with_settlement_currency(
328        signal: Arc<AtomicBool>,
329        cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
330        raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
331        out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
332        subscriptions: SubscriptionState,
333        settlement_currency: Currency,
334        book_snapshot_timeout: Duration,
335    ) -> Self {
336        Self {
337            clock: get_atomic_clock_realtime(),
338            signal,
339            inner: None,
340            cmd_rx,
341            cmd_tx: None,
342            raw_rx,
343            out_tx,
344            subscriptions,
345            retry_manager: create_websocket_retry_manager(),
346            pending_messages: VecDeque::new(),
347            pending_subs: VecDeque::new(),
348            inflight_subs: AHashMap::new(),
349            subscription_attempts: AHashMap::new(),
350            subscription_retries: FuturesUnordered::new(),
351            ignored_completions: AHashMap::new(),
352            next_subscription_generation: 1,
353            instruments: AHashMap::new(),
354            book: BookSyncTracker::default(),
355            book_snapshot_timeout,
356            last_candles: AHashMap::new(),
357            exec_account: None,
358            account_state_reconciler: LighterAccountStateReconciler::new_with_settlement_currency(
359                settlement_currency,
360            ),
361        }
362    }
363
364    pub(super) fn send(&self, msg: NautilusWsMessage) -> Result<(), String> {
365        self.out_tx
366            .send(msg)
367            .map_err(|e| format!("Failed to send message: {e}"))
368    }
369
370    pub(super) fn set_command_sender(
371        &mut self,
372        cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
373    ) {
374        self.cmd_tx = Some(cmd_tx);
375    }
376
377    pub(super) fn is_stopped(&self) -> bool {
378        self.signal.load(Ordering::Relaxed)
379    }
380
381    async fn send_with_retry(&self, payload: String) -> Result<(), LighterWsError> {
382        self.send_secret_with_retry(SecretString::from(payload))
383            .await
384    }
385
386    async fn send_secret_with_retry(&self, payload: SecretString) -> Result<(), LighterWsError> {
387        if let Some(client) = &self.inner {
388            self.retry_manager
389                .invocation(
390                    "websocket_send",
391                    || {
392                        let payload = payload.clone();
393                        async move {
394                            client
395                                .send_text(
396                                    payload.expose_secret().to_owned(),
397                                    Some(LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY.as_slice()),
398                                )
399                                .await
400                                .map_err(LighterWsError::Transport)
401                        }
402                    },
403                    should_retry_lighter_ws_error,
404                    |e| create_lighter_ws_timeout_error(e.to_string()),
405                )
406                .execute()
407                .await
408        } else {
409            Err(LighterWsError::Client(
410                "no active WebSocket client".to_string(),
411            ))
412        }
413    }
414
415    // Single-shot: sendTx payloads carry a signed nonce; transport-layer
416    // retry could double-submit if the original landed and only the ack was lost.
417    async fn send_once(
418        &self,
419        payload: String,
420        connection_epoch: u64,
421    ) -> Result<(), LighterWsError> {
422        if let Some(client) = &self.inner {
423            match client
424                .send_text_on_connection(payload, None, connection_epoch)
425                .await
426            {
427                Err(SendError::BrokenPipe(message)) => {
428                    Err(LighterWsError::SendTxOutcomeUnknown(message))
429                }
430                result => result.map_err(LighterWsError::Transport),
431            }
432        } else {
433            Err(LighterWsError::Client(
434                "no active WebSocket client".to_string(),
435            ))
436        }
437    }
438
439    async fn dispatch_subscribe(
440        &self,
441        channel: LighterWsChannel,
442        auth: Option<SecretString>,
443    ) -> Result<(), String> {
444        let topic = channel.topic_key();
445
446        let authed = auth.is_some();
447        let mut request = match auth {
448            Some(token) => LighterWsRequest::subscribe_auth(
449                channel.subscription_channel(),
450                token.expose_secret(),
451            ),
452            None => LighterWsRequest::subscribe(channel.subscription_channel()),
453        };
454
455        let payload = serde_json::to_string(&request);
456        request.zeroize();
457
458        match payload {
459            Ok(payload) => {
460                // Avoid logging the serialized payload for authenticated channels;
461                // it embeds a live Lighter L2 bearer token.
462                log::debug!("Sending Lighter subscribe: topic={topic} authed={authed}");
463                let result = if authed {
464                    self.send_secret_with_retry(SecretString::from(payload))
465                        .await
466                } else {
467                    self.send_with_retry(payload).await
468                };
469
470                if let Err(e) = result {
471                    log::error!("Error subscribing to {topic}: {e}");
472                    Err(e.to_string())
473                } else {
474                    Ok(())
475                }
476            }
477            Err(e) => {
478                log::error!("Error serializing subscription for {topic}: {e}");
479                Err(format!("failed to serialize subscription for {topic}: {e}"))
480            }
481        }
482    }
483
484    async fn dispatch_send_tx(
485        &self,
486        tx_type: u8,
487        tx_info: Box<serde_json::value::RawValue>,
488        connection_epoch: u64,
489    ) -> Result<(), LighterWsError> {
490        let request = LighterWsRequest::SendTx {
491            data: super::messages::LighterWsSendTx { tx_type, tx_info },
492        };
493
494        match serde_json::to_string(&request) {
495            Ok(payload) => {
496                log::debug!(
497                    "Sending Lighter sendTx: tx_type={tx_type} ({} bytes)",
498                    payload.len(),
499                );
500
501                match self.send_once(payload, connection_epoch).await {
502                    Ok(()) => Ok(()),
503                    Err(e) => {
504                        log::debug!("Error dispatching Lighter sendTx (tx_type={tx_type}): {e}");
505                        Err(e)
506                    }
507                }
508            }
509            Err(e) => {
510                log::error!("Error serializing Lighter sendTx (tx_type={tx_type}): {e}");
511                Err(LighterWsError::Client(format!(
512                    "failed to serialize Lighter sendTx: {e}"
513                )))
514            }
515        }
516    }
517
518    async fn dispatch_unsubscribe(&self, channel: LighterWsChannel) {
519        let topic = channel.topic_key();
520        self.subscriptions.mark_unsubscribe(&topic);
521
522        let request = LighterWsRequest::unsubscribe(channel.subscription_channel());
523        match serde_json::to_string(&request) {
524            Ok(payload) => {
525                log::debug!("Sending Lighter unsubscribe ({} bytes)", payload.len());
526                if let Err(e) = self.send_with_retry(payload).await {
527                    log::error!("Error unsubscribing from {topic}: {e}");
528                }
529            }
530            Err(e) => {
531                log::error!("Error serializing unsubscription for {topic}: {e}");
532            }
533        }
534    }
535
536    pub(super) async fn next(&mut self) -> Option<NautilusWsMessage> {
537        if let Some(msg) = self.pending_messages.pop_front() {
538            return Some(msg);
539        }
540
541        loop {
542            // Pump each iteration: the select below drains acks that free slots
543            self.pump_pending_subscribes().await;
544
545            tokio::select! {
546                biased;
547                Some(work) = self.book.work.next(), if !self.book.work.is_empty() => {
548                    match work {
549                        BookWorkResult::Sent { market_index, generation, cancel, write, result } => {
550                            self.complete_book_send(market_index, generation, cancel, write, result);
551                        }
552                        BookWorkResult::Initial { market_index, cancel } => {
553                            if !cancel.is_cancelled() {
554                                self.start_book_recovery(market_index);
555                            }
556                        }
557                        // Recovery ends once a snapshot is accepted or the episode is cancelled
558                        BookWorkResult::Recovery => {}
559                    }
560                }
561                Some(cmd) = self.cmd_rx.recv() => {
562                    match cmd {
563                        HandlerCommand::SetClient(client) => {
564                            log::debug!("Setting WebSocket client in Lighter handler");
565                            self.inner = Some(Arc::new(client));
566                        }
567                        HandlerCommand::Disconnect => {
568                            log::debug!("Lighter handler received disconnect");
569                            if let Some(ref client) = self.inner {
570                                client.disconnect().await;
571                            }
572                            self.signal.store(true, Ordering::SeqCst);
573                            return None;
574                        }
575                        HandlerCommand::Subscribe {
576                            channel,
577                            auth,
578                            response_tx,
579                        } => {
580                            self.queue_subscribe(channel, auth, response_tx);
581                        }
582                        HandlerCommand::Unsubscribe { channel } => {
583                            // Drop a queued-but-unsent subscribe for this channel so a freed
584                            // slot cannot resubscribe after the caller unsubscribed.
585                            let topic = channel.topic_key();
586                            self.cancel_subscription_attempt(
587                                &topic,
588                                "subscription cancelled before venue acknowledgement",
589                            );
590
591                            if let LighterWsChannel::OrderBook(market_index) = &channel {
592                                self.book.cancel(*market_index);
593                                self.book.clear_cached_order_book(*market_index);
594                            }
595                            self.dispatch_unsubscribe(channel).await;
596                        }
597                        HandlerCommand::RecoverBook { market_index, cancel, gate, completion } => {
598                            self.queue_book_replacement(market_index, BookWrite { cancel, gate, completion });
599                        }
600                        HandlerCommand::InitializeInstruments(instruments) => {
601                            self.instruments.clear();
602                            for (market_index, inst) in instruments {
603                                self.instruments.insert(market_index, inst);
604                            }
605                        }
606                        HandlerCommand::UpdateInstrument { market_index, instrument } => {
607                            self.instruments.insert(market_index, instrument);
608                        }
609                        HandlerCommand::SetBookDeltasSub { market_index, subscribed } => {
610                            if subscribed {
611                                let inserted = self.book.delta_subs.insert(market_index);
612                                if inserted
613                                    && let Some(first) = self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_deltas_snapshot(market_index, instrument, self.clock.get_time_ns()))
614                                {
615                                    return Some(first);
616                                }
617                            } else {
618                                self.book.delta_subs.remove(&market_index);
619                                self.cancel_orphaned_book_recovery(market_index);
620                            }
621                        }
622                        HandlerCommand::SetDepthSub { market_index, subscribed } => {
623                            if subscribed {
624                                let inserted = self.book.depth_subs.insert(market_index);
625                                if inserted
626                                    && let Some(first) =
627                                        self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_depth_snapshot(market_index, instrument, self.clock.get_time_ns()))
628                                {
629                                    return Some(first);
630                                }
631                            } else {
632                                self.book.depth_subs.remove(&market_index);
633                                self.cancel_orphaned_book_recovery(market_index);
634                            }
635                        }
636                        HandlerCommand::SetExecutionContext { account_id, account_index } => {
637                            self.exec_account = Some((account_id, account_index));
638                        }
639                        HandlerCommand::SendTxBatch { data, connection_epoch, response_tx } => {
640                            let result = match serde_json::to_string(&LighterWsRequest::SendTxBatch { data }) {
641                                Ok(payload) => self.send_once(payload, connection_epoch).await,
642                                Err(e) => Err(LighterWsError::Client(format!("failed to serialize Lighter sendTxBatch: {e}"))),
643                            };
644                            let _ = response_tx.send(result);
645                        }
646                        HandlerCommand::SendTx {
647                            tx_type,
648                            tx_info,
649                            connection_epoch,
650                            response_tx,
651                        } => {
652                            let result = self
653                                .dispatch_send_tx(tx_type, tx_info, connection_epoch)
654                                .await;
655
656                            if response_tx.send(result).is_err() {
657                                log::debug!("Lighter sendTx result receiver dropped");
658                            }
659                        }
660                    }
661                }
662                Some((topic, generation)) = self.subscription_retries.next(),
663                    if !self.subscription_retries.is_empty() =>
664                {
665                    self.queue_subscription_retry(topic, generation);
666                }
667                Some((connection_epoch, raw_msg)) = self.raw_rx.recv() => {
668                    match raw_msg {
669                        Message::Text(text) => {
670                            if text == RECONNECTED {
671                                log::debug!("Received Lighter WebSocket RECONNECTED sentinel");
672                                self.book.reset_on_reconnect();
673                                // Resubscribe replays a fresh `subscribed/candle`; pre-disconnect cache is stale.
674                                self.last_candles.clear();
675                                // The new socket starts with zero inflight and restore_subscriptions
676                                // re-queues every topic. Retire stale generations before accepting
677                                // replay commands so late work cannot complete a replacement attempt.
678                                self.reset_subscription_attempts_after_reconnect();
679                                self.account_state_reconciler.reset();
680                                return Some(NautilusWsMessage::Reconnected {
681                                    connection_epoch,
682                                });
683                            }
684
685                            let ts_init = self.clock.get_time_ns();
686                            let subscribed_topic = typed_subscribe_topic(&text);
687
688                            if let Ok(frame) = serde_json::from_str::<LighterWsFrame>(&text) {
689                                if let LighterWsFrame::Shutdown { close_in_ms } = frame {
690                                    if let Some(ws_client) = &self.inner
691                                        && ws_client.connection_epoch() == connection_epoch
692                                        && ws_client.request_reconnect()
693                                    {
694                                        log::info!("Lighter server shutdown in {close_in_ms} ms, reconnect requested");
695                                    }
696                                    continue;
697                                }
698
699                                if let Some(topic) = subscribed_topic
700                                    && !self.complete_typed_subscription(topic, connection_epoch)
701                                {
702                                    continue;
703                                }
704                                let messages = self
705                                    .handle_frame(frame, ts_init)
706                                    .into_iter()
707                                    .map(|msg| msg.with_connection_epoch(connection_epoch))
708                                    .collect();
709
710                                if let Some(first) = self.dispatch_results(messages) {
711                                    return Some(first);
712                                }
713                            } else if let Ok(value) =
714                                serde_json::from_str::<serde_json::Value>(&text)
715                            {
716                                // The venue acknowledged the subscription even
717                                // though the snapshot body is unparsable.
718                                // Complete from the header so subscribers never
719                                // hang on a malformed confirmation frame.
720                                if let Some(topic) = subscribed_topic {
721                                    self.complete_typed_subscription(topic, connection_epoch);
722                                }
723
724                                let (matched, msg) = self.handle_control_value(&value);
725                                if let Some(first) = msg {
726                                    return Some(first.with_connection_epoch(connection_epoch));
727                                }
728
729                                if !matched {
730                                    // Neither a typed frame nor a recognized
731                                    // control type. Surface raw so venue
732                                    // errors (bad signature, margin rejection,
733                                    // ...) don't drop silently.
734                                    log::warn!("Lighter WS unparsed frame: {value}");
735                                    return Some(NautilusWsMessage::Raw(value));
736                                }
737                            } else {
738                                log::warn!("Lighter WS non-JSON text: {text}");
739                            }
740                        }
741                        Message::Ping(data) => {
742                            if let Some(ref client) = self.inner
743                                && let Err(e) = client.send_pong(data.to_vec()).await {
744                                log::error!("Error sending Lighter pong: {e}");
745                            }
746                        }
747                        Message::Close(frame) => {
748                            log::debug!("Received Lighter WebSocket close frame: {frame:?}");
749                            return None;
750                        }
751                        _ => {}
752                    }
753                }
754                else => {
755                    log::debug!("Lighter handler shutting down: stream ended or command channel closed");
756                    return None;
757                }
758            }
759        }
760    }
761
762    fn dispatch_results(
763        &mut self,
764        mut messages: Vec<NautilusWsMessage>,
765    ) -> Option<NautilusWsMessage> {
766        if messages.is_empty() {
767            return None;
768        }
769        let first = messages.remove(0);
770        for extra in messages {
771            self.pending_messages.push_back(extra);
772        }
773        Some(first)
774    }
775
776    // Non-blocking count check, not a permit await: this task also drains the
777    // acks that free slots, so awaiting a permit here would deadlock.
778    async fn pump_pending_subscribes(&mut self) {
779        while self.inflight_subs.len() < SUBSCRIBE_INFLIGHT_MAX {
780            let Some((topic, generation)) = self.pending_subs.pop_front() else {
781                break;
782            };
783
784            let Some(attempt) = self.subscription_attempts.get(&topic) else {
785                continue;
786            };
787
788            if attempt.generation != generation {
789                continue;
790            }
791
792            let channel = attempt.channel.clone();
793            let auth = attempt.auth.clone();
794            self.inflight_subs.insert(topic, generation);
795            if let LighterWsChannel::OrderBook(market_index) = channel {
796                self.send_book_subscribe(market_index, generation);
797            } else if let Err(message) = self.dispatch_subscribe(channel, auth).await {
798                self.schedule_subscription_retry(topic, generation, &message);
799            }
800        }
801    }
802
803    fn queue_subscribe(
804        &mut self,
805        channel: LighterWsChannel,
806        auth: Option<SecretString>,
807        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
808    ) {
809        let topic = Ustr::from(channel.topic_key().as_str());
810
811        if let LighterWsChannel::OrderBook(market_index) = channel
812            && !self.book.writes.contains_key(&market_index)
813            && self
814                .book
815                .recovery
816                .get(&market_index)
817                .and_then(BookRecoveryState::current)
818                .is_some_and(|recovery| !recovery.is_accepted())
819        {
820            return;
821        }
822
823        if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
824            attempt.channel = channel;
825            if matches!(attempt.channel, LighterWsChannel::OrderBook(_))
826                && !self.inflight_subs.contains_key(&topic)
827                && !self
828                    .pending_subs
829                    .iter()
830                    .any(|(pending, _)| *pending == topic)
831            {
832                self.pending_subs.push_back((topic, attempt.generation));
833            }
834
835            let effective_auth = attempt
836                .pending_auth
837                .as_ref()
838                .or(attempt.auth.as_ref())
839                .map(SecretString::expose_secret);
840            let auth_changed = auth
841                .as_ref()
842                .map(SecretString::expose_secret)
843                .is_some_and(|auth| Some(auth) != effective_auth);
844
845            if auth_changed {
846                let is_inflight = self.inflight_subs.get(&topic) == Some(&attempt.generation);
847                if is_inflight {
848                    attempt.pending_auth = auth;
849                    if let Some(response_tx) = response_tx {
850                        attempt.pending_response_txs.push(response_tx);
851                    }
852                } else {
853                    debug_assert!(attempt.pending_auth.is_none());
854                    debug_assert!(attempt.pending_response_txs.is_empty());
855                    attempt.auth = auth;
856                    if let Some(response_tx) = response_tx {
857                        attempt.response_txs.push(response_tx);
858                    }
859                }
860            } else if let Some(response_tx) = response_tx {
861                if auth.is_some() && attempt.pending_auth.is_some() {
862                    attempt.pending_response_txs.push(response_tx);
863                } else {
864                    attempt.response_txs.push(response_tx);
865                }
866            }
867            return;
868        }
869
870        let newly_pending = self.subscriptions.try_mark_subscribe(topic.as_str());
871        if !newly_pending
872            && auth.is_none()
873            && !self
874                .subscriptions
875                .pending_subscribe_topics()
876                .iter()
877                .any(|pending| pending == topic.as_str())
878        {
879            if let Some(response_tx) = response_tx {
880                let _ = response_tx.send(Ok(()));
881            }
882            return;
883        }
884
885        let generation = self.take_subscription_generation();
886        self.subscription_attempts.insert(
887            topic,
888            SubscriptionAttempt {
889                channel,
890                auth,
891                pending_auth: None,
892                generation,
893                retries: 0,
894                response_txs: response_tx.into_iter().collect(),
895                pending_response_txs: Vec::new(),
896            },
897        );
898        self.pending_subs.push_back((topic, generation));
899    }
900
901    fn queue_subscription_retry(&mut self, topic: Ustr, generation: u64) {
902        if self
903            .subscription_attempts
904            .get(&topic)
905            .is_some_and(|attempt| attempt.generation == generation)
906            && self.inflight_subs.get(&topic) != Some(&generation)
907            && !self
908                .pending_subs
909                .iter()
910                .any(|pending| *pending == (topic, generation))
911        {
912            self.pending_subs.push_back((topic, generation));
913        }
914    }
915
916    fn complete_subscription(&mut self, topic: &str, kind: CompletionKind) -> bool {
917        let topic = Ustr::from(topic);
918
919        // Swallow the predecessor's trailing frame before the inflight guard,
920        // so it cannot complete a same-topic generation dispatched between the
921        // venue's control ack and its typed snapshot - whether that generation
922        // is a serialized successor, a fresh attempt, queued, or in backoff.
923        if self.ignored_completions.get(&topic) == Some(&kind) {
924            self.ignored_completions.remove(&topic);
925            return false;
926        }
927
928        let Some(generation) = self.inflight_subs.get(&topic).copied() else {
929            return false;
930        };
931
932        if !self
933            .subscription_attempts
934            .get(&topic)
935            .is_some_and(|attempt| attempt.generation == generation)
936        {
937            return false;
938        }
939
940        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
941            && !self
942                .book
943                .expected
944                .get(&market_index)
945                .is_some_and(|expected| expected.0 == generation)
946        {
947            return false;
948        }
949
950        self.inflight_subs.remove(&topic);
951        self.subscriptions.confirm_subscribe(topic.as_str());
952
953        // The venue contract is control ack first, typed snapshot second, so a
954        // control-ack completion owes one trailing typed frame. Typed and
955        // already-subscribed completions have no established trailing frame.
956        if kind == CompletionKind::ControlAck {
957            if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
958                && let Some(expected) = self.book.expected.get(&market_index)
959                && expected.0 == generation
960            {
961                self.book.trailing.entry(market_index).or_insert(*expected);
962            }
963
964            self.ignored_completions
965                .insert(topic, CompletionKind::Typed);
966        }
967        let mut attempt = self
968            .subscription_attempts
969            .remove(&topic)
970            .expect("matching subscription attempt disappeared");
971        for response_tx in std::mem::take(&mut attempt.response_txs) {
972            let _ = response_tx.send(Ok(()));
973        }
974
975        if let Some(auth) = attempt.pending_auth.take() {
976            let generation = self.take_subscription_generation();
977            attempt.auth = Some(auth);
978            attempt.generation = generation;
979            attempt.retries = 0;
980            attempt.response_txs = std::mem::take(&mut attempt.pending_response_txs);
981            self.subscription_attempts.insert(topic, attempt);
982            self.pending_subs.push_back((topic, generation));
983        } else {
984            debug_assert!(attempt.pending_response_txs.is_empty());
985        }
986        true
987    }
988
989    fn retry_inflight_subscriptions(&mut self, message: &str) {
990        let mut inflight: Vec<(Ustr, u64)> = self
991            .inflight_subs
992            .iter()
993            .map(|(topic, generation)| (*topic, *generation))
994            .collect();
995        inflight.sort_unstable_by_key(|(topic, _)| *topic);
996
997        for (topic, generation) in inflight {
998            self.schedule_subscription_retry(topic, generation, message);
999        }
1000    }
1001
1002    fn schedule_subscription_retry(&mut self, topic: Ustr, generation: u64, message: &str) {
1003        if self.inflight_subs.get(&topic) != Some(&generation) {
1004            return;
1005        }
1006
1007        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1008            self.reject_book_subscription(
1009                market_index,
1010                generation,
1011                LighterWsError::Network(message.into()),
1012            );
1013            return;
1014        }
1015
1016        self.inflight_subs.remove(&topic);
1017        self.subscriptions.mark_failure(topic.as_str());
1018
1019        let Some(attempt) = self.subscription_attempts.get_mut(&topic) else {
1020            return;
1021        };
1022
1023        if attempt.generation != generation {
1024            return;
1025        }
1026
1027        attempt.fold_pending_auth();
1028        if attempt.retries >= SUBSCRIBE_RETRY_MAX {
1029            self.fail_subscription_attempt(topic, message);
1030            return;
1031        }
1032
1033        attempt.retries += 1;
1034        let retry = attempt.retries;
1035        let next_generation = self.take_subscription_generation();
1036        let attempt = self
1037            .subscription_attempts
1038            .get_mut(&topic)
1039            .expect("subscription attempt disappeared before retry");
1040        attempt.generation = next_generation;
1041
1042        let delay = SUBSCRIBE_RETRY_BASE_BACKOFF.saturating_mul(1_u32 << (retry - 1));
1043        self.subscription_retries.push(Box::pin(async move {
1044            tokio::time::sleep(delay).await;
1045            (topic, next_generation)
1046        }));
1047    }
1048
1049    fn fail_subscription_attempt(&mut self, topic: Ustr, message: &str) {
1050        self.inflight_subs.remove(&topic);
1051        self.pending_subs.retain(|(pending, _)| *pending != topic);
1052        self.subscriptions.mark_unsubscribe(topic.as_str());
1053        self.subscriptions.confirm_unsubscribe(topic.as_str());
1054
1055        if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1056            let attempts = attempt.retries + 1;
1057            for response_tx in attempt
1058                .response_txs
1059                .into_iter()
1060                .chain(attempt.pending_response_txs)
1061            {
1062                let _ = response_tx.send(Err(format!(
1063                    "subscription {topic} failed after {attempts} attempts: {message}",
1064                )));
1065            }
1066        }
1067    }
1068
1069    fn cancel_subscription_attempt(&mut self, topic: &str, message: &str) {
1070        let topic = Ustr::from(topic);
1071        self.inflight_subs.remove(&topic);
1072        self.pending_subs.retain(|(pending, _)| *pending != topic);
1073        if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1074            for response_tx in attempt
1075                .response_txs
1076                .into_iter()
1077                .chain(attempt.pending_response_txs)
1078            {
1079                let _ = response_tx.send(Err(format!("{message}: {topic}")));
1080            }
1081        }
1082    }
1083
1084    fn reset_subscription_attempts_after_reconnect(&mut self) {
1085        for topic in self.subscriptions.all_topics() {
1086            self.subscriptions.mark_failure(&topic);
1087        }
1088        self.pending_subs.clear();
1089        self.inflight_subs.clear();
1090        // Completion frames from the old connection must not be applied to -
1091        // or swallowed from - the new connection.
1092        self.ignored_completions.clear();
1093
1094        let mut topics: Vec<Ustr> = self.subscription_attempts.keys().copied().collect();
1095        topics.sort_unstable();
1096        for topic in topics {
1097            let generation = self.take_subscription_generation();
1098            let attempt = self
1099                .subscription_attempts
1100                .get_mut(&topic)
1101                .expect("subscription attempt disappeared during reconnect");
1102            attempt.fold_pending_auth();
1103            attempt.generation = generation;
1104            attempt.retries = 0;
1105
1106            if order_book_market_index_from_topic(topic.as_str()).is_none() {
1107                self.pending_subs.push_back((topic, generation));
1108            }
1109        }
1110    }
1111
1112    fn take_subscription_generation(&mut self) -> u64 {
1113        let generation = self.next_subscription_generation;
1114        self.next_subscription_generation =
1115            self.next_subscription_generation.wrapping_add(1).max(1);
1116        generation
1117    }
1118
1119    /// Returns `(matched, msg)` where `matched=true` means the frame's
1120    /// `type` was recognized as a known control type (whether or not a
1121    /// message is emitted); `matched=false` lets the caller surface the
1122    /// raw frame so venue errors with unknown shapes aren't lost.
1123    fn handle_control_value(
1124        &mut self,
1125        value: &serde_json::Value,
1126    ) -> (bool, Option<NautilusWsMessage>) {
1127        if value.get("type").and_then(|v| v.as_str()) == Some("jsonapi/sendtxbatch")
1128            || value
1129                .get("id")
1130                .and_then(|v| v.as_str())
1131                .is_some_and(|id| id.starts_with(TX_BATCH_ID_PREFIX))
1132        {
1133            let body = value.get("error").unwrap_or(value);
1134            let id = value.get("id").and_then(|v| v.as_str());
1135            let code = body.get("code").and_then(|v| v.as_i64());
1136
1137            let (Some(id), Some(code)) = (id, code) else {
1138                log::warn!("Ignoring malformed Lighter sendTxBatch response");
1139                return (true, None);
1140            };
1141
1142            return (
1143                true,
1144                Some(NautilusWsMessage::SendTxBatchResult {
1145                    connection_epoch: 0,
1146                    id: id.to_string(),
1147                    code,
1148                    message: body
1149                        .get("message")
1150                        .and_then(|v| v.as_str())
1151                        .unwrap_or("batch rejected")
1152                        .to_string(),
1153                    tx_hashes: value
1154                        .get("tx_hash")
1155                        .and_then(|v| serde_json::from_value(v.clone()).ok())
1156                        .unwrap_or_default(),
1157                }),
1158            );
1159        }
1160
1161        if let Some(error) = already_subscribed_error(value) {
1162            self.confirm_already_subscribed(error);
1163            return (true, None);
1164        }
1165        let subscription_code = subscription_error_code(value);
1166
1167        if subscription_code == Some(LIGHTER_ERROR_CODE_WS_RATE_LIMITED) {
1168            self.retry_inflight_subscriptions(&format!(
1169                "venue rejected the WebSocket subscribe with code \
1170                 {LIGHTER_ERROR_CODE_WS_RATE_LIMITED}",
1171            ));
1172            return (true, None);
1173        }
1174
1175        if subscription_code == Some(LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED) {
1176            self.handle_subscribe_failure(value);
1177            return (true, None);
1178        }
1179
1180        let kind = value.get("type").and_then(|v| v.as_str()).unwrap_or("");
1181
1182        match kind {
1183            CTRL_TYPE_CONNECTED => {
1184                log::debug!("Lighter WebSocket handshake complete");
1185                (true, None)
1186            }
1187            CTRL_TYPE_PING | CTRL_TYPE_PONG => (true, None),
1188            CTRL_TYPE_SEND_TX => {
1189                let raw_code = value.get("code").and_then(|v| v.as_u64());
1190                match raw_code {
1191                    Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) => {
1192                        log_integrator_not_approved();
1193                        (
1194                            true,
1195                            Some(send_tx_rejected_from_value(
1196                                value,
1197                                SendTxRejectionSource::Ack,
1198                            )),
1199                        )
1200                    }
1201                    Some(200) => {
1202                        log::debug!("Lighter WebSocket sendTx ack: {value}");
1203                        let tx_hash = value
1204                            .get("tx_hash")
1205                            .and_then(|v| v.as_str())
1206                            .map(str::to_string);
1207                        (
1208                            true,
1209                            Some(NautilusWsMessage::SendTxAck {
1210                                connection_epoch: 0,
1211                                tx_hash,
1212                                code: 200,
1213                            }),
1214                        )
1215                    }
1216                    Some(_) => (
1217                        true,
1218                        Some(send_tx_rejected_from_value(
1219                            value,
1220                            SendTxRejectionSource::Ack,
1221                        )),
1222                    ),
1223                    None => {
1224                        log::warn!(
1225                            "Ignoring malformed Lighter sendTx response without numeric code: {value}",
1226                        );
1227                        (true, None)
1228                    }
1229                }
1230            }
1231            CTRL_TYPE_SUBSCRIBED | CTRL_TYPE_UNSUBSCRIBED => {
1232                if let Some(topic) = value.get("channel").and_then(|v| v.as_str()) {
1233                    if kind == CTRL_TYPE_SUBSCRIBED {
1234                        self.complete_subscription(topic, CompletionKind::ControlAck);
1235                    } else {
1236                        let was_pending_unsubscribe = self
1237                            .subscriptions
1238                            .pending_unsubscribe_topics()
1239                            .iter()
1240                            .any(|pending| pending == topic);
1241                        self.subscriptions.confirm_unsubscribe(topic);
1242
1243                        if was_pending_unsubscribe {
1244                            // Only matched unsubscribe ACKs should reset stream state
1245                            if let Some(market_index) = order_book_market_index_from_topic(topic) {
1246                                self.book.clear_cached_order_book(market_index);
1247                            }
1248
1249                            if let Some(key) = candle_market_and_resolution_from_topic(topic) {
1250                                self.last_candles.remove(&key);
1251                            }
1252                        }
1253                    }
1254                }
1255                (true, None)
1256            }
1257            CTRL_TYPE_ERROR => {
1258                let code = value.get("code").and_then(|v| v.as_u64());
1259                if code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1260                    log_integrator_not_approved();
1261                } else {
1262                    log::warn!("Lighter WebSocket error frame: {value}");
1263                }
1264
1265                if is_sendtx_error_code(code) {
1266                    (
1267                        true,
1268                        Some(send_tx_rejected_from_value(
1269                            value,
1270                            SendTxRejectionSource::BareError,
1271                        )),
1272                    )
1273                } else {
1274                    (true, None)
1275                }
1276            }
1277            _ => {
1278                if let Some(error) = value.get("error") {
1279                    let nested_code = error.get("code").and_then(|v| v.as_u64());
1280                    if nested_code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1281                        log_integrator_not_approved();
1282                    } else {
1283                        log::warn!("Lighter WebSocket error frame: {value}");
1284                    }
1285                    let rejected = is_sendtx_error_code(nested_code).then(|| {
1286                        send_tx_rejected_from_nested_error(error, SendTxRejectionSource::BareError)
1287                    });
1288                    return (true, rejected);
1289                }
1290                (false, None)
1291            }
1292        }
1293    }
1294
1295    fn handle_subscribe_failure(&mut self, value: &serde_json::Value) {
1296        let message = format!(
1297            "venue rejected the WebSocket subscribe with code \
1298             {LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED}",
1299        );
1300
1301        let channel = value
1302            .get("channel")
1303            .and_then(serde_json::Value::as_str)
1304            .or_else(|| {
1305                value
1306                    .get("error")
1307                    .and_then(|e| e.get("channel"))
1308                    .and_then(serde_json::Value::as_str)
1309            })
1310            .filter(|channel| !channel.is_empty());
1311
1312        let Some(channel) = channel else {
1313            log::warn!(
1314                "Lighter subscribe error code={LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED} channel=none: retrying inflight subscriptions"
1315            );
1316            self.retry_inflight_subscriptions(&message);
1317            return;
1318        };
1319
1320        let topic = Ustr::from(channel.replace('/', ":").as_str());
1321
1322        let Some(generation) = self.inflight_subs.get(&topic).copied() else {
1323            log::warn!(
1324                "Lighter subscribe error code={LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED} channel={channel}: no matching inflight subscription"
1325            );
1326            return;
1327        };
1328
1329        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1330            self.reject_book_subscription(
1331                market_index,
1332                generation,
1333                LighterWsError::Client(message),
1334            );
1335        } else {
1336            self.fail_subscription_attempt(topic, &message);
1337        }
1338    }
1339
1340    fn confirm_already_subscribed(&mut self, error: &serde_json::Value) {
1341        let Some(topic) = error
1342            .get("message")
1343            .and_then(|value| value.as_str())
1344            .and_then(|message| message.strip_prefix("Already Subscribed to : "))
1345        else {
1346            log::debug!(
1347                "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1348            );
1349            return;
1350        };
1351
1352        if !self.complete_subscription(topic, CompletionKind::AlreadySubscribed) {
1353            log::debug!(
1354                "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1355            );
1356            return;
1357        }
1358
1359        log::debug!(
1360            "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}, topic={topic}",
1361        );
1362    }
1363
1364    fn handle_frame(
1365        &mut self,
1366        frame: LighterWsFrame,
1367        ts_init: UnixNanos,
1368    ) -> Vec<NautilusWsMessage> {
1369        match frame {
1370            LighterWsFrame::OrderBookSnapshot {
1371                channel,
1372                order_book,
1373                timestamp,
1374                ..
1375            } => self.handle_order_book(channel, &order_book, timestamp, true, ts_init),
1376            LighterWsFrame::OrderBook {
1377                channel,
1378                order_book,
1379                timestamp,
1380                ..
1381            } => self.handle_order_book(channel, &order_book, timestamp, false, ts_init),
1382            LighterWsFrame::TickerSnapshot {
1383                channel,
1384                ticker,
1385                timestamp,
1386                ..
1387            }
1388            | LighterWsFrame::Ticker {
1389                channel,
1390                ticker,
1391                timestamp,
1392                ..
1393            } => self.handle_ticker(channel, &ticker, timestamp, ts_init),
1394            LighterWsFrame::TradeSnapshot {
1395                trades,
1396                liquidation_trades,
1397                ..
1398            }
1399            | LighterWsFrame::Trade {
1400                trades,
1401                liquidation_trades,
1402                ..
1403            } => self.handle_trades(&trades, &liquidation_trades, ts_init),
1404            LighterWsFrame::AccountTxSubscribed { .. } => Vec::new(),
1405            LighterWsFrame::AccountTx { channel, txs } => self.handle_account_tx(channel, txs),
1406            LighterWsFrame::AccountOrders { ref orders, .. }
1407            | LighterWsFrame::AccountAllOrders { ref orders, .. } => {
1408                if self.exec_account.is_none() {
1409                    return raw_message(&frame);
1410                }
1411                let mut msgs = self.handle_account_orders(orders, ts_init);
1412                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1413                    AccountStream::Orders,
1414                ));
1415                msgs
1416            }
1417            // Lighter publishes the historical fill replay as `subscribed/`
1418            // rather than `update/`. When typed execution routing is active
1419            // we drop the snapshot to avoid re-emitting prior executions as
1420            // fresh fill reports on every reconnect (HTTP reconciliation
1421            // owns historical state recovery). When no execution context is
1422            // set we still forward as `Raw` to preserve the prior contract
1423            // for unauthenticated subscribers.
1424            LighterWsFrame::AccountAllTradesSnapshot { .. } => {
1425                if self.exec_account.is_none() {
1426                    return raw_message(&frame);
1427                }
1428                log::debug!(
1429                    "Skipping Lighter account_all_trades snapshot frame; \
1430                     reconcile historical fills via HTTP",
1431                );
1432                vec![NautilusWsMessage::AccountStreamFirstFrame(
1433                    AccountStream::Trades,
1434                )]
1435            }
1436            LighterWsFrame::AccountAllTrades { ref trades, .. } => {
1437                if self.exec_account.is_none() {
1438                    return raw_message(&frame);
1439                }
1440                let mut msgs = self.handle_account_trades(trades.values().flatten(), ts_init);
1441                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1442                    AccountStream::Trades,
1443                ));
1444                msgs
1445            }
1446            LighterWsFrame::AccountAllPositionsSnapshot { ref positions, .. } => {
1447                if self.exec_account.is_none() {
1448                    return raw_message(&frame);
1449                }
1450                let mut msgs =
1451                    self.handle_account_positions(positions, ts_init, PositionFrameType::Snapshot);
1452                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1453                    AccountStream::Positions,
1454                ));
1455                msgs
1456            }
1457            LighterWsFrame::AccountAllPositions { ref positions, .. } => {
1458                if self.exec_account.is_none() {
1459                    return raw_message(&frame);
1460                }
1461                self.handle_account_positions(positions, ts_init, PositionFrameType::Update)
1462            }
1463            LighterWsFrame::AccountAllAssets {
1464                ref assets,
1465                timestamp,
1466                ..
1467            } => {
1468                if self.exec_account.is_none() {
1469                    return raw_message(&frame);
1470                }
1471                let mut msgs = self.handle_account_assets(assets, timestamp, ts_init);
1472                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1473                    AccountStream::Assets,
1474                ));
1475                msgs
1476            }
1477            LighterWsFrame::UserStats {
1478                ref stats,
1479                timestamp,
1480                ..
1481            } => {
1482                if self.exec_account.is_none() {
1483                    return raw_message(&frame);
1484                }
1485                let mut msgs = self.handle_user_stats(stats, timestamp, ts_init);
1486                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1487                    AccountStream::UserStats,
1488                ));
1489                msgs
1490            }
1491            LighterWsFrame::MarketStats {
1492                ref market_stats,
1493                timestamp,
1494                ..
1495            } => self.handle_market_stats(market_stats, timestamp, ts_init),
1496            LighterWsFrame::SpotMarketStats {
1497                ref spot_market_stats,
1498                timestamp,
1499                ..
1500            } => self.handle_spot_market_stats(spot_market_stats, timestamp, ts_init),
1501            LighterWsFrame::CandleSnapshot {
1502                channel,
1503                ref candles,
1504                ..
1505            }
1506            | LighterWsFrame::Candle {
1507                channel,
1508                ref candles,
1509                ..
1510            } => self.handle_candles(channel, candles, ts_init),
1511            LighterWsFrame::Shutdown { .. } => Vec::new(),
1512            LighterWsFrame::Height { .. } => raw_message(&frame),
1513        }
1514    }
1515
1516    fn handle_account_tx(&self, channel: Ustr, txs: Vec<LighterWsTx>) -> Vec<NautilusWsMessage> {
1517        let Some((_, account_index)) = self.exec_account else {
1518            return raw_message(&LighterWsFrame::AccountTx { channel, txs });
1519        };
1520
1521        if channel.as_str() != format!("account_tx:{account_index}") {
1522            log::warn!("Ignoring Lighter transaction update for unexpected channel {channel}");
1523            return Vec::new();
1524        }
1525
1526        vec![NautilusWsMessage::AccountTx {
1527            connection_epoch: 0,
1528            txs,
1529        }]
1530    }
1531
1532    fn handle_order_book(
1533        &mut self,
1534        channel: Ustr,
1535        book: &LighterWsOrderBook,
1536        timestamp: u64,
1537        is_snapshot: bool,
1538        ts_init: UnixNanos,
1539    ) -> Vec<NautilusWsMessage> {
1540        let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1541            log::debug!("Lighter order_book frame missing market index in channel '{channel}'");
1542            return Vec::new();
1543        };
1544
1545        let Some(instrument) = self.instruments.get(&market_index) else {
1546            log::debug!("No instrument cached for Lighter market_index={market_index}");
1547            return Vec::new();
1548        };
1549
1550        match self.book.validate_sequence(market_index, book, is_snapshot) {
1551            BookSequenceOutcome::Accept => self.book.apply(
1552                market_index,
1553                instrument,
1554                book,
1555                timestamp,
1556                is_snapshot,
1557                ts_init,
1558            ),
1559            BookSequenceOutcome::Suppress => Vec::new(),
1560            BookSequenceOutcome::Recover => {
1561                self.start_book_recovery(market_index);
1562                Vec::new()
1563            }
1564        }
1565    }
1566
1567    fn queue_book_replacement(&mut self, market_index: i64, write: BookWrite) {
1568        if write.cancel.is_cancelled() || !self.order_book_stream_is_referenced(market_index) {
1569            return;
1570        }
1571
1572        self.book.clear_cached_order_book(market_index);
1573        self.book.expected.remove(&market_index);
1574        let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1575        self.retire_book_attempt(&topic);
1576        self.subscriptions.mark_failure(&topic);
1577        self.book.writes.insert(market_index, write);
1578        self.queue_subscribe(LighterWsChannel::OrderBook(market_index), None, None);
1579    }
1580
1581    fn retire_book_attempt(&mut self, topic: &str) {
1582        let topic = Ustr::from(topic);
1583        self.inflight_subs.remove(&topic);
1584        self.pending_subs.retain(|(pending, _)| *pending != topic);
1585        if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
1586            // Keep callers waiting for the eventual replacement acknowledgement.
1587            attempt.generation = self.next_subscription_generation;
1588            self.next_subscription_generation =
1589                self.next_subscription_generation.wrapping_add(1).max(1);
1590            self.pending_subs.push_back((topic, attempt.generation));
1591        }
1592    }
1593
1594    fn reject_book_subscription(
1595        &mut self,
1596        market_index: i64,
1597        generation: u64,
1598        error: LighterWsError,
1599    ) {
1600        if !self
1601            .book
1602            .expected
1603            .get(&market_index)
1604            .is_some_and(|expected| expected.0 == generation)
1605            || self
1606                .book
1607                .recovery
1608                .get(&market_index)
1609                .and_then(BookRecoveryState::current)
1610                .is_some_and(|recovery| recovery.gate.lock().is_closed())
1611        {
1612            return;
1613        }
1614
1615        self.reject_book(market_index, error);
1616    }
1617
1618    fn reject_book(&mut self, market_index: i64, error: LighterWsError) {
1619        self.book.expected.remove(&market_index);
1620        self.book.clear_cached_order_book(market_index);
1621        let topic = Ustr::from(
1622            LighterWsChannel::OrderBook(market_index)
1623                .topic_key()
1624                .as_str(),
1625        );
1626
1627        let waiting = matches!(error, LighterWsError::Client(_))
1628            && self
1629                .subscription_attempts
1630                .get(&topic)
1631                .is_some_and(|attempt| {
1632                    !attempt.response_txs.is_empty() || !attempt.pending_response_txs.is_empty()
1633                });
1634
1635        if waiting {
1636            // The waiting subscriber owns the failure and releases its stream reference; an expired
1637            // initial wait must not start recovery before that release.
1638            self.book.initial.remove(&market_index);
1639            self.cancel_subscription_attempt(&topic, &error.to_string());
1640            log::error!(
1641                "Lighter book subscription rejected for market_index={market_index}: {error}"
1642            );
1643        }
1644
1645        if let Some(recovery) = self
1646            .book
1647            .recovery
1648            .get(&market_index)
1649            .and_then(BookRecoveryState::current)
1650            && !recovery.is_accepted()
1651        {
1652            // Frees the slot during backoff; the next replacement takes a new generation
1653            self.inflight_subs.remove(&topic);
1654            recovery.gate.lock().close();
1655            recovery
1656                .outcome
1657                .send_replace(BookRecoveryOutcome::Rejected(error));
1658        } else if !waiting {
1659            self.start_book_recovery(market_index);
1660        }
1661    }
1662
1663    fn start_book_recovery(&mut self, market_index: i64) {
1664        if !self.order_book_stream_is_referenced(market_index) {
1665            return;
1666        }
1667
1668        let Some(cmd_tx) = self.cmd_tx.clone() else {
1669            return;
1670        };
1671
1672        let Some(recovery) = self.book.recovery.entry(market_index).or_default().claim() else {
1673            return;
1674        };
1675
1676        recovery.gate.lock().close();
1677        self.book.initial.remove(&market_index);
1678        self.book.expected.remove(&market_index);
1679        self.book.clear_cached_order_book(market_index);
1680        let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1681        self.inflight_subs.remove(&Ustr::from(topic.as_str()));
1682        self.pending_subs
1683            .retain(|(pending, _)| pending.as_str() != topic);
1684        self.book.work.push(recovery::recover(
1685            market_index,
1686            recovery,
1687            cmd_tx,
1688            self.book_snapshot_timeout,
1689        ));
1690    }
1691
1692    // A rejected subscriber releases its stream reference without an unsubscribe, so the last
1693    // consumer's release must end the recovery.
1694    fn cancel_orphaned_book_recovery(&mut self, market_index: i64) {
1695        if !self.order_book_stream_is_referenced(market_index) {
1696            self.book.recovery.remove(&market_index);
1697        }
1698    }
1699
1700    fn complete_typed_subscription(&mut self, topic: &str, epoch: u64) -> bool {
1701        if let Some(market_index) = order_book_market_index_from_topic(topic)
1702            && self
1703                .book
1704                .expected
1705                .get(&market_index)
1706                .is_some_and(|expected| expected.1 != epoch)
1707        {
1708            return false;
1709        }
1710
1711        if !self.book_snapshot_matches(topic, epoch) {
1712            if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1713                self.ignored_completions.remove(&Ustr::from(topic));
1714            }
1715
1716            return false;
1717        }
1718
1719        self.complete_subscription(topic, CompletionKind::Typed);
1720        true
1721    }
1722
1723    fn book_snapshot_matches(&mut self, topic: &str, epoch: u64) -> bool {
1724        let Some(market_index) = order_book_market_index_from_topic(topic) else {
1725            return true;
1726        };
1727
1728        let source =
1729            if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1730                self.book.trailing.remove(&market_index)
1731            } else {
1732                self.inflight_subs
1733                    .get(&Ustr::from(topic))
1734                    .map(|generation| (*generation, epoch))
1735            };
1736
1737        source.is_some_and(|source| {
1738            source.1 == epoch && Some(source) == self.book.expected.get(&market_index).copied()
1739        })
1740    }
1741
1742    fn send_book_subscribe(&mut self, market_index: i64, generation: u64) {
1743        let write = self.book.writes.remove(&market_index);
1744        let cancel = write
1745            .as_ref()
1746            .map_or_else(CancellationToken::new, |write| write.cancel.clone());
1747        if write.is_none() {
1748            self.book.initial.insert(
1749                market_index,
1750                PendingSnapshot {
1751                    deadline: None,
1752                    cancel: cancel.clone(),
1753                    gate: SnapshotGate::default(),
1754                },
1755            );
1756        }
1757
1758        let client = self.inner.clone();
1759        let subscriptions = self.subscriptions.clone();
1760        let book_snapshot_timeout = self.book_snapshot_timeout;
1761        self.book.work.push(recovery::subscribe(
1762            market_index,
1763            generation,
1764            cancel,
1765            write,
1766            client,
1767            subscriptions,
1768            book_snapshot_timeout,
1769        ));
1770    }
1771
1772    fn complete_book_send(
1773        &mut self,
1774        market_index: i64,
1775        generation: u64,
1776        cancel: CancellationToken,
1777        write: Option<BookWrite>,
1778        result: Result<u64, LighterWsError>,
1779    ) {
1780        let topic = Ustr::from(
1781            LighterWsChannel::OrderBook(market_index)
1782                .topic_key()
1783                .as_str(),
1784        );
1785
1786        if cancel.is_cancelled()
1787            || self.inflight_subs.get(&topic) != Some(&generation)
1788            || !self.order_book_stream_is_referenced(market_index)
1789        {
1790            return;
1791        }
1792
1793        match result {
1794            Ok(epoch) => {
1795                self.book.expected.insert(market_index, (generation, epoch));
1796
1797                if let Some(write) = write {
1798                    write.gate.open();
1799                    let _ = write.completion.send(Ok(()));
1800                } else {
1801                    self.book.work.push(recovery::wait_for_snapshot(
1802                        market_index,
1803                        cancel,
1804                        self.book_snapshot_timeout,
1805                    ));
1806                }
1807            }
1808            Err(e) => {
1809                if let Some(write) = write {
1810                    // Frees the slot during backoff; the next replacement takes a new generation
1811                    self.inflight_subs.remove(&topic);
1812                    let _ = write.completion.send(Err(e));
1813                } else {
1814                    self.reject_book(market_index, e);
1815                }
1816            }
1817        }
1818    }
1819
1820    fn order_book_stream_is_referenced(&self, market_index: i64) -> bool {
1821        let channel = LighterWsChannel::OrderBook(market_index);
1822        self.subscriptions.get_reference_count(&channel.topic_key()) > 0
1823            && (self.book.delta_subs.contains(&market_index)
1824                || self.book.depth_subs.contains(&market_index))
1825    }
1826
1827    fn handle_ticker(
1828        &self,
1829        channel: Ustr,
1830        ticker: &super::messages::LighterTicker,
1831        timestamp: u64,
1832        ts_init: UnixNanos,
1833    ) -> Vec<NautilusWsMessage> {
1834        // Resolve the instrument from the channel's `ticker:N` market index,
1835        // not the ticker payload's `s` symbol. Lighter sends the unsuffixed
1836        // venue symbol (e.g. "ETH"), while cached `raw_symbol` carries the
1837        // -PERP / -SPOT suffix, so a symbol-name match never resolves and
1838        // would conflate spot vs perp listings of the same asset.
1839        let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1840            log::debug!("Lighter ticker frame missing market index in channel '{channel}'");
1841            return Vec::new();
1842        };
1843
1844        let Some(instrument) = self.instruments.get(&market_index) else {
1845            log::debug!("No instrument cached for Lighter ticker market_index={market_index}");
1846            return Vec::new();
1847        };
1848
1849        match parse_ws_quote_tick(ticker, instrument, timestamp, ts_init) {
1850            Ok(Some(quote)) => vec![NautilusWsMessage::Quote(quote)],
1851            Ok(None) => {
1852                log::debug!(
1853                    "Skipping Lighter ticker for market_index={market_index}: one-sided book",
1854                );
1855                Vec::new()
1856            }
1857            Err(e) => {
1858                log::error!("Error parsing Lighter ticker frame: {e}");
1859                Vec::new()
1860            }
1861        }
1862    }
1863
1864    fn handle_trades(
1865        &self,
1866        trades: &[crate::http::models::LighterTrade],
1867        liquidation_trades: &[crate::http::models::LighterTrade],
1868        ts_init: UnixNanos,
1869    ) -> Vec<NautilusWsMessage> {
1870        // Lighter splits trade prints across `trades` and `liquidation_trades`
1871        // on the same `update/trade` frame; both share the LighterTrade model
1872        // and are public executions, so emit them together.
1873        let Some(market_index) = trades
1874            .first()
1875            .or_else(|| liquidation_trades.first())
1876            .map(|t| t.market_id)
1877        else {
1878            return Vec::new();
1879        };
1880
1881        let Some(instrument) = self.instruments.get(&market_index) else {
1882            log::debug!("No instrument cached for Lighter trade market_index={market_index}");
1883            return Vec::new();
1884        };
1885
1886        let mut ticks = Vec::with_capacity(trades.len() + liquidation_trades.len());
1887        for trade in trades.iter().chain(liquidation_trades.iter()) {
1888            match parse_ws_trade_tick(trade, instrument, ts_init) {
1889                Ok(tick) => ticks.push(tick),
1890                Err(e) => log::error!("Error parsing Lighter trade tick: {e}"),
1891            }
1892        }
1893
1894        if ticks.is_empty() {
1895            Vec::new()
1896        } else {
1897            vec![NautilusWsMessage::Trades(ticks)]
1898        }
1899    }
1900
1901    fn handle_market_stats(
1902        &self,
1903        payload: &super::messages::LighterMarketStatsPayload,
1904        timestamp: u64,
1905        ts_init: UnixNanos,
1906    ) -> Vec<NautilusWsMessage> {
1907        match payload {
1908            super::messages::LighterMarketStatsPayload::All(stats) => stats
1909                .values()
1910                .flat_map(|stats| self.handle_one_market_stats(stats, timestamp, ts_init))
1911                .collect(),
1912            super::messages::LighterMarketStatsPayload::One(stats) => {
1913                self.handle_one_market_stats(stats, timestamp, ts_init)
1914            }
1915        }
1916    }
1917
1918    fn handle_one_market_stats(
1919        &self,
1920        stats: &super::messages::LighterMarketStats,
1921        timestamp: u64,
1922        ts_init: UnixNanos,
1923    ) -> Vec<NautilusWsMessage> {
1924        let Some(instrument) = self.instruments.get(&stats.market_id) else {
1925            log::debug!(
1926                "No instrument cached for Lighter market_stats market_id={}",
1927                stats.market_id,
1928            );
1929            return Vec::new();
1930        };
1931
1932        let mut messages = Vec::with_capacity(3);
1933
1934        match parse_ws_mark_price_update(stats, instrument, timestamp, ts_init) {
1935            Ok(mark_price) => messages.push(NautilusWsMessage::MarkPrice(mark_price)),
1936            Err(e) => log::error!("Error parsing Lighter mark price: {e}"),
1937        }
1938
1939        match parse_ws_index_price_update(stats, instrument, timestamp, ts_init) {
1940            Ok(index_price) => messages.push(NautilusWsMessage::IndexPrice(index_price)),
1941            Err(e) => log::error!("Error parsing Lighter index price: {e}"),
1942        }
1943
1944        match parse_ws_funding_rate_update(stats, instrument, timestamp, ts_init) {
1945            Ok(funding_rate) => messages.push(NautilusWsMessage::FundingRate(funding_rate)),
1946            Err(e) => log::error!("Error parsing Lighter funding rate: {e}"),
1947        }
1948
1949        messages
1950    }
1951
1952    fn handle_spot_market_stats(
1953        &self,
1954        payload: &super::messages::LighterSpotMarketStatsPayload,
1955        timestamp: u64,
1956        ts_init: UnixNanos,
1957    ) -> Vec<NautilusWsMessage> {
1958        match payload {
1959            super::messages::LighterSpotMarketStatsPayload::All(stats) => stats
1960                .values()
1961                .filter_map(|stats| self.handle_one_spot_market_stats(stats, timestamp, ts_init))
1962                .collect(),
1963            super::messages::LighterSpotMarketStatsPayload::One(stats) => self
1964                .handle_one_spot_market_stats(stats, timestamp, ts_init)
1965                .into_iter()
1966                .collect(),
1967        }
1968    }
1969
1970    fn handle_one_spot_market_stats(
1971        &self,
1972        stats: &super::messages::LighterSpotMarketStats,
1973        timestamp: u64,
1974        ts_init: UnixNanos,
1975    ) -> Option<NautilusWsMessage> {
1976        let Some(instrument) = self.instruments.get(&stats.market_id) else {
1977            log::debug!(
1978                "No instrument cached for Lighter spot_market_stats market_id={}",
1979                stats.market_id,
1980            );
1981            return None;
1982        };
1983
1984        match parse_ws_spot_index_price_update(stats, instrument, timestamp, ts_init) {
1985            Ok(index_price) => Some(NautilusWsMessage::IndexPrice(index_price)),
1986            Err(e) => {
1987                log::error!("Error parsing Lighter spot index price: {e}");
1988                None
1989            }
1990        }
1991    }
1992
1993    fn handle_candles(
1994        &mut self,
1995        channel: Ustr,
1996        candles: &[LighterWsCandle],
1997        ts_init: UnixNanos,
1998    ) -> Vec<NautilusWsMessage> {
1999        let Some((market_index, resolution)) =
2000            candle_market_and_resolution_from_topic(channel.as_str())
2001        else {
2002            log::warn!("Lighter candle frame with unparsable channel `{channel}`");
2003            return Vec::new();
2004        };
2005
2006        let Some(instrument) = self.instruments.get(&market_index) else {
2007            log::debug!("No instrument cached for Lighter candle market_index={market_index}");
2008            return Vec::new();
2009        };
2010
2011        let key = (market_index, resolution);
2012        let mut emitted = Vec::new();
2013
2014        for candle in candles {
2015            let previous = self.last_candles.get(&key).cloned();
2016            match previous {
2017                None => {}
2018                Some(prev) if candle.t > prev.t => {
2019                    // `t` advanced; previous candle is closed.
2020                    match parse_ws_bar(instrument, &prev, resolution, ts_init) {
2021                        Ok(bar) => emitted.push(NautilusWsMessage::Bar(bar)),
2022                        Err(e) => log::error!("Error parsing Lighter candle bar: {e}"),
2023                    }
2024                }
2025                Some(prev) if candle.t < prev.t => continue,
2026                Some(_) => {}
2027            }
2028            self.last_candles.insert(key, candle.clone());
2029        }
2030
2031        emitted
2032    }
2033
2034    fn handle_account_orders(
2035        &self,
2036        orders_by_market: &AHashMap<Ustr, Vec<LighterOrder>>,
2037        _ts_init: UnixNanos,
2038    ) -> Vec<NautilusWsMessage> {
2039        if self.exec_account.is_none() {
2040            log::debug!("Lighter account_orders frame skipped: no execution context set");
2041            return Vec::new();
2042        }
2043
2044        let mut reports = Vec::new();
2045
2046        for orders in orders_by_market.values() {
2047            for order in orders {
2048                if !self.instruments.contains_key(&order.market_index) {
2049                    log::debug!(
2050                        "No instrument cached for Lighter order market_index={}",
2051                        order.market_index,
2052                    );
2053                    continue;
2054                }
2055
2056                reports.push(ExecutionReport::Order(order.clone()));
2057            }
2058        }
2059
2060        if reports.is_empty() {
2061            Vec::new()
2062        } else {
2063            vec![NautilusWsMessage::ExecutionReports(reports)]
2064        }
2065    }
2066
2067    fn handle_account_trades<'a>(
2068        &self,
2069        trades: impl IntoIterator<Item = &'a LighterTrade>,
2070        _ts_init: UnixNanos,
2071    ) -> Vec<NautilusWsMessage> {
2072        let Some((_account_id, account_index)) = self.exec_account else {
2073            log::debug!("Lighter account_trades frame skipped: no execution context set");
2074            return Vec::new();
2075        };
2076
2077        let mut reports = Vec::new();
2078
2079        for trade in trades {
2080            if !self.instruments.contains_key(&trade.market_id) {
2081                log::debug!(
2082                    "No instrument cached for Lighter account trade market_id={}",
2083                    trade.market_id,
2084                );
2085                continue;
2086            }
2087
2088            // The venue shares the channel with crossed pairs the account is
2089            // not part of; gate the cheap account check at the handler so
2090            // unrelated traffic never reaches the execution consumer.
2091            if trade.bid_account_id != account_index && trade.ask_account_id != account_index {
2092                continue;
2093            }
2094
2095            reports.push(ExecutionReport::Fill(trade.clone()));
2096        }
2097
2098        if reports.is_empty() {
2099            Vec::new()
2100        } else {
2101            vec![NautilusWsMessage::ExecutionReports(reports)]
2102        }
2103    }
2104
2105    fn handle_account_positions(
2106        &self,
2107        positions: &AHashMap<Ustr, LighterPosition>,
2108        ts_init: UnixNanos,
2109        frame_type: PositionFrameType,
2110    ) -> Vec<NautilusWsMessage> {
2111        let Some((account_id, _)) = self.exec_account else {
2112            log::debug!("Lighter account_positions frame skipped: no execution context set");
2113            return Vec::new();
2114        };
2115
2116        // The position frame carries no top-level event timestamp, so stamp
2117        // ts_event with the wall-clock time the handler observed the frame.
2118        let ts_event = ts_init;
2119
2120        let mut reports = Vec::new();
2121        let mut skipped_market_ids = Vec::new();
2122        let mut closed_market_ids = Vec::new();
2123
2124        for position in positions.values() {
2125            if position.position.is_zero() {
2126                if matches!(frame_type, PositionFrameType::Update) {
2127                    closed_market_ids.push(position.market_id);
2128                }
2129                continue;
2130            }
2131
2132            let Some(instrument) = self.instruments.get(&position.market_id) else {
2133                log::debug!(
2134                    "No instrument cached for Lighter position market_id={}",
2135                    position.market_id,
2136                );
2137
2138                skipped_market_ids.push(position.market_id);
2139                continue;
2140            };
2141
2142            match parse_ws_position_status_report(
2143                position, instrument, account_id, ts_event, ts_init,
2144            ) {
2145                Ok(report) => reports.push(report),
2146                Err(e) => {
2147                    skipped_market_ids.push(position.market_id);
2148                    log::error!("Error parsing Lighter position status report: {e}");
2149                }
2150            }
2151        }
2152
2153        match frame_type {
2154            PositionFrameType::Snapshot => {
2155                // Emit even when empty: signals the last position closed.
2156                vec![NautilusWsMessage::PositionSnapshot {
2157                    reports,
2158                    skipped_market_ids,
2159                }]
2160            }
2161            PositionFrameType::Update => vec![NautilusWsMessage::PositionUpdate {
2162                reports,
2163                closed_market_ids,
2164                skipped_market_ids,
2165            }],
2166        }
2167    }
2168
2169    fn handle_account_assets(
2170        &self,
2171        assets: &AHashMap<Ustr, LighterAsset>,
2172        timestamp_ms: u64,
2173        ts_init: UnixNanos,
2174    ) -> Vec<NautilusWsMessage> {
2175        let Some((account_id, _)) = self.exec_account else {
2176            log::debug!("Lighter account_assets frame skipped: no execution context set");
2177            return Vec::new();
2178        };
2179
2180        let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2181            Ok(ts) => ts,
2182            Err(e) => {
2183                log::error!("Invalid Lighter account_assets timestamp {timestamp_ms}: {e}");
2184                return Vec::new();
2185            }
2186        };
2187
2188        self.account_state_reconciler.update_assets(assets);
2189        self.emit_unified_account_state(account_id, ts_event, ts_init)
2190    }
2191
2192    fn handle_user_stats(
2193        &self,
2194        stats: &LighterUserStats,
2195        timestamp_ms: u64,
2196        ts_init: UnixNanos,
2197    ) -> Vec<NautilusWsMessage> {
2198        let Some((account_id, _)) = self.exec_account else {
2199            log::debug!("Lighter user_stats frame skipped: no execution context set");
2200            return Vec::new();
2201        };
2202
2203        let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2204            Ok(ts) => ts,
2205            Err(e) => {
2206                log::error!("Invalid Lighter user_stats timestamp {timestamp_ms}: {e}");
2207                return Vec::new();
2208            }
2209        };
2210
2211        self.account_state_reconciler.update_user_stats(stats);
2212        self.emit_unified_account_state(account_id, ts_event, ts_init)
2213    }
2214
2215    /// Asks the reconciler for a merged [`AccountState`] and wraps it as a
2216    /// `NautilusWsMessage`. Returns an empty vec when the second stream
2217    /// hasn't arrived yet; the AccountStreamFirstFrame marker still gets
2218    /// pushed by the caller so the startup gate progresses.
2219    fn emit_unified_account_state(
2220        &self,
2221        account_id: AccountId,
2222        ts_event: UnixNanos,
2223        ts_init: UnixNanos,
2224    ) -> Vec<NautilusWsMessage> {
2225        match self
2226            .account_state_reconciler
2227            .build_state(account_id, ts_event, ts_init)
2228        {
2229            Some(Ok(state)) => vec![NautilusWsMessage::AccountState(Box::new(state))],
2230            Some(Err(e)) => {
2231                log::error!("Error building unified Lighter account state: {e}");
2232                Vec::new()
2233            }
2234            None => Vec::new(),
2235        }
2236    }
2237}
2238
2239#[derive(Clone, Copy)]
2240enum PositionFrameType {
2241    Snapshot,
2242    Update,
2243}
2244
2245fn raw_message(frame: &LighterWsFrame) -> Vec<NautilusWsMessage> {
2246    let value = serde_json::to_value(frame).unwrap_or(serde_json::Value::Null);
2247    vec![NautilusWsMessage::Raw(value)]
2248}
2249
2250// Codes outside the transaction range (e.g. 30003 "Already Subscribed")
2251// would falsely reject a live order; see `LIGHTER_ERROR_CODE_TX_RANGE`.
2252fn is_sendtx_error_code(code: Option<u64>) -> bool {
2253    code.is_some_and(|c| LIGHTER_ERROR_CODE_TX_RANGE.contains(&c))
2254}
2255
2256fn already_subscribed_error(value: &serde_json::Value) -> Option<&serde_json::Value> {
2257    if value.get("code").and_then(|code| code.as_u64())
2258        == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2259    {
2260        Some(value)
2261    } else {
2262        value.get("error").filter(|e| {
2263            e.get("code").and_then(|code| code.as_u64())
2264                == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2265        })
2266    }
2267}
2268
2269fn subscription_error_code(value: &serde_json::Value) -> Option<u64> {
2270    value
2271        .get("code")
2272        .and_then(|code| code.as_u64())
2273        .or_else(|| {
2274            value
2275                .get("error")
2276                .and_then(|e| e.get("code"))
2277                .and_then(|code| code.as_u64())
2278        })
2279}
2280
2281fn typed_subscribe_topic(text: &str) -> Option<&str> {
2282    let header = serde_json::from_str::<LighterWsFrameHeader<'_>>(text).ok()?;
2283    header
2284        .kind
2285        .starts_with("subscribed/")
2286        .then_some(header.channel?)
2287}
2288
2289// SendTxRejected from a top-level `{code, message}` frame: non-200 sendTx
2290// ACK or `{"type":"error",...}`.
2291fn send_tx_rejected_from_value(
2292    value: &serde_json::Value,
2293    source: SendTxRejectionSource,
2294) -> NautilusWsMessage {
2295    let code = value.get("code").and_then(|v| v.as_i64());
2296    let message = value
2297        .get("message")
2298        .and_then(|v| v.as_str())
2299        .unwrap_or("")
2300        .to_string();
2301    let tx_hash = value
2302        .get("tx_hash")
2303        .and_then(|v| v.as_str())
2304        .map(str::to_string);
2305    NautilusWsMessage::SendTxRejected {
2306        connection_epoch: 0,
2307        source,
2308        code,
2309        message,
2310        tx_hash,
2311    }
2312}
2313
2314// SendTxRejected from a nested `{"error":{"code":N,"message":...}}` frame;
2315// these frames never carry a `tx_hash`.
2316fn send_tx_rejected_from_nested_error(
2317    error: &serde_json::Value,
2318    source: SendTxRejectionSource,
2319) -> NautilusWsMessage {
2320    let code = error.get("code").and_then(|v| v.as_i64());
2321    let message = error
2322        .get("message")
2323        .and_then(|v| v.as_str())
2324        .unwrap_or("")
2325        .to_string();
2326    NautilusWsMessage::SendTxRejected {
2327        connection_epoch: 0,
2328        source,
2329        code,
2330        message,
2331        tx_hash: None,
2332    }
2333}
2334
2335fn log_integrator_not_approved() {
2336    log::error!(
2337        "Lighter venue rejected with code {LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED} \
2338         'integrator is not approved'.\n\
2339         Tagged orders require Nautilus integrator approval. \
2340         See: {LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL}",
2341    );
2342}
2343
2344fn market_index_from_topic(topic: &str) -> Option<i64> {
2345    let (_, rest) = topic.split_once(':')?;
2346    rest.parse::<i64>().ok()
2347}
2348
2349fn candle_market_and_resolution_from_topic(topic: &str) -> Option<(i64, LighterCandleResolution)> {
2350    let (channel, rest) = topic.split_once(':')?;
2351    if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::Candle) {
2352        return None;
2353    }
2354    let (market, res) = rest.split_once(':')?;
2355    let market_index = market.parse::<i64>().ok()?;
2356    let resolution = res.parse::<LighterCandleResolution>().ok()?;
2357    Some((market_index, resolution))
2358}
2359
2360fn order_book_market_index_from_topic(topic: &str) -> Option<i64> {
2361    let (channel, rest) = topic.split_once(':')?;
2362    if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::OrderBook) {
2363        return None;
2364    }
2365
2366    rest.parse::<i64>().ok()
2367}
2368
2369pub(crate) fn should_retry_lighter_ws_error(error: &LighterWsError) -> bool {
2370    match error {
2371        LighterWsError::Network(_) => true,
2372        // Closed and BrokenPipe are terminal on this client; only Timeout
2373        // (wait_for_active) can recover if the connection comes up. WriteTimeout
2374        // is not retryable: the write was cancelled after it began, so the peer
2375        // may already have the message and a retry could duplicate it.
2376        LighterWsError::Transport(send_error) => match send_error {
2377            SendError::Timeout => true,
2378            SendError::InvalidInput(_)
2379            | SendError::BufferFull
2380            | SendError::Closed
2381            | SendError::ConnectionChanged
2382            | SendError::BrokenPipe(_)
2383            | SendError::WriteTimeout => false,
2384        },
2385        LighterWsError::Authentication(_)
2386        | LighterWsError::Parse(_)
2387        | LighterWsError::Client(_)
2388        | LighterWsError::SendTxOutcomeUnknown(_) => false,
2389    }
2390}
2391
2392pub(crate) fn create_lighter_ws_timeout_error(_msg: String) -> LighterWsError {
2393    // Structured variant so the classifier retries; the retry manager
2394    // already logs the textual timeout context.
2395    LighterWsError::Transport(SendError::Timeout)
2396}
2397
2398#[cfg(test)]
2399mod tests {
2400    use std::time::Duration;
2401
2402    use futures_util::FutureExt;
2403    use log::{Level, LevelFilter, Log, Metadata, Record};
2404    use nautilus_model::{
2405        enums::{AccountType, BookAction, RecordFlag},
2406        identifiers::{InstrumentId, Symbol, Venue},
2407        instruments::{CryptoPerpetual, CurrencyPair},
2408        types::{Currency, Money, Price, Quantity},
2409    };
2410    use parking_lot::Mutex;
2411    use rstest::rstest;
2412    use rust_decimal::Decimal;
2413    use serde_json::json;
2414
2415    use super::*;
2416    use crate::{
2417        common::enums::{LighterCandleResolution, LighterTxStatus, LighterTxType},
2418        websocket::messages::{LighterMarketSelection, LighterWsCandle, LighterWsChannel},
2419    };
2420
2421    const SECRET_MARKER: &str = "426426426";
2422
2423    struct OutboundLogCapture {
2424        messages: Mutex<Vec<String>>,
2425    }
2426
2427    static OUTBOUND_LOG_CAPTURE: OutboundLogCapture = OutboundLogCapture {
2428        messages: Mutex::new(Vec::new()),
2429    };
2430
2431    impl OutboundLogCapture {
2432        fn clear(&self) {
2433            self.messages.lock().clear();
2434        }
2435
2436        fn messages(&self) -> Vec<String> {
2437            self.messages.lock().clone()
2438        }
2439    }
2440
2441    impl Log for OutboundLogCapture {
2442        fn enabled(&self, metadata: &Metadata<'_>) -> bool {
2443            matches!(metadata.level(), Level::Debug | Level::Warn)
2444                && metadata.target() == "nautilus_lighter::websocket::handler"
2445        }
2446
2447        fn log(&self, record: &Record<'_>) {
2448            if self.enabled(record.metadata()) {
2449                let message = record.args().to_string();
2450                if (record.level() == Level::Debug
2451                    && message.starts_with("Sending Lighter unsubscribe"))
2452                    || (record.level() == Level::Warn
2453                        && message.starts_with("Lighter subscribe error"))
2454                {
2455                    self.messages.lock().push(message);
2456                }
2457            }
2458        }
2459
2460        fn flush(&self) {}
2461    }
2462
2463    const WS_ACCOUNT_ORDERS_UPDATE: &str =
2464        include_str!("../../test_data/ws_account_orders_update.json");
2465    const WS_ACCOUNT_ALL_TRADES_UPDATE: &str =
2466        include_str!("../../test_data/ws_account_all_trades_update.json");
2467    const WS_ACCOUNT_ALL_POSITIONS_UPDATE: &str =
2468        include_str!("../../test_data/ws_account_all_positions_update.json");
2469    const WS_ACCOUNT_ALL_ASSETS_UPDATE: &str =
2470        include_str!("../../test_data/ws_account_all_assets_update.json");
2471    const WS_USER_STATS_UPDATE: &str = include_str!("../../test_data/ws_user_stats_update.json");
2472    const WS_ACCOUNT_ALL_ASSETS_WITH_POSITION: &str =
2473        include_str!("../../test_data/ws_account_all_assets_with_position.json");
2474    const WS_USER_STATS_WITH_POSITION: &str =
2475        include_str!("../../test_data/ws_user_stats_with_position.json");
2476    const WS_MARKET_STATS_UPDATE_SINGLE: &str =
2477        include_str!("../../test_data/ws_market_stats_update_single.json");
2478    const WS_MARKET_STATS_UPDATE_ALL: &str =
2479        include_str!("../../test_data/ws_market_stats_update_all.json");
2480    const WS_SPOT_MARKET_STATS_UPDATE_SINGLE: &str =
2481        include_str!("../../test_data/ws_spot_market_stats_update_single.json");
2482    const WS_SPOT_MARKET_STATS_UPDATE_ALL: &str =
2483        include_str!("../../test_data/ws_spot_market_stats_update_all.json");
2484    const WS_CANDLE_SUBSCRIBED: &str = include_str!("../../test_data/ws_candle_subscribed.json");
2485    const WS_SPOT_STATS_SUBSCRIBED_BAD_BODY: &str =
2486        include_str!("../../test_data/ws_spot_market_stats_subscribed_single_bad_body.json");
2487    const WS_BOOK_SUBSCRIBED_BAD_BODY: &str =
2488        include_str!("../../test_data/ws_order_book_subscribed_bad_body.json");
2489    const WS_SUBSCRIBE_FAILED: &str =
2490        include_str!("../../test_data/ws_order_book_subscribe_failed.json");
2491    const BOOK_SUBSCRIBE_REJECTION: &str =
2492        "client error: venue rejected the WebSocket subscribe with code 30012: order_book:0";
2493
2494    fn handle_control_text(
2495        handler: &mut FeedHandler,
2496        text: &str,
2497    ) -> (bool, Option<NautilusWsMessage>) {
2498        let Ok(value) = serde_json::from_str::<serde_json::Value>(text) else {
2499            return (false, None);
2500        };
2501        handler.handle_control_value(&value)
2502    }
2503
2504    fn stub_eth_perp_instrument() -> InstrumentAny {
2505        let instrument_id = InstrumentId::new(Symbol::new("ETH-PERP"), Venue::new("LIGHTER"));
2506        InstrumentAny::CryptoPerpetual(
2507            CryptoPerpetual::builder()
2508                .instrument_id(instrument_id)
2509                .raw_symbol(Symbol::new("ETH-PERP"))
2510                .base_currency(Currency::from("ETH"))
2511                .quote_currency(Currency::from("USDC"))
2512                .settlement_currency(Currency::from("USDC"))
2513                .is_inverse(false)
2514                .price_precision(2)
2515                .size_precision(4)
2516                .price_increment(Price::from("0.01"))
2517                .size_increment(Quantity::from("0.0001"))
2518                .ts_event(UnixNanos::default())
2519                .ts_init(UnixNanos::default())
2520                .build()
2521                .unwrap(),
2522        )
2523    }
2524
2525    fn stub_eth_spot_instrument() -> InstrumentAny {
2526        let instrument_id = InstrumentId::new(Symbol::new("ETH-SPOT"), Venue::new("LIGHTER"));
2527        InstrumentAny::CurrencyPair(
2528            CurrencyPair::builder()
2529                .instrument_id(instrument_id)
2530                .raw_symbol(Symbol::new("ETH-SPOT"))
2531                .base_currency(Currency::from("ETH"))
2532                .quote_currency(Currency::from("USDC"))
2533                .price_precision(2)
2534                .size_precision(4)
2535                .price_increment(Price::from("0.01"))
2536                .size_increment(Quantity::from("0.0001"))
2537                .ts_event(UnixNanos::default())
2538                .ts_init(UnixNanos::default())
2539                .build()
2540                .unwrap(),
2541        )
2542    }
2543
2544    fn make_handler_with_account() -> FeedHandler {
2545        let signal = Arc::new(AtomicBool::new(false));
2546        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
2547        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
2548        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
2549        let mut handler =
2550            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2551        handler.instruments.insert(0, stub_eth_perp_instrument());
2552        handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
2553        handler
2554    }
2555
2556    // A recovery past its retry budget keeps running, so a late snapshot still restores the book
2557    #[rstest]
2558    fn book_late_snapshot_completes_running_recovery() {
2559        let mut handler = make_handler_with_account();
2560        handler.book.delta_subs.insert(0);
2561        let frame: LighterWsFrame = serde_json::from_str(include_str!(
2562            "../../test_data/ws_order_book_subscribed.json"
2563        ))
2564        .unwrap();
2565        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2566        handler.subscriptions.add_reference("order_book:0");
2567        assert!(recovery.begin_replacement());
2568
2569        let during_write = handler.handle_frame(frame.clone(), UnixNanos::from(1));
2570        recovery.gate.open();
2571        let after_write = handler.handle_frame(frame, UnixNanos::from(2));
2572
2573        assert!(during_write.is_empty());
2574        assert_eq!(after_write.len(), 1);
2575        assert!(recovery.is_accepted());
2576        assert!(handler.book.snapshots_seen.contains(&0));
2577    }
2578
2579    #[rstest]
2580    #[case::unconfirmed_permanent(false, false, false)]
2581    #[case::closed_gate_permanent(true, false, false)]
2582    #[case::confirmed_permanent(true, true, false)]
2583    #[case::unconfirmed_retry(false, false, true)]
2584    #[case::closed_gate_retry(true, false, true)]
2585    #[case::confirmed_retry(true, true, true)]
2586    fn book_venue_rejection_requires_confirmed_write(
2587        #[case] confirmed: bool,
2588        #[case] gate_open: bool,
2589        #[case] retry: bool,
2590    ) {
2591        let mut handler = make_handler_with_account();
2592        let (topic, generation) =
2593            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2594        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2595        assert!(recovery.begin_replacement());
2596
2597        if gate_open {
2598            recovery.gate.open();
2599        }
2600
2601        if !confirmed {
2602            handler.book.expected.remove(&0);
2603        }
2604
2605        if retry {
2606            handler.retry_inflight_subscriptions("venue rejection");
2607        } else {
2608            let value = serde_json::json!({"code": 30012, "channel": "order_book:0"});
2609            handler.handle_control_value(&value);
2610        }
2611
2612        let rejected = confirmed && gate_open;
2613        assert_eq!(
2614            handler.inflight_subs.get(&topic),
2615            (!rejected).then_some(&generation)
2616        );
2617        assert!(!recovery.cancellation.is_cancelled());
2618        assert!(recovery.gate.lock().is_closed());
2619
2620        match &*recovery.outcome.borrow() {
2621            BookRecoveryOutcome::Pending => assert!(!confirmed || !gate_open),
2622            BookRecoveryOutcome::Rejected(LighterWsError::Network(message)) => {
2623                assert!(confirmed && gate_open && retry);
2624                assert_eq!(message, "venue rejection");
2625            }
2626            BookRecoveryOutcome::Rejected(LighterWsError::Client(message)) => {
2627                assert!(confirmed && gate_open && !retry);
2628                assert_eq!(
2629                    message,
2630                    "venue rejected the WebSocket subscribe with code 30012"
2631                );
2632            }
2633            outcome => panic!("unexpected recovery outcome: {outcome:?}"),
2634        }
2635    }
2636
2637    // A waiting subscriber owns a rejection and releases the stream; a replayed subscription
2638    // recovers instead of failing the book.
2639    #[rstest]
2640    #[case::waiting_subscriber(true)]
2641    #[case::replayed(false)]
2642    fn book_client_rejection_recovers_only_without_waiting_subscriber(#[case] waiting: bool) {
2643        let mut handler = make_handler_with_account();
2644        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2645        handler.cmd_tx = Some(cmd_tx);
2646        handler.book.delta_subs.insert(0);
2647        handler.subscriptions.add_reference("order_book:0");
2648        let (response, mut result) = tokio::sync::oneshot::channel();
2649        let (topic, generation) = mark_subscription_inflight(
2650            &mut handler,
2651            LighterWsChannel::OrderBook(0),
2652            waiting.then_some(response),
2653        );
2654        let initial = CancellationToken::new();
2655        handler.book.initial.insert(
2656            0,
2657            PendingSnapshot {
2658                deadline: None,
2659                cancel: initial.clone(),
2660                gate: SnapshotGate::default(),
2661            },
2662        );
2663
2664        handler.reject_book_subscription(
2665            0,
2666            generation,
2667            LighterWsError::Client("invalid market".into()),
2668        );
2669
2670        let recovering = handler
2671            .book
2672            .recovery
2673            .get(&0)
2674            .is_some_and(BookRecoveryState::is_running);
2675
2676        if waiting {
2677            assert!(
2678                matches!(result.try_recv(), Ok(Err(message)) if message.contains("invalid market"))
2679            );
2680        } else {
2681            assert_eq!(
2682                result.try_recv(),
2683                Err(tokio::sync::oneshot::error::TryRecvError::Closed)
2684            );
2685        }
2686
2687        assert_eq!(recovering, !waiting);
2688        assert_eq!(handler.book.work.len(), usize::from(!waiting));
2689        assert_eq!(handler.subscription_attempts.contains_key(&topic), !waiting);
2690        assert!(!handler.inflight_subs.contains_key(&topic));
2691        assert!(!handler.book.initial.contains_key(&0));
2692        assert!(initial.is_cancelled());
2693    }
2694
2695    // A rejected replacement answers the subscriber still waiting on the initial subscribe, and
2696    // the recovery keeps its gate closed until the subscriber releases the book.
2697    #[rstest]
2698    #[case::response_txs(false)]
2699    #[case::pending_response_txs(true)]
2700    #[tokio::test]
2701    async fn book_client_rejection_during_recovery_answers_waiting_subscriber(
2702        #[case] pending: bool,
2703    ) {
2704        let (mut handler, _cmd_tx, mut result) =
2705            recovering_book_with_waiting_subscriber(false).await;
2706        let topic = Ustr::from("order_book:0");
2707
2708        if pending {
2709            // Book subscribes carry no auth, so seed a waiter behind a pending auth change directly
2710            let attempt = handler.subscription_attempts.get_mut(&topic).unwrap();
2711            attempt.pending_auth = Some(SecretString::from("token"));
2712            attempt.pending_response_txs = std::mem::take(&mut attempt.response_txs);
2713        }
2714
2715        let recovery = Arc::clone(handler.book.recovery[&0].current().unwrap());
2716        let gate_opened = !recovery.gate.lock().is_closed();
2717        let pending = result.try_recv();
2718
2719        handle_control_text(&mut handler, WS_SUBSCRIBE_FAILED);
2720
2721        assert!(gate_opened);
2722        assert_eq!(
2723            pending,
2724            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2725        );
2726        assert_eq!(
2727            result.try_recv(),
2728            Ok(Err(BOOK_SUBSCRIBE_REJECTION.to_string()))
2729        );
2730        assert!(!handler.subscription_attempts.contains_key(&topic));
2731        assert!(!handler.inflight_subs.contains_key(&topic));
2732        assert!(!handler.book.expected.contains_key(&0));
2733        assert!(recovery.is_running());
2734        assert!(recovery.gate.lock().is_closed());
2735        assert!(matches!(
2736            &*recovery.outcome.borrow(),
2737            BookRecoveryOutcome::Rejected(LighterWsError::Client(message))
2738                if message == "venue rejected the WebSocket subscribe with code 30012"
2739        ));
2740    }
2741
2742    // The rejected subscriber's release ends the recovery unless another consumer still references
2743    // the book, whose next replacement then queues a fresh attempt; either way a late snapshot
2744    // cannot reopen the book.
2745    #[rstest]
2746    #[case::deltas_last_reference(false, false)]
2747    #[case::deltas_other_consumer(false, true)]
2748    #[case::depth_last_reference(true, false)]
2749    #[case::depth_other_consumer(true, true)]
2750    #[tokio::test]
2751    async fn book_rejected_subscriber_release_cancels_orphaned_recovery(
2752        #[case] depth: bool,
2753        #[case] other_consumer: bool,
2754    ) {
2755        let (mut handler, cmd_tx, mut result) =
2756            recovering_book_with_waiting_subscriber(depth).await;
2757        let topic = Ustr::from("order_book:0");
2758
2759        if other_consumer {
2760            let other = if depth {
2761                &mut handler.book.delta_subs
2762            } else {
2763                &mut handler.book.depth_subs
2764            };
2765
2766            other.insert(0);
2767            assert!(!handler.subscriptions.add_reference("order_book:0"));
2768        }
2769
2770        let recovery = Arc::clone(handler.book.recovery[&0].current().unwrap());
2771        handle_control_text(&mut handler, WS_SUBSCRIBE_FAILED);
2772        let answer = result.try_recv();
2773
2774        // Release as the rejected client does: drop the reference, then clear the consumer flag
2775        let released = handler.subscriptions.remove_reference("order_book:0");
2776
2777        let release = if depth {
2778            HandlerCommand::SetDepthSub {
2779                market_index: 0,
2780                subscribed: false,
2781            }
2782        } else {
2783            HandlerCommand::SetBookDeltasSub {
2784                market_index: 0,
2785                subscribed: false,
2786            }
2787        };
2788
2789        cmd_tx.send(release).unwrap();
2790        cmd_tx.send(HandlerCommand::Disconnect).unwrap();
2791        assert!(handler.next().await.is_none());
2792
2793        let frame: LighterWsFrame = serde_json::from_str(include_str!(
2794            "../../test_data/ws_order_book_subscribed.json"
2795        ))
2796        .unwrap();
2797        let late = handler.handle_frame(frame, UnixNanos::from(1));
2798        let (completion, _completed) = tokio::sync::oneshot::channel();
2799        handler.queue_book_replacement(
2800            0,
2801            BookWrite {
2802                cancel: recovery.cancellation.child_token(),
2803                gate: recovery.gate.clone(),
2804                completion,
2805            },
2806        );
2807
2808        let replacement = handler.subscription_attempts.get(&topic).map(|attempt| {
2809            (
2810                attempt.response_txs.len(),
2811                handler.pending_subs.contains(&(topic, attempt.generation)),
2812            )
2813        });
2814
2815        assert_eq!(answer, Ok(Err(BOOK_SUBSCRIBE_REJECTION.to_string())));
2816        assert_eq!(released, !other_consumer);
2817        assert_eq!(recovery.is_running(), other_consumer);
2818        assert_eq!(handler.book.recovery.contains_key(&0), other_consumer);
2819        assert_eq!(handler.book.work.len(), usize::from(other_consumer));
2820        assert!(!recovery.is_accepted());
2821        assert!(recovery.gate.lock().is_closed());
2822        assert!(late.is_empty());
2823        assert!(!handler.book.snapshots_seen.contains(&0));
2824        assert_eq!(handler.book.writes.contains_key(&0), other_consumer);
2825        assert_eq!(replacement, other_consumer.then_some((0, true)));
2826    }
2827
2828    // Rate limits the initial book subscribe, which starts a recovery, then confirms the
2829    // recovery's replacement write while the initial subscriber still waits.
2830    async fn recovering_book_with_waiting_subscriber(
2831        depth: bool,
2832    ) -> (
2833        FeedHandler,
2834        tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2835        tokio::sync::oneshot::Receiver<Result<(), String>>,
2836    ) {
2837        let signal = Arc::new(AtomicBool::new(false));
2838        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2839        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
2840        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
2841        let mut handler =
2842            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2843        handler.set_command_sender(cmd_tx.clone());
2844        handler.instruments.insert(0, stub_eth_perp_instrument());
2845
2846        if depth {
2847            handler.book.depth_subs.insert(0);
2848        } else {
2849            handler.book.delta_subs.insert(0);
2850        }
2851
2852        handler.subscriptions.add_reference("order_book:0");
2853        let (response, result) = tokio::sync::oneshot::channel();
2854        mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(response));
2855        handle_control_text(
2856            &mut handler,
2857            r#"{"type":"error","code":30009,"message":"rate limit exceeded"}"#,
2858        );
2859
2860        // The first attempt requests its replacement on the first poll
2861        assert!(handler.book.work.next().now_or_never().is_none());
2862
2863        let Ok(HandlerCommand::RecoverBook {
2864            market_index,
2865            cancel,
2866            gate,
2867            completion,
2868        }) = handler.cmd_rx.try_recv()
2869        else {
2870            panic!("expected book replacement request");
2871        };
2872
2873        handler.queue_book_replacement(
2874            market_index,
2875            BookWrite {
2876                cancel,
2877                gate,
2878                completion,
2879            },
2880        );
2881
2882        handler.pump_pending_subscribes().await;
2883
2884        // The send fails without a socket, so confirm the write as the connection would
2885        let Some(BookWorkResult::Sent {
2886            market_index,
2887            generation,
2888            cancel,
2889            write,
2890            ..
2891        }) = handler.book.work.next().await
2892        else {
2893            panic!("expected replacement send");
2894        };
2895
2896        handler.complete_book_send(market_index, generation, cancel, write, Ok(0));
2897
2898        (handler, cmd_tx, result)
2899    }
2900
2901    // A failed replacement frees its slot during backoff so other subscriptions can proceed
2902    #[rstest]
2903    #[case::rejected(true)]
2904    #[case::send_failed(false)]
2905    #[tokio::test]
2906    async fn book_failed_replacement_releases_subscription_slot(#[case] rejected: bool) {
2907        let mut handler = make_handler_with_account();
2908        handler.book.delta_subs.insert(0);
2909        handler.subscriptions.add_reference("order_book:0");
2910        let (topic, generation) =
2911            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2912
2913        for i in 0..SUBSCRIBE_INFLIGHT_MAX - 1 {
2914            handler
2915                .inflight_subs
2916                .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
2917        }
2918
2919        handler.queue_subscribe(LighterWsChannel::OrderBook(1), None, None);
2920        let other = Ustr::from("order_book:1");
2921        handler.pump_pending_subscribes().await;
2922        let blocked = !handler.inflight_subs.contains_key(&other);
2923        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2924        assert!(recovery.begin_replacement());
2925        recovery.gate.open();
2926        let (completion, completed) = tokio::sync::oneshot::channel();
2927
2928        if rejected {
2929            handler.reject_book_subscription(
2930                0,
2931                generation,
2932                LighterWsError::Client("venue rejection".into()),
2933            );
2934        } else {
2935            handler.complete_book_send(
2936                0,
2937                generation,
2938                recovery.cancellation.child_token(),
2939                Some(BookWrite {
2940                    cancel: recovery.cancellation.child_token(),
2941                    gate: recovery.gate.clone(),
2942                    completion,
2943                }),
2944                Err(LighterWsError::Network("send failed".into())),
2945            );
2946        }
2947
2948        handler.pump_pending_subscribes().await;
2949
2950        assert!(blocked);
2951        assert!(!handler.inflight_subs.contains_key(&topic));
2952        assert!(handler.inflight_subs.contains_key(&other));
2953        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
2954        assert!(recovery.is_running());
2955
2956        if rejected {
2957            assert!(matches!(
2958                &*recovery.outcome.borrow(),
2959                BookRecoveryOutcome::Rejected(LighterWsError::Client(message))
2960                    if message == "venue rejection"
2961            ));
2962        } else {
2963            assert!(
2964                matches!(completed.await, Ok(Err(LighterWsError::Network(message))) if message == "send failed")
2965            );
2966        }
2967    }
2968
2969    #[tokio::test]
2970    async fn book_queued_write_skips_after_last_reference_disappears() {
2971        let mut handler = make_handler_with_account();
2972        handler.book.delta_subs.insert(0);
2973        let channel = LighterWsChannel::OrderBook(0);
2974        handler.subscriptions.add_reference(&channel.topic_key());
2975        let (_, generation) = mark_subscription_inflight(&mut handler, channel.clone(), None);
2976        handler.send_book_subscribe(0, generation);
2977        assert!(handler.subscriptions.remove_reference(&channel.topic_key()));
2978
2979        let BookWorkResult::Sent { result, .. } = handler.book.work.next().await.unwrap() else {
2980            panic!("expected send completion");
2981        };
2982
2983        assert_eq!(
2984            result.unwrap_err().to_string(),
2985            "client error: book subscription cancelled"
2986        );
2987        assert!(handler.inner.is_none());
2988    }
2989
2990    #[rstest]
2991    fn book_wrong_epoch_snapshot_preserves_current_completion() {
2992        let mut handler = make_handler_with_account();
2993        let (tx, mut rx) = tokio::sync::oneshot::channel();
2994        let (topic, generation) =
2995            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
2996        handler.book.expected.insert(0, (generation, 12));
2997
2998        assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
2999        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
3000        assert_eq!(
3001            rx.try_recv(),
3002            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
3003        );
3004        assert!(handler.complete_typed_subscription(topic.as_str(), 12));
3005        assert_eq!(rx.try_recv(), Ok(Ok(())));
3006        assert!(!handler.inflight_subs.contains_key(&topic));
3007    }
3008
3009    #[rstest]
3010    fn book_control_ack_preserves_snapshot_but_not_successor_completion() {
3011        let mut handler = make_handler_with_account();
3012        let (topic, old_generation) =
3013            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3014        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3015        assert!(handler.complete_typed_subscription(topic.as_str(), 0));
3016        assert_eq!(handler.book.expected[&0], (old_generation, 0));
3017
3018        handler.subscriptions.mark_failure(topic.as_str());
3019        let (_, generation) =
3020            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3021        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3022        handler.subscriptions.mark_failure(topic.as_str());
3023        let (tx, mut rx) = tokio::sync::oneshot::channel();
3024        let (_, successor) =
3025            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
3026
3027        assert_ne!(generation, successor);
3028        assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
3029        assert_eq!(handler.inflight_subs.get(&topic), Some(&successor));
3030        assert_eq!(
3031            rx.try_recv(),
3032            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
3033        );
3034        assert!(handler.complete_typed_subscription(topic.as_str(), 0));
3035        assert_eq!(rx.try_recv(), Ok(Ok(())));
3036    }
3037
3038    #[rstest]
3039    fn book_old_epoch_does_not_consume_current_trailing_snapshot() {
3040        let mut handler = make_handler_with_account();
3041        let (topic, generation) =
3042            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3043        handler.book.expected.insert(0, (generation, 12));
3044        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3045
3046        assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
3047        assert_eq!(handler.book.trailing.get(&0), Some(&(generation, 12)));
3048        assert_eq!(
3049            handler.ignored_completions.get(&topic),
3050            Some(&CompletionKind::Typed)
3051        );
3052        assert!(handler.complete_typed_subscription(topic.as_str(), 12));
3053        assert!(!handler.book.trailing.contains_key(&0));
3054        assert!(!handler.ignored_completions.contains_key(&topic));
3055    }
3056
3057    #[rstest]
3058    fn book_ack_before_write_cannot_complete_subscription() {
3059        let mut handler = make_handler_with_account();
3060        let (topic, generation) =
3061            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3062        handler.book.expected.clear();
3063
3064        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3065        assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
3066        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
3067    }
3068
3069    #[rstest]
3070    fn book_empty_snapshot_replaces_cached_levels_for_both_consumers() {
3071        let mut handler = make_handler_with_account();
3072        handler.book.delta_subs.insert(0);
3073        handler.book.depth_subs.insert(0);
3074        let frame: LighterWsFrame = serde_json::from_str(include_str!(
3075            "../../test_data/ws_order_book_subscribed.json"
3076        ))
3077        .unwrap();
3078        assert_eq!(
3079            handler
3080                .handle_frame(frame.clone(), UnixNanos::from(1))
3081                .len(),
3082            2
3083        );
3084
3085        let LighterWsFrame::OrderBookSnapshot { order_book, .. } = &frame else {
3086            panic!("expected book snapshot");
3087        };
3088
3089        let mut empty = order_book.clone();
3090        empty.bids.clear();
3091        empty.asks.clear();
3092        empty.nonce += 1;
3093        let messages = handler.handle_order_book(
3094            Ustr::from("order_book:0"),
3095            &empty,
3096            123,
3097            true,
3098            UnixNanos::from(2),
3099        );
3100
3101        assert_eq!(messages.len(), 2);
3102
3103        let NautilusWsMessage::Deltas(deltas) = &messages[0] else {
3104            panic!("expected deltas");
3105        };
3106
3107        assert_eq!(deltas.deltas.len(), 1);
3108        assert_eq!(deltas.deltas[0].action, BookAction::Clear);
3109        assert_eq!(deltas.deltas[0].sequence, empty.nonce as u64);
3110        assert_eq!(
3111            deltas.deltas[0].flags,
3112            RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8
3113        );
3114
3115        let NautilusWsMessage::Depth(depth) = &messages[1] else {
3116            panic!("expected depth");
3117        };
3118
3119        assert!(depth.bids.is_empty());
3120        assert!(depth.asks.is_empty());
3121        assert!(depth.bid_counts.is_empty());
3122        assert!(depth.ask_counts.is_empty());
3123        assert!(handler.book.states[&0].book.bids.is_empty());
3124        assert!(handler.book.states[&0].book.asks.is_empty());
3125    }
3126
3127    #[tokio::test]
3128    async fn book_initial_deadline_is_cancelled_by_replacement_or_unsubscribe() {
3129        let mut handler = make_handler_with_account();
3130        handler.book.delta_subs.insert(0);
3131        let channel = LighterWsChannel::OrderBook(0);
3132        handler.subscriptions.add_reference(&channel.topic_key());
3133        let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
3134        let cancel = CancellationToken::new();
3135        handler.book.initial.insert(
3136            0,
3137            PendingSnapshot {
3138                deadline: None,
3139                cancel: cancel.clone(),
3140                gate: SnapshotGate::default(),
3141            },
3142        );
3143
3144        handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
3145        handler.book.cancel(0);
3146
3147        let BookWorkResult::Initial {
3148            cancel: completed,
3149            market_index,
3150        } = handler.book.work.next().await.unwrap()
3151        else {
3152            panic!("expected deadline completion");
3153        };
3154
3155        assert_eq!(market_index, 0);
3156        assert!(completed.is_cancelled());
3157        assert!(cancel.is_cancelled());
3158        assert!(handler.book.recovery.is_empty());
3159        assert!(handler.book.expected.is_empty());
3160    }
3161
3162    #[tokio::test]
3163    async fn book_initial_deadline_expires_without_cancellation() {
3164        let mut handler = make_handler_with_account();
3165        handler.book_snapshot_timeout = Duration::from_millis(50);
3166        handler.book.delta_subs.insert(0);
3167        let channel = LighterWsChannel::OrderBook(0);
3168        handler.subscriptions.add_reference(&channel.topic_key());
3169        let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
3170        let cancel = CancellationToken::new();
3171        handler.book.initial.insert(
3172            0,
3173            PendingSnapshot {
3174                deadline: None,
3175                cancel: cancel.clone(),
3176                gate: SnapshotGate::default(),
3177            },
3178        );
3179
3180        handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
3181
3182        let BookWorkResult::Initial {
3183            market_index,
3184            cancel: completed,
3185        } = handler.book.work.next().await.unwrap()
3186        else {
3187            panic!("expected deadline completion");
3188        };
3189
3190        assert_eq!(market_index, 0);
3191        assert!(!completed.is_cancelled());
3192        assert!(!cancel.is_cancelled());
3193    }
3194
3195    #[tokio::test]
3196    async fn book_reconnect_retains_recovery_owner_and_retires_old_writes() {
3197        let signal = Arc::new(AtomicBool::new(false));
3198        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
3199        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
3200        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
3201        let mut handler =
3202            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3203        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
3204        handler.book.expected.insert(0, (3, 9));
3205        raw_tx
3206            .send((10, Message::Text(RECONNECTED.into())))
3207            .unwrap();
3208        assert!(matches!(
3209            handler.next().await,
3210            Some(NautilusWsMessage::Reconnected {
3211                connection_epoch: 10
3212            })
3213        ));
3214
3215        assert!(Arc::ptr_eq(
3216            handler.book.recovery[&0].current().unwrap(),
3217            &recovery
3218        ));
3219        assert!(!recovery.cancellation.is_cancelled());
3220        assert!(recovery.gate.lock().is_closed());
3221        assert!(matches!(
3222            *recovery.outcome.borrow(),
3223            BookRecoveryOutcome::Rejected(LighterWsError::Network(_))
3224        ));
3225        assert!(handler.book.expected.is_empty());
3226        assert!(handler.book.writes.is_empty());
3227    }
3228
3229    #[rstest]
3230    fn book_recovery_is_scoped_to_handler_and_market() {
3231        let mut first = make_handler_with_account();
3232        let mut second = make_handler_with_account();
3233        let recovery = first.book.recovery.entry(0).or_default().claim().unwrap();
3234        let other_market = first.book.recovery.entry(1).or_default().claim().unwrap();
3235        let other_socket = second.book.recovery.entry(0).or_default().claim().unwrap();
3236        first.book.cancel(0);
3237
3238        assert!(recovery.cancellation.is_cancelled());
3239        assert!(!other_market.cancellation.is_cancelled());
3240        assert!(!other_socket.cancellation.is_cancelled());
3241    }
3242
3243    #[rstest]
3244    #[tokio::test]
3245    async fn test_subscription_logs_omit_payload_body() {
3246        log::set_logger(&OUTBOUND_LOG_CAPTURE).expect("test logger already installed");
3247        log::set_max_level(LevelFilter::Debug);
3248
3249        let mut handler = make_handler_with_account();
3250        let account_index = SECRET_MARKER.parse::<i64>().unwrap();
3251        let channel = LighterWsChannel::AccountAll(account_index);
3252        let payload_len = serde_json::to_string(&LighterWsRequest::unsubscribe(
3253            channel.subscription_channel(),
3254        ))
3255        .unwrap()
3256        .len();
3257        OUTBOUND_LOG_CAPTURE.clear();
3258
3259        handler.dispatch_unsubscribe(channel).await;
3260        handler.handle_control_value(&json!({"code": 30012, "auth": SECRET_MARKER}));
3261        handler.handle_control_value(&json!({
3262            "code": 30012, "channel": "trade/999", "auth": SECRET_MARKER,
3263        }));
3264
3265        let messages = OUTBOUND_LOG_CAPTURE.messages();
3266
3267        assert!(
3268            messages
3269                .iter()
3270                .all(|message| !message.contains(SECRET_MARKER)),
3271            "outbound logs exposed the secret marker: {messages:?}"
3272        );
3273        assert!(
3274            messages.iter().any(|message| {
3275                message == &format!("Sending Lighter unsubscribe ({payload_len} bytes)")
3276            }),
3277            "unsubscribe metadata missing or inaccurate: {messages:?}"
3278        );
3279        assert!(messages.iter().any(|message| {
3280            message == "Lighter subscribe error code=30012 channel=none: retrying inflight subscriptions"
3281        }));
3282        assert!(messages.iter().any(|message| {
3283            message == "Lighter subscribe error code=30012 channel=trade/999: no matching inflight subscription"
3284        }));
3285    }
3286
3287    fn mark_subscription_inflight(
3288        handler: &mut FeedHandler,
3289        channel: LighterWsChannel,
3290        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
3291    ) -> (Ustr, u64) {
3292        handler.queue_subscribe(channel, None, response_tx);
3293        let (topic, generation) = handler
3294            .pending_subs
3295            .pop_front()
3296            .expect("subscription was not queued");
3297        handler.inflight_subs.insert(topic, generation);
3298        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
3299            handler.book.expected.insert(market_index, (generation, 0));
3300        }
3301
3302        (topic, generation)
3303    }
3304
3305    fn saturate_subscription_gate(handler: &mut FeedHandler) {
3306        for i in 0..SUBSCRIBE_INFLIGHT_MAX {
3307            handler
3308                .inflight_subs
3309                .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
3310        }
3311    }
3312
3313    /// Strip the trailing `AccountStreamFirstFrame` marker from a handler
3314    /// emission so account-stream tests can assert on the typed payload
3315    /// alone. The marker contract is pinned in
3316    /// `handle_frame_emits_account_stream_first_frame_marker_per_variant`.
3317    fn strip_account_marker(mut msgs: Vec<NautilusWsMessage>) -> Vec<NautilusWsMessage> {
3318        if matches!(
3319            msgs.last(),
3320            Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
3321        ) {
3322            msgs.pop();
3323        }
3324        msgs
3325    }
3326
3327    #[rstest]
3328    #[case::execution_context(true)]
3329    #[case::raw_without_context(false)]
3330    fn account_tx_routes_execution_results(#[case] execution_context: bool) {
3331        let mut handler = make_handler_with_account();
3332        if !execution_context {
3333            handler.exec_account = None;
3334        }
3335
3336        let frame: LighterWsFrame =
3337            serde_json::from_str(include_str!("../../test_data/ws_account_tx_update.json"))
3338                .unwrap();
3339        let messages = handler.handle_frame(frame.clone(), UnixNanos::from(11));
3340        assert_eq!(messages.len(), 1);
3341        let message = messages
3342            .into_iter()
3343            .next()
3344            .unwrap()
3345            .with_connection_epoch(7);
3346
3347        if execution_context {
3348            let NautilusWsMessage::AccountTx {
3349                connection_epoch,
3350                txs,
3351            } = message
3352            else {
3353                panic!("expected account transaction results");
3354            };
3355
3356            assert_eq!(connection_epoch, 7);
3357            assert_eq!(
3358                txs,
3359                vec![LighterWsTx {
3360                    hash: "abc123".to_string(),
3361                    status: LighterTxStatus::Executed,
3362                    event_info: "{\"ae\":\"\"}".to_string(),
3363                }]
3364            );
3365        } else {
3366            let NautilusWsMessage::Raw(value) = message else {
3367                panic!("expected raw transaction update");
3368            };
3369
3370            assert_eq!(value, serde_json::to_value(frame).unwrap());
3371        }
3372    }
3373
3374    #[rstest]
3375    fn account_tx_rejects_wrong_account_channel() {
3376        let mut handler = make_handler_with_account();
3377        let mut frame: LighterWsFrame =
3378            serde_json::from_str(include_str!("../../test_data/ws_account_tx_update.json"))
3379                .unwrap();
3380
3381        let LighterWsFrame::AccountTx { channel, .. } = &mut frame else {
3382            unreachable!()
3383        };
3384
3385        *channel = Ustr::from("account_tx:1235");
3386        assert!(handler.handle_frame(frame, UnixNanos::from(11)).is_empty());
3387    }
3388
3389    #[rstest]
3390    fn handle_frame_routes_account_orders_to_execution_reports() {
3391        let mut handler = make_handler_with_account();
3392        let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3393
3394        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3395
3396        assert_eq!(messages.len(), 1);
3397        match &messages[0] {
3398            NautilusWsMessage::ExecutionReports(reports) => {
3399                assert_eq!(reports.len(), 1);
3400                match &reports[0] {
3401                    super::ExecutionReport::Order(order) => {
3402                        assert_eq!(order.order_id, "281476929510110");
3403                        assert_eq!(order.client_order_id, "42");
3404                    }
3405                    other => panic!("expected order report, was {other:?}"),
3406                }
3407            }
3408            other => panic!("expected execution reports, was {other:?}"),
3409        }
3410    }
3411
3412    #[rstest]
3413    fn handle_frame_routes_account_trades_to_execution_reports() {
3414        let mut handler = make_handler_with_account();
3415        let frame: super::LighterWsFrame =
3416            serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
3417
3418        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3419
3420        assert_eq!(messages.len(), 1);
3421        match &messages[0] {
3422            NautilusWsMessage::ExecutionReports(reports) => {
3423                assert_eq!(reports.len(), 1);
3424                match &reports[0] {
3425                    super::ExecutionReport::Fill(fill) => {
3426                        // Bid side is the user's account in this fixture
3427                        // (see handle_frame_routes_account_trades_to_execution_reports
3428                        // fixture: bid_account_id == 1234). The raw payload
3429                        // carries the bid order id as the venue id; the
3430                        // execution loop derives `venue_order_id` from it.
3431                        assert_eq!(fill.bid_id_str.as_deref(), Some("562947905631053"),);
3432                    }
3433                    other => panic!("expected fill report, was {other:?}"),
3434                }
3435            }
3436            other => panic!("expected execution reports, was {other:?}"),
3437        }
3438    }
3439
3440    #[rstest]
3441    fn handle_frame_routes_account_positions_to_update_without_readiness_marker() {
3442        let mut handler = make_handler_with_account();
3443        let frame: super::LighterWsFrame =
3444            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3445
3446        let messages = handler.handle_frame(frame, UnixNanos::from(11));
3447
3448        assert_eq!(messages.len(), 1);
3449        match &messages[0] {
3450            NautilusWsMessage::PositionUpdate {
3451                reports,
3452                closed_market_ids,
3453                skipped_market_ids,
3454            } => {
3455                assert!(closed_market_ids.is_empty());
3456                assert!(skipped_market_ids.is_empty());
3457                assert_eq!(reports.len(), 1);
3458                assert_eq!(reports[0].quantity, Quantity::from("1.5000"));
3459            }
3460            other => panic!("expected position update, was {other:?}"),
3461        }
3462    }
3463
3464    #[rstest]
3465    fn handle_frame_tolerates_unknown_position_margin_mode() {
3466        let mut handler = make_handler_with_account();
3467        let mut frame_json: serde_json::Value =
3468            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3469        frame_json["positions"]["0"]["margin_mode"] = json!(99);
3470        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3471
3472        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3473
3474        assert_eq!(messages.len(), 1);
3475        match &messages[0] {
3476            NautilusWsMessage::PositionUpdate {
3477                reports,
3478                closed_market_ids,
3479                skipped_market_ids,
3480            } => {
3481                assert!(closed_market_ids.is_empty());
3482                assert!(skipped_market_ids.is_empty());
3483                assert_eq!(reports.len(), 1);
3484            }
3485            other => panic!("expected position update, was {other:?}"),
3486        }
3487    }
3488
3489    #[rstest]
3490    fn handle_frame_routes_empty_account_positions_to_empty_update() {
3491        let mut handler = make_handler_with_account();
3492        let frame_json = serde_json::json!({
3493            "type": "update/account_all_positions",
3494            "channel": "account_all_positions:1234",
3495            "positions": {},
3496            "shares": [],
3497            "last_funding_round": null,
3498            "last_funding_discount": null,
3499        });
3500        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3501
3502        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3503
3504        assert_eq!(messages.len(), 1);
3505        match &messages[0] {
3506            NautilusWsMessage::PositionUpdate {
3507                reports,
3508                closed_market_ids,
3509                skipped_market_ids,
3510            } => {
3511                assert!(closed_market_ids.is_empty());
3512                assert!(skipped_market_ids.is_empty());
3513                assert!(reports.is_empty());
3514            }
3515            other => panic!("expected empty position update, was {other:?}"),
3516        }
3517    }
3518
3519    #[rstest]
3520    fn handle_frame_routes_zero_account_position_to_closed_update() {
3521        let mut handler = make_handler_with_account();
3522        let mut frame_json: serde_json::Value =
3523            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3524        frame_json["positions"]["0"]["position"] = json!("0.0000");
3525        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3526
3527        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3528
3529        assert_eq!(messages.len(), 1);
3530        match &messages[0] {
3531            NautilusWsMessage::PositionUpdate {
3532                reports,
3533                closed_market_ids,
3534                skipped_market_ids,
3535            } => {
3536                assert!(reports.is_empty());
3537                assert!(skipped_market_ids.is_empty());
3538                assert_eq!(closed_market_ids, &[0]);
3539            }
3540            other => panic!("expected closed position update, was {other:?}"),
3541        }
3542    }
3543
3544    #[rstest]
3545    fn handle_frame_marks_account_positions_incomplete_when_position_instrument_uncached() {
3546        let mut handler = make_handler_with_account();
3547        let mut frame_json: serde_json::Value =
3548            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3549        frame_json["type"] = json!("subscribed/account_all_positions");
3550        frame_json["positions"]["0"]["market_id"] = json!(999);
3551        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3552
3553        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3554
3555        assert_eq!(messages.len(), 1);
3556        match &messages[0] {
3557            NautilusWsMessage::PositionSnapshot {
3558                reports,
3559                skipped_market_ids,
3560            } => {
3561                assert_eq!(skipped_market_ids, &[999]);
3562                assert!(reports.is_empty());
3563            }
3564            other => panic!("expected incomplete position snapshot, was {other:?}"),
3565        }
3566    }
3567
3568    #[rstest]
3569    fn handle_frame_marks_account_positions_incomplete_when_position_parse_fails() {
3570        let mut handler = make_handler_with_account();
3571        let mut frame_json: serde_json::Value =
3572            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3573        frame_json["type"] = json!("subscribed/account_all_positions");
3574        frame_json["positions"]["0"]["position"] = json!("-1.5000");
3575        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3576
3577        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3578
3579        assert_eq!(messages.len(), 1);
3580        match &messages[0] {
3581            NautilusWsMessage::PositionSnapshot {
3582                reports,
3583                skipped_market_ids,
3584            } => {
3585                assert_eq!(skipped_market_ids, &[0]);
3586                assert!(reports.is_empty());
3587            }
3588            other => panic!("expected incomplete position snapshot, was {other:?}"),
3589        }
3590    }
3591
3592    #[rstest]
3593    fn handle_frame_marks_position_update_incomplete_when_position_parse_fails() {
3594        let mut handler = make_handler_with_account();
3595        let mut frame_json: serde_json::Value =
3596            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3597        frame_json["positions"]["0"]["position"] = json!("-1.5000");
3598        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3599
3600        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3601
3602        assert_eq!(messages.len(), 1);
3603        match &messages[0] {
3604            NautilusWsMessage::PositionUpdate {
3605                reports,
3606                closed_market_ids,
3607                skipped_market_ids,
3608            } => {
3609                assert!(reports.is_empty());
3610                assert!(closed_market_ids.is_empty());
3611                assert_eq!(skipped_market_ids, &[0]);
3612            }
3613            other => panic!("expected incomplete position update, was {other:?}"),
3614        }
3615    }
3616
3617    #[rstest]
3618    fn handle_frame_routes_subscribed_account_all_positions_snapshot() {
3619        let mut handler = make_handler_with_account();
3620        let frame_json = serde_json::json!({
3621            "type": "subscribed/account_all_positions",
3622            "channel": "account_all_positions:1234",
3623            "positions": {
3624                "0": {
3625                    "allocated_margin": "0.000000",
3626                    "avg_entry_price": "0.111230",
3627                    "initial_margin_fraction": "10.00",
3628                    "liquidation_price": "0.100598",
3629                    "margin_mode": 0,
3630                    "market_id": 0,
3631                    "open_order_count": 0,
3632                    "pending_order_count": 0,
3633                    "position": "100",
3634                    "position_tied_order_count": 0,
3635                    "position_value": "11.123000",
3636                    "realized_pnl": "0.000000",
3637                    "sign": 1,
3638                    "symbol": "ETH",
3639                    "total_discount": "0.000000",
3640                    "total_funding_paid_out": "0.000000",
3641                    "unrealized_pnl": "0.000000"
3642                }
3643            },
3644        });
3645        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3646
3647        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3648
3649        assert_eq!(messages.len(), 1);
3650        match &messages[0] {
3651            NautilusWsMessage::PositionSnapshot {
3652                reports,
3653                skipped_market_ids,
3654            } => {
3655                assert!(skipped_market_ids.is_empty());
3656                assert_eq!(reports.len(), 1);
3657                assert_eq!(reports[0].quantity, Quantity::from("100"));
3658            }
3659            other => panic!("expected position snapshot, was {other:?}"),
3660        }
3661    }
3662
3663    #[rstest]
3664    #[case::connected(serde_json::json!({"type": "connected", "session_id": "x"}), true, false)]
3665    #[case::ping(serde_json::json!({"type": "ping"}), true, false)]
3666    #[case::pong(serde_json::json!({"type": "pong"}), true, false)]
3667    #[case::send_tx_ack(
3668        serde_json::json!({"type": "jsonapi/sendtx", "code": 200, "tx_hash": "abc"}),
3669        true,
3670        true,
3671    )]
3672    #[case::send_tx_without_code(
3673        serde_json::json!({"type": "jsonapi/sendtx", "tx_hash": "abc"}),
3674        true,
3675        false,
3676    )]
3677    #[case::send_tx_with_nonnumeric_code(
3678        serde_json::json!({"type": "jsonapi/sendtx", "code": "200", "tx_hash": "abc"}),
3679        true,
3680        false,
3681    )]
3682    #[case::error_frame(
3683        serde_json::json!({"type": "error", "code": 21727, "message": "invalid client order index"}),
3684        true,
3685        true,
3686    )]
3687    #[case::error_frame_integrator_not_approved(
3688        serde_json::json!({"type": "error", "code": 21149, "message": "integrator is not approved"}),
3689        true,
3690        true,
3691    )]
3692    #[case::send_tx_ack_integrator_not_approved(
3693        serde_json::json!({"type": "jsonapi/sendtx", "code": 21149, "message": "integrator is not approved"}),
3694        true,
3695        true,
3696    )]
3697    #[case::wrapped_error_integrator_not_approved(
3698        serde_json::json!({"error": {"code": 21149, "message": "integrator is not approved"}}),
3699        true,
3700        true,
3701    )]
3702    #[case::subscription_error_frame(
3703        serde_json::json!({"type": "error", "code": 30003, "message": "Already Subscribed to : ticker:3"}),
3704        true,
3705        false,
3706    )]
3707    #[case::wrapped_subscription_error(
3708        serde_json::json!({"error": {"code": 30003, "message": "Already Subscribed to : ticker:3"}}),
3709        true,
3710        false,
3711    )]
3712    #[case::codeless_error_frame(
3713        serde_json::json!({"type": "error", "message": "unclassifiable"}),
3714        true,
3715        false,
3716    )]
3717    #[case::unknown_type(
3718        serde_json::json!({"type": "something_unexpected", "payload": "x"}),
3719        false,
3720        false,
3721    )]
3722    #[case::no_type_field(
3723        serde_json::json!({"error": {"code": 21702, "message": "invalid price"}}),
3724        true,
3725        true,
3726    )]
3727    fn handle_control_text_tri_state(
3728        #[case] payload: serde_json::Value,
3729        #[case] expected_matched: bool,
3730        #[case] expected_has_msg: bool,
3731    ) {
3732        // Pins the (matched, msg) contract that lets the WS receive loop
3733        // distinguish "known control type, consume silently" from
3734        // "surface a typed/Raw message to the consumer". sendTx success ACKs
3735        // and venue-error frames now produce typed SendTxAck/SendTxRejected
3736        // variants instead of Raw, so the execution loop can correlate them
3737        // back to in-flight cloids.
3738        let mut handler = make_handler_with_account();
3739        let text = payload.to_string();
3740        let (matched, msg) = handle_control_text(&mut handler, &text);
3741        assert_eq!(matched, expected_matched, "matched flag");
3742        assert_eq!(msg.is_some(), expected_has_msg, "msg presence");
3743    }
3744
3745    #[rstest]
3746    #[case::ack(serde_json::json!({"type":"jsonapi/sendtxbatch","id":"tx-batch:abc","code":200,"tx_hash":["abc","def"]}), 200, "batch rejected", vec!["abc", "def"])]
3747    #[case::error(serde_json::json!({"id":"tx-batch:abc","error":{"code":21104,"message":"invalid nonce"}}), 21104, "invalid nonce", vec![])]
3748    fn batch_response_preserves_correlation_and_epoch(
3749        #[case] value: serde_json::Value,
3750        #[case] expected_code: i64,
3751        #[case] expected_message: &str,
3752        #[case] expected_hashes: Vec<&str>,
3753    ) {
3754        let mut handler = make_handler_with_account();
3755        let (matched, message) = handler.handle_control_value(&value);
3756        assert!(matched);
3757
3758        match message.unwrap().with_connection_epoch(7) {
3759            NautilusWsMessage::SendTxBatchResult {
3760                connection_epoch,
3761                id,
3762                code,
3763                message,
3764                tx_hashes,
3765            } => {
3766                assert_eq!(connection_epoch, 7);
3767                assert_eq!(id, "tx-batch:abc");
3768                assert_eq!(code, expected_code);
3769                assert_eq!(message, expected_message);
3770                assert_eq!(tx_hashes, expected_hashes);
3771            }
3772            message => panic!("expected batch response, was {message:?}"),
3773        }
3774    }
3775
3776    #[rstest]
3777    fn handle_control_text_sendtx_success_emits_typed_ack() {
3778        let mut handler = make_handler_with_account();
3779        let payload = serde_json::json!({
3780            "type": "jsonapi/sendtx",
3781            "code": 200,
3782            "tx_hash": "0000abcd",
3783        })
3784        .to_string();
3785
3786        let (_, msg) = handle_control_text(&mut handler, &payload);
3787
3788        match msg.expect("SendTxAck emitted") {
3789            NautilusWsMessage::SendTxAck { tx_hash, code, .. } => {
3790                assert_eq!(code, 200);
3791                assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3792            }
3793            other => panic!("expected SendTxAck, was {other:?}"),
3794        }
3795    }
3796
3797    #[rstest]
3798    fn handle_control_text_sendtx_failure_emits_ack_sourced_rejection() {
3799        let mut handler = make_handler_with_account();
3800        let payload = serde_json::json!({
3801            "type": "jsonapi/sendtx",
3802            "code": 21727,
3803            "message": "invalid client order index",
3804        })
3805        .to_string();
3806
3807        let (_, msg) = handle_control_text(&mut handler, &payload);
3808
3809        match msg.expect("SendTxRejected emitted") {
3810            NautilusWsMessage::SendTxRejected {
3811                source,
3812                code,
3813                message,
3814                tx_hash,
3815                ..
3816            } => {
3817                assert_eq!(source, SendTxRejectionSource::Ack);
3818                assert_eq!(code, Some(21727));
3819                assert_eq!(message, "invalid client order index");
3820                assert_eq!(tx_hash, None);
3821            }
3822            other => panic!("expected SendTxRejected, was {other:?}"),
3823        }
3824    }
3825
3826    #[rstest]
3827    fn handle_control_text_sendtx_failure_carries_echoed_tx_hash() {
3828        let mut handler = make_handler_with_account();
3829        let payload = serde_json::json!({
3830            "type": "jsonapi/sendtx",
3831            "code": 21727,
3832            "message": "invalid client order index",
3833            "tx_hash": "0000abcd",
3834        })
3835        .to_string();
3836
3837        let (_, msg) = handle_control_text(&mut handler, &payload);
3838
3839        match msg.expect("SendTxRejected emitted") {
3840            NautilusWsMessage::SendTxRejected { tx_hash, .. } => {
3841                assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3842            }
3843            other => panic!("expected SendTxRejected, was {other:?}"),
3844        }
3845    }
3846
3847    #[rstest]
3848    fn handle_control_text_bare_error_frame_emits_bare_error_rejection() {
3849        let mut handler = make_handler_with_account();
3850        let payload = serde_json::json!({
3851            "type": "error",
3852            "code": 21702,
3853            "message": "invalid price",
3854        })
3855        .to_string();
3856
3857        let (_, msg) = handle_control_text(&mut handler, &payload);
3858
3859        match msg.expect("SendTxRejected emitted") {
3860            NautilusWsMessage::SendTxRejected {
3861                source,
3862                code,
3863                message,
3864                tx_hash,
3865                ..
3866            } => {
3867                assert_eq!(source, SendTxRejectionSource::BareError);
3868                assert_eq!(code, Some(21702));
3869                assert_eq!(message, "invalid price");
3870                assert_eq!(tx_hash, None);
3871            }
3872            other => panic!("expected SendTxRejected, was {other:?}"),
3873        }
3874    }
3875
3876    #[rstest]
3877    fn handle_control_text_wrapped_error_emits_bare_error_rejection() {
3878        // No top-level `type` field; the rejection sits under `error.{code,message}`.
3879        let mut handler = make_handler_with_account();
3880        let payload = serde_json::json!({
3881            "error": {"code": 21149, "message": "integrator is not approved"},
3882        })
3883        .to_string();
3884
3885        let (_, msg) = handle_control_text(&mut handler, &payload);
3886
3887        match msg.expect("SendTxRejected emitted") {
3888            NautilusWsMessage::SendTxRejected {
3889                source,
3890                code,
3891                message,
3892                tx_hash,
3893                ..
3894            } => {
3895                assert_eq!(source, SendTxRejectionSource::BareError);
3896                assert_eq!(code, Some(21149));
3897                assert_eq!(message, "integrator is not approved");
3898                assert_eq!(tx_hash, None);
3899            }
3900            other => panic!("expected SendTxRejected, was {other:?}"),
3901        }
3902    }
3903
3904    #[rstest]
3905    fn handle_frame_emits_no_account_state_until_both_streams_seen() {
3906        // Reconciler refuses to emit until both `account_all_assets` and
3907        // `user_stats` have delivered. An assets frame alone produces only
3908        // the AccountStreamFirstFrame marker, no AccountState payload.
3909        let mut handler = make_handler_with_account();
3910        let assets_only: super::LighterWsFrame =
3911            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3912
3913        let messages = strip_account_marker(handler.handle_frame(assets_only, UnixNanos::from(11)));
3914
3915        assert!(
3916            messages.is_empty(),
3917            "expected no AccountState before user_stats arrives, received {messages:?}"
3918        );
3919    }
3920
3921    #[rstest]
3922    fn handle_frame_routes_account_assets_and_user_stats_to_unified_state() {
3923        // The snapshot holds 50 USDC in total and reports 40 USDC available
3924        let mut handler = make_handler_with_account();
3925        let assets_frame: super::LighterWsFrame =
3926            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3927        let user_stats_frame: super::LighterWsFrame =
3928            serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
3929
3930        let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
3931        let messages =
3932            strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
3933
3934        assert_eq!(messages.len(), 1);
3935        match &messages[0] {
3936            NautilusWsMessage::AccountState(state) => {
3937                let usdc = Currency::get_or_create_crypto("USDC");
3938                assert_eq!(state.account_type, AccountType::Margin);
3939                assert_eq!(state.base_currency, None);
3940                assert_eq!(state.balances.len(), 1);
3941                assert_eq!(state.balances[0].currency, usdc);
3942                assert_eq!(state.balances[0].total, Money::from("50.000000 USDC"));
3943                assert_eq!(state.balances[0].locked, Money::from("10 USDC"));
3944                assert_eq!(state.balances[0].free, Money::from("40.000000 USDC"));
3945                assert_eq!(state.margins.len(), 1);
3946                assert_eq!(state.margins[0].currency, usdc);
3947                assert_eq!(state.margins[0].initial, Money::from("0 USDC"));
3948                assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
3949                assert!(state.margins[0].instrument_id.is_none());
3950                assert!(state.is_reported);
3951            }
3952            other => panic!("expected account state, was {other:?}"),
3953        }
3954    }
3955
3956    #[rstest]
3957    fn handle_frame_unified_state_reflects_open_position() {
3958        // Fixture pair captured live with one open ETH-LONG position:
3959        //   account_all_assets[USDC] = {balance:10, locked:0, margin_balance:39.995369556}
3960        //   user_stats              = {collateral:39.995369, available:39.168314, margin_usage:2.07}
3961        //
3962        // The 5 mUSDC haircut on margin_balance is the entry fee. The
3963        // 0.827055 USDC gap between collateral and available_balance is
3964        // the initial margin pledged to the open position. AccountBalance
3965        // also includes funds outside the venue's reported availability.
3966        // Maintenance is zero because user_stats does not publish it.
3967        let mut handler = make_handler_with_account();
3968        let assets_frame: super::LighterWsFrame =
3969            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_WITH_POSITION).unwrap();
3970        let user_stats_frame: super::LighterWsFrame =
3971            serde_json::from_str(WS_USER_STATS_WITH_POSITION).unwrap();
3972
3973        let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
3974        let messages =
3975            strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
3976
3977        assert_eq!(messages.len(), 1);
3978        match &messages[0] {
3979            NautilusWsMessage::AccountState(state) => {
3980                assert_eq!(state.account_type, AccountType::Margin);
3981                assert_eq!(state.base_currency, None);
3982                assert_eq!(state.balances.len(), 1);
3983                // total = 10 + 39.995369556 = 49.99536956 (Money truncates
3984                // the trailing digit to 8 decimals, matching the
3985                // production log).
3986                assert_eq!(state.balances[0].total, Money::from("49.99536956 USDC"));
3987                assert_eq!(state.balances[0].locked, Money::from("10.82705556 USDC"));
3988                assert_eq!(state.balances[0].free, Money::from("39.16831400 USDC"));
3989                assert_eq!(state.margins.len(), 1);
3990                // initial = collateral 39.995369 - available 39.168314 = 0.827055
3991                assert_eq!(state.margins[0].initial, Money::from("0.82705500 USDC"));
3992                assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
3993                assert!(state.margins[0].instrument_id.is_none());
3994            }
3995            other => panic!("expected account state, was {other:?}"),
3996        }
3997    }
3998
3999    #[rstest]
4000    fn handle_frame_emits_account_stream_first_frame_marker_per_variant() {
4001        // Pins the strict-await wiring: the handler appends an
4002        // `AccountStreamFirstFrame` marker after any typed reports so the
4003        // execution consumption loop can mark readiness only after the
4004        // accompanying state has been applied. The marker fires even for
4005        // empty content frames that produce no typed reports.
4006        let mut handler = make_handler_with_account();
4007        let orders_frame: super::LighterWsFrame =
4008            serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
4009        let trades_frame: super::LighterWsFrame =
4010            serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
4011        let positions_frame: super::LighterWsFrame = serde_json::from_value({
4012            let mut value: serde_json::Value =
4013                serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
4014            value["type"] = json!("subscribed/account_all_positions");
4015            value
4016        })
4017        .unwrap();
4018        let assets_frame: super::LighterWsFrame =
4019            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
4020        let user_stats_frame: super::LighterWsFrame =
4021            serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
4022
4023        let cases = [
4024            (orders_frame, AccountStream::Orders),
4025            (trades_frame, AccountStream::Trades),
4026            (positions_frame, AccountStream::Positions),
4027            (assets_frame, AccountStream::Assets),
4028            (user_stats_frame, AccountStream::UserStats),
4029        ];
4030
4031        for (frame, expected) in cases {
4032            let msgs = handler.handle_frame(frame, UnixNanos::from(11));
4033            let marker = msgs
4034                .iter()
4035                .find(|m| matches!(m, NautilusWsMessage::AccountStreamFirstFrame(_)))
4036                .unwrap_or_else(|| panic!("missing marker for {expected:?}"));
4037            match marker {
4038                NautilusWsMessage::AccountStreamFirstFrame(stream) => {
4039                    assert_eq!(*stream, expected);
4040                }
4041                other => panic!("expected AccountStreamFirstFrame, was {other:?}"),
4042            }
4043            // The marker must trail any typed reports the handler emitted
4044            // so the consumption loop applies state first.
4045            assert!(
4046                matches!(
4047                    msgs.last(),
4048                    Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
4049                ),
4050                "marker must trail typed reports for {expected:?}",
4051            );
4052        }
4053    }
4054
4055    #[rstest]
4056    fn handle_frame_account_orders_without_context_falls_back_to_raw() {
4057        let signal = Arc::new(AtomicBool::new(false));
4058        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4059        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4060        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4061        let mut handler =
4062            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4063        handler.instruments.insert(0, stub_eth_perp_instrument());
4064        // exec_account intentionally left unset; the handler must preserve
4065        // the prior `Raw` forwarding contract for unauthenticated subscribers.
4066
4067        let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
4068        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4069
4070        assert_eq!(messages.len(), 1);
4071        match &messages[0] {
4072            NautilusWsMessage::Raw(value) => {
4073                assert_eq!(value["type"], "update/account_orders");
4074            }
4075            other => panic!("expected raw fallback, was {other:?}"),
4076        }
4077    }
4078
4079    #[rstest]
4080    fn handle_frame_account_orders_skips_unknown_market() {
4081        // Build a handler with the execution context but no instrument
4082        // cached for the order's market_index; the handler should log and
4083        // emit no execution reports (the trailing readiness marker still
4084        // fires so `connect()` does not stall when the venue resubscribes
4085        // before instrument bootstrap completes).
4086        let signal = Arc::new(AtomicBool::new(false));
4087        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4088        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4089        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4090        let mut handler =
4091            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4092        handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
4093        // No instrument inserted for market_index=0.
4094
4095        let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
4096        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4097
4098        assert!(messages.is_empty());
4099    }
4100
4101    #[rstest]
4102    fn handle_frame_account_assets_invalid_timestamp_returns_empty() {
4103        let mut handler = make_handler_with_account();
4104        // u64::MAX millis overflows the seconds-to-nanos conversion in
4105        // `parse_millis_to_nanos`; the handler must log and emit no typed
4106        // account state rather than surface a partially constructed one.
4107        // The trailing readiness marker is still emitted so `connect()`
4108        // does not stall if the venue ever sends a malformed timestamp on
4109        // the initial frame.
4110        let frame_json = r#"{
4111            "type": "update/account_all_assets",
4112            "channel": "account_all_assets:1234",
4113            "timestamp": 18446744073709551615,
4114            "assets": {
4115                "0": {
4116                    "symbol": "USDC",
4117                    "asset_id": 0,
4118                    "balance": "100.000000",
4119                    "locked_balance": "1.000000"
4120                }
4121            }
4122        }"#;
4123        let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
4124
4125        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4126
4127        assert!(messages.is_empty());
4128    }
4129
4130    #[rstest]
4131    fn handle_frame_account_all_orders_routes_to_execution_reports() {
4132        // The `update/account_all_orders` variant lacks the per-account
4133        // top-level `account` and `nonce` fields that `account_orders`
4134        // carries, but the handler should treat them identically.
4135        let mut handler = make_handler_with_account();
4136        let frame_json = r#"{
4137            "type": "update/account_all_orders",
4138            "channel": "account_all_orders:1234",
4139            "orders": {
4140                "0": [{
4141                    "order_index": 281476929510110,
4142                    "client_order_index": 42,
4143                    "order_id": "281476929510110",
4144                    "client_order_id": "42",
4145                    "market_index": 0,
4146                    "owner_account_index": 1234,
4147                    "initial_base_amount": "0.0050",
4148                    "price": "2352.74",
4149                    "nonce": 9182390020,
4150                    "remaining_base_amount": "0.0050",
4151                    "is_ask": true,
4152                    "base_size": 50,
4153                    "base_price": 235274,
4154                    "filled_base_amount": "0.0000",
4155                    "filled_quote_amount": "0.000000",
4156                    "side": "sell",
4157                    "type": "limit",
4158                    "time_in_force": "good-till-time",
4159                    "reduce_only": false,
4160                    "trigger_price": "0.00",
4161                    "order_expiry": 1780360584479,
4162                    "status": "open",
4163                    "trigger_status": "na",
4164                    "trigger_time": 0,
4165                    "parent_order_index": 0,
4166                    "parent_order_id": "0",
4167                    "to_trigger_order_id_0": "0",
4168                    "to_trigger_order_id_1": "0",
4169                    "to_cancel_order_id_0": "0",
4170                    "integrator_fee_collector_index": "0",
4171                    "integrator_taker_fee": "0",
4172                    "integrator_maker_fee": "0",
4173                    "block_height": 227535532,
4174                    "timestamp": 1777941383576,
4175                    "created_at": 1777941383576,
4176                    "updated_at": 1777941383576,
4177                    "transaction_time": 1777941383576735
4178                }]
4179            }
4180        }"#;
4181        let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
4182
4183        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4184
4185        assert_eq!(messages.len(), 1);
4186        match &messages[0] {
4187            NautilusWsMessage::ExecutionReports(reports) => {
4188                assert_eq!(reports.len(), 1);
4189                match &reports[0] {
4190                    super::ExecutionReport::Order(order) => {
4191                        assert_eq!(order.order_id, "281476929510110");
4192                    }
4193                    other => panic!("expected order report, was {other:?}"),
4194                }
4195            }
4196            other => panic!("expected execution reports, was {other:?}"),
4197        }
4198    }
4199
4200    fn snapshot_trade_frame_json() -> &'static str {
4201        r#"{
4202            "type": "subscribed/account_all_trades",
4203            "channel": "account_all_trades:1234",
4204            "trades": [{
4205                "trade_id": 19209006902,
4206                "trade_id_str": "19209006902",
4207                "tx_hash": "000000128b1ee814",
4208                "type": "trade",
4209                "market_id": 0,
4210                "size": "0.1336",
4211                "price": "2352.73",
4212                "usd_amount": "314.324728",
4213                "ask_id": 281476929510102,
4214                "bid_id": 562947905631053,
4215                "ask_client_id": 0,
4216                "bid_client_id": 7001011966,
4217                "ask_account_id": 91249,
4218                "bid_account_id": 1234,
4219                "is_maker_ask": true,
4220                "block_height": 227535535,
4221                "timestamp": 1777941384181,
4222                "transaction_time": 1777941384181586
4223            }],
4224            "total_volume": "100.0",
4225            "monthly_volume": "100.0",
4226            "weekly_volume": "100.0",
4227            "daily_volume": "100.0"
4228        }"#
4229    }
4230
4231    #[rstest]
4232    fn handle_frame_account_all_trades_snapshot_is_dropped_with_context() {
4233        let mut handler = make_handler_with_account();
4234        // With execution context active the snapshot must be dropped to
4235        // avoid replaying historical fills as live FillReports on reconnect.
4236        // The trailing readiness marker is still emitted so `connect()`
4237        // observes the trades stream as having delivered.
4238        let frame: super::LighterWsFrame =
4239            serde_json::from_str(snapshot_trade_frame_json()).unwrap();
4240
4241        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4242
4243        assert!(messages.is_empty());
4244    }
4245
4246    #[rstest]
4247    fn handle_frame_account_all_trades_snapshot_falls_back_to_raw_without_context() {
4248        let signal = Arc::new(AtomicBool::new(false));
4249        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4250        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4251        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4252        let mut handler =
4253            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4254        handler.instruments.insert(0, stub_eth_perp_instrument());
4255        // No exec_account: preserve prior Raw forwarding so unauthenticated
4256        // subscribers still see the snapshot frame.
4257
4258        let frame: super::LighterWsFrame =
4259            serde_json::from_str(snapshot_trade_frame_json()).unwrap();
4260        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4261
4262        assert_eq!(messages.len(), 1);
4263        match &messages[0] {
4264            NautilusWsMessage::Raw(value) => {
4265                assert_eq!(value["type"], "subscribed/account_all_trades");
4266            }
4267            other => panic!("expected raw fallback, was {other:?}"),
4268        }
4269    }
4270
4271    #[rstest]
4272    fn handle_frame_market_stats_emits_mark_index_and_funding_updates() {
4273        let mut handler = make_handler_with_account();
4274        let frame: super::LighterWsFrame =
4275            serde_json::from_str(WS_MARKET_STATS_UPDATE_SINGLE).unwrap();
4276
4277        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4278
4279        assert_eq!(messages.len(), 3);
4280        match &messages[0] {
4281            NautilusWsMessage::MarkPrice(update) => {
4282                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4283                assert_eq!(update.value, Price::from("2064.47"));
4284                assert_eq!(update.ts_event, UnixNanos::from(1_774_883_844_933_000_000));
4285            }
4286            event => panic!("expected mark price update, was {event:?}"),
4287        }
4288
4289        match &messages[1] {
4290            NautilusWsMessage::IndexPrice(update) => {
4291                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4292                assert_eq!(update.value, Price::from("2064.48"));
4293            }
4294            event => panic!("expected index price update, was {event:?}"),
4295        }
4296
4297        match &messages[2] {
4298            NautilusWsMessage::FundingRate(update) => {
4299                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4300                assert_eq!(update.rate, Decimal::new(1, 8));
4301                assert_eq!(update.next_funding_ns, None);
4302            }
4303            event => panic!("expected funding rate update, was {event:?}"),
4304        }
4305    }
4306
4307    #[rstest]
4308    fn handle_frame_market_stats_all_emits_mark_index_and_funding_updates() {
4309        let mut handler = make_handler_with_account();
4310        let frame: super::LighterWsFrame =
4311            serde_json::from_str(WS_MARKET_STATS_UPDATE_ALL).unwrap();
4312
4313        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4314
4315        assert_eq!(messages.len(), 3);
4316        assert!(matches!(&messages[0], NautilusWsMessage::MarkPrice(_)));
4317        assert!(matches!(&messages[1], NautilusWsMessage::IndexPrice(_)));
4318        assert!(matches!(&messages[2], NautilusWsMessage::FundingRate(_)));
4319
4320        match &messages[0] {
4321            NautilusWsMessage::MarkPrice(update) => {
4322                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4323                assert_eq!(update.value, Price::from("2064.47"));
4324            }
4325            event => panic!("expected mark price update, was {event:?}"),
4326        }
4327
4328        match &messages[2] {
4329            NautilusWsMessage::FundingRate(update) => {
4330                assert_eq!(update.rate, Decimal::new(1, 8));
4331                assert_eq!(update.next_funding_ns, None);
4332            }
4333            event => panic!("expected funding rate update, was {event:?}"),
4334        }
4335    }
4336
4337    #[rstest]
4338    fn handle_frame_spot_market_stats_emits_index_update() {
4339        let signal = Arc::new(AtomicBool::new(false));
4340        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4341        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4342        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4343        let mut handler =
4344            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4345        handler.instruments.insert(2048, stub_eth_spot_instrument());
4346        let frame: super::LighterWsFrame =
4347            serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_SINGLE).unwrap();
4348
4349        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4350
4351        assert_eq!(messages.len(), 1);
4352        match &messages[0] {
4353            NautilusWsMessage::IndexPrice(update) => {
4354                assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
4355                assert_eq!(update.value, Price::from("1.00"));
4356            }
4357            event => panic!("expected spot index price update, was {event:?}"),
4358        }
4359    }
4360
4361    #[rstest]
4362    fn handle_frame_spot_market_stats_all_emits_index_update() {
4363        let signal = Arc::new(AtomicBool::new(false));
4364        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4365        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4366        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4367        let mut handler =
4368            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4369        handler.instruments.insert(2048, stub_eth_spot_instrument());
4370        let frame: super::LighterWsFrame =
4371            serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_ALL).unwrap();
4372
4373        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4374
4375        assert_eq!(messages.len(), 1);
4376        match &messages[0] {
4377            NautilusWsMessage::IndexPrice(update) => {
4378                assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
4379                assert_eq!(update.value, Price::from("1.00"));
4380            }
4381            event => panic!("expected spot index price update, was {event:?}"),
4382        }
4383    }
4384
4385    #[rstest]
4386    #[case(LighterWsChannel::OrderBook(0), "order_book:0", "order_book/0")]
4387    #[case(LighterWsChannel::Trade(7), "trade:7", "trade/7")]
4388    #[case(LighterWsChannel::Ticker(2), "ticker:2", "ticker/2")]
4389    #[case(LighterWsChannel::Height, "height", "height")]
4390    #[case(
4391        LighterWsChannel::MarketStats(LighterMarketSelection::All),
4392        "market_stats:all",
4393        "market_stats/all"
4394    )]
4395    #[case(
4396        LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(2048)),
4397        "spot_market_stats:2048",
4398        "spot_market_stats/2048"
4399    )]
4400    #[case(
4401        LighterWsChannel::OrderBook(4095),
4402        "order_book:4095",
4403        "order_book/4095"
4404    )]
4405    #[case(
4406        LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(50_000)),
4407        "spot_market_stats:50000",
4408        "spot_market_stats/50000"
4409    )]
4410    #[case(
4411        LighterWsChannel::Candle {
4412            market_index: 40_000,
4413            resolution: LighterCandleResolution::OneHour,
4414        },
4415        "candle:40000:1h",
4416        "candle/40000/1h"
4417    )]
4418    #[case(
4419        LighterWsChannel::AccountOrders { market_index: 0, account_index: 1234 },
4420        "account_orders:0:1234",
4421        "account_orders/0/1234",
4422    )]
4423    fn topic_and_subscription_round_trip(
4424        #[case] channel: LighterWsChannel,
4425        #[case] expected_topic: &str,
4426        #[case] expected_subscription: &str,
4427    ) {
4428        assert_eq!(channel.topic_key(), expected_topic);
4429        assert_eq!(channel.subscription_channel(), expected_subscription);
4430    }
4431
4432    #[rstest]
4433    #[case("order_book:0", Some(0))]
4434    #[case("trade:42", Some(42))]
4435    #[case("order_book:4095", Some(4095))]
4436    #[case("trade:40000", Some(40_000))]
4437    #[case("market_stats:50000", Some(50_000))]
4438    #[case("height", None)]
4439    #[case("malformed", None)]
4440    fn market_index_extraction(#[case] topic: &str, #[case] expected: Option<i64>) {
4441        assert_eq!(market_index_from_topic(topic), expected);
4442    }
4443
4444    #[rstest]
4445    #[case("order_book:0", Some(0))]
4446    #[case("order_book:42", Some(42))]
4447    #[case("order_book:40000", Some(40_000))]
4448    #[case("trade:42", None)]
4449    #[case("ticker:2", None)]
4450    #[case("market_stats:0", None)]
4451    #[case("height", None)]
4452    #[case("order_book:not-an-int", None)]
4453    fn order_book_market_index_only_matches_order_book_channel(
4454        #[case] topic: &str,
4455        #[case] expected: Option<i64>,
4456    ) {
4457        assert_eq!(order_book_market_index_from_topic(topic), expected);
4458    }
4459
4460    #[rstest]
4461    #[case(LighterWsChannel::AccountAll(1234), true)]
4462    #[case(LighterWsChannel::OrderBook(0), false)]
4463    #[case(LighterWsChannel::AccountAllPositions(1), true)]
4464    #[case(LighterWsChannel::Trade(0), false)]
4465    fn requires_auth_classification(#[case] channel: LighterWsChannel, #[case] expected: bool) {
4466        assert_eq!(channel.requires_auth(), expected);
4467    }
4468
4469    #[rstest]
4470    fn handler_command_subscribe_debug_redacts_auth_token() {
4471        let token = "schnorr-signature-bytes-do-not-leak";
4472        let cmd = HandlerCommand::Subscribe {
4473            channel: LighterWsChannel::AccountAll(1234),
4474            auth: Some(SecretString::from(token)),
4475            response_tx: None,
4476        };
4477
4478        let dbg = format!("{cmd:?}");
4479
4480        assert!(
4481            !dbg.contains(token),
4482            "Debug output must not contain the auth token, found: {dbg}",
4483        );
4484        assert!(dbg.contains(REDACTED));
4485    }
4486
4487    #[tokio::test]
4488    async fn send_tx_command_returns_handler_send_error_without_active_client() {
4489        let signal = Arc::new(AtomicBool::new(false));
4490        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4491        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4492        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4493        let mut handler =
4494            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4495        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
4496
4497        cmd_tx
4498            .send(HandlerCommand::SendTx {
4499                tx_type: LighterTxType::CreateOrder as u8,
4500                tx_info: serde_json::value::RawValue::from_string(
4501                    r#"{"AccountIndex":12345,"Nonce":42}"#.to_string(),
4502                )
4503                .unwrap(),
4504                connection_epoch: 0,
4505                response_tx,
4506            })
4507            .unwrap();
4508        drop(cmd_tx);
4509        drop(raw_tx);
4510
4511        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4512            .await
4513            .expect("timed out waiting for handler to drain command");
4514        let result = response_rx.await.expect("sendTx response channel closed");
4515
4516        assert!(next.is_none());
4517        let Err(LighterWsError::Client(message)) = result else {
4518            panic!("expected client send error, was {result:?}");
4519        };
4520        assert!(message.contains("no active WebSocket client"));
4521    }
4522
4523    #[tokio::test]
4524    async fn resubscribe_order_book_command_skips_when_reference_removed() {
4525        let signal = Arc::new(AtomicBool::new(false));
4526        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4527        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4528        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4529        let subscriptions = SubscriptionState::new(':');
4530        let topic = LighterWsChannel::OrderBook(0).topic_key();
4531        assert!(subscriptions.add_reference(&topic));
4532        assert!(subscriptions.remove_reference(&topic));
4533
4534        let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions.clone());
4535        handler.book.delta_subs.insert(0);
4536
4537        let (completion, result) = tokio::sync::oneshot::channel();
4538        cmd_tx
4539            .send(HandlerCommand::RecoverBook {
4540                market_index: 0,
4541                cancel: CancellationToken::new(),
4542                gate: SnapshotGate::default(),
4543                completion,
4544            })
4545            .expect("queue resync");
4546        drop(cmd_tx);
4547        drop(raw_tx);
4548
4549        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4550            .await
4551            .expect("timed out waiting for handler to drain command");
4552
4553        assert!(next.is_none());
4554        assert!(subscriptions.pending_subscribe_topics().is_empty());
4555        assert!(subscriptions.pending_unsubscribe_topics().is_empty());
4556        assert!(result.await.is_err());
4557    }
4558
4559    #[tokio::test]
4560    async fn resubscribe_order_book_queues_when_inflight_is_at_cap() {
4561        let signal = Arc::new(AtomicBool::new(false));
4562        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4563        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4564        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4565        let subscriptions = SubscriptionState::new(':');
4566        let topic = LighterWsChannel::OrderBook(0).topic_key();
4567        assert!(subscriptions.add_reference(&topic));
4568
4569        let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions);
4570        handler.book.delta_subs.insert(0);
4571        saturate_subscription_gate(&mut handler);
4572
4573        let (completion, _rx) = tokio::sync::oneshot::channel();
4574        handler.queue_book_replacement(
4575            0,
4576            BookWrite {
4577                cancel: CancellationToken::new(),
4578                gate: SnapshotGate::default(),
4579                completion,
4580            },
4581        );
4582
4583        handler.pump_pending_subscribes().await;
4584
4585        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4586        assert!(
4587            handler
4588                .pending_subs
4589                .iter()
4590                .any(|(topic, _)| *topic == Ustr::from("order_book:0")),
4591        );
4592    }
4593
4594    fn stub_candle(
4595        t: i64,
4596        open: i64,
4597        high: i64,
4598        low: i64,
4599        close: i64,
4600        volume_ticks: i64,
4601    ) -> LighterWsCandle {
4602        LighterWsCandle {
4603            t,
4604            o: Decimal::new(open, 2),
4605            h: Decimal::new(high, 2),
4606            l: Decimal::new(low, 2),
4607            c: Decimal::new(close, 2),
4608            v: Decimal::new(volume_ticks, 4),
4609            quote_volume: Decimal::ZERO,
4610            i: 0,
4611        }
4612    }
4613
4614    fn candle_frame(channel: &str, candle: LighterWsCandle, is_snapshot: bool) -> LighterWsFrame {
4615        if is_snapshot {
4616            LighterWsFrame::CandleSnapshot {
4617                channel: Ustr::from(channel),
4618                candles: vec![candle],
4619                timestamp: 0,
4620            }
4621        } else {
4622            LighterWsFrame::Candle {
4623                channel: Ustr::from(channel),
4624                candles: vec![candle],
4625                timestamp: 0,
4626            }
4627        }
4628    }
4629
4630    #[rstest]
4631    fn handle_candles_first_observation_caches_without_emit() {
4632        let mut handler = make_handler_with_account();
4633        let frame = candle_frame(
4634            "candle:0:1m",
4635            stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
4636            true,
4637        );
4638
4639        let messages = handler.handle_frame(frame, UnixNanos::from(99));
4640
4641        assert!(messages.is_empty(), "first observation must not emit");
4642        let key = (0_i64, LighterCandleResolution::OneMinute);
4643        assert_eq!(handler.last_candles.get(&key).map(|c| c.t), Some(1_000_000));
4644    }
4645
4646    #[rstest]
4647    fn handle_candles_t_advance_emits_bar_for_previous_candle() {
4648        let mut handler = make_handler_with_account();
4649        let prev = stub_candle(1_000_000, 10_000, 11_000, 9_900, 10_500, 10_000);
4650        let next = stub_candle(1_060_000, 10_500, 10_600, 10_450, 10_550, 20_000);
4651        let next_t = next.t;
4652        handler.handle_frame(candle_frame("candle:0:1m", prev, true), UnixNanos::from(1));
4653
4654        let messages =
4655            handler.handle_frame(candle_frame("candle:0:1m", next, false), UnixNanos::from(2));
4656
4657        assert_eq!(messages.len(), 1);
4658        match &messages[0] {
4659            NautilusWsMessage::Bar(bar) => {
4660                // The emitted bar reflects the previous (closed) candle, not the new one.
4661                assert_eq!(bar.open, Price::from("100.00"));
4662                assert_eq!(bar.high, Price::from("110.00"));
4663                assert_eq!(bar.low, Price::from("99.00"));
4664                assert_eq!(bar.close, Price::from("105.00"));
4665                assert_eq!(bar.volume, Quantity::from("1.0000"));
4666                assert_eq!(bar.ts_event, UnixNanos::from(1_000_000 * 1_000_000));
4667            }
4668            other => panic!("expected Bar message, was {other:?}"),
4669        }
4670        let cached = handler
4671            .last_candles
4672            .get(&(0_i64, LighterCandleResolution::OneMinute))
4673            .expect("cache populated");
4674        assert_eq!(cached.t, next_t);
4675    }
4676
4677    #[rstest]
4678    fn handle_candles_same_t_updates_cache_without_emit() {
4679        let mut handler = make_handler_with_account();
4680        let initial = stub_candle(1_000_000, 10_000, 10_050, 9_950, 10_025, 5_000);
4681        let same_t_updated = stub_candle(1_000_000, 10_000, 10_100, 9_950, 10_075, 7_500);
4682        let same_t_h = same_t_updated.h;
4683        let same_t_c = same_t_updated.c;
4684        handler.handle_frame(
4685            candle_frame("candle:0:1m", initial, true),
4686            UnixNanos::from(1),
4687        );
4688
4689        let messages = handler.handle_frame(
4690            candle_frame("candle:0:1m", same_t_updated, false),
4691            UnixNanos::from(2),
4692        );
4693
4694        assert!(messages.is_empty(), "same-`t` update must not emit");
4695        let cached = handler
4696            .last_candles
4697            .get(&(0_i64, LighterCandleResolution::OneMinute))
4698            .expect("cache populated");
4699        assert_eq!(cached.h, same_t_h);
4700        assert_eq!(cached.c, same_t_c);
4701    }
4702
4703    #[rstest]
4704    fn handle_candles_regressed_t_is_skipped() {
4705        let mut handler = make_handler_with_account();
4706        let initial = stub_candle(2_000_000, 10_000, 10_000, 10_000, 10_000, 5_000);
4707        let regressed = stub_candle(1_000_000, 9_000, 9_000, 9_000, 9_000, 5_000);
4708        let initial_t = initial.t;
4709        handler.handle_frame(
4710            candle_frame("candle:0:1m", initial, true),
4711            UnixNanos::from(1),
4712        );
4713
4714        let messages = handler.handle_frame(
4715            candle_frame("candle:0:1m", regressed, false),
4716            UnixNanos::from(2),
4717        );
4718
4719        assert!(messages.is_empty(), "regressed `t` must not emit");
4720        let cached = handler
4721            .last_candles
4722            .get(&(0_i64, LighterCandleResolution::OneMinute))
4723            .expect("cache populated");
4724        // Regressed frame is skipped entirely; cache stays on the original entry.
4725        assert_eq!(cached.t, initial_t);
4726    }
4727
4728    #[rstest]
4729    fn handle_candles_unknown_market_returns_empty() {
4730        let mut handler = make_handler_with_account();
4731        let frame = candle_frame(
4732            "candle:99:1m",
4733            stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 5_000),
4734            true,
4735        );
4736
4737        let messages = handler.handle_frame(frame, UnixNanos::from(1));
4738
4739        assert!(messages.is_empty());
4740    }
4741
4742    #[rstest]
4743    fn handle_unsubscribe_ack_clears_only_matching_candle_key() {
4744        let mut handler = make_handler_with_account();
4745        handler.last_candles.insert(
4746            (0, LighterCandleResolution::OneMinute),
4747            stub_candle(1, 0, 0, 0, 0, 0),
4748        );
4749        handler.last_candles.insert(
4750            (0, LighterCandleResolution::FiveMinute),
4751            stub_candle(2, 0, 0, 0, 0, 0),
4752        );
4753        handler.subscriptions.mark_unsubscribe("candle:0:1m");
4754
4755        let payload = json!({"type": "unsubscribed", "channel": "candle:0:1m"});
4756        let (matched, _) = handle_control_text(&mut handler, &payload.to_string());
4757
4758        assert!(matched);
4759        assert!(
4760            handler
4761                .last_candles
4762                .get(&(0, LighterCandleResolution::OneMinute))
4763                .is_none(),
4764        );
4765        assert!(
4766            handler
4767                .last_candles
4768                .get(&(0, LighterCandleResolution::FiveMinute))
4769                .is_some(),
4770        );
4771    }
4772
4773    #[rstest]
4774    #[case::well_formed("candle:0:1m", Some((0, LighterCandleResolution::OneMinute)))]
4775    #[case::weekly("candle:3:1w", Some((3, LighterCandleResolution::OneWeek)))]
4776    #[case::widened_id("candle:40000:1h", Some((40_000, LighterCandleResolution::OneHour)))]
4777    #[case::other_kind("order_book:0", None)]
4778    #[case::missing_resolution("candle:0", None)]
4779    #[case::bad_market("candle:notanint:1m", None)]
4780    #[case::bad_resolution("candle:0:bogus", None)]
4781    fn test_candle_market_and_resolution_from_topic(
4782        #[case] topic: &str,
4783        #[case] expected: Option<(i64, LighterCandleResolution)>,
4784    ) {
4785        assert_eq!(candle_market_and_resolution_from_topic(topic), expected);
4786    }
4787
4788    #[rstest]
4789    #[case::network_retries(LighterWsError::Network("disconnected".into()), true)]
4790    #[case::auth_does_not_retry(LighterWsError::Authentication("bad token".into()), false)]
4791    #[case::parse_does_not_retry(LighterWsError::Parse("bad json".into()), false)]
4792    #[case::client_does_not_retry(LighterWsError::Client("no active WebSocket client".into()), false)]
4793    #[case::transport_closed_does_not_retry(LighterWsError::Transport(SendError::Closed), false)]
4794    #[case::transport_invalid_input_does_not_retry(
4795        LighterWsError::Transport(SendError::InvalidInput("pong payload too large".into())),
4796        false
4797    )]
4798    #[case::transport_connection_changed_does_not_retry(
4799        LighterWsError::Transport(SendError::ConnectionChanged),
4800        false
4801    )]
4802    #[case::transport_timeout_retries(LighterWsError::Transport(SendError::Timeout), true)]
4803    #[case::transport_write_timeout_does_not_retry(
4804        LighterWsError::Transport(SendError::WriteTimeout),
4805        false
4806    )]
4807    #[case::transport_broken_pipe_does_not_retry(
4808        LighterWsError::Transport(SendError::BrokenPipe(
4809            "writer closed".into(),
4810        )),
4811        false,
4812    )]
4813    #[case::buffer_full(LighterWsError::Transport(SendError::BufferFull), false)]
4814    fn test_should_retry_lighter_ws_error(#[case] error: LighterWsError, #[case] expected: bool) {
4815        assert_eq!(should_retry_lighter_ws_error(&error), expected);
4816    }
4817
4818    // Pins the `#[from] SendError` derive that `.map_err(LighterWsError::Transport)` relies on.
4819    #[rstest]
4820    #[case::closed(SendError::Closed)]
4821    #[case::invalid_input(SendError::InvalidInput("pong payload too large".into()))]
4822    #[case::timeout(SendError::Timeout)]
4823    #[case::write_timeout(SendError::WriteTimeout)]
4824    #[case::connection_changed(SendError::ConnectionChanged)]
4825    #[case::broken_pipe(SendError::BrokenPipe("writer dropped".into()))]
4826    fn send_error_converts_into_transport_variant(#[case] send_error: SendError) {
4827        let err: LighterWsError = send_error.into();
4828        assert!(
4829            matches!(err, LighterWsError::Transport(_)),
4830            "expected Transport variant, was {err:?}",
4831        );
4832    }
4833
4834    #[tokio::test]
4835    async fn subscribe_command_parks_in_pending_subs_when_inflight_at_cap() {
4836        let signal = Arc::new(AtomicBool::new(false));
4837        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4838        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4839        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4840        let mut handler =
4841            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4842
4843        // Saturate the gate so the pump cannot dispatch the queued subscribe
4844        saturate_subscription_gate(&mut handler);
4845
4846        cmd_tx
4847            .send(HandlerCommand::Subscribe {
4848                channel: LighterWsChannel::Candle {
4849                    market_index: 0,
4850                    resolution: LighterCandleResolution::OneMinute,
4851                },
4852                auth: None,
4853                response_tx: None,
4854            })
4855            .expect("queue subscribe");
4856        drop(cmd_tx);
4857        drop(raw_tx);
4858
4859        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4860            .await
4861            .expect("timed out waiting for handler to drain command");
4862
4863        assert!(next.is_none());
4864        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4865        assert_eq!(handler.pending_subs.len(), 1);
4866        assert_eq!(handler.subscription_attempts.len(), 1);
4867        assert_eq!(handler.pending_subs[0].0, Ustr::from("candle:0:1m"));
4868    }
4869
4870    #[tokio::test]
4871    async fn unsubscribe_drops_queued_subscribe_while_gate_full() {
4872        let signal = Arc::new(AtomicBool::new(false));
4873        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4874        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4875        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4876        let mut handler =
4877            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4878
4879        // Saturate the gate so the queued subscribe cannot dispatch before the unsubscribe
4880        saturate_subscription_gate(&mut handler);
4881
4882        cmd_tx
4883            .send(HandlerCommand::Subscribe {
4884                channel: LighterWsChannel::Trade(0),
4885                auth: None,
4886                response_tx: None,
4887            })
4888            .expect("queue subscribe");
4889        cmd_tx
4890            .send(HandlerCommand::Unsubscribe {
4891                channel: LighterWsChannel::Trade(0),
4892            })
4893            .expect("queue unsubscribe");
4894        drop(cmd_tx);
4895        drop(raw_tx);
4896
4897        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4898            .await
4899            .expect("timed out waiting for handler to drain commands");
4900
4901        assert!(next.is_none());
4902        assert!(handler.pending_subs.is_empty());
4903        assert!(handler.subscription_attempts.is_empty());
4904    }
4905
4906    #[tokio::test]
4907    async fn reconnect_requeues_attempt_with_new_generation() {
4908        let signal = Arc::new(AtomicBool::new(false));
4909        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4910        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4911        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4912        let mut handler =
4913            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4914
4915        saturate_subscription_gate(&mut handler);
4916        handler.queue_subscribe(LighterWsChannel::Trade(0), None, None);
4917        let old_generation = handler.pending_subs[0].1;
4918
4919        raw_tx
4920            .send((7, Message::Text(RECONNECTED.to_string().into())))
4921            .expect("queue reconnect sentinel");
4922
4923        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4924            .await
4925            .expect("timed out waiting for reconnect");
4926
4927        assert!(matches!(next, Some(NautilusWsMessage::Reconnected { .. })));
4928        assert!(handler.inflight_subs.is_empty());
4929        assert_eq!(handler.pending_subs.len(), 1);
4930        assert_eq!(handler.pending_subs[0].0, Ustr::from("trade:0"));
4931        assert_ne!(handler.pending_subs[0].1, old_generation);
4932        assert_eq!(
4933            handler.subscription_attempts[&Ustr::from("trade:0")].generation,
4934            handler.pending_subs[0].1,
4935        );
4936    }
4937
4938    #[rstest]
4939    #[tokio::test]
4940    async fn pump_releases_inflight_slot_and_schedules_send_failure_retry() {
4941        let signal = Arc::new(AtomicBool::new(false));
4942        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4943        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4944        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4945        let mut handler =
4946            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4947
4948        // No active client, so every dispatch fails; the gate must not leak slots
4949        for market_index in 0..3 {
4950            handler.queue_subscribe(LighterWsChannel::Trade(market_index), None, None);
4951        }
4952
4953        handler.pump_pending_subscribes().await;
4954
4955        assert!(handler.pending_subs.is_empty());
4956        assert!(handler.inflight_subs.is_empty());
4957        assert_eq!(handler.subscription_attempts.len(), 3);
4958        assert!(
4959            handler
4960                .subscription_attempts
4961                .values()
4962                .all(|attempt| attempt.retries == 1),
4963        );
4964        assert_eq!(handler.subscription_retries.len(), 3);
4965    }
4966
4967    #[rstest]
4968    fn subscribed_control_frame_releases_inflight_slot() {
4969        let mut handler = make_handler_with_account();
4970        mark_subscription_inflight(
4971            &mut handler,
4972            LighterWsChannel::Candle {
4973                market_index: 0,
4974                resolution: LighterCandleResolution::OneMinute,
4975            },
4976            None,
4977        );
4978
4979        let (matched, msg) = handle_control_text(
4980            &mut handler,
4981            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
4982        );
4983
4984        assert!(matched);
4985        assert!(msg.is_none());
4986        assert!(
4987            !handler
4988                .inflight_subs
4989                .contains_key(&Ustr::from("candle:0:1m"))
4990        );
4991        assert!(handler.subscription_attempts.is_empty());
4992    }
4993
4994    #[rstest]
4995    #[case::top_level(
4996        r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_orders:12345"}"#
4997    )]
4998    #[case::nested(
4999        r#"{"error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
5000    )]
5001    #[case::nested_typed(
5002        r#"{"type":"error","error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
5003    )]
5004    fn already_subscribed_confirms_matching_inflight_topic(#[case] payload: &str) {
5005        let mut handler = make_handler_with_account();
5006        mark_subscription_inflight(
5007            &mut handler,
5008            LighterWsChannel::AccountAllOrders(12345),
5009            None,
5010        );
5011
5012        let (matched, msg) = handle_control_text(&mut handler, payload);
5013
5014        assert!(matched);
5015        assert!(msg.is_none());
5016        assert!(handler.inflight_subs.is_empty());
5017        assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
5018        assert_eq!(handler.subscriptions.len(), 1);
5019    }
5020
5021    #[rstest]
5022    fn already_subscribed_does_not_confirm_unmatched_topic() {
5023        let mut handler = make_handler_with_account();
5024        let inflight = "account_all_orders:12345";
5025        mark_subscription_inflight(
5026            &mut handler,
5027            LighterWsChannel::AccountAllOrders(12345),
5028            None,
5029        );
5030
5031        let (matched, msg) = handle_control_text(
5032            &mut handler,
5033            r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_trades:12345"}"#,
5034        );
5035
5036        assert!(matched);
5037        assert!(msg.is_none());
5038        assert!(handler.inflight_subs.contains_key(&Ustr::from(inflight)));
5039        assert_eq!(
5040            handler.subscriptions.pending_subscribe_topics(),
5041            vec![inflight]
5042        );
5043        assert_eq!(handler.subscriptions.len(), 0);
5044    }
5045
5046    #[tokio::test]
5047    async fn duplicate_queued_and_inflight_topics_share_one_generation() {
5048        let mut handler = make_handler_with_account();
5049        let (response_tx_1, mut response_rx_1) = tokio::sync::oneshot::channel();
5050        let (response_tx_2, mut response_rx_2) = tokio::sync::oneshot::channel();
5051        let (response_tx_3, mut response_rx_3) = tokio::sync::oneshot::channel();
5052        let channel = LighterWsChannel::Trade(7);
5053
5054        handler.queue_subscribe(channel.clone(), None, Some(response_tx_1));
5055        let generation = handler.pending_subs[0].1;
5056        handler.queue_subscribe(channel.clone(), None, Some(response_tx_2));
5057
5058        assert_eq!(handler.pending_subs.len(), 1);
5059        assert_eq!(handler.subscription_attempts.len(), 1);
5060        assert_eq!(
5061            handler.subscription_attempts[&Ustr::from("trade:7")]
5062                .response_txs
5063                .len(),
5064            2,
5065        );
5066
5067        let (topic, queued_generation) = handler.pending_subs.pop_front().unwrap();
5068        handler.inflight_subs.insert(topic, queued_generation);
5069        handler.queue_subscribe(channel, None, Some(response_tx_3));
5070
5071        assert_eq!(queued_generation, generation);
5072        assert!(handler.pending_subs.is_empty());
5073        assert_eq!(handler.inflight_subs.len(), 1);
5074        assert_eq!(
5075            handler.subscription_attempts[&Ustr::from("trade:7")]
5076                .response_txs
5077                .len(),
5078            3,
5079        );
5080        assert!(matches!(
5081            response_rx_1.try_recv(),
5082            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5083        ));
5084        assert!(matches!(
5085            response_rx_2.try_recv(),
5086            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5087        ));
5088        assert!(matches!(
5089            response_rx_3.try_recv(),
5090            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5091        ));
5092
5093        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5094
5095        assert_eq!(response_rx_1.await.unwrap(), Ok(()));
5096        assert_eq!(response_rx_2.await.unwrap(), Ok(()));
5097        assert_eq!(response_rx_3.await.unwrap(), Ok(()));
5098        assert!(handler.inflight_subs.is_empty());
5099        assert!(handler.subscription_attempts.is_empty());
5100    }
5101
5102    #[tokio::test]
5103    async fn changed_inflight_auth_waits_for_serialized_successor() {
5104        let mut handler = make_handler_with_account();
5105        let channel = LighterWsChannel::AccountAllOrders(12345);
5106        let topic = Ustr::from(channel.topic_key().as_str());
5107        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5108        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5109
5110        handler.queue_subscribe(
5111            channel.clone(),
5112            Some(SecretString::from("old-token")),
5113            Some(old_tx),
5114        );
5115        let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
5116        handler.inflight_subs.insert(queued_topic, old_generation);
5117        handler.queue_subscribe(
5118            channel,
5119            Some(SecretString::from("fresh-token")),
5120            Some(fresh_tx),
5121        );
5122
5123        let attempt = &handler.subscription_attempts[&topic];
5124        assert_eq!(
5125            attempt.auth.as_ref().map(SecretString::expose_secret),
5126            Some("old-token")
5127        );
5128        assert_eq!(
5129            attempt
5130                .pending_auth
5131                .as_ref()
5132                .map(SecretString::expose_secret),
5133            Some("fresh-token")
5134        );
5135        assert_eq!(attempt.response_txs.len(), 1);
5136        assert_eq!(attempt.pending_response_txs.len(), 1);
5137
5138        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5139        assert_eq!(old_rx.await.unwrap(), Ok(()));
5140        assert!(matches!(
5141            fresh_rx.try_recv(),
5142            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5143        ));
5144        let attempt = &handler.subscription_attempts[&topic];
5145        assert_ne!(attempt.generation, old_generation);
5146        assert_eq!(
5147            attempt.auth.as_ref().map(SecretString::expose_secret),
5148            Some("fresh-token")
5149        );
5150        assert!(attempt.pending_auth.is_none());
5151        assert_eq!(
5152            handler.pending_subs.front(),
5153            Some(&(topic, attempt.generation)),
5154        );
5155
5156        let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
5157        handler.inflight_subs.insert(topic, fresh_generation);
5158
5159        // The predecessor's trailing typed snapshot must not complete the
5160        // dispatched successor.
5161        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
5162        assert!(matches!(
5163            fresh_rx.try_recv(),
5164            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5165        ));
5166        assert_eq!(
5167            handler.subscription_attempts[&topic].generation,
5168            fresh_generation,
5169        );
5170
5171        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5172        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5173        assert!(handler.subscription_attempts.is_empty());
5174    }
5175
5176    #[tokio::test]
5177    async fn stale_typed_frame_does_not_complete_fresh_attempt_after_predecessor_removed() {
5178        let mut handler = make_handler_with_account();
5179        let channel = LighterWsChannel::AccountAllOrders(12345);
5180        let topic = Ustr::from(channel.topic_key().as_str());
5181        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5182        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5183
5184        // Predecessor completes on its control ack and is removed before any
5185        // pending auth exists.
5186        handler.queue_subscribe(
5187            channel.clone(),
5188            Some(SecretString::from("old-token")),
5189            Some(old_tx),
5190        );
5191        let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
5192        handler.inflight_subs.insert(queued_topic, old_generation);
5193        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5194        assert_eq!(old_rx.await.unwrap(), Ok(()));
5195        assert!(handler.subscription_attempts.is_empty());
5196
5197        // The fresh command then creates and dispatches a brand-new attempt.
5198        handler.queue_subscribe(
5199            channel,
5200            Some(SecretString::from("fresh-token")),
5201            Some(fresh_tx),
5202        );
5203        let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
5204        handler.inflight_subs.insert(topic, fresh_generation);
5205
5206        // The predecessor's trailing typed snapshot must not complete it.
5207        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
5208        assert!(matches!(
5209            fresh_rx.try_recv(),
5210            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5211        ));
5212        assert_eq!(
5213            handler.subscription_attempts[&topic].generation,
5214            fresh_generation,
5215        );
5216
5217        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5218        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5219        assert!(handler.subscription_attempts.is_empty());
5220    }
5221
5222    #[tokio::test]
5223    async fn changed_queued_auth_updates_existing_generation_before_dispatch() {
5224        let mut handler = make_handler_with_account();
5225        let channel = LighterWsChannel::AccountAllOrders(12345);
5226        let topic = Ustr::from(channel.topic_key().as_str());
5227        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5228        let (fresh_tx, fresh_rx) = tokio::sync::oneshot::channel();
5229
5230        handler.queue_subscribe(
5231            channel.clone(),
5232            Some(SecretString::from("old-token")),
5233            Some(old_tx),
5234        );
5235        let generation = handler.pending_subs[0].1;
5236        handler.queue_subscribe(
5237            channel,
5238            Some(SecretString::from("fresh-token")),
5239            Some(fresh_tx),
5240        );
5241
5242        assert_eq!(handler.pending_subs.len(), 1);
5243        assert_eq!(handler.pending_subs[0], (topic, generation));
5244        let attempt = &handler.subscription_attempts[&topic];
5245        assert_eq!(attempt.generation, generation);
5246        assert_eq!(
5247            attempt.auth.as_ref().map(SecretString::expose_secret),
5248            Some("fresh-token")
5249        );
5250        assert!(attempt.pending_auth.is_none());
5251        assert_eq!(attempt.response_txs.len(), 2);
5252        assert!(attempt.pending_response_txs.is_empty());
5253
5254        handler.pending_subs.pop_front();
5255        handler.inflight_subs.insert(topic, generation);
5256        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5257        assert_eq!(old_rx.await.unwrap(), Ok(()));
5258        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5259    }
5260
5261    #[rstest]
5262    fn second_reconnect_folds_pending_auth_into_requeued_generation() {
5263        let mut handler = make_handler_with_account();
5264        let channel = LighterWsChannel::AccountAllOrders(12345);
5265        let topic = Ustr::from(channel.topic_key().as_str());
5266        let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
5267        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5268
5269        handler.queue_subscribe(
5270            channel.clone(),
5271            Some(SecretString::from("old-token")),
5272            Some(old_tx),
5273        );
5274        handler.reset_subscription_attempts_after_reconnect();
5275        let (_, replay_generation) = handler.pending_subs.pop_front().unwrap();
5276        handler.inflight_subs.insert(topic, replay_generation);
5277        handler.queue_subscribe(
5278            channel,
5279            Some(SecretString::from("fresh-token")),
5280            Some(fresh_tx),
5281        );
5282
5283        let attempt = &handler.subscription_attempts[&topic];
5284        assert_eq!(
5285            attempt.auth.as_ref().map(SecretString::expose_secret),
5286            Some("old-token")
5287        );
5288        assert_eq!(
5289            attempt
5290                .pending_auth
5291                .as_ref()
5292                .map(SecretString::expose_secret),
5293            Some("fresh-token")
5294        );
5295        assert_eq!(attempt.response_txs.len(), 1);
5296        assert_eq!(attempt.pending_response_txs.len(), 1);
5297
5298        handler.reset_subscription_attempts_after_reconnect();
5299
5300        let attempt = &handler.subscription_attempts[&topic];
5301        assert_ne!(attempt.generation, replay_generation);
5302        assert_eq!(
5303            attempt.auth.as_ref().map(SecretString::expose_secret),
5304            Some("fresh-token")
5305        );
5306        assert!(attempt.pending_auth.is_none());
5307        assert_eq!(attempt.response_txs.len(), 2);
5308        assert!(attempt.pending_response_txs.is_empty());
5309        assert_eq!(handler.pending_subs.len(), 1);
5310        assert_eq!(handler.pending_subs[0], (topic, attempt.generation));
5311        assert!(handler.inflight_subs.is_empty());
5312        assert!(matches!(
5313            old_rx.try_recv(),
5314            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5315        ));
5316        assert!(matches!(
5317            fresh_rx.try_recv(),
5318            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5319        ));
5320
5321        let (_, generation) = handler.pending_subs.pop_front().unwrap();
5322        handler.inflight_subs.insert(topic, generation);
5323        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5324
5325        assert_eq!(old_rx.try_recv(), Ok(Ok(())));
5326        assert_eq!(fresh_rx.try_recv(), Ok(Ok(())));
5327        assert!(handler.subscription_attempts.is_empty());
5328    }
5329
5330    #[tokio::test]
5331    async fn retry_folds_pending_auth_without_resetting_retry_budget() {
5332        let mut handler = make_handler_with_account();
5333        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5334        handler.set_command_sender(cmd_tx);
5335        let channel = LighterWsChannel::AccountAllOrders(12345);
5336        let topic = Ustr::from(channel.topic_key().as_str());
5337        let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
5338        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5339
5340        handler.queue_subscribe(
5341            channel.clone(),
5342            Some(SecretString::from("old-token")),
5343            Some(old_tx),
5344        );
5345        let (_, generation) = handler.pending_subs.pop_front().unwrap();
5346        handler.inflight_subs.insert(topic, generation);
5347        handler
5348            .subscription_attempts
5349            .get_mut(&topic)
5350            .unwrap()
5351            .retries = 2;
5352        handler.queue_subscribe(
5353            channel,
5354            Some(SecretString::from("fresh-token")),
5355            Some(fresh_tx),
5356        );
5357
5358        handler.schedule_subscription_retry(topic, generation, "retry");
5359
5360        let attempt = &handler.subscription_attempts[&topic];
5361        assert_eq!(attempt.retries, 3);
5362        assert_ne!(attempt.generation, generation);
5363        assert_eq!(
5364            attempt.auth.as_ref().map(SecretString::expose_secret),
5365            Some("fresh-token")
5366        );
5367        assert!(attempt.pending_auth.is_none());
5368        assert_eq!(attempt.response_txs.len(), 2);
5369        assert!(attempt.pending_response_txs.is_empty());
5370        assert!(handler.pending_subs.is_empty());
5371        assert!(handler.inflight_subs.is_empty());
5372        assert_eq!(handler.subscription_retries.len(), 1);
5373        assert!(matches!(
5374            old_rx.try_recv(),
5375            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5376        ));
5377        assert!(matches!(
5378            fresh_rx.try_recv(),
5379            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5380        ));
5381    }
5382
5383    #[tokio::test]
5384    async fn confirmed_authenticated_topic_opens_new_generation() {
5385        let mut handler = make_handler_with_account();
5386        let channel = LighterWsChannel::AccountAllOrders(12345);
5387        let topic = channel.topic_key();
5388        handler.subscriptions.mark_subscribe(&topic);
5389        handler.subscriptions.confirm_subscribe(&topic);
5390        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5391
5392        handler.queue_subscribe(
5393            channel,
5394            Some(SecretString::from("rotated-auth-token")),
5395            Some(response_tx),
5396        );
5397
5398        assert_eq!(handler.pending_subs.len(), 1);
5399        assert_eq!(handler.subscription_attempts.len(), 1);
5400        assert_eq!(handler.subscriptions.len(), 1);
5401        assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
5402
5403        let (topic, generation) = handler.pending_subs.pop_front().unwrap();
5404        handler.inflight_subs.insert(topic, generation);
5405        handle_control_text(
5406            &mut handler,
5407            r#"{"type":"subscribed","channel":"account_all_orders:12345"}"#,
5408        );
5409
5410        assert_eq!(response_rx.await.unwrap(), Ok(()));
5411        assert!(handler.subscription_attempts.is_empty());
5412        assert_eq!(handler.subscriptions.len(), 1);
5413    }
5414
5415    #[tokio::test]
5416    async fn typed_update_before_ack_does_not_complete_subscription_generation() {
5417        let mut handler = make_handler_with_account();
5418        let (response_tx, mut response_rx) = tokio::sync::oneshot::channel();
5419        let (topic, generation) = mark_subscription_inflight(
5420            &mut handler,
5421            LighterWsChannel::Candle {
5422                market_index: 0,
5423                resolution: LighterCandleResolution::OneMinute,
5424            },
5425            Some(response_tx),
5426        );
5427
5428        handler.handle_frame(
5429            candle_frame(
5430                "candle:0:1m",
5431                stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
5432                false,
5433            ),
5434            UnixNanos::from(1),
5435        );
5436
5437        assert_eq!(
5438            handler
5439                .last_candles
5440                .get(&(0, LighterCandleResolution::OneMinute))
5441                .map(|candle| candle.t),
5442            Some(1_000_000),
5443        );
5444        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5445        assert_eq!(
5446            handler.subscriptions.pending_subscribe_topics(),
5447            vec!["candle:0:1m"],
5448        );
5449        assert_eq!(handler.subscriptions.len(), 0);
5450        assert!(matches!(
5451            response_rx.try_recv(),
5452            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5453        ));
5454
5455        handle_control_text(
5456            &mut handler,
5457            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5458        );
5459
5460        assert_eq!(response_rx.await.unwrap(), Ok(()));
5461        assert_eq!(handler.subscriptions.len(), 1);
5462    }
5463
5464    #[tokio::test]
5465    async fn typed_subscribed_frame_completes_subscription_generation() {
5466        let signal = Arc::new(AtomicBool::new(false));
5467        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5468        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5469        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5470        let mut handler =
5471            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5472        handler.instruments.insert(0, stub_eth_perp_instrument());
5473        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5474        mark_subscription_inflight(
5475            &mut handler,
5476            LighterWsChannel::Candle {
5477                market_index: 0,
5478                resolution: LighterCandleResolution::OneMinute,
5479            },
5480            Some(response_tx),
5481        );
5482
5483        raw_tx
5484            .send((7, Message::Text(WS_CANDLE_SUBSCRIBED.into())))
5485            .expect("typed subscribed frame");
5486        drop(raw_tx);
5487        drop(cmd_tx);
5488
5489        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5490            .await
5491            .expect("handler did not process typed subscribed frame");
5492
5493        assert!(next.is_none());
5494        assert_eq!(response_rx.await.unwrap(), Ok(()));
5495        assert!(handler.inflight_subs.is_empty());
5496        assert!(handler.subscription_attempts.is_empty());
5497        assert_eq!(handler.subscriptions.len(), 1);
5498    }
5499
5500    #[tokio::test]
5501    async fn unparsable_subscribed_frame_still_completes_subscription() {
5502        let signal = Arc::new(AtomicBool::new(false));
5503        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5504        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5505        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5506        let mut handler =
5507            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5508        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5509        mark_subscription_inflight(
5510            &mut handler,
5511            LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(4098)),
5512            Some(response_tx),
5513        );
5514
5515        raw_tx
5516            .send((7, Message::Text(WS_SPOT_STATS_SUBSCRIBED_BAD_BODY.into())))
5517            .expect("unparsable subscribed frame");
5518        drop(raw_tx);
5519        drop(cmd_tx);
5520
5521        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5522            .await
5523            .expect("handler did not process unparsable subscribed frame");
5524
5525        assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5526        let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5527            .await
5528            .expect("subscription was not completed");
5529        assert_eq!(response.unwrap(), Ok(()));
5530        assert!(handler.inflight_subs.is_empty());
5531        assert!(handler.subscription_attempts.is_empty());
5532        assert_eq!(handler.subscriptions.len(), 1);
5533    }
5534
5535    #[tokio::test]
5536    async fn unparsable_book_subscribed_frame_still_completes_subscription() {
5537        let signal = Arc::new(AtomicBool::new(false));
5538        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5539        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5540        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5541        let mut handler =
5542            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5543        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5544        mark_subscription_inflight(
5545            &mut handler,
5546            LighterWsChannel::OrderBook(4095),
5547            Some(response_tx),
5548        );
5549
5550        // Epoch 0 matches the generation entry seeded by the helper, so the
5551        // book-specific snapshot matching accepts this confirmation.
5552        raw_tx
5553            .send((0, Message::Text(WS_BOOK_SUBSCRIBED_BAD_BODY.into())))
5554            .expect("unparsable book subscribed frame");
5555        drop(raw_tx);
5556        drop(cmd_tx);
5557
5558        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5559            .await
5560            .expect("handler did not process unparsable book subscribed frame");
5561
5562        assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5563        let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5564            .await
5565            .expect("book subscription was not completed");
5566        assert_eq!(response.unwrap(), Ok(()));
5567        assert!(handler.inflight_subs.is_empty());
5568        assert!(handler.subscription_attempts.is_empty());
5569        assert_eq!(handler.subscriptions.len(), 1);
5570    }
5571
5572    #[rstest]
5573    #[case::nonbook_top_level(false, false, "market_stats:0")]
5574    #[case::nonbook_wrapped(false, true, "market_stats/0")]
5575    #[case::book_top_level(true, false, "order_book/0")]
5576    #[case::book_wrapped(true, true, "order_book:0")]
5577    #[tokio::test]
5578    async fn subscribe_failure_preserves_distinct_inflight_subscription(
5579        #[case] book: bool,
5580        #[case] wrapped: bool,
5581        #[case] channel: &str,
5582    ) {
5583        let mut handler = make_handler_with_account();
5584        let (failed_tx, mut failed_rx) = tokio::sync::oneshot::channel();
5585        let (valid_tx, mut valid_rx) = tokio::sync::oneshot::channel();
5586
5587        let failed_channel = if book {
5588            LighterWsChannel::OrderBook(0)
5589        } else {
5590            LighterWsChannel::MarketStats(LighterMarketSelection::Market(0))
5591        };
5592
5593        let (failed_topic, _) =
5594            mark_subscription_inflight(&mut handler, failed_channel, Some(failed_tx));
5595        let (valid_topic, generation) =
5596            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(valid_tx));
5597        let error = serde_json::json!({"code": 30012, "channel": channel});
5598
5599        let value = if wrapped {
5600            serde_json::json!({"error": error})
5601        } else {
5602            error
5603        };
5604
5605        let (matched, message) = handler.handle_control_value(&value);
5606        let pending = valid_rx.try_recv();
5607        let inflight = handler.inflight_subs.get(&valid_topic).copied();
5608        let retries = handler.subscription_attempts[&valid_topic].retries;
5609        let rejected = failed_rx.try_recv();
5610        let failure_retained = handler.subscription_attempts.contains_key(&failed_topic);
5611        handler.handle_control_value(&serde_json::json!({
5612            "type": "subscribed", "channel": "trade:7"
5613        }));
5614
5615        let expected_error = if book {
5616            BOOK_SUBSCRIBE_REJECTION.to_string()
5617        } else {
5618            "subscription market_stats:0 failed after 1 attempts: venue rejected the WebSocket \
5619             subscribe with code 30012"
5620                .to_string()
5621        };
5622
5623        assert!(matched);
5624        assert!(message.is_none());
5625        assert_eq!(
5626            pending,
5627            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
5628        );
5629        assert_eq!(inflight, Some(generation));
5630        assert_eq!(retries, 0);
5631        assert_eq!(rejected, Ok(Err(expected_error)));
5632        assert!(!failure_retained);
5633        assert_eq!(valid_rx.try_recv(), Ok(Ok(())));
5634        assert!(handler.inflight_subs.is_empty());
5635        assert!(handler.subscription_attempts.is_empty());
5636        assert_eq!(handler.subscriptions.len(), 1);
5637    }
5638
5639    #[rstest]
5640    #[case::unknown("trade:999")]
5641    #[case::confirmed("trade:7")]
5642    #[case::queued("market_stats:0")]
5643    #[tokio::test]
5644    async fn subscribe_failure_ignores_channel_without_inflight_attempt(#[case] channel: &str) {
5645        let mut handler = make_handler_with_account();
5646        let (response, mut result) = tokio::sync::oneshot::channel();
5647        let (topic, generation) =
5648            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(3), Some(response));
5649        handler.subscriptions.mark_subscribe("trade:7");
5650        handler.subscriptions.confirm_subscribe("trade:7");
5651        handler.queue_subscribe(
5652            LighterWsChannel::MarketStats(LighterMarketSelection::Market(0)),
5653            None,
5654            None,
5655        );
5656        let (queued_topic, queued_generation) = *handler.pending_subs.front().unwrap();
5657
5658        handler.handle_control_value(&serde_json::json!({"code": 30012, "channel": channel}));
5659
5660        assert_eq!(
5661            result.try_recv(),
5662            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
5663        );
5664        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5665        assert_eq!(
5666            handler.subscription_attempts[&queued_topic].generation,
5667            queued_generation
5668        );
5669        assert_eq!(handler.subscription_attempts[&queued_topic].retries, 0);
5670        assert_eq!(
5671            handler.pending_subs.iter().copied().collect::<Vec<_>>(),
5672            [(queued_topic, queued_generation)]
5673        );
5674        assert!(handler.subscription_retries.is_empty());
5675        assert_eq!(handler.subscriptions.len(), 1);
5676    }
5677
5678    #[tokio::test]
5679    async fn failed_subscribe_error_fails_every_waiter_for_the_generation() {
5680        let mut handler = make_handler_with_account();
5681        let (response_tx_1, response_rx_1) = tokio::sync::oneshot::channel();
5682        let (response_tx_2, response_rx_2) = tokio::sync::oneshot::channel();
5683        let channel = LighterWsChannel::MarketStats(LighterMarketSelection::Market(0));
5684        mark_subscription_inflight(&mut handler, channel.clone(), Some(response_tx_1));
5685        handler.queue_subscribe(channel, None, Some(response_tx_2));
5686
5687        let (matched, msg) = handle_control_text(
5688            &mut handler,
5689            &serde_json::json!({
5690                "type": "error", "code": 30012, "channel": "market_stats:0"
5691            })
5692            .to_string(),
5693        );
5694
5695        assert!(matched);
5696        assert!(msg.is_none());
5697        let error_1 = response_rx_1.await.unwrap().unwrap_err();
5698        let error_2 = response_rx_2.await.unwrap().unwrap_err();
5699        assert!(error_1.contains("market_stats:0"));
5700        assert!(error_1.contains("30012"));
5701        assert_eq!(error_1, error_2);
5702        assert!(handler.inflight_subs.is_empty());
5703        assert!(handler.subscription_attempts.is_empty());
5704        assert!(handler.subscriptions.is_empty());
5705    }
5706
5707    #[rstest]
5708    #[case::rate_limit(30009)]
5709    #[case::unattributed_failure(30012)]
5710    #[tokio::test]
5711    async fn subscription_retry_exhaustion_fails_waiter_and_clears_intent(#[case] code: u64) {
5712        let mut handler = make_handler_with_account();
5713        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5714        let (topic, _) =
5715            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(response_tx));
5716        handler
5717            .subscription_attempts
5718            .get_mut(&topic)
5719            .expect("subscription attempt")
5720            .retries = SUBSCRIBE_RETRY_MAX;
5721
5722        let (matched, msg) =
5723            handle_control_text(&mut handler, &serde_json::json!({"code": code}).to_string());
5724
5725        assert!(matched);
5726        assert!(msg.is_none());
5727        assert_eq!(
5728            response_rx.await.unwrap(),
5729            Err(format!(
5730                "subscription trade:7 failed after 6 attempts: venue rejected the WebSocket \
5731                 subscribe with code {code}"
5732            )),
5733        );
5734        assert!(handler.inflight_subs.is_empty());
5735        assert!(handler.subscription_attempts.is_empty());
5736        assert!(handler.subscriptions.is_empty());
5737    }
5738
5739    #[rstest]
5740    #[case::rate_limit(serde_json::json!({"code": 30009}))]
5741    #[case::unattributed_failure(serde_json::json!({"code": 30012}))]
5742    #[case::wrapped_failure(serde_json::json!({"error": {"code": 30012}}))]
5743    #[case::invalid_channel(serde_json::json!({"code": 30012, "channel": 7}))]
5744    #[case::empty_channel(serde_json::json!({"code": 30012, "channel": ""}))]
5745    #[tokio::test]
5746    async fn subscription_error_retries_each_inflight_topic_with_new_generation(
5747        #[case] value: serde_json::Value,
5748    ) {
5749        let mut handler = make_handler_with_account();
5750        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5751        handler.set_command_sender(cmd_tx);
5752        let (trade_tx, mut trade_rx) = tokio::sync::oneshot::channel();
5753        let (candle_tx, mut candle_rx) = tokio::sync::oneshot::channel();
5754        let (trade_topic, trade_generation) =
5755            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(trade_tx));
5756        let (candle_topic, candle_generation) = mark_subscription_inflight(
5757            &mut handler,
5758            LighterWsChannel::Candle {
5759                market_index: 0,
5760                resolution: LighterCandleResolution::OneMinute,
5761            },
5762            Some(candle_tx),
5763        );
5764
5765        let (matched, msg) = handler.handle_control_value(&value);
5766
5767        assert!(matched);
5768        assert!(msg.is_none());
5769        assert!(handler.inflight_subs.is_empty());
5770        assert!(handler.pending_subs.is_empty());
5771        let trade_retry = &handler.subscription_attempts[&trade_topic];
5772        let candle_retry = &handler.subscription_attempts[&candle_topic];
5773        assert_eq!(trade_retry.retries, 1);
5774        assert_eq!(candle_retry.retries, 1);
5775        assert_ne!(trade_retry.generation, trade_generation);
5776        assert_ne!(candle_retry.generation, candle_generation);
5777        assert_eq!(handler.subscription_retries.len(), 2);
5778        assert!(matches!(
5779            trade_rx.try_recv(),
5780            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5781        ));
5782        assert!(matches!(
5783            candle_rx.try_recv(),
5784            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5785        ));
5786
5787        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5788        assert_eq!(handler.subscription_attempts.len(), 2);
5789        assert_eq!(handler.subscriptions.len(), 0);
5790
5791        let retry_generations = [
5792            (
5793                trade_topic,
5794                handler.subscription_attempts[&trade_topic].generation,
5795            ),
5796            (
5797                candle_topic,
5798                handler.subscription_attempts[&candle_topic].generation,
5799            ),
5800        ];
5801
5802        for (topic, generation) in retry_generations {
5803            handler.queue_subscription_retry(topic, generation);
5804            let queued = handler.pending_subs.pop_front().unwrap();
5805            assert_eq!(queued, (topic, generation));
5806            handler.inflight_subs.insert(topic, generation);
5807        }
5808
5809        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5810        handle_control_text(
5811            &mut handler,
5812            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5813        );
5814
5815        assert_eq!(trade_rx.await.unwrap(), Ok(()));
5816        assert_eq!(candle_rx.await.unwrap(), Ok(()));
5817        assert!(handler.subscription_attempts.is_empty());
5818        assert_eq!(handler.subscriptions.len(), 2);
5819    }
5820}