Skip to main content

nautilus_lighter/websocket/
handler.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Inner WebSocket feed handler running on a dedicated tokio task.
17
18use std::{
19    collections::VecDeque,
20    fmt::Debug,
21    future::Future,
22    pin::Pin,
23    sync::{
24        Arc,
25        atomic::{AtomicBool, Ordering},
26    },
27    time::Duration,
28};
29
30use ahash::AHashMap;
31use futures_util::{StreamExt, stream::FuturesUnordered};
32use nautilus_core::{
33    AtomicTime,
34    nanos::UnixNanos,
35    string::secret::{REDACTED, SecretString},
36    time::get_atomic_clock_realtime,
37};
38#[cfg(test)]
39use nautilus_live::book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS;
40use nautilus_live::book::{
41    BookSequenceOutcome,
42    recovery::{BookRecoveryOutcome, BookRecoveryState},
43    snapshot::{PendingSnapshot, SnapshotGate},
44};
45use nautilus_model::{identifiers::AccountId, instruments::InstrumentAny, types::Currency};
46use nautilus_network::{
47    RECONNECTED,
48    error::SendError,
49    retry::{RetryManager, create_websocket_retry_manager},
50    websocket::{SubscriptionState, WebSocketClient},
51};
52use tokio_tungstenite::tungstenite::Message;
53use tokio_util::sync::CancellationToken;
54use ustr::Ustr;
55use zeroize::Zeroize;
56
57use super::{
58    account_state::LighterAccountStateReconciler,
59    error::LighterWsError,
60    messages::{
61        AccountStream, ExecutionReport, LighterAsset, LighterPosition, LighterUserStats,
62        LighterWsCandle, LighterWsChannel, LighterWsChannelKind, LighterWsFrame,
63        LighterWsOrderBook, LighterWsRequest, LighterWsTx, NautilusWsMessage,
64        SendTxRejectionSource, TX_BATCH_ID_PREFIX,
65    },
66    parse::{
67        parse_ws_bar, parse_ws_funding_rate_update, parse_ws_index_price_update,
68        parse_ws_mark_price_update, parse_ws_position_status_report, parse_ws_quote_tick,
69        parse_ws_spot_index_price_update, parse_ws_trade_tick,
70    },
71};
72use crate::{
73    book::{
74        recovery::{self, BookWorkResult, BookWrite},
75        sync::BookSyncTracker,
76    },
77    common::{
78        consts::{
79            LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED, LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED,
80            LIGHTER_ERROR_CODE_TX_RANGE, LIGHTER_ERROR_CODE_WS_RATE_LIMITED,
81            LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED, LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL,
82            SUBSCRIBE_INFLIGHT_MAX, SUBSCRIBE_RETRY_BASE_BACKOFF, SUBSCRIBE_RETRY_MAX,
83        },
84        enums::LighterCandleResolution,
85        rate_limit::LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY,
86    },
87    http::models::{LighterOrder, LighterTrade},
88};
89
90// Lighter control-frame `type` field values that fall outside the typed
91// `LighterWsFrame` variants. These are protocol-level frames the handler
92// inspects via the typed-frame parse fallback in `handle_control_value`.
93const CTRL_TYPE_CONNECTED: &str = "connected";
94const CTRL_TYPE_SUBSCRIBED: &str = "subscribed";
95const CTRL_TYPE_UNSUBSCRIBED: &str = "unsubscribed";
96const CTRL_TYPE_PING: &str = "ping";
97const CTRL_TYPE_PONG: &str = "pong";
98const CTRL_TYPE_ERROR: &str = "error";
99const CTRL_TYPE_SEND_TX: &str = "jsonapi/sendtx";
100
101#[derive(serde::Deserialize)]
102struct LighterWsFrameHeader<'a> {
103    #[serde(rename = "type", borrow)]
104    kind: &'a str,
105    #[serde(borrow)]
106    channel: Option<&'a str>,
107}
108
109/// Commands sent from the outer client to the inner feed handler.
110#[expect(
111    clippy::large_enum_variant,
112    reason = "commands are ephemeral and immediately consumed"
113)]
114pub enum HandlerCommand {
115    /// Hand the live `WebSocketClient` to the handler after the outer client
116    /// completes the network connect.
117    SetClient(WebSocketClient),
118    /// Drain the queue and shut the handler down.
119    Disconnect,
120    /// Subscribe to a channel. `auth` carries an optional auth token for the
121    /// account-scoped channels.
122    Subscribe {
123        channel: LighterWsChannel,
124        auth: Option<SecretString>,
125        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
126    },
127    /// Unsubscribe from a channel.
128    Unsubscribe { channel: LighterWsChannel },
129    /// Replace a book subscription for the current recovery owner.
130    RecoverBook {
131        market_index: i64,
132        cancel: CancellationToken,
133        gate: SnapshotGate,
134        completion: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
135    },
136    /// Replace the handler's instrument cache (used on initial connect).
137    InitializeInstruments(Vec<(i64, InstrumentAny)>),
138    /// Insert or replace a single instrument by `market_index`.
139    UpdateInstrument {
140        market_index: i64,
141        instrument: InstrumentAny,
142    },
143    /// Toggle whether `update/order_book` frames for `market_index` should
144    /// be emitted as [`NautilusWsMessage::Deltas`].
145    SetBookDeltasSub { market_index: i64, subscribed: bool },
146    /// Toggle whether `update/order_book` frames for `market_index` should
147    /// also be emitted as a [`NautilusWsMessage::Depth`] snapshot.
148    SetDepthSub { market_index: i64, subscribed: bool },
149    /// Provide the execution context (`AccountId` and venue `account_index`)
150    /// the handler stamps onto reports parsed from `account_*` frames.
151    /// Without this context the handler cannot construct typed reports and
152    /// account frames are forwarded as raw JSON instead.
153    SetExecutionContext {
154        account_id: AccountId,
155        account_index: i64,
156    },
157    /// Dispatch a signed L2 transaction over the WebSocket.
158    ///
159    /// The handler serializes a [`LighterWsRequest::SendTx`] frame; the venue
160    /// confirms acceptance via the `account_*` streams (and reports any
161    /// rejection via a control-plane `error` frame).
162    SendTx {
163        tx_type: u8,
164        tx_info: Box<serde_json::value::RawValue>,
165        connection_epoch: u64,
166        response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
167    },
168    SendTxBatch {
169        data: super::messages::LighterWsSendTxBatch,
170        connection_epoch: u64,
171        response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
172    },
173}
174
175impl Debug for HandlerCommand {
176    /// Custom `Debug` that redacts the `auth` field of `Subscribe`. The
177    /// command is routed in-process via mpsc and may be printed by error
178    /// paths or trace logging; deriving `Debug` would otherwise leak the
179    /// venue bearer token verbatim.
180    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181        match self {
182            Self::SetClient(_) => f.write_str("SetClient(<WebSocketClient>)"),
183            Self::Disconnect => f.write_str("Disconnect"),
184            Self::Subscribe { channel, auth, .. } => f
185                .debug_struct(stringify!(Subscribe))
186                .field("channel", channel)
187                .field("auth", auth)
188                .finish(),
189            Self::Unsubscribe { channel } => f
190                .debug_struct(stringify!(Unsubscribe))
191                .field("channel", channel)
192                .finish(),
193            Self::RecoverBook { market_index, .. } => f
194                .debug_struct(stringify!(RecoverBook))
195                .field("market_index", market_index)
196                .finish(),
197            Self::InitializeInstruments(instruments) => f
198                .debug_tuple(stringify!(InitializeInstruments))
199                .field(&instruments.len())
200                .finish(),
201            Self::UpdateInstrument { market_index, .. } => f
202                .debug_struct(stringify!(UpdateInstrument))
203                .field("market_index", market_index)
204                .finish(),
205            Self::SetBookDeltasSub {
206                market_index,
207                subscribed,
208            } => f
209                .debug_struct(stringify!(SetBookDeltasSub))
210                .field("market_index", market_index)
211                .field("subscribed", subscribed)
212                .finish(),
213            Self::SetDepthSub {
214                market_index,
215                subscribed,
216            } => f
217                .debug_struct(stringify!(SetDepthSub))
218                .field("market_index", market_index)
219                .field("subscribed", subscribed)
220                .finish(),
221            Self::SetExecutionContext {
222                account_id,
223                account_index,
224            } => f
225                .debug_struct(stringify!(SetExecutionContext))
226                .field("account_id", account_id)
227                .field("account_index", account_index)
228                .finish(),
229            Self::SendTxBatch { .. } => f.write_str("SendTxBatch(<redacted>)"),
230            Self::SendTx { tx_type, .. } => f
231                .debug_struct(stringify!(SendTx))
232                .field("tx_type", tx_type)
233                .field("tx_info", &REDACTED)
234                .finish(),
235        }
236    }
237}
238
239/// Inner feed handler. Owns the [`WebSocketClient`] exclusively and routes
240/// raw frames into the venue-message channel.
241///
242/// Subscribe dispatch is gated by a closed-loop inflight count: the venue caps
243/// unacknowledged client messages per connection, and acknowledgement latency
244/// is multi-second, so queued subscribes are held back until acks free a slot.
245pub(super) struct FeedHandler {
246    clock: &'static AtomicTime,
247    signal: Arc<AtomicBool>,
248    inner: Option<Arc<WebSocketClient>>,
249    cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
250    cmd_tx: Option<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>,
251    raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
252    out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
253    subscriptions: SubscriptionState,
254    retry_manager: RetryManager<LighterWsError>,
255    pending_messages: VecDeque<NautilusWsMessage>,
256    pending_subs: VecDeque<(Ustr, u64)>,
257    inflight_subs: AHashMap<Ustr, u64>,
258    subscription_attempts: AHashMap<Ustr, SubscriptionAttempt>,
259    subscription_retries: FuturesUnordered<SubscriptionRetry>,
260    ignored_completions: AHashMap<Ustr, CompletionKind>,
261    next_subscription_generation: u64,
262    instruments: AHashMap<i64, InstrumentAny>,
263    book: BookSyncTracker,
264    book_snapshot_timeout: Duration,
265    last_candles: AHashMap<(i64, LighterCandleResolution), LighterWsCandle>,
266    exec_account: Option<(AccountId, i64)>,
267    account_state_reconciler: LighterAccountStateReconciler,
268}
269
270type SubscriptionRetry = Pin<Box<dyn Future<Output = (Ustr, u64)> + Send + Sync + 'static>>;
271
272struct SubscriptionAttempt {
273    channel: LighterWsChannel,
274    auth: Option<SecretString>,
275    pending_auth: Option<SecretString>,
276    generation: u64,
277    retries: u8,
278    response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
279    pending_response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
280}
281
282impl SubscriptionAttempt {
283    fn fold_pending_auth(&mut self) {
284        if let Some(auth) = self.pending_auth.take() {
285            self.auth = Some(auth);
286            self.response_txs.append(&mut self.pending_response_txs);
287        } else {
288            debug_assert!(self.pending_response_txs.is_empty());
289        }
290    }
291}
292
293/// Which venue frame produced a subscription completion.
294///
295/// The venue answers a subscribe with a generic control ack followed by a
296/// typed `subscribed/<channel>` snapshot, and both complete by topic. The kind
297/// lets `complete_subscription` swallow the predecessor's trailing frame so a
298/// same-topic generation dispatched between the two cannot be completed by a
299/// response that belongs to the earlier request.
300#[derive(Clone, Copy, Debug, PartialEq, Eq)]
301enum CompletionKind {
302    ControlAck,
303    Typed,
304    AlreadySubscribed,
305}
306
307impl FeedHandler {
308    #[cfg(test)]
309    pub(super) fn new(
310        signal: Arc<AtomicBool>,
311        cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
312        raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
313        out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
314        subscriptions: SubscriptionState,
315    ) -> Self {
316        Self::new_with_settlement_currency(
317            signal,
318            cmd_rx,
319            raw_rx,
320            out_tx,
321            subscriptions,
322            Currency::get_or_create_crypto("USDC"),
323            Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
324        )
325    }
326
327    pub(super) fn new_with_settlement_currency(
328        signal: Arc<AtomicBool>,
329        cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
330        raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
331        out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
332        subscriptions: SubscriptionState,
333        settlement_currency: Currency,
334        book_snapshot_timeout: Duration,
335    ) -> Self {
336        Self {
337            clock: get_atomic_clock_realtime(),
338            signal,
339            inner: None,
340            cmd_rx,
341            cmd_tx: None,
342            raw_rx,
343            out_tx,
344            subscriptions,
345            retry_manager: create_websocket_retry_manager(),
346            pending_messages: VecDeque::new(),
347            pending_subs: VecDeque::new(),
348            inflight_subs: AHashMap::new(),
349            subscription_attempts: AHashMap::new(),
350            subscription_retries: FuturesUnordered::new(),
351            ignored_completions: AHashMap::new(),
352            next_subscription_generation: 1,
353            instruments: AHashMap::new(),
354            book: BookSyncTracker::default(),
355            book_snapshot_timeout,
356            last_candles: AHashMap::new(),
357            exec_account: None,
358            account_state_reconciler: LighterAccountStateReconciler::new_with_settlement_currency(
359                settlement_currency,
360            ),
361        }
362    }
363
364    pub(super) fn send(&self, msg: NautilusWsMessage) -> Result<(), String> {
365        self.out_tx
366            .send(msg)
367            .map_err(|e| format!("Failed to send message: {e}"))
368    }
369
370    pub(super) fn set_command_sender(
371        &mut self,
372        cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
373    ) {
374        self.cmd_tx = Some(cmd_tx);
375    }
376
377    pub(super) fn is_stopped(&self) -> bool {
378        self.signal.load(Ordering::Relaxed)
379    }
380
381    async fn send_with_retry(&self, payload: String) -> Result<(), LighterWsError> {
382        self.send_secret_with_retry(SecretString::from(payload))
383            .await
384    }
385
386    async fn send_secret_with_retry(&self, payload: SecretString) -> Result<(), LighterWsError> {
387        if let Some(client) = &self.inner {
388            self.retry_manager
389                .invocation(
390                    "websocket_send",
391                    || {
392                        let payload = payload.clone();
393                        async move {
394                            client
395                                .send_text(
396                                    payload.expose_secret().to_owned(),
397                                    Some(LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY.as_slice()),
398                                )
399                                .await
400                                .map_err(LighterWsError::Transport)
401                        }
402                    },
403                    should_retry_lighter_ws_error,
404                    |e| create_lighter_ws_timeout_error(e.to_string()),
405                )
406                .execute()
407                .await
408        } else {
409            Err(LighterWsError::Client(
410                "no active WebSocket client".to_string(),
411            ))
412        }
413    }
414
415    // Single-shot: sendTx payloads carry a signed nonce; transport-layer
416    // retry could double-submit if the original landed and only the ack was lost.
417    async fn send_once(
418        &self,
419        payload: String,
420        connection_epoch: u64,
421    ) -> Result<(), LighterWsError> {
422        if let Some(client) = &self.inner {
423            match client
424                .send_text_on_connection(payload, None, connection_epoch)
425                .await
426            {
427                Err(SendError::BrokenPipe(message)) => {
428                    Err(LighterWsError::SendTxOutcomeUnknown(message))
429                }
430                result => result.map_err(LighterWsError::Transport),
431            }
432        } else {
433            Err(LighterWsError::Client(
434                "no active WebSocket client".to_string(),
435            ))
436        }
437    }
438
439    async fn dispatch_subscribe(
440        &self,
441        channel: LighterWsChannel,
442        auth: Option<SecretString>,
443    ) -> Result<(), String> {
444        let topic = channel.topic_key();
445
446        let authed = auth.is_some();
447        let mut request = match auth {
448            Some(token) => LighterWsRequest::subscribe_auth(
449                channel.subscription_channel(),
450                token.expose_secret(),
451            ),
452            None => LighterWsRequest::subscribe(channel.subscription_channel()),
453        };
454
455        let payload = serde_json::to_string(&request);
456        request.zeroize();
457
458        match payload {
459            Ok(payload) => {
460                // Avoid logging the serialized payload for authenticated channels;
461                // it embeds a live Lighter L2 bearer token.
462                log::debug!("Sending Lighter subscribe: topic={topic} authed={authed}");
463                let result = if authed {
464                    self.send_secret_with_retry(SecretString::from(payload))
465                        .await
466                } else {
467                    self.send_with_retry(payload).await
468                };
469
470                if let Err(e) = result {
471                    log::error!("Error subscribing to {topic}: {e}");
472                    Err(e.to_string())
473                } else {
474                    Ok(())
475                }
476            }
477            Err(e) => {
478                log::error!("Error serializing subscription for {topic}: {e}");
479                Err(format!("failed to serialize subscription for {topic}: {e}"))
480            }
481        }
482    }
483
484    async fn dispatch_send_tx(
485        &self,
486        tx_type: u8,
487        tx_info: Box<serde_json::value::RawValue>,
488        connection_epoch: u64,
489    ) -> Result<(), LighterWsError> {
490        let request = LighterWsRequest::SendTx {
491            data: super::messages::LighterWsSendTx { tx_type, tx_info },
492        };
493
494        match serde_json::to_string(&request) {
495            Ok(payload) => {
496                log::debug!(
497                    "Sending Lighter sendTx: tx_type={tx_type} ({} bytes)",
498                    payload.len(),
499                );
500
501                match self.send_once(payload, connection_epoch).await {
502                    Ok(()) => Ok(()),
503                    Err(e) => {
504                        log::debug!("Error dispatching Lighter sendTx (tx_type={tx_type}): {e}");
505                        Err(e)
506                    }
507                }
508            }
509            Err(e) => {
510                log::error!("Error serializing Lighter sendTx (tx_type={tx_type}): {e}");
511                Err(LighterWsError::Client(format!(
512                    "failed to serialize Lighter sendTx: {e}"
513                )))
514            }
515        }
516    }
517
518    async fn dispatch_unsubscribe(&self, channel: LighterWsChannel) {
519        let topic = channel.topic_key();
520        self.subscriptions.mark_unsubscribe(&topic);
521
522        let request = LighterWsRequest::unsubscribe(channel.subscription_channel());
523        match serde_json::to_string(&request) {
524            Ok(payload) => {
525                log::debug!("Sending Lighter unsubscribe ({} bytes)", payload.len());
526                if let Err(e) = self.send_with_retry(payload).await {
527                    log::error!("Error unsubscribing from {topic}: {e}");
528                }
529            }
530            Err(e) => {
531                log::error!("Error serializing unsubscription for {topic}: {e}");
532            }
533        }
534    }
535
536    pub(super) async fn next(&mut self) -> Option<NautilusWsMessage> {
537        if let Some(msg) = self.pending_messages.pop_front() {
538            return Some(msg);
539        }
540
541        loop {
542            // Pump each iteration: the select below drains acks that free slots
543            self.pump_pending_subscribes().await;
544
545            tokio::select! {
546                biased;
547                Some(work) = self.book.work.next(), if !self.book.work.is_empty() => {
548                    match work {
549                        BookWorkResult::Sent { market_index, generation, cancel, write, result } => {
550                            self.complete_book_send(market_index, generation, cancel, write, result);
551                        }
552                        BookWorkResult::Initial { market_index, cancel } => {
553                            if !cancel.is_cancelled() {
554                                self.start_book_recovery(market_index);
555                            }
556                        }
557                        // Recovery ends once a snapshot is accepted or the episode is cancelled
558                        BookWorkResult::Recovery => {}
559                    }
560                }
561                Some(cmd) = self.cmd_rx.recv() => {
562                    match cmd {
563                        HandlerCommand::SetClient(client) => {
564                            log::debug!("Setting WebSocket client in Lighter handler");
565                            self.inner = Some(Arc::new(client));
566                        }
567                        HandlerCommand::Disconnect => {
568                            log::debug!("Lighter handler received disconnect");
569                            if let Some(ref client) = self.inner {
570                                client.disconnect().await;
571                            }
572                            self.signal.store(true, Ordering::SeqCst);
573                            return None;
574                        }
575                        HandlerCommand::Subscribe {
576                            channel,
577                            auth,
578                            response_tx,
579                        } => {
580                            self.queue_subscribe(channel, auth, response_tx);
581                        }
582                        HandlerCommand::Unsubscribe { channel } => {
583                            // Drop a queued-but-unsent subscribe for this channel so a freed
584                            // slot cannot resubscribe after the caller unsubscribed.
585                            let topic = channel.topic_key();
586                            self.cancel_subscription_attempt(
587                                &topic,
588                                "subscription cancelled before venue acknowledgement",
589                            );
590
591                            if let LighterWsChannel::OrderBook(market_index) = &channel {
592                                self.book.cancel(*market_index);
593                                self.book.clear_cached_order_book(*market_index);
594                            }
595                            self.dispatch_unsubscribe(channel).await;
596                        }
597                        HandlerCommand::RecoverBook { market_index, cancel, gate, completion } => {
598                            self.queue_book_replacement(market_index, BookWrite { cancel, gate, completion });
599                        }
600                        HandlerCommand::InitializeInstruments(instruments) => {
601                            self.instruments.clear();
602                            for (market_index, inst) in instruments {
603                                self.instruments.insert(market_index, inst);
604                            }
605                        }
606                        HandlerCommand::UpdateInstrument { market_index, instrument } => {
607                            self.instruments.insert(market_index, instrument);
608                        }
609                        HandlerCommand::SetBookDeltasSub { market_index, subscribed } => {
610                            if subscribed {
611                                let inserted = self.book.delta_subs.insert(market_index);
612                                if inserted
613                                    && let Some(first) = self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_deltas_snapshot(market_index, instrument, self.clock.get_time_ns()))
614                                {
615                                    return Some(first);
616                                }
617                            } else {
618                                self.book.delta_subs.remove(&market_index);
619                                self.cancel_orphaned_book_recovery(market_index);
620                            }
621                        }
622                        HandlerCommand::SetDepthSub { market_index, subscribed } => {
623                            if subscribed {
624                                let inserted = self.book.depth_subs.insert(market_index);
625                                if inserted
626                                    && let Some(first) =
627                                        self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_depth_snapshot(market_index, instrument, self.clock.get_time_ns()))
628                                {
629                                    return Some(first);
630                                }
631                            } else {
632                                self.book.depth_subs.remove(&market_index);
633                                self.cancel_orphaned_book_recovery(market_index);
634                            }
635                        }
636                        HandlerCommand::SetExecutionContext { account_id, account_index } => {
637                            self.exec_account = Some((account_id, account_index));
638                        }
639                        HandlerCommand::SendTxBatch { data, connection_epoch, response_tx } => {
640                            let result = match serde_json::to_string(&LighterWsRequest::SendTxBatch { data }) {
641                                Ok(payload) => self.send_once(payload, connection_epoch).await,
642                                Err(e) => Err(LighterWsError::Client(format!("failed to serialize Lighter sendTxBatch: {e}"))),
643                            };
644                            let _ = response_tx.send(result);
645                        }
646                        HandlerCommand::SendTx {
647                            tx_type,
648                            tx_info,
649                            connection_epoch,
650                            response_tx,
651                        } => {
652                            let result = self
653                                .dispatch_send_tx(tx_type, tx_info, connection_epoch)
654                                .await;
655
656                            if response_tx.send(result).is_err() {
657                                log::debug!("Lighter sendTx result receiver dropped");
658                            }
659                        }
660                    }
661                }
662                Some((topic, generation)) = self.subscription_retries.next(),
663                    if !self.subscription_retries.is_empty() =>
664                {
665                    self.queue_subscription_retry(topic, generation);
666                }
667                Some((connection_epoch, raw_msg)) = self.raw_rx.recv() => {
668                    match raw_msg {
669                        Message::Text(text) => {
670                            if text == RECONNECTED {
671                                log::debug!("Received Lighter WebSocket RECONNECTED sentinel");
672                                self.book.reset_on_reconnect();
673                                // Resubscribe replays a fresh `subscribed/candle`; pre-disconnect cache is stale.
674                                self.last_candles.clear();
675                                // The new socket starts with zero inflight and restore_subscriptions
676                                // re-queues every topic. Retire stale generations before accepting
677                                // replay commands so late work cannot complete a replacement attempt.
678                                self.reset_subscription_attempts_after_reconnect();
679                                self.account_state_reconciler.reset();
680                                return Some(NautilusWsMessage::Reconnected {
681                                    connection_epoch,
682                                });
683                            }
684
685                            let ts_init = self.clock.get_time_ns();
686                            let subscribed_topic = typed_subscribe_topic(&text);
687
688                            if let Ok(frame) = serde_json::from_str::<LighterWsFrame>(&text) {
689                                if let LighterWsFrame::Shutdown { close_in_ms } = frame {
690                                    if let Some(ws_client) = &self.inner
691                                        && ws_client.connection_epoch() == connection_epoch
692                                        && ws_client.request_reconnect()
693                                    {
694                                        log::info!("Lighter server shutdown in {close_in_ms} ms, reconnect requested");
695                                    }
696                                    continue;
697                                }
698
699                                if let Some(topic) = subscribed_topic
700                                    && !self.complete_typed_subscription(topic, connection_epoch)
701                                {
702                                    continue;
703                                }
704                                let messages = self
705                                    .route_frame(frame, &text, ts_init)
706                                    .into_iter()
707                                    .map(|msg| msg.with_connection_epoch(connection_epoch))
708                                    .collect();
709
710                                if let Some(first) = self.dispatch_results(messages) {
711                                    return Some(first);
712                                }
713                            } else if let Ok(value) =
714                                serde_json::from_str::<serde_json::Value>(&text)
715                            {
716                                // The venue acknowledged the subscription even
717                                // though the snapshot body is unparsable.
718                                // Complete from the header so subscribers never
719                                // hang on a malformed confirmation frame.
720                                if let Some(topic) = subscribed_topic {
721                                    self.complete_typed_subscription(topic, connection_epoch);
722                                }
723
724                                let (matched, msg) = self.handle_control_value(&value);
725                                if let Some(first) = msg {
726                                    return Some(first.with_connection_epoch(connection_epoch));
727                                }
728
729                                if !matched {
730                                    // Neither a typed frame nor a recognized
731                                    // control type. Surface raw so venue
732                                    // errors (bad signature, margin rejection,
733                                    // ...) don't drop silently.
734                                    log::warn!("Lighter WS unparsed frame: {value}");
735                                    return Some(NautilusWsMessage::Raw(value));
736                                }
737                            } else {
738                                log::warn!("Lighter WS non-JSON text: {text}");
739                            }
740                        }
741                        Message::Ping(data) => {
742                            if let Some(ref client) = self.inner
743                                && let Err(e) = client.send_pong(data.to_vec()).await {
744                                log::error!("Error sending Lighter pong: {e}");
745                            }
746                        }
747                        Message::Close(frame) => {
748                            log::debug!("Received Lighter WebSocket close frame: {frame:?}");
749                            return None;
750                        }
751                        _ => {}
752                    }
753                }
754                else => {
755                    log::debug!("Lighter handler shutting down: stream ended or command channel closed");
756                    return None;
757                }
758            }
759        }
760    }
761
762    fn dispatch_results(
763        &mut self,
764        mut messages: Vec<NautilusWsMessage>,
765    ) -> Option<NautilusWsMessage> {
766        if messages.is_empty() {
767            return None;
768        }
769        let first = messages.remove(0);
770        for extra in messages {
771            self.pending_messages.push_back(extra);
772        }
773        Some(first)
774    }
775
776    // Non-blocking count check, not a permit await: this task also drains the
777    // acks that free slots, so awaiting a permit here would deadlock.
778    async fn pump_pending_subscribes(&mut self) {
779        while self.inflight_subs.len() < SUBSCRIBE_INFLIGHT_MAX {
780            let Some((topic, generation)) = self.pending_subs.pop_front() else {
781                break;
782            };
783
784            let Some(attempt) = self.subscription_attempts.get(&topic) else {
785                continue;
786            };
787
788            if attempt.generation != generation {
789                continue;
790            }
791
792            let channel = attempt.channel.clone();
793            let auth = attempt.auth.clone();
794            self.inflight_subs.insert(topic, generation);
795            if let LighterWsChannel::OrderBook(market_index) = channel {
796                self.send_book_subscribe(market_index, generation);
797            } else if let Err(message) = self.dispatch_subscribe(channel, auth).await {
798                self.schedule_subscription_retry(topic, generation, &message);
799            }
800        }
801    }
802
803    fn queue_subscribe(
804        &mut self,
805        channel: LighterWsChannel,
806        auth: Option<SecretString>,
807        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
808    ) {
809        let topic = Ustr::from(channel.topic_key().as_str());
810
811        if let LighterWsChannel::OrderBook(market_index) = channel
812            && !self.book.writes.contains_key(&market_index)
813            && self
814                .book
815                .recovery
816                .get(&market_index)
817                .and_then(BookRecoveryState::current)
818                .is_some_and(|recovery| !recovery.is_accepted())
819        {
820            return;
821        }
822
823        if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
824            attempt.channel = channel;
825            if matches!(attempt.channel, LighterWsChannel::OrderBook(_))
826                && !self.inflight_subs.contains_key(&topic)
827                && !self
828                    .pending_subs
829                    .iter()
830                    .any(|(pending, _)| *pending == topic)
831            {
832                self.pending_subs.push_back((topic, attempt.generation));
833            }
834
835            let effective_auth = attempt
836                .pending_auth
837                .as_ref()
838                .or(attempt.auth.as_ref())
839                .map(SecretString::expose_secret);
840            let auth_changed = auth
841                .as_ref()
842                .map(SecretString::expose_secret)
843                .is_some_and(|auth| Some(auth) != effective_auth);
844
845            if auth_changed {
846                let is_inflight = self.inflight_subs.get(&topic) == Some(&attempt.generation);
847                if is_inflight {
848                    attempt.pending_auth = auth;
849                    if let Some(response_tx) = response_tx {
850                        attempt.pending_response_txs.push(response_tx);
851                    }
852                } else {
853                    debug_assert!(attempt.pending_auth.is_none());
854                    debug_assert!(attempt.pending_response_txs.is_empty());
855                    attempt.auth = auth;
856                    if let Some(response_tx) = response_tx {
857                        attempt.response_txs.push(response_tx);
858                    }
859                }
860            } else if let Some(response_tx) = response_tx {
861                if auth.is_some() && attempt.pending_auth.is_some() {
862                    attempt.pending_response_txs.push(response_tx);
863                } else {
864                    attempt.response_txs.push(response_tx);
865                }
866            }
867            return;
868        }
869
870        let newly_pending = self.subscriptions.try_mark_subscribe(topic.as_str());
871        if !newly_pending
872            && auth.is_none()
873            && !self
874                .subscriptions
875                .pending_subscribe_topics()
876                .iter()
877                .any(|pending| pending == topic.as_str())
878        {
879            if let Some(response_tx) = response_tx {
880                let _ = response_tx.send(Ok(()));
881            }
882            return;
883        }
884
885        let generation = self.take_subscription_generation();
886        self.subscription_attempts.insert(
887            topic,
888            SubscriptionAttempt {
889                channel,
890                auth,
891                pending_auth: None,
892                generation,
893                retries: 0,
894                response_txs: response_tx.into_iter().collect(),
895                pending_response_txs: Vec::new(),
896            },
897        );
898        self.pending_subs.push_back((topic, generation));
899    }
900
901    fn queue_subscription_retry(&mut self, topic: Ustr, generation: u64) {
902        if self
903            .subscription_attempts
904            .get(&topic)
905            .is_some_and(|attempt| attempt.generation == generation)
906            && self.inflight_subs.get(&topic) != Some(&generation)
907            && !self
908                .pending_subs
909                .iter()
910                .any(|pending| *pending == (topic, generation))
911        {
912            self.pending_subs.push_back((topic, generation));
913        }
914    }
915
916    fn complete_subscription(&mut self, topic: &str, kind: CompletionKind) -> bool {
917        let topic = Ustr::from(topic);
918
919        // Swallow the predecessor's trailing frame before the inflight guard,
920        // so it cannot complete a same-topic generation dispatched between the
921        // venue's control ack and its typed snapshot - whether that generation
922        // is a serialized successor, a fresh attempt, queued, or in backoff.
923        if self.ignored_completions.get(&topic) == Some(&kind) {
924            self.ignored_completions.remove(&topic);
925            return false;
926        }
927
928        let Some(generation) = self.inflight_subs.get(&topic).copied() else {
929            return false;
930        };
931
932        if !self
933            .subscription_attempts
934            .get(&topic)
935            .is_some_and(|attempt| attempt.generation == generation)
936        {
937            return false;
938        }
939
940        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
941            && !self
942                .book
943                .expected
944                .get(&market_index)
945                .is_some_and(|expected| expected.0 == generation)
946        {
947            return false;
948        }
949
950        self.inflight_subs.remove(&topic);
951        self.subscriptions.confirm_subscribe(topic.as_str());
952
953        // The venue contract is control ack first, typed snapshot second, so a
954        // control-ack completion owes one trailing typed frame. Typed and
955        // already-subscribed completions have no established trailing frame.
956        if kind == CompletionKind::ControlAck {
957            if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
958                && let Some(expected) = self.book.expected.get(&market_index)
959                && expected.0 == generation
960            {
961                self.book.trailing.entry(market_index).or_insert(*expected);
962            }
963
964            self.ignored_completions
965                .insert(topic, CompletionKind::Typed);
966        }
967        let mut attempt = self
968            .subscription_attempts
969            .remove(&topic)
970            .expect("matching subscription attempt disappeared");
971        for response_tx in std::mem::take(&mut attempt.response_txs) {
972            let _ = response_tx.send(Ok(()));
973        }
974
975        if let Some(auth) = attempt.pending_auth.take() {
976            let generation = self.take_subscription_generation();
977            attempt.auth = Some(auth);
978            attempt.generation = generation;
979            attempt.retries = 0;
980            attempt.response_txs = std::mem::take(&mut attempt.pending_response_txs);
981            self.subscription_attempts.insert(topic, attempt);
982            self.pending_subs.push_back((topic, generation));
983        } else {
984            debug_assert!(attempt.pending_response_txs.is_empty());
985        }
986        true
987    }
988
989    fn retry_inflight_subscriptions(&mut self, message: &str) {
990        let mut inflight: Vec<(Ustr, u64)> = self
991            .inflight_subs
992            .iter()
993            .map(|(topic, generation)| (*topic, *generation))
994            .collect();
995        inflight.sort_unstable_by_key(|(topic, _)| *topic);
996
997        for (topic, generation) in inflight {
998            self.schedule_subscription_retry(topic, generation, message);
999        }
1000    }
1001
1002    fn schedule_subscription_retry(&mut self, topic: Ustr, generation: u64, message: &str) {
1003        if self.inflight_subs.get(&topic) != Some(&generation) {
1004            return;
1005        }
1006
1007        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1008            self.reject_book_subscription(
1009                market_index,
1010                generation,
1011                LighterWsError::Network(message.into()),
1012            );
1013            return;
1014        }
1015
1016        self.inflight_subs.remove(&topic);
1017        self.subscriptions.mark_failure(topic.as_str());
1018
1019        let Some(attempt) = self.subscription_attempts.get_mut(&topic) else {
1020            return;
1021        };
1022
1023        if attempt.generation != generation {
1024            return;
1025        }
1026
1027        attempt.fold_pending_auth();
1028        if attempt.retries >= SUBSCRIBE_RETRY_MAX {
1029            self.fail_subscription_attempt(topic, message);
1030            return;
1031        }
1032
1033        attempt.retries += 1;
1034        let retry = attempt.retries;
1035        let next_generation = self.take_subscription_generation();
1036        let attempt = self
1037            .subscription_attempts
1038            .get_mut(&topic)
1039            .expect("subscription attempt disappeared before retry");
1040        attempt.generation = next_generation;
1041
1042        let delay = SUBSCRIBE_RETRY_BASE_BACKOFF.saturating_mul(1_u32 << (retry - 1));
1043        self.subscription_retries.push(Box::pin(async move {
1044            tokio::time::sleep(delay).await;
1045            (topic, next_generation)
1046        }));
1047    }
1048
1049    fn fail_subscription_attempt(&mut self, topic: Ustr, message: &str) {
1050        self.inflight_subs.remove(&topic);
1051        self.pending_subs.retain(|(pending, _)| *pending != topic);
1052        self.subscriptions.mark_unsubscribe(topic.as_str());
1053        self.subscriptions.confirm_unsubscribe(topic.as_str());
1054
1055        if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1056            let attempts = attempt.retries + 1;
1057            for response_tx in attempt
1058                .response_txs
1059                .into_iter()
1060                .chain(attempt.pending_response_txs)
1061            {
1062                let _ = response_tx.send(Err(format!(
1063                    "subscription {topic} failed after {attempts} attempts: {message}",
1064                )));
1065            }
1066        }
1067    }
1068
1069    fn cancel_subscription_attempt(&mut self, topic: &str, message: &str) {
1070        let topic = Ustr::from(topic);
1071        self.inflight_subs.remove(&topic);
1072        self.pending_subs.retain(|(pending, _)| *pending != topic);
1073        if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1074            for response_tx in attempt
1075                .response_txs
1076                .into_iter()
1077                .chain(attempt.pending_response_txs)
1078            {
1079                let _ = response_tx.send(Err(format!("{message}: {topic}")));
1080            }
1081        }
1082    }
1083
1084    fn reset_subscription_attempts_after_reconnect(&mut self) {
1085        for topic in self.subscriptions.all_topics() {
1086            self.subscriptions.mark_failure(&topic);
1087        }
1088        self.pending_subs.clear();
1089        self.inflight_subs.clear();
1090        // Completion frames from the old connection must not be applied to -
1091        // or swallowed from - the new connection.
1092        self.ignored_completions.clear();
1093
1094        let mut topics: Vec<Ustr> = self.subscription_attempts.keys().copied().collect();
1095        topics.sort_unstable();
1096        for topic in topics {
1097            let generation = self.take_subscription_generation();
1098            let attempt = self
1099                .subscription_attempts
1100                .get_mut(&topic)
1101                .expect("subscription attempt disappeared during reconnect");
1102            attempt.fold_pending_auth();
1103            attempt.generation = generation;
1104            attempt.retries = 0;
1105
1106            if order_book_market_index_from_topic(topic.as_str()).is_none() {
1107                self.pending_subs.push_back((topic, generation));
1108            }
1109        }
1110    }
1111
1112    fn take_subscription_generation(&mut self) -> u64 {
1113        let generation = self.next_subscription_generation;
1114        self.next_subscription_generation =
1115            self.next_subscription_generation.wrapping_add(1).max(1);
1116        generation
1117    }
1118
1119    /// Returns `(matched, msg)` where `matched=true` means the frame's
1120    /// `type` was recognized as a known control type (whether or not a
1121    /// message is emitted); `matched=false` lets the caller surface the
1122    /// raw frame so venue errors with unknown shapes aren't lost.
1123    fn handle_control_value(
1124        &mut self,
1125        value: &serde_json::Value,
1126    ) -> (bool, Option<NautilusWsMessage>) {
1127        if value.get("type").and_then(|v| v.as_str()) == Some("jsonapi/sendtxbatch")
1128            || value
1129                .get("id")
1130                .and_then(|v| v.as_str())
1131                .is_some_and(|id| id.starts_with(TX_BATCH_ID_PREFIX))
1132        {
1133            let body = value.get("error").unwrap_or(value);
1134            let id = value.get("id").and_then(|v| v.as_str());
1135            let code = body.get("code").and_then(|v| v.as_i64());
1136
1137            let (Some(id), Some(code)) = (id, code) else {
1138                log::warn!("Ignoring malformed Lighter sendTxBatch response");
1139                return (true, None);
1140            };
1141
1142            return (
1143                true,
1144                Some(NautilusWsMessage::SendTxBatchResult {
1145                    connection_epoch: 0,
1146                    id: id.to_string(),
1147                    code,
1148                    message: body
1149                        .get("message")
1150                        .and_then(|v| v.as_str())
1151                        .unwrap_or("batch rejected")
1152                        .to_string(),
1153                    tx_hashes: value
1154                        .get("tx_hash")
1155                        .and_then(|v| serde_json::from_value(v.clone()).ok())
1156                        .unwrap_or_default(),
1157                }),
1158            );
1159        }
1160
1161        if let Some(error) = already_subscribed_error(value) {
1162            self.confirm_already_subscribed(error);
1163            return (true, None);
1164        }
1165        let subscription_code = subscription_error_code(value);
1166
1167        if subscription_code == Some(LIGHTER_ERROR_CODE_WS_RATE_LIMITED) {
1168            self.retry_inflight_subscriptions(&format!(
1169                "venue rejected the WebSocket subscribe with code \
1170                 {LIGHTER_ERROR_CODE_WS_RATE_LIMITED}",
1171            ));
1172            return (true, None);
1173        }
1174
1175        if subscription_code == Some(LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED) {
1176            self.handle_subscribe_failure(value);
1177            return (true, None);
1178        }
1179
1180        let kind = value.get("type").and_then(|v| v.as_str()).unwrap_or("");
1181
1182        match kind {
1183            CTRL_TYPE_CONNECTED => {
1184                log::debug!("Lighter WebSocket handshake complete");
1185                (true, None)
1186            }
1187            CTRL_TYPE_PING | CTRL_TYPE_PONG => (true, None),
1188            CTRL_TYPE_SEND_TX => {
1189                let raw_code = value.get("code").and_then(|v| v.as_u64());
1190                match raw_code {
1191                    Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) => {
1192                        log_integrator_not_approved();
1193                        (
1194                            true,
1195                            Some(send_tx_rejected_from_value(
1196                                value,
1197                                SendTxRejectionSource::Ack,
1198                            )),
1199                        )
1200                    }
1201                    Some(200) => {
1202                        log::debug!("Lighter WebSocket sendTx ack: {value}");
1203                        let tx_hash = value
1204                            .get("tx_hash")
1205                            .and_then(|v| v.as_str())
1206                            .map(str::to_string);
1207                        (
1208                            true,
1209                            Some(NautilusWsMessage::SendTxAck {
1210                                connection_epoch: 0,
1211                                tx_hash,
1212                                code: 200,
1213                            }),
1214                        )
1215                    }
1216                    Some(_) => (
1217                        true,
1218                        Some(send_tx_rejected_from_value(
1219                            value,
1220                            SendTxRejectionSource::Ack,
1221                        )),
1222                    ),
1223                    None => {
1224                        log::warn!(
1225                            "Ignoring malformed Lighter sendTx response without numeric code: {value}",
1226                        );
1227                        (true, None)
1228                    }
1229                }
1230            }
1231            CTRL_TYPE_SUBSCRIBED | CTRL_TYPE_UNSUBSCRIBED => {
1232                if let Some(topic) = value.get("channel").and_then(|v| v.as_str()) {
1233                    if kind == CTRL_TYPE_SUBSCRIBED {
1234                        self.complete_subscription(topic, CompletionKind::ControlAck);
1235                    } else {
1236                        let was_pending_unsubscribe = self
1237                            .subscriptions
1238                            .pending_unsubscribe_topics()
1239                            .iter()
1240                            .any(|pending| pending == topic);
1241                        self.subscriptions.confirm_unsubscribe(topic);
1242
1243                        if was_pending_unsubscribe {
1244                            // Only matched unsubscribe ACKs should reset stream state
1245                            if let Some(market_index) = order_book_market_index_from_topic(topic) {
1246                                self.book.clear_cached_order_book(market_index);
1247                            }
1248
1249                            if let Some(key) = candle_market_and_resolution_from_topic(topic) {
1250                                self.last_candles.remove(&key);
1251                            }
1252                        }
1253                    }
1254                }
1255                (true, None)
1256            }
1257            CTRL_TYPE_ERROR => {
1258                let code = value.get("code").and_then(|v| v.as_u64());
1259                if code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1260                    log_integrator_not_approved();
1261                } else {
1262                    log::warn!("Lighter WebSocket error frame: {value}");
1263                }
1264
1265                if is_sendtx_error_code(code) {
1266                    (
1267                        true,
1268                        Some(send_tx_rejected_from_value(
1269                            value,
1270                            SendTxRejectionSource::BareError,
1271                        )),
1272                    )
1273                } else {
1274                    (true, None)
1275                }
1276            }
1277            _ => {
1278                if let Some(error) = value.get("error") {
1279                    let nested_code = error.get("code").and_then(|v| v.as_u64());
1280                    if nested_code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1281                        log_integrator_not_approved();
1282                    } else {
1283                        log::warn!("Lighter WebSocket error frame: {value}");
1284                    }
1285                    let rejected = is_sendtx_error_code(nested_code).then(|| {
1286                        send_tx_rejected_from_nested_error(error, SendTxRejectionSource::BareError)
1287                    });
1288                    return (true, rejected);
1289                }
1290                (false, None)
1291            }
1292        }
1293    }
1294
1295    fn handle_subscribe_failure(&mut self, value: &serde_json::Value) {
1296        let message = format!(
1297            "venue rejected the WebSocket subscribe with code \
1298             {LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED}",
1299        );
1300
1301        let channel = value
1302            .get("channel")
1303            .and_then(serde_json::Value::as_str)
1304            .or_else(|| {
1305                value
1306                    .get("error")
1307                    .and_then(|e| e.get("channel"))
1308                    .and_then(serde_json::Value::as_str)
1309            })
1310            .filter(|channel| !channel.is_empty());
1311
1312        let Some(channel) = channel else {
1313            log::warn!(
1314                "Lighter subscribe error code={LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED} channel=none: retrying inflight subscriptions"
1315            );
1316            self.retry_inflight_subscriptions(&message);
1317            return;
1318        };
1319
1320        let topic = Ustr::from(channel.replace('/', ":").as_str());
1321
1322        let Some(generation) = self.inflight_subs.get(&topic).copied() else {
1323            log::warn!(
1324                "Lighter subscribe error code={LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED} channel={channel}: no matching inflight subscription"
1325            );
1326            return;
1327        };
1328
1329        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1330            self.reject_book_subscription(
1331                market_index,
1332                generation,
1333                LighterWsError::Client(message),
1334            );
1335        } else {
1336            self.fail_subscription_attempt(topic, &message);
1337        }
1338    }
1339
1340    fn confirm_already_subscribed(&mut self, error: &serde_json::Value) {
1341        let Some(topic) = error
1342            .get("message")
1343            .and_then(|value| value.as_str())
1344            .and_then(|message| message.strip_prefix("Already Subscribed to : "))
1345        else {
1346            log::debug!(
1347                "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1348            );
1349            return;
1350        };
1351
1352        if !self.complete_subscription(topic, CompletionKind::AlreadySubscribed) {
1353            log::debug!(
1354                "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1355            );
1356            return;
1357        }
1358
1359        log::debug!(
1360            "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}, topic={topic}",
1361        );
1362    }
1363
1364    fn route_frame(
1365        &mut self,
1366        frame: LighterWsFrame,
1367        text: &str,
1368        ts_init: UnixNanos,
1369    ) -> Vec<NautilusWsMessage> {
1370        if self.forwards_raw(&frame) {
1371            return raw_message(text);
1372        }
1373
1374        self.handle_frame(frame, ts_init)
1375    }
1376
1377    // Account frames are typed only with an execution context; height has no typed consumer
1378    fn forwards_raw(&self, frame: &LighterWsFrame) -> bool {
1379        match frame {
1380            LighterWsFrame::Height { .. } => true,
1381            LighterWsFrame::AccountOrders { .. }
1382            | LighterWsFrame::AccountAllOrders { .. }
1383            | LighterWsFrame::AccountTx { .. }
1384            | LighterWsFrame::AccountAllTradesSnapshot { .. }
1385            | LighterWsFrame::AccountAllTrades { .. }
1386            | LighterWsFrame::AccountAllPositionsSnapshot { .. }
1387            | LighterWsFrame::AccountAllPositions { .. }
1388            | LighterWsFrame::AccountAllAssets { .. }
1389            | LighterWsFrame::UserStats { .. } => self.exec_account.is_none(),
1390            _ => false,
1391        }
1392    }
1393
1394    fn handle_frame(
1395        &mut self,
1396        frame: LighterWsFrame,
1397        ts_init: UnixNanos,
1398    ) -> Vec<NautilusWsMessage> {
1399        match frame {
1400            LighterWsFrame::OrderBookSnapshot {
1401                channel,
1402                order_book,
1403                timestamp,
1404                ..
1405            } => self.handle_order_book(channel, &order_book, timestamp, true, ts_init),
1406            LighterWsFrame::OrderBook {
1407                channel,
1408                order_book,
1409                timestamp,
1410                ..
1411            } => self.handle_order_book(channel, &order_book, timestamp, false, ts_init),
1412            LighterWsFrame::TickerSnapshot {
1413                channel,
1414                ticker,
1415                timestamp,
1416                ..
1417            }
1418            | LighterWsFrame::Ticker {
1419                channel,
1420                ticker,
1421                timestamp,
1422                ..
1423            } => self.handle_ticker(channel, &ticker, timestamp, ts_init),
1424            LighterWsFrame::TradeSnapshot {
1425                trades,
1426                liquidation_trades,
1427                ..
1428            }
1429            | LighterWsFrame::Trade {
1430                trades,
1431                liquidation_trades,
1432                ..
1433            } => self.handle_trades(&trades, &liquidation_trades, ts_init),
1434            LighterWsFrame::AccountTxSubscribed { .. } => Vec::new(),
1435            LighterWsFrame::AccountTx { channel, txs } => self.handle_account_tx(channel, txs),
1436            LighterWsFrame::AccountOrders { ref orders, .. }
1437            | LighterWsFrame::AccountAllOrders { ref orders, .. } => {
1438                let mut msgs = self.handle_account_orders(orders, ts_init);
1439                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1440                    AccountStream::Orders,
1441                ));
1442                msgs
1443            }
1444            // Lighter publishes the historical fill replay as `subscribed/`
1445            // rather than `update/`. When typed execution routing is active
1446            // we drop the snapshot to avoid re-emitting prior executions as
1447            // fresh fill reports on every reconnect (HTTP reconciliation
1448            // owns historical state recovery).
1449            LighterWsFrame::AccountAllTradesSnapshot { .. } => {
1450                log::debug!(
1451                    "Skipping Lighter account_all_trades snapshot frame; \
1452                     reconcile historical fills via HTTP",
1453                );
1454                vec![NautilusWsMessage::AccountStreamFirstFrame(
1455                    AccountStream::Trades,
1456                )]
1457            }
1458            LighterWsFrame::AccountAllTrades { ref trades, .. } => {
1459                let mut msgs = self.handle_account_trades(trades.values().flatten(), ts_init);
1460                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1461                    AccountStream::Trades,
1462                ));
1463                msgs
1464            }
1465            LighterWsFrame::AccountAllPositionsSnapshot { ref positions, .. } => {
1466                let mut msgs =
1467                    self.handle_account_positions(positions, ts_init, PositionFrameType::Snapshot);
1468                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1469                    AccountStream::Positions,
1470                ));
1471                msgs
1472            }
1473            LighterWsFrame::AccountAllPositions { ref positions, .. } => {
1474                self.handle_account_positions(positions, ts_init, PositionFrameType::Update)
1475            }
1476            LighterWsFrame::AccountAllAssets {
1477                ref assets,
1478                timestamp,
1479                ..
1480            } => {
1481                let mut msgs = self.handle_account_assets(assets, timestamp, ts_init);
1482                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1483                    AccountStream::Assets,
1484                ));
1485                msgs
1486            }
1487            LighterWsFrame::UserStats {
1488                ref stats,
1489                timestamp,
1490                ..
1491            } => {
1492                let mut msgs = self.handle_user_stats(stats, timestamp, ts_init);
1493                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1494                    AccountStream::UserStats,
1495                ));
1496                msgs
1497            }
1498            LighterWsFrame::MarketStats {
1499                ref market_stats,
1500                timestamp,
1501                ..
1502            } => self.handle_market_stats(market_stats, timestamp, ts_init),
1503            LighterWsFrame::SpotMarketStats {
1504                ref spot_market_stats,
1505                timestamp,
1506                ..
1507            } => self.handle_spot_market_stats(spot_market_stats, timestamp, ts_init),
1508            LighterWsFrame::CandleSnapshot {
1509                channel,
1510                ref candles,
1511                ..
1512            }
1513            | LighterWsFrame::Candle {
1514                channel,
1515                ref candles,
1516                ..
1517            } => self.handle_candles(channel, candles, ts_init),
1518            LighterWsFrame::Shutdown { .. } | LighterWsFrame::Height { .. } => Vec::new(),
1519        }
1520    }
1521
1522    fn handle_account_tx(&self, channel: Ustr, txs: Vec<LighterWsTx>) -> Vec<NautilusWsMessage> {
1523        let Some((_, account_index)) = self.exec_account else {
1524            return Vec::new();
1525        };
1526
1527        if channel.as_str() != format!("account_tx:{account_index}") {
1528            log::warn!("Ignoring Lighter transaction update for unexpected channel {channel}");
1529            return Vec::new();
1530        }
1531
1532        vec![NautilusWsMessage::AccountTx {
1533            connection_epoch: 0,
1534            txs,
1535        }]
1536    }
1537
1538    fn handle_order_book(
1539        &mut self,
1540        channel: Ustr,
1541        book: &LighterWsOrderBook,
1542        timestamp: u64,
1543        is_snapshot: bool,
1544        ts_init: UnixNanos,
1545    ) -> Vec<NautilusWsMessage> {
1546        let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1547            log::debug!("Lighter order_book frame missing market index in channel '{channel}'");
1548            return Vec::new();
1549        };
1550
1551        let Some(instrument) = self.instruments.get(&market_index) else {
1552            log::debug!("No instrument cached for Lighter market_index={market_index}");
1553            return Vec::new();
1554        };
1555
1556        match self.book.validate_sequence(market_index, book, is_snapshot) {
1557            BookSequenceOutcome::Accept => self.book.apply(
1558                market_index,
1559                instrument,
1560                book,
1561                timestamp,
1562                is_snapshot,
1563                ts_init,
1564            ),
1565            BookSequenceOutcome::Suppress => Vec::new(),
1566            BookSequenceOutcome::Recover => {
1567                self.start_book_recovery(market_index);
1568                Vec::new()
1569            }
1570        }
1571    }
1572
1573    fn queue_book_replacement(&mut self, market_index: i64, write: BookWrite) {
1574        if write.cancel.is_cancelled() || !self.order_book_stream_is_referenced(market_index) {
1575            return;
1576        }
1577
1578        self.book.clear_cached_order_book(market_index);
1579        self.book.expected.remove(&market_index);
1580        let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1581        self.retire_book_attempt(&topic);
1582        self.subscriptions.mark_failure(&topic);
1583        self.book.writes.insert(market_index, write);
1584        self.queue_subscribe(LighterWsChannel::OrderBook(market_index), None, None);
1585    }
1586
1587    fn retire_book_attempt(&mut self, topic: &str) {
1588        let topic = Ustr::from(topic);
1589        self.inflight_subs.remove(&topic);
1590        self.pending_subs.retain(|(pending, _)| *pending != topic);
1591        if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
1592            // Keep callers waiting for the eventual replacement acknowledgement.
1593            attempt.generation = self.next_subscription_generation;
1594            self.next_subscription_generation =
1595                self.next_subscription_generation.wrapping_add(1).max(1);
1596            self.pending_subs.push_back((topic, attempt.generation));
1597        }
1598    }
1599
1600    fn reject_book_subscription(
1601        &mut self,
1602        market_index: i64,
1603        generation: u64,
1604        error: LighterWsError,
1605    ) {
1606        if !self
1607            .book
1608            .expected
1609            .get(&market_index)
1610            .is_some_and(|expected| expected.0 == generation)
1611            || self
1612                .book
1613                .recovery
1614                .get(&market_index)
1615                .and_then(BookRecoveryState::current)
1616                .is_some_and(|recovery| recovery.gate.lock().is_closed())
1617        {
1618            return;
1619        }
1620
1621        self.reject_book(market_index, error);
1622    }
1623
1624    fn reject_book(&mut self, market_index: i64, error: LighterWsError) {
1625        self.book.expected.remove(&market_index);
1626        self.book.clear_cached_order_book(market_index);
1627        let topic = Ustr::from(
1628            LighterWsChannel::OrderBook(market_index)
1629                .topic_key()
1630                .as_str(),
1631        );
1632
1633        let waiting = matches!(error, LighterWsError::Client(_))
1634            && self
1635                .subscription_attempts
1636                .get(&topic)
1637                .is_some_and(|attempt| {
1638                    !attempt.response_txs.is_empty() || !attempt.pending_response_txs.is_empty()
1639                });
1640
1641        if waiting {
1642            // The waiting subscriber owns the failure and releases its stream reference; an expired
1643            // initial wait must not start recovery before that release.
1644            self.book.initial.remove(&market_index);
1645            self.cancel_subscription_attempt(&topic, &error.to_string());
1646            log::error!(
1647                "Lighter book subscription rejected for market_index={market_index}: {error}"
1648            );
1649        }
1650
1651        if let Some(recovery) = self
1652            .book
1653            .recovery
1654            .get(&market_index)
1655            .and_then(BookRecoveryState::current)
1656            && !recovery.is_accepted()
1657        {
1658            // Frees the slot during backoff; the next replacement takes a new generation
1659            self.inflight_subs.remove(&topic);
1660            recovery.gate.lock().close();
1661            recovery
1662                .outcome
1663                .send_replace(BookRecoveryOutcome::Rejected(error));
1664        } else if !waiting {
1665            self.start_book_recovery(market_index);
1666        }
1667    }
1668
1669    fn start_book_recovery(&mut self, market_index: i64) {
1670        if !self.order_book_stream_is_referenced(market_index) {
1671            return;
1672        }
1673
1674        let Some(cmd_tx) = self.cmd_tx.clone() else {
1675            return;
1676        };
1677
1678        let Some(recovery) = self.book.recovery.entry(market_index).or_default().claim() else {
1679            return;
1680        };
1681
1682        recovery.gate.lock().close();
1683        self.book.initial.remove(&market_index);
1684        self.book.expected.remove(&market_index);
1685        self.book.clear_cached_order_book(market_index);
1686        let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1687        self.inflight_subs.remove(&Ustr::from(topic.as_str()));
1688        self.pending_subs
1689            .retain(|(pending, _)| pending.as_str() != topic);
1690        self.book.work.push(recovery::recover(
1691            market_index,
1692            recovery,
1693            cmd_tx,
1694            self.book_snapshot_timeout,
1695        ));
1696    }
1697
1698    // A rejected subscriber releases its stream reference without an unsubscribe, so the last
1699    // consumer's release must end the recovery.
1700    fn cancel_orphaned_book_recovery(&mut self, market_index: i64) {
1701        if !self.order_book_stream_is_referenced(market_index) {
1702            self.book.recovery.remove(&market_index);
1703        }
1704    }
1705
1706    fn complete_typed_subscription(&mut self, topic: &str, epoch: u64) -> bool {
1707        if let Some(market_index) = order_book_market_index_from_topic(topic)
1708            && self
1709                .book
1710                .expected
1711                .get(&market_index)
1712                .is_some_and(|expected| expected.1 != epoch)
1713        {
1714            return false;
1715        }
1716
1717        if !self.book_snapshot_matches(topic, epoch) {
1718            if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1719                self.ignored_completions.remove(&Ustr::from(topic));
1720            }
1721
1722            return false;
1723        }
1724
1725        self.complete_subscription(topic, CompletionKind::Typed);
1726        true
1727    }
1728
1729    fn book_snapshot_matches(&mut self, topic: &str, epoch: u64) -> bool {
1730        let Some(market_index) = order_book_market_index_from_topic(topic) else {
1731            return true;
1732        };
1733
1734        let source =
1735            if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1736                self.book.trailing.remove(&market_index)
1737            } else {
1738                self.inflight_subs
1739                    .get(&Ustr::from(topic))
1740                    .map(|generation| (*generation, epoch))
1741            };
1742
1743        source.is_some_and(|source| {
1744            source.1 == epoch && Some(source) == self.book.expected.get(&market_index).copied()
1745        })
1746    }
1747
1748    fn send_book_subscribe(&mut self, market_index: i64, generation: u64) {
1749        let write = self.book.writes.remove(&market_index);
1750        let cancel = write
1751            .as_ref()
1752            .map_or_else(CancellationToken::new, |write| write.cancel.clone());
1753        if write.is_none() {
1754            self.book.initial.insert(
1755                market_index,
1756                PendingSnapshot {
1757                    deadline: None,
1758                    cancel: cancel.clone(),
1759                    gate: SnapshotGate::default(),
1760                },
1761            );
1762        }
1763
1764        let client = self.inner.clone();
1765        let subscriptions = self.subscriptions.clone();
1766        let book_snapshot_timeout = self.book_snapshot_timeout;
1767        self.book.work.push(recovery::subscribe(
1768            market_index,
1769            generation,
1770            cancel,
1771            write,
1772            client,
1773            subscriptions,
1774            book_snapshot_timeout,
1775        ));
1776    }
1777
1778    fn complete_book_send(
1779        &mut self,
1780        market_index: i64,
1781        generation: u64,
1782        cancel: CancellationToken,
1783        write: Option<BookWrite>,
1784        result: Result<u64, LighterWsError>,
1785    ) {
1786        let topic = Ustr::from(
1787            LighterWsChannel::OrderBook(market_index)
1788                .topic_key()
1789                .as_str(),
1790        );
1791
1792        if cancel.is_cancelled()
1793            || self.inflight_subs.get(&topic) != Some(&generation)
1794            || !self.order_book_stream_is_referenced(market_index)
1795        {
1796            return;
1797        }
1798
1799        match result {
1800            Ok(epoch) => {
1801                self.book.expected.insert(market_index, (generation, epoch));
1802
1803                if let Some(write) = write {
1804                    write.gate.open();
1805                    let _ = write.completion.send(Ok(()));
1806                } else {
1807                    self.book.work.push(recovery::wait_for_snapshot(
1808                        market_index,
1809                        cancel,
1810                        self.book_snapshot_timeout,
1811                    ));
1812                }
1813            }
1814            Err(e) => {
1815                if let Some(write) = write {
1816                    // Frees the slot during backoff; the next replacement takes a new generation
1817                    self.inflight_subs.remove(&topic);
1818                    let _ = write.completion.send(Err(e));
1819                } else {
1820                    self.reject_book(market_index, e);
1821                }
1822            }
1823        }
1824    }
1825
1826    fn order_book_stream_is_referenced(&self, market_index: i64) -> bool {
1827        let channel = LighterWsChannel::OrderBook(market_index);
1828        self.subscriptions.get_reference_count(&channel.topic_key()) > 0
1829            && (self.book.delta_subs.contains(&market_index)
1830                || self.book.depth_subs.contains(&market_index))
1831    }
1832
1833    fn handle_ticker(
1834        &self,
1835        channel: Ustr,
1836        ticker: &super::messages::LighterTicker,
1837        timestamp: u64,
1838        ts_init: UnixNanos,
1839    ) -> Vec<NautilusWsMessage> {
1840        // Resolve the instrument from the channel's `ticker:N` market index,
1841        // not the ticker payload's `s` symbol. Lighter sends the unsuffixed
1842        // venue symbol (e.g. "ETH"), while cached `raw_symbol` carries the
1843        // -PERP / -SPOT suffix, so a symbol-name match never resolves and
1844        // would conflate spot vs perp listings of the same asset.
1845        let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1846            log::debug!("Lighter ticker frame missing market index in channel '{channel}'");
1847            return Vec::new();
1848        };
1849
1850        let Some(instrument) = self.instruments.get(&market_index) else {
1851            log::debug!("No instrument cached for Lighter ticker market_index={market_index}");
1852            return Vec::new();
1853        };
1854
1855        match parse_ws_quote_tick(ticker, instrument, timestamp, ts_init) {
1856            Ok(Some(quote)) => vec![NautilusWsMessage::Quote(quote)],
1857            Ok(None) => {
1858                log::debug!(
1859                    "Skipping Lighter ticker for market_index={market_index}: one-sided book",
1860                );
1861                Vec::new()
1862            }
1863            Err(e) => {
1864                log::error!("Error parsing Lighter ticker frame: {e}");
1865                Vec::new()
1866            }
1867        }
1868    }
1869
1870    fn handle_trades(
1871        &self,
1872        trades: &[crate::http::models::LighterTrade],
1873        liquidation_trades: &[crate::http::models::LighterTrade],
1874        ts_init: UnixNanos,
1875    ) -> Vec<NautilusWsMessage> {
1876        // Lighter splits trade prints across `trades` and `liquidation_trades`
1877        // on the same `update/trade` frame; both share the LighterTrade model
1878        // and are public executions, so emit them together.
1879        let Some(market_index) = trades
1880            .first()
1881            .or_else(|| liquidation_trades.first())
1882            .map(|t| t.market_id)
1883        else {
1884            return Vec::new();
1885        };
1886
1887        let Some(instrument) = self.instruments.get(&market_index) else {
1888            log::debug!("No instrument cached for Lighter trade market_index={market_index}");
1889            return Vec::new();
1890        };
1891
1892        let mut ticks = Vec::with_capacity(trades.len() + liquidation_trades.len());
1893        for trade in trades.iter().chain(liquidation_trades.iter()) {
1894            match parse_ws_trade_tick(trade, instrument, ts_init) {
1895                Ok(tick) => ticks.push(tick),
1896                Err(e) => log::error!("Error parsing Lighter trade tick: {e}"),
1897            }
1898        }
1899
1900        if ticks.is_empty() {
1901            Vec::new()
1902        } else {
1903            vec![NautilusWsMessage::Trades(ticks)]
1904        }
1905    }
1906
1907    fn handle_market_stats(
1908        &self,
1909        payload: &super::messages::LighterMarketStatsPayload,
1910        timestamp: u64,
1911        ts_init: UnixNanos,
1912    ) -> Vec<NautilusWsMessage> {
1913        match payload {
1914            super::messages::LighterMarketStatsPayload::All(stats) => stats
1915                .values()
1916                .flat_map(|stats| self.handle_one_market_stats(stats, timestamp, ts_init))
1917                .collect(),
1918            super::messages::LighterMarketStatsPayload::One(stats) => {
1919                self.handle_one_market_stats(stats, timestamp, ts_init)
1920            }
1921        }
1922    }
1923
1924    fn handle_one_market_stats(
1925        &self,
1926        stats: &super::messages::LighterMarketStats,
1927        timestamp: u64,
1928        ts_init: UnixNanos,
1929    ) -> Vec<NautilusWsMessage> {
1930        let Some(instrument) = self.instruments.get(&stats.market_id) else {
1931            log::debug!(
1932                "No instrument cached for Lighter market_stats market_id={}",
1933                stats.market_id,
1934            );
1935            return Vec::new();
1936        };
1937
1938        let mut messages = Vec::with_capacity(3);
1939
1940        match parse_ws_mark_price_update(stats, instrument, timestamp, ts_init) {
1941            Ok(mark_price) => messages.push(NautilusWsMessage::MarkPrice(mark_price)),
1942            Err(e) => log::error!("Error parsing Lighter mark price: {e}"),
1943        }
1944
1945        match parse_ws_index_price_update(stats, instrument, timestamp, ts_init) {
1946            Ok(index_price) => messages.push(NautilusWsMessage::IndexPrice(index_price)),
1947            Err(e) => log::error!("Error parsing Lighter index price: {e}"),
1948        }
1949
1950        match parse_ws_funding_rate_update(stats, instrument, timestamp, ts_init) {
1951            Ok(funding_rate) => messages.push(NautilusWsMessage::FundingRate(funding_rate)),
1952            Err(e) => log::error!("Error parsing Lighter funding rate: {e}"),
1953        }
1954
1955        messages
1956    }
1957
1958    fn handle_spot_market_stats(
1959        &self,
1960        payload: &super::messages::LighterSpotMarketStatsPayload,
1961        timestamp: u64,
1962        ts_init: UnixNanos,
1963    ) -> Vec<NautilusWsMessage> {
1964        match payload {
1965            super::messages::LighterSpotMarketStatsPayload::All(stats) => stats
1966                .values()
1967                .filter_map(|stats| self.handle_one_spot_market_stats(stats, timestamp, ts_init))
1968                .collect(),
1969            super::messages::LighterSpotMarketStatsPayload::One(stats) => self
1970                .handle_one_spot_market_stats(stats, timestamp, ts_init)
1971                .into_iter()
1972                .collect(),
1973        }
1974    }
1975
1976    fn handle_one_spot_market_stats(
1977        &self,
1978        stats: &super::messages::LighterSpotMarketStats,
1979        timestamp: u64,
1980        ts_init: UnixNanos,
1981    ) -> Option<NautilusWsMessage> {
1982        let Some(instrument) = self.instruments.get(&stats.market_id) else {
1983            log::debug!(
1984                "No instrument cached for Lighter spot_market_stats market_id={}",
1985                stats.market_id,
1986            );
1987            return None;
1988        };
1989
1990        match parse_ws_spot_index_price_update(stats, instrument, timestamp, ts_init) {
1991            Ok(index_price) => Some(NautilusWsMessage::IndexPrice(index_price)),
1992            Err(e) => {
1993                log::error!("Error parsing Lighter spot index price: {e}");
1994                None
1995            }
1996        }
1997    }
1998
1999    fn handle_candles(
2000        &mut self,
2001        channel: Ustr,
2002        candles: &[LighterWsCandle],
2003        ts_init: UnixNanos,
2004    ) -> Vec<NautilusWsMessage> {
2005        let Some((market_index, resolution)) =
2006            candle_market_and_resolution_from_topic(channel.as_str())
2007        else {
2008            log::warn!("Lighter candle frame with unparsable channel `{channel}`");
2009            return Vec::new();
2010        };
2011
2012        let Some(instrument) = self.instruments.get(&market_index) else {
2013            log::debug!("No instrument cached for Lighter candle market_index={market_index}");
2014            return Vec::new();
2015        };
2016
2017        let key = (market_index, resolution);
2018        let mut emitted = Vec::new();
2019
2020        for candle in candles {
2021            let previous = self.last_candles.get(&key).cloned();
2022            match previous {
2023                None => {}
2024                Some(prev) if candle.t > prev.t => {
2025                    // `t` advanced; previous candle is closed.
2026                    match parse_ws_bar(instrument, &prev, resolution, ts_init) {
2027                        Ok(bar) => emitted.push(NautilusWsMessage::Bar(bar)),
2028                        Err(e) => log::error!("Error parsing Lighter candle bar: {e}"),
2029                    }
2030                }
2031                Some(prev) if candle.t < prev.t => continue,
2032                Some(_) => {}
2033            }
2034            self.last_candles.insert(key, candle.clone());
2035        }
2036
2037        emitted
2038    }
2039
2040    fn handle_account_orders(
2041        &self,
2042        orders_by_market: &AHashMap<Ustr, Vec<LighterOrder>>,
2043        _ts_init: UnixNanos,
2044    ) -> Vec<NautilusWsMessage> {
2045        if self.exec_account.is_none() {
2046            log::debug!("Lighter account_orders frame skipped: no execution context set");
2047            return Vec::new();
2048        }
2049
2050        let mut reports = Vec::new();
2051
2052        for orders in orders_by_market.values() {
2053            for order in orders {
2054                if !self.instruments.contains_key(&order.market_index) {
2055                    log::debug!(
2056                        "No instrument cached for Lighter order market_index={}",
2057                        order.market_index,
2058                    );
2059                    continue;
2060                }
2061
2062                reports.push(ExecutionReport::Order(order.clone()));
2063            }
2064        }
2065
2066        if reports.is_empty() {
2067            Vec::new()
2068        } else {
2069            vec![NautilusWsMessage::ExecutionReports(reports)]
2070        }
2071    }
2072
2073    fn handle_account_trades<'a>(
2074        &self,
2075        trades: impl IntoIterator<Item = &'a LighterTrade>,
2076        _ts_init: UnixNanos,
2077    ) -> Vec<NautilusWsMessage> {
2078        let Some((_account_id, account_index)) = self.exec_account else {
2079            log::debug!("Lighter account_trades frame skipped: no execution context set");
2080            return Vec::new();
2081        };
2082
2083        let mut reports = Vec::new();
2084
2085        for trade in trades {
2086            if !self.instruments.contains_key(&trade.market_id) {
2087                log::debug!(
2088                    "No instrument cached for Lighter account trade market_id={}",
2089                    trade.market_id,
2090                );
2091                continue;
2092            }
2093
2094            // The venue shares the channel with crossed pairs the account is
2095            // not part of; gate the cheap account check at the handler so
2096            // unrelated traffic never reaches the execution consumer.
2097            if trade.bid_account_id != account_index && trade.ask_account_id != account_index {
2098                continue;
2099            }
2100
2101            reports.push(ExecutionReport::Fill(trade.clone()));
2102        }
2103
2104        if reports.is_empty() {
2105            Vec::new()
2106        } else {
2107            vec![NautilusWsMessage::ExecutionReports(reports)]
2108        }
2109    }
2110
2111    fn handle_account_positions(
2112        &self,
2113        positions: &AHashMap<Ustr, LighterPosition>,
2114        ts_init: UnixNanos,
2115        frame_type: PositionFrameType,
2116    ) -> Vec<NautilusWsMessage> {
2117        let Some((account_id, _)) = self.exec_account else {
2118            log::debug!("Lighter account_positions frame skipped: no execution context set");
2119            return Vec::new();
2120        };
2121
2122        // Account transaction times stay on raw frames; reports use arrival time
2123        let ts_event = ts_init;
2124
2125        let mut reports = Vec::new();
2126        let mut skipped_market_ids = Vec::new();
2127        let mut closed_market_ids = Vec::new();
2128
2129        for position in positions.values() {
2130            if position.position.is_zero() {
2131                if matches!(frame_type, PositionFrameType::Update) {
2132                    closed_market_ids.push(position.market_id);
2133                }
2134                continue;
2135            }
2136
2137            let Some(instrument) = self.instruments.get(&position.market_id) else {
2138                log::debug!(
2139                    "No instrument cached for Lighter position market_id={}",
2140                    position.market_id,
2141                );
2142
2143                skipped_market_ids.push(position.market_id);
2144                continue;
2145            };
2146
2147            match parse_ws_position_status_report(
2148                position, instrument, account_id, ts_event, ts_init,
2149            ) {
2150                Ok(report) => reports.push(report),
2151                Err(e) => {
2152                    skipped_market_ids.push(position.market_id);
2153                    log::error!("Error parsing Lighter position status report: {e}");
2154                }
2155            }
2156        }
2157
2158        match frame_type {
2159            PositionFrameType::Snapshot => {
2160                // Emit even when empty: signals the last position closed.
2161                vec![NautilusWsMessage::PositionSnapshot {
2162                    reports,
2163                    skipped_market_ids,
2164                }]
2165            }
2166            PositionFrameType::Update => vec![NautilusWsMessage::PositionUpdate {
2167                reports,
2168                closed_market_ids,
2169                skipped_market_ids,
2170            }],
2171        }
2172    }
2173
2174    fn handle_account_assets(
2175        &self,
2176        assets: &AHashMap<Ustr, LighterAsset>,
2177        timestamp_ms: u64,
2178        ts_init: UnixNanos,
2179    ) -> Vec<NautilusWsMessage> {
2180        let Some((account_id, _)) = self.exec_account else {
2181            log::debug!("Lighter account_assets frame skipped: no execution context set");
2182            return Vec::new();
2183        };
2184
2185        let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2186            Ok(ts) => ts,
2187            Err(e) => {
2188                log::error!("Invalid Lighter account_assets timestamp {timestamp_ms}: {e}");
2189                return Vec::new();
2190            }
2191        };
2192
2193        self.account_state_reconciler.update_assets(assets);
2194        self.emit_unified_account_state(account_id, ts_event, ts_init)
2195    }
2196
2197    fn handle_user_stats(
2198        &self,
2199        stats: &LighterUserStats,
2200        timestamp_ms: u64,
2201        ts_init: UnixNanos,
2202    ) -> Vec<NautilusWsMessage> {
2203        let Some((account_id, _)) = self.exec_account else {
2204            log::debug!("Lighter user_stats frame skipped: no execution context set");
2205            return Vec::new();
2206        };
2207
2208        let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2209            Ok(ts) => ts,
2210            Err(e) => {
2211                log::error!("Invalid Lighter user_stats timestamp {timestamp_ms}: {e}");
2212                return Vec::new();
2213            }
2214        };
2215
2216        self.account_state_reconciler.update_user_stats(stats);
2217        self.emit_unified_account_state(account_id, ts_event, ts_init)
2218    }
2219
2220    /// Asks the reconciler for a merged [`AccountState`] and wraps it as a
2221    /// `NautilusWsMessage`. Returns an empty vec when the second stream
2222    /// hasn't arrived yet; the AccountStreamFirstFrame marker still gets
2223    /// pushed by the caller so the startup gate progresses.
2224    fn emit_unified_account_state(
2225        &self,
2226        account_id: AccountId,
2227        ts_event: UnixNanos,
2228        ts_init: UnixNanos,
2229    ) -> Vec<NautilusWsMessage> {
2230        match self
2231            .account_state_reconciler
2232            .build_state(account_id, ts_event, ts_init)
2233        {
2234            Some(Ok(state)) => vec![NautilusWsMessage::AccountState(Box::new(state))],
2235            Some(Err(e)) => {
2236                log::error!("Error building unified Lighter account state: {e}");
2237                Vec::new()
2238            }
2239            None => Vec::new(),
2240        }
2241    }
2242}
2243
2244#[derive(Clone, Copy)]
2245enum PositionFrameType {
2246    Snapshot,
2247    Update,
2248}
2249
2250fn raw_message(text: &str) -> Vec<NautilusWsMessage> {
2251    match serde_json::from_str(text) {
2252        Ok(value) => vec![NautilusWsMessage::Raw(value)],
2253        Err(e) => {
2254            log::error!("Failed to forward raw Lighter WS frame: {e}");
2255            Vec::new()
2256        }
2257    }
2258}
2259
2260// Codes outside the transaction range (e.g. 30003 "Already Subscribed")
2261// would falsely reject a live order; see `LIGHTER_ERROR_CODE_TX_RANGE`.
2262fn is_sendtx_error_code(code: Option<u64>) -> bool {
2263    code.is_some_and(|c| LIGHTER_ERROR_CODE_TX_RANGE.contains(&c))
2264}
2265
2266fn already_subscribed_error(value: &serde_json::Value) -> Option<&serde_json::Value> {
2267    if value.get("code").and_then(|code| code.as_u64())
2268        == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2269    {
2270        Some(value)
2271    } else {
2272        value.get("error").filter(|e| {
2273            e.get("code").and_then(|code| code.as_u64())
2274                == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2275        })
2276    }
2277}
2278
2279fn subscription_error_code(value: &serde_json::Value) -> Option<u64> {
2280    value
2281        .get("code")
2282        .and_then(|code| code.as_u64())
2283        .or_else(|| {
2284            value
2285                .get("error")
2286                .and_then(|e| e.get("code"))
2287                .and_then(|code| code.as_u64())
2288        })
2289}
2290
2291fn typed_subscribe_topic(text: &str) -> Option<&str> {
2292    let header = serde_json::from_str::<LighterWsFrameHeader<'_>>(text).ok()?;
2293    header
2294        .kind
2295        .starts_with("subscribed/")
2296        .then_some(header.channel?)
2297}
2298
2299// SendTxRejected from a top-level `{code, message}` frame: non-200 sendTx
2300// ACK or `{"type":"error",...}`.
2301fn send_tx_rejected_from_value(
2302    value: &serde_json::Value,
2303    source: SendTxRejectionSource,
2304) -> NautilusWsMessage {
2305    let code = value.get("code").and_then(|v| v.as_i64());
2306    let message = value
2307        .get("message")
2308        .and_then(|v| v.as_str())
2309        .unwrap_or("")
2310        .to_string();
2311    let tx_hash = value
2312        .get("tx_hash")
2313        .and_then(|v| v.as_str())
2314        .map(str::to_string);
2315    NautilusWsMessage::SendTxRejected {
2316        connection_epoch: 0,
2317        source,
2318        code,
2319        message,
2320        tx_hash,
2321    }
2322}
2323
2324// SendTxRejected from a nested `{"error":{"code":N,"message":...}}` frame;
2325// these frames never carry a `tx_hash`.
2326fn send_tx_rejected_from_nested_error(
2327    error: &serde_json::Value,
2328    source: SendTxRejectionSource,
2329) -> NautilusWsMessage {
2330    let code = error.get("code").and_then(|v| v.as_i64());
2331    let message = error
2332        .get("message")
2333        .and_then(|v| v.as_str())
2334        .unwrap_or("")
2335        .to_string();
2336    NautilusWsMessage::SendTxRejected {
2337        connection_epoch: 0,
2338        source,
2339        code,
2340        message,
2341        tx_hash: None,
2342    }
2343}
2344
2345fn log_integrator_not_approved() {
2346    log::error!(
2347        "Lighter venue rejected with code {LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED} \
2348         'integrator is not approved'.\n\
2349         Tagged orders require Nautilus integrator approval. \
2350         See: {LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL}",
2351    );
2352}
2353
2354fn market_index_from_topic(topic: &str) -> Option<i64> {
2355    let (_, rest) = topic.split_once(':')?;
2356    rest.parse::<i64>().ok()
2357}
2358
2359fn candle_market_and_resolution_from_topic(topic: &str) -> Option<(i64, LighterCandleResolution)> {
2360    let (channel, rest) = topic.split_once(':')?;
2361    if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::Candle) {
2362        return None;
2363    }
2364    let (market, res) = rest.split_once(':')?;
2365    let market_index = market.parse::<i64>().ok()?;
2366    let resolution = res.parse::<LighterCandleResolution>().ok()?;
2367    Some((market_index, resolution))
2368}
2369
2370fn order_book_market_index_from_topic(topic: &str) -> Option<i64> {
2371    let (channel, rest) = topic.split_once(':')?;
2372    if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::OrderBook) {
2373        return None;
2374    }
2375
2376    rest.parse::<i64>().ok()
2377}
2378
2379pub(crate) fn should_retry_lighter_ws_error(error: &LighterWsError) -> bool {
2380    match error {
2381        LighterWsError::Network(_) => true,
2382        // Closed and BrokenPipe are terminal on this client; only Timeout
2383        // (wait_for_active) can recover if the connection comes up. WriteTimeout
2384        // is not retryable: the write was cancelled after it began, so the peer
2385        // may already have the message and a retry could duplicate it.
2386        LighterWsError::Transport(send_error) => match send_error {
2387            SendError::Timeout => true,
2388            SendError::InvalidInput(_)
2389            | SendError::BufferFull
2390            | SendError::Closed
2391            | SendError::ConnectionChanged
2392            | SendError::BrokenPipe(_)
2393            | SendError::WriteTimeout => false,
2394        },
2395        LighterWsError::Authentication(_)
2396        | LighterWsError::Parse(_)
2397        | LighterWsError::Client(_)
2398        | LighterWsError::SendTxOutcomeUnknown(_) => false,
2399    }
2400}
2401
2402pub(crate) fn create_lighter_ws_timeout_error(_msg: String) -> LighterWsError {
2403    // Structured variant so the classifier retries; the retry manager
2404    // already logs the textual timeout context.
2405    LighterWsError::Transport(SendError::Timeout)
2406}
2407
2408#[cfg(test)]
2409mod tests {
2410    use std::time::Duration;
2411
2412    use futures_util::FutureExt;
2413    use log::{Level, LevelFilter, Log, Metadata, Record};
2414    use nautilus_model::{
2415        enums::{AccountType, BookAction, RecordFlag},
2416        identifiers::{InstrumentId, Symbol, Venue},
2417        instruments::{CryptoPerpetual, CurrencyPair},
2418        types::{Currency, Money, Price, Quantity},
2419    };
2420    use parking_lot::Mutex;
2421    use rstest::rstest;
2422    use rust_decimal::Decimal;
2423    use serde_json::json;
2424
2425    use super::*;
2426    use crate::{
2427        common::enums::{LighterCandleResolution, LighterTxStatus, LighterTxType},
2428        websocket::messages::{LighterMarketSelection, LighterWsCandle, LighterWsChannel},
2429    };
2430
2431    const SECRET_MARKER: &str = "426426426";
2432
2433    struct OutboundLogCapture {
2434        messages: Mutex<Vec<String>>,
2435    }
2436
2437    static OUTBOUND_LOG_CAPTURE: OutboundLogCapture = OutboundLogCapture {
2438        messages: Mutex::new(Vec::new()),
2439    };
2440
2441    impl OutboundLogCapture {
2442        fn clear(&self) {
2443            self.messages.lock().clear();
2444        }
2445
2446        fn messages(&self) -> Vec<String> {
2447            self.messages.lock().clone()
2448        }
2449    }
2450
2451    impl Log for OutboundLogCapture {
2452        fn enabled(&self, metadata: &Metadata<'_>) -> bool {
2453            matches!(metadata.level(), Level::Debug | Level::Warn)
2454                && metadata.target() == "nautilus_lighter::websocket::handler"
2455        }
2456
2457        fn log(&self, record: &Record<'_>) {
2458            if self.enabled(record.metadata()) {
2459                let message = record.args().to_string();
2460                if (record.level() == Level::Debug
2461                    && message.starts_with("Sending Lighter unsubscribe"))
2462                    || (record.level() == Level::Warn
2463                        && message.starts_with("Lighter subscribe error"))
2464                {
2465                    self.messages.lock().push(message);
2466                }
2467            }
2468        }
2469
2470        fn flush(&self) {}
2471    }
2472
2473    const WS_ACCOUNT_ORDERS_UPDATE: &str =
2474        include_str!("../../test_data/ws_account_orders_update.json");
2475    const WS_ACCOUNT_ALL_TRADES_UPDATE: &str =
2476        include_str!("../../test_data/ws_account_all_trades_update.json");
2477    const WS_ACCOUNT_ALL_POSITIONS_UPDATE: &str =
2478        include_str!("../../test_data/ws_account_all_positions_update.json");
2479    const WS_ACCOUNT_ALL_ASSETS_UPDATE: &str =
2480        include_str!("../../test_data/ws_account_all_assets_update.json");
2481    const WS_USER_STATS_UPDATE: &str = include_str!("../../test_data/ws_user_stats_update.json");
2482    const WS_ACCOUNT_ALL_ASSETS_WITH_POSITION: &str =
2483        include_str!("../../test_data/ws_account_all_assets_with_position.json");
2484    const WS_USER_STATS_WITH_POSITION: &str =
2485        include_str!("../../test_data/ws_user_stats_with_position.json");
2486    const WS_MARKET_STATS_UPDATE_SINGLE: &str =
2487        include_str!("../../test_data/ws_market_stats_update_single.json");
2488    const WS_MARKET_STATS_UPDATE_ALL: &str =
2489        include_str!("../../test_data/ws_market_stats_update_all.json");
2490    const WS_SPOT_MARKET_STATS_UPDATE_SINGLE: &str =
2491        include_str!("../../test_data/ws_spot_market_stats_update_single.json");
2492    const WS_SPOT_MARKET_STATS_UPDATE_ALL: &str =
2493        include_str!("../../test_data/ws_spot_market_stats_update_all.json");
2494    const WS_CANDLE_SUBSCRIBED: &str = include_str!("../../test_data/ws_candle_subscribed.json");
2495    const WS_SPOT_STATS_SUBSCRIBED_BAD_BODY: &str =
2496        include_str!("../../test_data/ws_spot_market_stats_subscribed_single_bad_body.json");
2497    const WS_BOOK_SUBSCRIBED_BAD_BODY: &str =
2498        include_str!("../../test_data/ws_order_book_subscribed_bad_body.json");
2499    const WS_SUBSCRIBE_FAILED: &str =
2500        include_str!("../../test_data/ws_order_book_subscribe_failed.json");
2501    const BOOK_SUBSCRIBE_REJECTION: &str =
2502        "client error: venue rejected the WebSocket subscribe with code 30012: order_book:0";
2503
2504    fn handle_control_text(
2505        handler: &mut FeedHandler,
2506        text: &str,
2507    ) -> (bool, Option<NautilusWsMessage>) {
2508        let Ok(value) = serde_json::from_str::<serde_json::Value>(text) else {
2509            return (false, None);
2510        };
2511        handler.handle_control_value(&value)
2512    }
2513
2514    fn stub_eth_perp_instrument() -> InstrumentAny {
2515        let instrument_id = InstrumentId::new(Symbol::new("ETH-PERP"), Venue::new("LIGHTER"));
2516        InstrumentAny::CryptoPerpetual(
2517            CryptoPerpetual::builder()
2518                .instrument_id(instrument_id)
2519                .raw_symbol(Symbol::new("ETH-PERP"))
2520                .base_currency(Currency::from("ETH"))
2521                .quote_currency(Currency::from("USDC"))
2522                .settlement_currency(Currency::from("USDC"))
2523                .is_inverse(false)
2524                .price_precision(2)
2525                .size_precision(4)
2526                .price_increment(Price::from("0.01"))
2527                .size_increment(Quantity::from("0.0001"))
2528                .ts_event(UnixNanos::default())
2529                .ts_init(UnixNanos::default())
2530                .build()
2531                .unwrap(),
2532        )
2533    }
2534
2535    fn stub_eth_spot_instrument() -> InstrumentAny {
2536        let instrument_id = InstrumentId::new(Symbol::new("ETH-SPOT"), Venue::new("LIGHTER"));
2537        InstrumentAny::CurrencyPair(
2538            CurrencyPair::builder()
2539                .instrument_id(instrument_id)
2540                .raw_symbol(Symbol::new("ETH-SPOT"))
2541                .base_currency(Currency::from("ETH"))
2542                .quote_currency(Currency::from("USDC"))
2543                .price_precision(2)
2544                .size_precision(4)
2545                .price_increment(Price::from("0.01"))
2546                .size_increment(Quantity::from("0.0001"))
2547                .ts_event(UnixNanos::default())
2548                .ts_init(UnixNanos::default())
2549                .build()
2550                .unwrap(),
2551        )
2552    }
2553
2554    fn make_handler_with_account() -> FeedHandler {
2555        let signal = Arc::new(AtomicBool::new(false));
2556        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
2557        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
2558        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
2559        let mut handler =
2560            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2561        handler.instruments.insert(0, stub_eth_perp_instrument());
2562        handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
2563        handler
2564    }
2565
2566    // A recovery past its retry budget keeps running, so a late snapshot still restores the book
2567    #[rstest]
2568    fn book_late_snapshot_completes_running_recovery() {
2569        let mut handler = make_handler_with_account();
2570        handler.book.delta_subs.insert(0);
2571        let frame: LighterWsFrame = serde_json::from_str(include_str!(
2572            "../../test_data/ws_order_book_subscribed.json"
2573        ))
2574        .unwrap();
2575        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2576        handler.subscriptions.add_reference("order_book:0");
2577        assert!(recovery.begin_replacement());
2578
2579        let during_write = handler.handle_frame(frame.clone(), UnixNanos::from(1));
2580        recovery.gate.open();
2581        let after_write = handler.handle_frame(frame, UnixNanos::from(2));
2582
2583        assert!(during_write.is_empty());
2584        assert_eq!(after_write.len(), 1);
2585        assert!(recovery.is_accepted());
2586        assert!(handler.book.snapshots_seen.contains(&0));
2587    }
2588
2589    #[rstest]
2590    #[case::unconfirmed_permanent(false, false, false)]
2591    #[case::closed_gate_permanent(true, false, false)]
2592    #[case::confirmed_permanent(true, true, false)]
2593    #[case::unconfirmed_retry(false, false, true)]
2594    #[case::closed_gate_retry(true, false, true)]
2595    #[case::confirmed_retry(true, true, true)]
2596    fn book_venue_rejection_requires_confirmed_write(
2597        #[case] confirmed: bool,
2598        #[case] gate_open: bool,
2599        #[case] retry: bool,
2600    ) {
2601        let mut handler = make_handler_with_account();
2602        let (topic, generation) =
2603            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2604        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2605        assert!(recovery.begin_replacement());
2606
2607        if gate_open {
2608            recovery.gate.open();
2609        }
2610
2611        if !confirmed {
2612            handler.book.expected.remove(&0);
2613        }
2614
2615        if retry {
2616            handler.retry_inflight_subscriptions("venue rejection");
2617        } else {
2618            let value = serde_json::json!({"code": 30012, "channel": "order_book:0"});
2619            handler.handle_control_value(&value);
2620        }
2621
2622        let rejected = confirmed && gate_open;
2623        assert_eq!(
2624            handler.inflight_subs.get(&topic),
2625            (!rejected).then_some(&generation)
2626        );
2627        assert!(!recovery.cancellation.is_cancelled());
2628        assert!(recovery.gate.lock().is_closed());
2629
2630        match &*recovery.outcome.borrow() {
2631            BookRecoveryOutcome::Pending => assert!(!confirmed || !gate_open),
2632            BookRecoveryOutcome::Rejected(LighterWsError::Network(message)) => {
2633                assert!(confirmed && gate_open && retry);
2634                assert_eq!(message, "venue rejection");
2635            }
2636            BookRecoveryOutcome::Rejected(LighterWsError::Client(message)) => {
2637                assert!(confirmed && gate_open && !retry);
2638                assert_eq!(
2639                    message,
2640                    "venue rejected the WebSocket subscribe with code 30012"
2641                );
2642            }
2643            outcome => panic!("unexpected recovery outcome: {outcome:?}"),
2644        }
2645    }
2646
2647    // A waiting subscriber owns a rejection and releases the stream; a replayed subscription
2648    // recovers instead of failing the book.
2649    #[rstest]
2650    #[case::waiting_subscriber(true)]
2651    #[case::replayed(false)]
2652    fn book_client_rejection_recovers_only_without_waiting_subscriber(#[case] waiting: bool) {
2653        let mut handler = make_handler_with_account();
2654        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2655        handler.cmd_tx = Some(cmd_tx);
2656        handler.book.delta_subs.insert(0);
2657        handler.subscriptions.add_reference("order_book:0");
2658        let (response, mut result) = tokio::sync::oneshot::channel();
2659        let (topic, generation) = mark_subscription_inflight(
2660            &mut handler,
2661            LighterWsChannel::OrderBook(0),
2662            waiting.then_some(response),
2663        );
2664        let initial = CancellationToken::new();
2665        handler.book.initial.insert(
2666            0,
2667            PendingSnapshot {
2668                deadline: None,
2669                cancel: initial.clone(),
2670                gate: SnapshotGate::default(),
2671            },
2672        );
2673
2674        handler.reject_book_subscription(
2675            0,
2676            generation,
2677            LighterWsError::Client("invalid market".into()),
2678        );
2679
2680        let recovering = handler
2681            .book
2682            .recovery
2683            .get(&0)
2684            .is_some_and(BookRecoveryState::is_running);
2685
2686        if waiting {
2687            assert!(
2688                matches!(result.try_recv(), Ok(Err(message)) if message.contains("invalid market"))
2689            );
2690        } else {
2691            assert_eq!(
2692                result.try_recv(),
2693                Err(tokio::sync::oneshot::error::TryRecvError::Closed)
2694            );
2695        }
2696
2697        assert_eq!(recovering, !waiting);
2698        assert_eq!(handler.book.work.len(), usize::from(!waiting));
2699        assert_eq!(handler.subscription_attempts.contains_key(&topic), !waiting);
2700        assert!(!handler.inflight_subs.contains_key(&topic));
2701        assert!(!handler.book.initial.contains_key(&0));
2702        assert!(initial.is_cancelled());
2703    }
2704
2705    // A rejected replacement answers the subscriber still waiting on the initial subscribe, and
2706    // the recovery keeps its gate closed until the subscriber releases the book.
2707    #[rstest]
2708    #[case::response_txs(false)]
2709    #[case::pending_response_txs(true)]
2710    #[tokio::test]
2711    async fn book_client_rejection_during_recovery_answers_waiting_subscriber(
2712        #[case] pending: bool,
2713    ) {
2714        let (mut handler, _cmd_tx, mut result) =
2715            recovering_book_with_waiting_subscriber(false).await;
2716        let topic = Ustr::from("order_book:0");
2717
2718        if pending {
2719            // Book subscribes carry no auth, so seed a waiter behind a pending auth change directly
2720            let attempt = handler.subscription_attempts.get_mut(&topic).unwrap();
2721            attempt.pending_auth = Some(SecretString::from("token"));
2722            attempt.pending_response_txs = std::mem::take(&mut attempt.response_txs);
2723        }
2724
2725        let recovery = Arc::clone(handler.book.recovery[&0].current().unwrap());
2726        let gate_opened = !recovery.gate.lock().is_closed();
2727        let pending = result.try_recv();
2728
2729        handle_control_text(&mut handler, WS_SUBSCRIBE_FAILED);
2730
2731        assert!(gate_opened);
2732        assert_eq!(
2733            pending,
2734            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2735        );
2736        assert_eq!(
2737            result.try_recv(),
2738            Ok(Err(BOOK_SUBSCRIBE_REJECTION.to_string()))
2739        );
2740        assert!(!handler.subscription_attempts.contains_key(&topic));
2741        assert!(!handler.inflight_subs.contains_key(&topic));
2742        assert!(!handler.book.expected.contains_key(&0));
2743        assert!(recovery.is_running());
2744        assert!(recovery.gate.lock().is_closed());
2745        assert!(matches!(
2746            &*recovery.outcome.borrow(),
2747            BookRecoveryOutcome::Rejected(LighterWsError::Client(message))
2748                if message == "venue rejected the WebSocket subscribe with code 30012"
2749        ));
2750    }
2751
2752    // The rejected subscriber's release ends the recovery unless another consumer still references
2753    // the book, whose next replacement then queues a fresh attempt; either way a late snapshot
2754    // cannot reopen the book.
2755    #[rstest]
2756    #[case::deltas_last_reference(false, false)]
2757    #[case::deltas_other_consumer(false, true)]
2758    #[case::depth_last_reference(true, false)]
2759    #[case::depth_other_consumer(true, true)]
2760    #[tokio::test]
2761    async fn book_rejected_subscriber_release_cancels_orphaned_recovery(
2762        #[case] depth: bool,
2763        #[case] other_consumer: bool,
2764    ) {
2765        let (mut handler, cmd_tx, mut result) =
2766            recovering_book_with_waiting_subscriber(depth).await;
2767        let topic = Ustr::from("order_book:0");
2768
2769        if other_consumer {
2770            let other = if depth {
2771                &mut handler.book.delta_subs
2772            } else {
2773                &mut handler.book.depth_subs
2774            };
2775
2776            other.insert(0);
2777            assert!(!handler.subscriptions.add_reference("order_book:0"));
2778        }
2779
2780        let recovery = Arc::clone(handler.book.recovery[&0].current().unwrap());
2781        handle_control_text(&mut handler, WS_SUBSCRIBE_FAILED);
2782        let answer = result.try_recv();
2783
2784        // Release as the rejected client does: drop the reference, then clear the consumer flag
2785        let released = handler.subscriptions.remove_reference("order_book:0");
2786
2787        let release = if depth {
2788            HandlerCommand::SetDepthSub {
2789                market_index: 0,
2790                subscribed: false,
2791            }
2792        } else {
2793            HandlerCommand::SetBookDeltasSub {
2794                market_index: 0,
2795                subscribed: false,
2796            }
2797        };
2798
2799        cmd_tx.send(release).unwrap();
2800        cmd_tx.send(HandlerCommand::Disconnect).unwrap();
2801        assert!(handler.next().await.is_none());
2802
2803        let frame: LighterWsFrame = serde_json::from_str(include_str!(
2804            "../../test_data/ws_order_book_subscribed.json"
2805        ))
2806        .unwrap();
2807        let late = handler.handle_frame(frame, UnixNanos::from(1));
2808        let (completion, _completed) = tokio::sync::oneshot::channel();
2809        handler.queue_book_replacement(
2810            0,
2811            BookWrite {
2812                cancel: recovery.cancellation.child_token(),
2813                gate: recovery.gate.clone(),
2814                completion,
2815            },
2816        );
2817
2818        let replacement = handler.subscription_attempts.get(&topic).map(|attempt| {
2819            (
2820                attempt.response_txs.len(),
2821                handler.pending_subs.contains(&(topic, attempt.generation)),
2822            )
2823        });
2824
2825        assert_eq!(answer, Ok(Err(BOOK_SUBSCRIBE_REJECTION.to_string())));
2826        assert_eq!(released, !other_consumer);
2827        assert_eq!(recovery.is_running(), other_consumer);
2828        assert_eq!(handler.book.recovery.contains_key(&0), other_consumer);
2829        assert_eq!(handler.book.work.len(), usize::from(other_consumer));
2830        assert!(!recovery.is_accepted());
2831        assert!(recovery.gate.lock().is_closed());
2832        assert!(late.is_empty());
2833        assert!(!handler.book.snapshots_seen.contains(&0));
2834        assert_eq!(handler.book.writes.contains_key(&0), other_consumer);
2835        assert_eq!(replacement, other_consumer.then_some((0, true)));
2836    }
2837
2838    // Rate limits the initial book subscribe, which starts a recovery, then confirms the
2839    // recovery's replacement write while the initial subscriber still waits.
2840    async fn recovering_book_with_waiting_subscriber(
2841        depth: bool,
2842    ) -> (
2843        FeedHandler,
2844        tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2845        tokio::sync::oneshot::Receiver<Result<(), String>>,
2846    ) {
2847        let signal = Arc::new(AtomicBool::new(false));
2848        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2849        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
2850        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
2851        let mut handler =
2852            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2853        handler.set_command_sender(cmd_tx.clone());
2854        handler.instruments.insert(0, stub_eth_perp_instrument());
2855
2856        if depth {
2857            handler.book.depth_subs.insert(0);
2858        } else {
2859            handler.book.delta_subs.insert(0);
2860        }
2861
2862        handler.subscriptions.add_reference("order_book:0");
2863        let (response, result) = tokio::sync::oneshot::channel();
2864        mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(response));
2865        handle_control_text(
2866            &mut handler,
2867            r#"{"type":"error","code":30009,"message":"rate limit exceeded"}"#,
2868        );
2869
2870        // The first attempt requests its replacement on the first poll
2871        assert!(handler.book.work.next().now_or_never().is_none());
2872
2873        let Ok(HandlerCommand::RecoverBook {
2874            market_index,
2875            cancel,
2876            gate,
2877            completion,
2878        }) = handler.cmd_rx.try_recv()
2879        else {
2880            panic!("expected book replacement request");
2881        };
2882
2883        handler.queue_book_replacement(
2884            market_index,
2885            BookWrite {
2886                cancel,
2887                gate,
2888                completion,
2889            },
2890        );
2891
2892        handler.pump_pending_subscribes().await;
2893
2894        // The send fails without a socket, so confirm the write as the connection would
2895        let Some(BookWorkResult::Sent {
2896            market_index,
2897            generation,
2898            cancel,
2899            write,
2900            ..
2901        }) = handler.book.work.next().await
2902        else {
2903            panic!("expected replacement send");
2904        };
2905
2906        handler.complete_book_send(market_index, generation, cancel, write, Ok(0));
2907
2908        (handler, cmd_tx, result)
2909    }
2910
2911    // A failed replacement frees its slot during backoff so other subscriptions can proceed
2912    #[rstest]
2913    #[case::rejected(true)]
2914    #[case::send_failed(false)]
2915    #[tokio::test]
2916    async fn book_failed_replacement_releases_subscription_slot(#[case] rejected: bool) {
2917        let mut handler = make_handler_with_account();
2918        handler.book.delta_subs.insert(0);
2919        handler.subscriptions.add_reference("order_book:0");
2920        let (topic, generation) =
2921            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2922
2923        for i in 0..SUBSCRIBE_INFLIGHT_MAX - 1 {
2924            handler
2925                .inflight_subs
2926                .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
2927        }
2928
2929        handler.queue_subscribe(LighterWsChannel::OrderBook(1), None, None);
2930        let other = Ustr::from("order_book:1");
2931        handler.pump_pending_subscribes().await;
2932        let blocked = !handler.inflight_subs.contains_key(&other);
2933        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2934        assert!(recovery.begin_replacement());
2935        recovery.gate.open();
2936        let (completion, completed) = tokio::sync::oneshot::channel();
2937
2938        if rejected {
2939            handler.reject_book_subscription(
2940                0,
2941                generation,
2942                LighterWsError::Client("venue rejection".into()),
2943            );
2944        } else {
2945            handler.complete_book_send(
2946                0,
2947                generation,
2948                recovery.cancellation.child_token(),
2949                Some(BookWrite {
2950                    cancel: recovery.cancellation.child_token(),
2951                    gate: recovery.gate.clone(),
2952                    completion,
2953                }),
2954                Err(LighterWsError::Network("send failed".into())),
2955            );
2956        }
2957
2958        handler.pump_pending_subscribes().await;
2959
2960        assert!(blocked);
2961        assert!(!handler.inflight_subs.contains_key(&topic));
2962        assert!(handler.inflight_subs.contains_key(&other));
2963        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
2964        assert!(recovery.is_running());
2965
2966        if rejected {
2967            assert!(matches!(
2968                &*recovery.outcome.borrow(),
2969                BookRecoveryOutcome::Rejected(LighterWsError::Client(message))
2970                    if message == "venue rejection"
2971            ));
2972        } else {
2973            assert!(
2974                matches!(completed.await, Ok(Err(LighterWsError::Network(message))) if message == "send failed")
2975            );
2976        }
2977    }
2978
2979    #[tokio::test]
2980    async fn book_queued_write_skips_after_last_reference_disappears() {
2981        let mut handler = make_handler_with_account();
2982        handler.book.delta_subs.insert(0);
2983        let channel = LighterWsChannel::OrderBook(0);
2984        handler.subscriptions.add_reference(&channel.topic_key());
2985        let (_, generation) = mark_subscription_inflight(&mut handler, channel.clone(), None);
2986        handler.send_book_subscribe(0, generation);
2987        assert!(handler.subscriptions.remove_reference(&channel.topic_key()));
2988
2989        let BookWorkResult::Sent { result, .. } = handler.book.work.next().await.unwrap() else {
2990            panic!("expected send completion");
2991        };
2992
2993        assert_eq!(
2994            result.unwrap_err().to_string(),
2995            "client error: book subscription cancelled"
2996        );
2997        assert!(handler.inner.is_none());
2998    }
2999
3000    #[rstest]
3001    fn book_wrong_epoch_snapshot_preserves_current_completion() {
3002        let mut handler = make_handler_with_account();
3003        let (tx, mut rx) = tokio::sync::oneshot::channel();
3004        let (topic, generation) =
3005            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
3006        handler.book.expected.insert(0, (generation, 12));
3007
3008        assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
3009        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
3010        assert_eq!(
3011            rx.try_recv(),
3012            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
3013        );
3014        assert!(handler.complete_typed_subscription(topic.as_str(), 12));
3015        assert_eq!(rx.try_recv(), Ok(Ok(())));
3016        assert!(!handler.inflight_subs.contains_key(&topic));
3017    }
3018
3019    #[rstest]
3020    fn book_control_ack_preserves_snapshot_but_not_successor_completion() {
3021        let mut handler = make_handler_with_account();
3022        let (topic, old_generation) =
3023            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3024        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3025        assert!(handler.complete_typed_subscription(topic.as_str(), 0));
3026        assert_eq!(handler.book.expected[&0], (old_generation, 0));
3027
3028        handler.subscriptions.mark_failure(topic.as_str());
3029        let (_, generation) =
3030            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3031        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3032        handler.subscriptions.mark_failure(topic.as_str());
3033        let (tx, mut rx) = tokio::sync::oneshot::channel();
3034        let (_, successor) =
3035            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
3036
3037        assert_ne!(generation, successor);
3038        assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
3039        assert_eq!(handler.inflight_subs.get(&topic), Some(&successor));
3040        assert_eq!(
3041            rx.try_recv(),
3042            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
3043        );
3044        assert!(handler.complete_typed_subscription(topic.as_str(), 0));
3045        assert_eq!(rx.try_recv(), Ok(Ok(())));
3046    }
3047
3048    #[rstest]
3049    fn book_old_epoch_does_not_consume_current_trailing_snapshot() {
3050        let mut handler = make_handler_with_account();
3051        let (topic, generation) =
3052            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3053        handler.book.expected.insert(0, (generation, 12));
3054        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3055
3056        assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
3057        assert_eq!(handler.book.trailing.get(&0), Some(&(generation, 12)));
3058        assert_eq!(
3059            handler.ignored_completions.get(&topic),
3060            Some(&CompletionKind::Typed)
3061        );
3062        assert!(handler.complete_typed_subscription(topic.as_str(), 12));
3063        assert!(!handler.book.trailing.contains_key(&0));
3064        assert!(!handler.ignored_completions.contains_key(&topic));
3065    }
3066
3067    #[rstest]
3068    fn book_ack_before_write_cannot_complete_subscription() {
3069        let mut handler = make_handler_with_account();
3070        let (topic, generation) =
3071            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3072        handler.book.expected.clear();
3073
3074        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3075        assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
3076        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
3077    }
3078
3079    #[rstest]
3080    fn book_empty_snapshot_replaces_cached_levels_for_both_consumers() {
3081        let mut handler = make_handler_with_account();
3082        handler.book.delta_subs.insert(0);
3083        handler.book.depth_subs.insert(0);
3084        let frame: LighterWsFrame = serde_json::from_str(include_str!(
3085            "../../test_data/ws_order_book_subscribed.json"
3086        ))
3087        .unwrap();
3088        assert_eq!(
3089            handler
3090                .handle_frame(frame.clone(), UnixNanos::from(1))
3091                .len(),
3092            2
3093        );
3094
3095        let LighterWsFrame::OrderBookSnapshot { order_book, .. } = &frame else {
3096            panic!("expected book snapshot");
3097        };
3098
3099        let mut empty = order_book.clone();
3100        empty.bids.clear();
3101        empty.asks.clear();
3102        empty.nonce += 1;
3103        let messages = handler.handle_order_book(
3104            Ustr::from("order_book:0"),
3105            &empty,
3106            123,
3107            true,
3108            UnixNanos::from(2),
3109        );
3110
3111        assert_eq!(messages.len(), 2);
3112
3113        let NautilusWsMessage::Deltas(deltas) = &messages[0] else {
3114            panic!("expected deltas");
3115        };
3116
3117        assert_eq!(deltas.deltas.len(), 1);
3118        assert_eq!(deltas.deltas[0].action, BookAction::Clear);
3119        assert_eq!(deltas.deltas[0].sequence, empty.nonce as u64);
3120        assert_eq!(
3121            deltas.deltas[0].flags,
3122            RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8
3123        );
3124
3125        let NautilusWsMessage::Depth(depth) = &messages[1] else {
3126            panic!("expected depth");
3127        };
3128
3129        assert!(depth.bids.is_empty());
3130        assert!(depth.asks.is_empty());
3131        assert!(depth.bid_counts.is_empty());
3132        assert!(depth.ask_counts.is_empty());
3133        assert!(handler.book.states[&0].book.bids.is_empty());
3134        assert!(handler.book.states[&0].book.asks.is_empty());
3135    }
3136
3137    #[tokio::test]
3138    async fn book_initial_deadline_is_cancelled_by_replacement_or_unsubscribe() {
3139        let mut handler = make_handler_with_account();
3140        handler.book.delta_subs.insert(0);
3141        let channel = LighterWsChannel::OrderBook(0);
3142        handler.subscriptions.add_reference(&channel.topic_key());
3143        let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
3144        let cancel = CancellationToken::new();
3145        handler.book.initial.insert(
3146            0,
3147            PendingSnapshot {
3148                deadline: None,
3149                cancel: cancel.clone(),
3150                gate: SnapshotGate::default(),
3151            },
3152        );
3153
3154        handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
3155        handler.book.cancel(0);
3156
3157        let BookWorkResult::Initial {
3158            cancel: completed,
3159            market_index,
3160        } = handler.book.work.next().await.unwrap()
3161        else {
3162            panic!("expected deadline completion");
3163        };
3164
3165        assert_eq!(market_index, 0);
3166        assert!(completed.is_cancelled());
3167        assert!(cancel.is_cancelled());
3168        assert!(handler.book.recovery.is_empty());
3169        assert!(handler.book.expected.is_empty());
3170    }
3171
3172    #[tokio::test]
3173    async fn book_initial_deadline_expires_without_cancellation() {
3174        let mut handler = make_handler_with_account();
3175        handler.book_snapshot_timeout = Duration::from_millis(50);
3176        handler.book.delta_subs.insert(0);
3177        let channel = LighterWsChannel::OrderBook(0);
3178        handler.subscriptions.add_reference(&channel.topic_key());
3179        let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
3180        let cancel = CancellationToken::new();
3181        handler.book.initial.insert(
3182            0,
3183            PendingSnapshot {
3184                deadline: None,
3185                cancel: cancel.clone(),
3186                gate: SnapshotGate::default(),
3187            },
3188        );
3189
3190        handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
3191
3192        let BookWorkResult::Initial {
3193            market_index,
3194            cancel: completed,
3195        } = handler.book.work.next().await.unwrap()
3196        else {
3197            panic!("expected deadline completion");
3198        };
3199
3200        assert_eq!(market_index, 0);
3201        assert!(!completed.is_cancelled());
3202        assert!(!cancel.is_cancelled());
3203    }
3204
3205    #[tokio::test]
3206    async fn book_reconnect_retains_recovery_owner_and_retires_old_writes() {
3207        let signal = Arc::new(AtomicBool::new(false));
3208        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
3209        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
3210        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
3211        let mut handler =
3212            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3213        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
3214        handler.book.expected.insert(0, (3, 9));
3215        raw_tx
3216            .send((10, Message::Text(RECONNECTED.into())))
3217            .unwrap();
3218        assert!(matches!(
3219            handler.next().await,
3220            Some(NautilusWsMessage::Reconnected {
3221                connection_epoch: 10
3222            })
3223        ));
3224
3225        assert!(Arc::ptr_eq(
3226            handler.book.recovery[&0].current().unwrap(),
3227            &recovery
3228        ));
3229        assert!(!recovery.cancellation.is_cancelled());
3230        assert!(recovery.gate.lock().is_closed());
3231        assert!(matches!(
3232            *recovery.outcome.borrow(),
3233            BookRecoveryOutcome::Rejected(LighterWsError::Network(_))
3234        ));
3235        assert!(handler.book.expected.is_empty());
3236        assert!(handler.book.writes.is_empty());
3237    }
3238
3239    #[rstest]
3240    fn book_recovery_is_scoped_to_handler_and_market() {
3241        let mut first = make_handler_with_account();
3242        let mut second = make_handler_with_account();
3243        let recovery = first.book.recovery.entry(0).or_default().claim().unwrap();
3244        let other_market = first.book.recovery.entry(1).or_default().claim().unwrap();
3245        let other_socket = second.book.recovery.entry(0).or_default().claim().unwrap();
3246        first.book.cancel(0);
3247
3248        assert!(recovery.cancellation.is_cancelled());
3249        assert!(!other_market.cancellation.is_cancelled());
3250        assert!(!other_socket.cancellation.is_cancelled());
3251    }
3252
3253    #[rstest]
3254    #[tokio::test]
3255    async fn test_subscription_logs_omit_payload_body() {
3256        log::set_logger(&OUTBOUND_LOG_CAPTURE).expect("test logger already installed");
3257        log::set_max_level(LevelFilter::Debug);
3258
3259        let mut handler = make_handler_with_account();
3260        let account_index = SECRET_MARKER.parse::<i64>().unwrap();
3261        let channel = LighterWsChannel::AccountAll(account_index);
3262        let payload_len = serde_json::to_string(&LighterWsRequest::unsubscribe(
3263            channel.subscription_channel(),
3264        ))
3265        .unwrap()
3266        .len();
3267        OUTBOUND_LOG_CAPTURE.clear();
3268
3269        handler.dispatch_unsubscribe(channel).await;
3270        handler.handle_control_value(&json!({"code": 30012, "auth": SECRET_MARKER}));
3271        handler.handle_control_value(&json!({
3272            "code": 30012, "channel": "trade/999", "auth": SECRET_MARKER,
3273        }));
3274
3275        let messages = OUTBOUND_LOG_CAPTURE.messages();
3276
3277        assert!(
3278            messages
3279                .iter()
3280                .all(|message| !message.contains(SECRET_MARKER)),
3281            "outbound logs exposed the secret marker: {messages:?}"
3282        );
3283        assert!(
3284            messages.iter().any(|message| {
3285                message == &format!("Sending Lighter unsubscribe ({payload_len} bytes)")
3286            }),
3287            "unsubscribe metadata missing or inaccurate: {messages:?}"
3288        );
3289        assert!(messages.iter().any(|message| {
3290            message == "Lighter subscribe error code=30012 channel=none: retrying inflight subscriptions"
3291        }));
3292        assert!(messages.iter().any(|message| {
3293            message == "Lighter subscribe error code=30012 channel=trade/999: no matching inflight subscription"
3294        }));
3295    }
3296
3297    fn mark_subscription_inflight(
3298        handler: &mut FeedHandler,
3299        channel: LighterWsChannel,
3300        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
3301    ) -> (Ustr, u64) {
3302        handler.queue_subscribe(channel, None, response_tx);
3303        let (topic, generation) = handler
3304            .pending_subs
3305            .pop_front()
3306            .expect("subscription was not queued");
3307        handler.inflight_subs.insert(topic, generation);
3308        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
3309            handler.book.expected.insert(market_index, (generation, 0));
3310        }
3311
3312        (topic, generation)
3313    }
3314
3315    fn saturate_subscription_gate(handler: &mut FeedHandler) {
3316        for i in 0..SUBSCRIBE_INFLIGHT_MAX {
3317            handler
3318                .inflight_subs
3319                .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
3320        }
3321    }
3322
3323    /// Strip the trailing `AccountStreamFirstFrame` marker from a handler
3324    /// emission so account-stream tests can assert on the typed payload
3325    /// alone. The marker contract is pinned in
3326    /// `handle_frame_emits_account_stream_first_frame_marker_per_variant`.
3327    fn strip_account_marker(mut msgs: Vec<NautilusWsMessage>) -> Vec<NautilusWsMessage> {
3328        if matches!(
3329            msgs.last(),
3330            Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
3331        ) {
3332            msgs.pop();
3333        }
3334        msgs
3335    }
3336
3337    #[rstest]
3338    #[case::execution_context(true)]
3339    #[case::raw_without_context(false)]
3340    fn account_tx_routes_execution_results(#[case] execution_context: bool) {
3341        let mut handler = make_handler_with_account();
3342        if !execution_context {
3343            handler.exec_account = None;
3344        }
3345
3346        let text = include_str!("../../test_data/ws_account_tx_update.json");
3347        let frame: LighterWsFrame = serde_json::from_str(text).unwrap();
3348        let messages = handler.route_frame(frame, text, UnixNanos::from(11));
3349        assert_eq!(messages.len(), 1);
3350        let message = messages
3351            .into_iter()
3352            .next()
3353            .unwrap()
3354            .with_connection_epoch(7);
3355
3356        if execution_context {
3357            let NautilusWsMessage::AccountTx {
3358                connection_epoch,
3359                txs,
3360            } = message
3361            else {
3362                panic!("expected account transaction results");
3363            };
3364
3365            assert_eq!(connection_epoch, 7);
3366            assert_eq!(
3367                txs,
3368                vec![LighterWsTx {
3369                    hash: "abc123".to_string(),
3370                    status: LighterTxStatus::Executed,
3371                    event_info: "{\"ae\":\"\"}".to_string(),
3372                }]
3373            );
3374        } else {
3375            let NautilusWsMessage::Raw(value) = message else {
3376                panic!("expected raw transaction update");
3377            };
3378
3379            assert_eq!(
3380                value,
3381                serde_json::from_str::<serde_json::Value>(text).unwrap()
3382            );
3383        }
3384    }
3385
3386    #[rstest]
3387    fn account_tx_rejects_wrong_account_channel() {
3388        let mut handler = make_handler_with_account();
3389        let mut frame: LighterWsFrame =
3390            serde_json::from_str(include_str!("../../test_data/ws_account_tx_update.json"))
3391                .unwrap();
3392
3393        let LighterWsFrame::AccountTx { channel, .. } = &mut frame else {
3394            unreachable!()
3395        };
3396
3397        *channel = Ustr::from("account_tx:1235");
3398        assert!(handler.handle_frame(frame, UnixNanos::from(11)).is_empty());
3399    }
3400
3401    #[rstest]
3402    fn handle_frame_routes_account_orders_to_execution_reports() {
3403        let mut handler = make_handler_with_account();
3404        let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3405
3406        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3407
3408        assert_eq!(messages.len(), 1);
3409        match &messages[0] {
3410            NautilusWsMessage::ExecutionReports(reports) => {
3411                assert_eq!(reports.len(), 1);
3412                match &reports[0] {
3413                    super::ExecutionReport::Order(order) => {
3414                        assert_eq!(order.order_id, "281476929510110");
3415                        assert_eq!(order.client_order_id, "42");
3416                    }
3417                    other => panic!("expected order report, was {other:?}"),
3418                }
3419            }
3420            other => panic!("expected execution reports, was {other:?}"),
3421        }
3422    }
3423
3424    #[rstest]
3425    fn handle_frame_routes_account_trades_to_execution_reports() {
3426        let mut handler = make_handler_with_account();
3427        let frame: super::LighterWsFrame =
3428            serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
3429
3430        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3431
3432        assert_eq!(messages.len(), 1);
3433        match &messages[0] {
3434            NautilusWsMessage::ExecutionReports(reports) => {
3435                assert_eq!(reports.len(), 1);
3436                match &reports[0] {
3437                    super::ExecutionReport::Fill(fill) => {
3438                        // Bid side is the user's account in this fixture
3439                        // (see handle_frame_routes_account_trades_to_execution_reports
3440                        // fixture: bid_account_id == 1234). The raw payload
3441                        // carries the bid order id as the venue id; the
3442                        // execution loop derives `venue_order_id` from it.
3443                        assert_eq!(fill.bid_id_str.as_deref(), Some("562947905631053"),);
3444                    }
3445                    other => panic!("expected fill report, was {other:?}"),
3446                }
3447            }
3448            other => panic!("expected execution reports, was {other:?}"),
3449        }
3450    }
3451
3452    #[rstest]
3453    #[case::positions(include_str!("../../test_data/ws_account_all_positions_transaction_time.json"), true)]
3454    #[case::assets(include_str!("../../test_data/ws_account_all_assets_transaction_time.json"), false)]
3455    fn account_transaction_time_preserves_raw_frames_and_receive_order(
3456        #[case] fixture: &str,
3457        #[case] positions: bool,
3458    ) {
3459        let mut handler = make_handler_with_account();
3460        let mut raw_handler = make_handler_with_account();
3461        raw_handler.exec_account = None;
3462        let stats: LighterWsFrame = serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
3463        handler.handle_frame(stats, UnixNanos::from(10));
3464
3465        for (index, (transaction_time, amount)) in [
3466            (200, "1.5000"),
3467            (200, "2.5000"),
3468            (100, "3.5000"),
3469            (100, "3.5000"),
3470        ]
3471        .into_iter()
3472        .enumerate()
3473        {
3474            let mut value: serde_json::Value = serde_json::from_str(fixture).unwrap();
3475            value["transaction_time"] = json!(transaction_time);
3476            if positions {
3477                value["positions"]["0"]["position"] = json!(amount);
3478            } else {
3479                value["assets"]["3"]["balance"] = json!(amount);
3480            }
3481
3482            let frame: LighterWsFrame = serde_json::from_value(value.clone()).unwrap();
3483            let ts_init = UnixNanos::from(11 + index as u64);
3484            let raw = raw_handler.route_frame(frame.clone(), &value.to_string(), ts_init);
3485            let messages = strip_account_marker(handler.handle_frame(frame, ts_init));
3486
3487            assert_eq!(raw.len(), 1);
3488
3489            let NautilusWsMessage::Raw(raw_value) = &raw[0] else {
3490                panic!("expected raw account frame");
3491            };
3492
3493            assert_eq!(raw_value, &value);
3494            assert_eq!(messages.len(), 1);
3495
3496            match &messages[0] {
3497                NautilusWsMessage::PositionUpdate {
3498                    reports,
3499                    closed_market_ids,
3500                    skipped_market_ids,
3501                } => {
3502                    assert!(positions);
3503                    assert_eq!(reports.len(), 1);
3504                    assert_eq!(reports[0].quantity, Quantity::from(amount));
3505                    assert_eq!(reports[0].ts_last, ts_init);
3506                    assert_eq!(reports[0].ts_init, ts_init);
3507                    assert!(closed_market_ids.is_empty());
3508                    assert!(skipped_market_ids.is_empty());
3509                }
3510                NautilusWsMessage::AccountState(state) => {
3511                    assert!(!positions);
3512                    assert_eq!(state.balances.len(), 1);
3513                    assert_eq!(
3514                        state.balances[0].total.as_decimal(),
3515                        Decimal::from_str_exact(amount).unwrap() + Decimal::from(40),
3516                    );
3517                    assert_eq!(state.ts_event, UnixNanos::from(1_781_161_199_648_000_000));
3518                    assert_eq!(state.ts_init, ts_init);
3519                    assert_eq!(state.info, None);
3520                }
3521                other => panic!("expected account state or position update, was {other:?}"),
3522            }
3523        }
3524    }
3525
3526    #[rstest]
3527    fn handle_frame_routes_account_positions_to_update_without_readiness_marker() {
3528        let mut handler = make_handler_with_account();
3529        let frame: super::LighterWsFrame =
3530            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3531
3532        let messages = handler.handle_frame(frame, UnixNanos::from(11));
3533
3534        assert_eq!(messages.len(), 1);
3535        match &messages[0] {
3536            NautilusWsMessage::PositionUpdate {
3537                reports,
3538                closed_market_ids,
3539                skipped_market_ids,
3540            } => {
3541                assert!(closed_market_ids.is_empty());
3542                assert!(skipped_market_ids.is_empty());
3543                assert_eq!(reports.len(), 1);
3544                assert_eq!(reports[0].quantity, Quantity::from("1.5000"));
3545            }
3546            other => panic!("expected position update, was {other:?}"),
3547        }
3548    }
3549
3550    #[rstest]
3551    fn handle_frame_tolerates_unknown_position_margin_mode() {
3552        let mut handler = make_handler_with_account();
3553        let mut frame_json: serde_json::Value =
3554            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3555        frame_json["positions"]["0"]["margin_mode"] = json!(99);
3556        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3557
3558        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3559
3560        assert_eq!(messages.len(), 1);
3561        match &messages[0] {
3562            NautilusWsMessage::PositionUpdate {
3563                reports,
3564                closed_market_ids,
3565                skipped_market_ids,
3566            } => {
3567                assert!(closed_market_ids.is_empty());
3568                assert!(skipped_market_ids.is_empty());
3569                assert_eq!(reports.len(), 1);
3570            }
3571            other => panic!("expected position update, was {other:?}"),
3572        }
3573    }
3574
3575    #[rstest]
3576    fn handle_frame_routes_empty_account_positions_to_empty_update() {
3577        let mut handler = make_handler_with_account();
3578        let frame_json = serde_json::json!({
3579            "type": "update/account_all_positions",
3580            "channel": "account_all_positions:1234",
3581            "positions": {},
3582            "shares": [],
3583            "last_funding_round": null,
3584            "last_funding_discount": null,
3585        });
3586        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3587
3588        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3589
3590        assert_eq!(messages.len(), 1);
3591        match &messages[0] {
3592            NautilusWsMessage::PositionUpdate {
3593                reports,
3594                closed_market_ids,
3595                skipped_market_ids,
3596            } => {
3597                assert!(closed_market_ids.is_empty());
3598                assert!(skipped_market_ids.is_empty());
3599                assert!(reports.is_empty());
3600            }
3601            other => panic!("expected empty position update, was {other:?}"),
3602        }
3603    }
3604
3605    #[rstest]
3606    fn handle_frame_routes_zero_account_position_to_closed_update() {
3607        let mut handler = make_handler_with_account();
3608        let mut frame_json: serde_json::Value =
3609            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3610        frame_json["positions"]["0"]["position"] = json!("0.0000");
3611        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3612
3613        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3614
3615        assert_eq!(messages.len(), 1);
3616        match &messages[0] {
3617            NautilusWsMessage::PositionUpdate {
3618                reports,
3619                closed_market_ids,
3620                skipped_market_ids,
3621            } => {
3622                assert!(reports.is_empty());
3623                assert!(skipped_market_ids.is_empty());
3624                assert_eq!(closed_market_ids, &[0]);
3625            }
3626            other => panic!("expected closed position update, was {other:?}"),
3627        }
3628    }
3629
3630    #[rstest]
3631    fn handle_frame_marks_account_positions_incomplete_when_position_instrument_uncached() {
3632        let mut handler = make_handler_with_account();
3633        let mut frame_json: serde_json::Value =
3634            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3635        frame_json["type"] = json!("subscribed/account_all_positions");
3636        frame_json["positions"]["0"]["market_id"] = json!(999);
3637        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3638
3639        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3640
3641        assert_eq!(messages.len(), 1);
3642        match &messages[0] {
3643            NautilusWsMessage::PositionSnapshot {
3644                reports,
3645                skipped_market_ids,
3646            } => {
3647                assert_eq!(skipped_market_ids, &[999]);
3648                assert!(reports.is_empty());
3649            }
3650            other => panic!("expected incomplete position snapshot, was {other:?}"),
3651        }
3652    }
3653
3654    #[rstest]
3655    fn handle_frame_marks_account_positions_incomplete_when_position_parse_fails() {
3656        let mut handler = make_handler_with_account();
3657        let mut frame_json: serde_json::Value =
3658            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3659        frame_json["type"] = json!("subscribed/account_all_positions");
3660        frame_json["positions"]["0"]["position"] = json!("-1.5000");
3661        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3662
3663        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3664
3665        assert_eq!(messages.len(), 1);
3666        match &messages[0] {
3667            NautilusWsMessage::PositionSnapshot {
3668                reports,
3669                skipped_market_ids,
3670            } => {
3671                assert_eq!(skipped_market_ids, &[0]);
3672                assert!(reports.is_empty());
3673            }
3674            other => panic!("expected incomplete position snapshot, was {other:?}"),
3675        }
3676    }
3677
3678    #[rstest]
3679    fn handle_frame_marks_position_update_incomplete_when_position_parse_fails() {
3680        let mut handler = make_handler_with_account();
3681        let mut frame_json: serde_json::Value =
3682            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3683        frame_json["positions"]["0"]["position"] = json!("-1.5000");
3684        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3685
3686        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3687
3688        assert_eq!(messages.len(), 1);
3689        match &messages[0] {
3690            NautilusWsMessage::PositionUpdate {
3691                reports,
3692                closed_market_ids,
3693                skipped_market_ids,
3694            } => {
3695                assert!(reports.is_empty());
3696                assert!(closed_market_ids.is_empty());
3697                assert_eq!(skipped_market_ids, &[0]);
3698            }
3699            other => panic!("expected incomplete position update, was {other:?}"),
3700        }
3701    }
3702
3703    #[rstest]
3704    fn handle_frame_routes_subscribed_account_all_positions_snapshot() {
3705        let mut handler = make_handler_with_account();
3706        let frame_json = serde_json::json!({
3707            "type": "subscribed/account_all_positions",
3708            "channel": "account_all_positions:1234",
3709            "positions": {
3710                "0": {
3711                    "allocated_margin": "0.000000",
3712                    "avg_entry_price": "0.111230",
3713                    "initial_margin_fraction": "10.00",
3714                    "liquidation_price": "0.100598",
3715                    "margin_mode": 0,
3716                    "market_id": 0,
3717                    "open_order_count": 0,
3718                    "pending_order_count": 0,
3719                    "position": "100",
3720                    "position_tied_order_count": 0,
3721                    "position_value": "11.123000",
3722                    "realized_pnl": "0.000000",
3723                    "sign": 1,
3724                    "symbol": "ETH",
3725                    "total_discount": "0.000000",
3726                    "total_funding_paid_out": "0.000000",
3727                    "unrealized_pnl": "0.000000"
3728                }
3729            },
3730        });
3731        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3732
3733        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3734
3735        assert_eq!(messages.len(), 1);
3736        match &messages[0] {
3737            NautilusWsMessage::PositionSnapshot {
3738                reports,
3739                skipped_market_ids,
3740            } => {
3741                assert!(skipped_market_ids.is_empty());
3742                assert_eq!(reports.len(), 1);
3743                assert_eq!(reports[0].quantity, Quantity::from("100"));
3744            }
3745            other => panic!("expected position snapshot, was {other:?}"),
3746        }
3747    }
3748
3749    #[rstest]
3750    #[case::connected(serde_json::json!({"type": "connected", "session_id": "x"}), true, false)]
3751    #[case::ping(serde_json::json!({"type": "ping"}), true, false)]
3752    #[case::pong(serde_json::json!({"type": "pong"}), true, false)]
3753    #[case::send_tx_ack(
3754        serde_json::json!({"type": "jsonapi/sendtx", "code": 200, "tx_hash": "abc"}),
3755        true,
3756        true,
3757    )]
3758    #[case::send_tx_without_code(
3759        serde_json::json!({"type": "jsonapi/sendtx", "tx_hash": "abc"}),
3760        true,
3761        false,
3762    )]
3763    #[case::send_tx_with_nonnumeric_code(
3764        serde_json::json!({"type": "jsonapi/sendtx", "code": "200", "tx_hash": "abc"}),
3765        true,
3766        false,
3767    )]
3768    #[case::error_frame(
3769        serde_json::json!({"type": "error", "code": 21727, "message": "invalid client order index"}),
3770        true,
3771        true,
3772    )]
3773    #[case::error_frame_integrator_not_approved(
3774        serde_json::json!({"type": "error", "code": 21149, "message": "integrator is not approved"}),
3775        true,
3776        true,
3777    )]
3778    #[case::send_tx_ack_integrator_not_approved(
3779        serde_json::json!({"type": "jsonapi/sendtx", "code": 21149, "message": "integrator is not approved"}),
3780        true,
3781        true,
3782    )]
3783    #[case::wrapped_error_integrator_not_approved(
3784        serde_json::json!({"error": {"code": 21149, "message": "integrator is not approved"}}),
3785        true,
3786        true,
3787    )]
3788    #[case::subscription_error_frame(
3789        serde_json::json!({"type": "error", "code": 30003, "message": "Already Subscribed to : ticker:3"}),
3790        true,
3791        false,
3792    )]
3793    #[case::wrapped_subscription_error(
3794        serde_json::json!({"error": {"code": 30003, "message": "Already Subscribed to : ticker:3"}}),
3795        true,
3796        false,
3797    )]
3798    #[case::codeless_error_frame(
3799        serde_json::json!({"type": "error", "message": "unclassifiable"}),
3800        true,
3801        false,
3802    )]
3803    #[case::unknown_type(
3804        serde_json::json!({"type": "something_unexpected", "payload": "x"}),
3805        false,
3806        false,
3807    )]
3808    #[case::no_type_field(
3809        serde_json::json!({"error": {"code": 21702, "message": "invalid price"}}),
3810        true,
3811        true,
3812    )]
3813    fn handle_control_text_tri_state(
3814        #[case] payload: serde_json::Value,
3815        #[case] expected_matched: bool,
3816        #[case] expected_has_msg: bool,
3817    ) {
3818        // Pins the (matched, msg) contract that lets the WS receive loop
3819        // distinguish "known control type, consume silently" from
3820        // "surface a typed/Raw message to the consumer". sendTx success ACKs
3821        // and venue-error frames now produce typed SendTxAck/SendTxRejected
3822        // variants instead of Raw, so the execution loop can correlate them
3823        // back to in-flight cloids.
3824        let mut handler = make_handler_with_account();
3825        let text = payload.to_string();
3826        let (matched, msg) = handle_control_text(&mut handler, &text);
3827        assert_eq!(matched, expected_matched, "matched flag");
3828        assert_eq!(msg.is_some(), expected_has_msg, "msg presence");
3829    }
3830
3831    #[rstest]
3832    #[case::ack(serde_json::json!({"type":"jsonapi/sendtxbatch","id":"tx-batch:abc","code":200,"tx_hash":["abc","def"]}), 200, "batch rejected", vec!["abc", "def"])]
3833    #[case::error(serde_json::json!({"id":"tx-batch:abc","error":{"code":21104,"message":"invalid nonce"}}), 21104, "invalid nonce", vec![])]
3834    fn batch_response_preserves_correlation_and_epoch(
3835        #[case] value: serde_json::Value,
3836        #[case] expected_code: i64,
3837        #[case] expected_message: &str,
3838        #[case] expected_hashes: Vec<&str>,
3839    ) {
3840        let mut handler = make_handler_with_account();
3841        let (matched, message) = handler.handle_control_value(&value);
3842        assert!(matched);
3843
3844        match message.unwrap().with_connection_epoch(7) {
3845            NautilusWsMessage::SendTxBatchResult {
3846                connection_epoch,
3847                id,
3848                code,
3849                message,
3850                tx_hashes,
3851            } => {
3852                assert_eq!(connection_epoch, 7);
3853                assert_eq!(id, "tx-batch:abc");
3854                assert_eq!(code, expected_code);
3855                assert_eq!(message, expected_message);
3856                assert_eq!(tx_hashes, expected_hashes);
3857            }
3858            message => panic!("expected batch response, was {message:?}"),
3859        }
3860    }
3861
3862    #[rstest]
3863    fn handle_control_text_sendtx_success_emits_typed_ack() {
3864        let mut handler = make_handler_with_account();
3865        let payload = serde_json::json!({
3866            "type": "jsonapi/sendtx",
3867            "code": 200,
3868            "tx_hash": "0000abcd",
3869        })
3870        .to_string();
3871
3872        let (_, msg) = handle_control_text(&mut handler, &payload);
3873
3874        match msg.expect("SendTxAck emitted") {
3875            NautilusWsMessage::SendTxAck { tx_hash, code, .. } => {
3876                assert_eq!(code, 200);
3877                assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3878            }
3879            other => panic!("expected SendTxAck, was {other:?}"),
3880        }
3881    }
3882
3883    #[rstest]
3884    fn handle_control_text_sendtx_failure_emits_ack_sourced_rejection() {
3885        let mut handler = make_handler_with_account();
3886        let payload = serde_json::json!({
3887            "type": "jsonapi/sendtx",
3888            "code": 21727,
3889            "message": "invalid client order index",
3890        })
3891        .to_string();
3892
3893        let (_, msg) = handle_control_text(&mut handler, &payload);
3894
3895        match msg.expect("SendTxRejected emitted") {
3896            NautilusWsMessage::SendTxRejected {
3897                source,
3898                code,
3899                message,
3900                tx_hash,
3901                ..
3902            } => {
3903                assert_eq!(source, SendTxRejectionSource::Ack);
3904                assert_eq!(code, Some(21727));
3905                assert_eq!(message, "invalid client order index");
3906                assert_eq!(tx_hash, None);
3907            }
3908            other => panic!("expected SendTxRejected, was {other:?}"),
3909        }
3910    }
3911
3912    #[rstest]
3913    fn handle_control_text_sendtx_failure_carries_echoed_tx_hash() {
3914        let mut handler = make_handler_with_account();
3915        let payload = serde_json::json!({
3916            "type": "jsonapi/sendtx",
3917            "code": 21727,
3918            "message": "invalid client order index",
3919            "tx_hash": "0000abcd",
3920        })
3921        .to_string();
3922
3923        let (_, msg) = handle_control_text(&mut handler, &payload);
3924
3925        match msg.expect("SendTxRejected emitted") {
3926            NautilusWsMessage::SendTxRejected { tx_hash, .. } => {
3927                assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3928            }
3929            other => panic!("expected SendTxRejected, was {other:?}"),
3930        }
3931    }
3932
3933    #[rstest]
3934    fn handle_control_text_bare_error_frame_emits_bare_error_rejection() {
3935        let mut handler = make_handler_with_account();
3936        let payload = serde_json::json!({
3937            "type": "error",
3938            "code": 21702,
3939            "message": "invalid price",
3940        })
3941        .to_string();
3942
3943        let (_, msg) = handle_control_text(&mut handler, &payload);
3944
3945        match msg.expect("SendTxRejected emitted") {
3946            NautilusWsMessage::SendTxRejected {
3947                source,
3948                code,
3949                message,
3950                tx_hash,
3951                ..
3952            } => {
3953                assert_eq!(source, SendTxRejectionSource::BareError);
3954                assert_eq!(code, Some(21702));
3955                assert_eq!(message, "invalid price");
3956                assert_eq!(tx_hash, None);
3957            }
3958            other => panic!("expected SendTxRejected, was {other:?}"),
3959        }
3960    }
3961
3962    #[rstest]
3963    fn handle_control_text_wrapped_error_emits_bare_error_rejection() {
3964        // No top-level `type` field; the rejection sits under `error.{code,message}`.
3965        let mut handler = make_handler_with_account();
3966        let payload = serde_json::json!({
3967            "error": {"code": 21149, "message": "integrator is not approved"},
3968        })
3969        .to_string();
3970
3971        let (_, msg) = handle_control_text(&mut handler, &payload);
3972
3973        match msg.expect("SendTxRejected emitted") {
3974            NautilusWsMessage::SendTxRejected {
3975                source,
3976                code,
3977                message,
3978                tx_hash,
3979                ..
3980            } => {
3981                assert_eq!(source, SendTxRejectionSource::BareError);
3982                assert_eq!(code, Some(21149));
3983                assert_eq!(message, "integrator is not approved");
3984                assert_eq!(tx_hash, None);
3985            }
3986            other => panic!("expected SendTxRejected, was {other:?}"),
3987        }
3988    }
3989
3990    #[rstest]
3991    fn handle_frame_emits_no_account_state_until_both_streams_seen() {
3992        // Reconciler refuses to emit until both `account_all_assets` and
3993        // `user_stats` have delivered. An assets frame alone produces only
3994        // the AccountStreamFirstFrame marker, no AccountState payload.
3995        let mut handler = make_handler_with_account();
3996        let assets_only: super::LighterWsFrame =
3997            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3998
3999        let messages = strip_account_marker(handler.handle_frame(assets_only, UnixNanos::from(11)));
4000
4001        assert!(
4002            messages.is_empty(),
4003            "expected no AccountState before user_stats arrives, received {messages:?}"
4004        );
4005    }
4006
4007    #[rstest]
4008    fn handle_frame_routes_account_assets_and_user_stats_to_unified_state() {
4009        // The snapshot holds 50 USDC in total and reports 40 USDC available
4010        let mut handler = make_handler_with_account();
4011        let assets_frame: super::LighterWsFrame =
4012            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
4013        let user_stats_frame: super::LighterWsFrame =
4014            serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
4015
4016        let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
4017        let messages =
4018            strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
4019
4020        assert_eq!(messages.len(), 1);
4021        match &messages[0] {
4022            NautilusWsMessage::AccountState(state) => {
4023                let usdc = Currency::get_or_create_crypto("USDC");
4024                assert_eq!(state.account_type, AccountType::Margin);
4025                assert_eq!(state.base_currency, None);
4026                assert_eq!(state.balances.len(), 1);
4027                assert_eq!(state.balances[0].currency, usdc);
4028                assert_eq!(state.balances[0].total, Money::from("50.000000 USDC"));
4029                assert_eq!(state.balances[0].locked, Money::from("10 USDC"));
4030                assert_eq!(state.balances[0].free, Money::from("40.000000 USDC"));
4031                assert_eq!(state.margins.len(), 1);
4032                assert_eq!(state.margins[0].currency, usdc);
4033                assert_eq!(state.margins[0].initial, Money::from("0 USDC"));
4034                assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
4035                assert!(state.margins[0].instrument_id.is_none());
4036                assert!(state.is_reported);
4037            }
4038            other => panic!("expected account state, was {other:?}"),
4039        }
4040    }
4041
4042    #[rstest]
4043    fn handle_frame_unified_state_reflects_open_position() {
4044        // Fixture pair captured live with one open ETH-LONG position:
4045        //   account_all_assets[USDC] = {balance:10, locked:0, margin_balance:39.995369556}
4046        //   user_stats              = {collateral:39.995369, available:39.168314, margin_usage:2.07}
4047        //
4048        // The 5 mUSDC haircut on margin_balance is the entry fee. The
4049        // 0.827055 USDC gap between collateral and available_balance is
4050        // the initial margin pledged to the open position. AccountBalance
4051        // also includes funds outside the venue's reported availability.
4052        // Maintenance is zero because user_stats does not publish it.
4053        let mut handler = make_handler_with_account();
4054        let assets_frame: super::LighterWsFrame =
4055            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_WITH_POSITION).unwrap();
4056        let user_stats_frame: super::LighterWsFrame =
4057            serde_json::from_str(WS_USER_STATS_WITH_POSITION).unwrap();
4058
4059        let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
4060        let messages =
4061            strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
4062
4063        assert_eq!(messages.len(), 1);
4064        match &messages[0] {
4065            NautilusWsMessage::AccountState(state) => {
4066                assert_eq!(state.account_type, AccountType::Margin);
4067                assert_eq!(state.base_currency, None);
4068                assert_eq!(state.balances.len(), 1);
4069                // total = 10 + 39.995369556 = 49.99536956 (Money truncates
4070                // the trailing digit to 8 decimals, matching the
4071                // production log).
4072                assert_eq!(state.balances[0].total, Money::from("49.99536956 USDC"));
4073                assert_eq!(state.balances[0].locked, Money::from("10.82705556 USDC"));
4074                assert_eq!(state.balances[0].free, Money::from("39.16831400 USDC"));
4075                assert_eq!(state.margins.len(), 1);
4076                // initial = collateral 39.995369 - available 39.168314 = 0.827055
4077                assert_eq!(state.margins[0].initial, Money::from("0.82705500 USDC"));
4078                assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
4079                assert!(state.margins[0].instrument_id.is_none());
4080            }
4081            other => panic!("expected account state, was {other:?}"),
4082        }
4083    }
4084
4085    #[rstest]
4086    fn handle_frame_emits_account_stream_first_frame_marker_per_variant() {
4087        // Pins the strict-await wiring: the handler appends an
4088        // `AccountStreamFirstFrame` marker after any typed reports so the
4089        // execution consumption loop can mark readiness only after the
4090        // accompanying state has been applied. The marker fires even for
4091        // empty content frames that produce no typed reports.
4092        let mut handler = make_handler_with_account();
4093        let orders_frame: super::LighterWsFrame =
4094            serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
4095        let trades_frame: super::LighterWsFrame =
4096            serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
4097        let positions_frame: super::LighterWsFrame = serde_json::from_value({
4098            let mut value: serde_json::Value =
4099                serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
4100            value["type"] = json!("subscribed/account_all_positions");
4101            value
4102        })
4103        .unwrap();
4104        let assets_frame: super::LighterWsFrame =
4105            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
4106        let user_stats_frame: super::LighterWsFrame =
4107            serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
4108
4109        let cases = [
4110            (orders_frame, AccountStream::Orders),
4111            (trades_frame, AccountStream::Trades),
4112            (positions_frame, AccountStream::Positions),
4113            (assets_frame, AccountStream::Assets),
4114            (user_stats_frame, AccountStream::UserStats),
4115        ];
4116
4117        for (frame, expected) in cases {
4118            let msgs = handler.handle_frame(frame, UnixNanos::from(11));
4119            let marker = msgs
4120                .iter()
4121                .find(|m| matches!(m, NautilusWsMessage::AccountStreamFirstFrame(_)))
4122                .unwrap_or_else(|| panic!("missing marker for {expected:?}"));
4123            match marker {
4124                NautilusWsMessage::AccountStreamFirstFrame(stream) => {
4125                    assert_eq!(*stream, expected);
4126                }
4127                other => panic!("expected AccountStreamFirstFrame, was {other:?}"),
4128            }
4129            // The marker must trail any typed reports the handler emitted
4130            // so the consumption loop applies state first.
4131            assert!(
4132                matches!(
4133                    msgs.last(),
4134                    Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
4135                ),
4136                "marker must trail typed reports for {expected:?}",
4137            );
4138        }
4139    }
4140
4141    #[rstest]
4142    #[case::account_orders_update(WS_ACCOUNT_ORDERS_UPDATE, "update/account_orders")]
4143    #[case::account_orders_snapshot(WS_ACCOUNT_ORDERS_UPDATE, "subscribed/account_orders")]
4144    #[case::account_all_trades_update(WS_ACCOUNT_ALL_TRADES_UPDATE, "update/account_all_trades")]
4145    #[case::account_all_trades_snapshot(
4146        snapshot_trade_frame_json(),
4147        "subscribed/account_all_trades"
4148    )]
4149    #[case::account_all_positions_update(
4150        WS_ACCOUNT_ALL_POSITIONS_UPDATE,
4151        "update/account_all_positions"
4152    )]
4153    #[case::account_all_positions_snapshot(
4154        WS_ACCOUNT_ALL_POSITIONS_UPDATE,
4155        "subscribed/account_all_positions"
4156    )]
4157    #[case::account_all_assets_update(WS_ACCOUNT_ALL_ASSETS_UPDATE, "update/account_all_assets")]
4158    #[case::account_all_assets_snapshot(
4159        WS_ACCOUNT_ALL_ASSETS_UPDATE,
4160        "subscribed/account_all_assets"
4161    )]
4162    #[case::user_stats_update(WS_USER_STATS_UPDATE, "update/user_stats")]
4163    #[case::user_stats_snapshot(WS_USER_STATS_UPDATE, "subscribed/user_stats")]
4164    #[case::account_tx_update(
4165        include_str!("../../test_data/ws_account_tx_update.json"),
4166        "update/account_tx"
4167    )]
4168    #[case::height_update(include_str!("../../test_data/ws_height_update.json"), "update/height")]
4169    #[case::height_snapshot(
4170        include_str!("../../test_data/ws_height_update.json"),
4171        "subscribed/height"
4172    )]
4173    fn route_frame_without_context_forwards_original_frame(
4174        #[case] fixture: &str,
4175        #[case] frame_type: &str,
4176    ) {
4177        let mut handler = make_handler_with_account();
4178        handler.exec_account = None;
4179        let mut original: serde_json::Value = serde_json::from_str(fixture).unwrap();
4180        original["type"] = json!(frame_type);
4181        original["unmodeled_sentinel"] = json!({"kept": [1, "two"]});
4182        let text = original.to_string();
4183        let frame: LighterWsFrame = serde_json::from_str(&text).unwrap();
4184
4185        let messages = handler.route_frame(frame, &text, UnixNanos::from(11));
4186
4187        assert_eq!(messages.len(), 1);
4188
4189        let NautilusWsMessage::Raw(value) = &messages[0] else {
4190            panic!("expected raw frame, was {:?}", messages[0]);
4191        };
4192
4193        assert_eq!(value, &original);
4194    }
4195
4196    #[rstest]
4197    #[case::account_orders_snapshot(WS_ACCOUNT_ORDERS_UPDATE, "subscribed/account_orders", 0)]
4198    #[case::account_all_trades_update(WS_ACCOUNT_ALL_TRADES_UPDATE, "update/account_all_trades", 1)]
4199    #[case::account_all_trades_snapshot(
4200        snapshot_trade_frame_json(),
4201        "subscribed/account_all_trades",
4202        0
4203    )]
4204    #[case::account_all_assets_snapshot(
4205        WS_ACCOUNT_ALL_ASSETS_UPDATE,
4206        "subscribed/account_all_assets",
4207        0
4208    )]
4209    #[case::user_stats_snapshot(WS_USER_STATS_UPDATE, "subscribed/user_stats", 0)]
4210    fn route_frame_with_context_keeps_typed_routing(
4211        #[case] fixture: &str,
4212        #[case] frame_type: &str,
4213        #[case] expected_fills: usize,
4214    ) {
4215        let mut handler = make_handler_with_account();
4216        let mut typed_handler = make_handler_with_account();
4217        let mut original: serde_json::Value = serde_json::from_str(fixture).unwrap();
4218        original["type"] = json!(frame_type);
4219        original["unmodeled_sentinel"] = json!("ignored");
4220        let text = original.to_string();
4221        let frame: LighterWsFrame = serde_json::from_str(&text).unwrap();
4222
4223        let routed = handler.route_frame(frame.clone(), &text, UnixNanos::from(11));
4224        let typed = typed_handler.handle_frame(frame, UnixNanos::from(11));
4225
4226        let fills = routed
4227            .iter()
4228            .filter_map(|msg| match msg {
4229                NautilusWsMessage::ExecutionReports(reports) => Some(reports),
4230                _ => None,
4231            })
4232            .flatten()
4233            .filter(|report| matches!(report, super::ExecutionReport::Fill(_)))
4234            .count();
4235
4236        assert_eq!(
4237            routed
4238                .iter()
4239                .map(std::mem::discriminant)
4240                .collect::<Vec<_>>(),
4241            typed.iter().map(std::mem::discriminant).collect::<Vec<_>>(),
4242        );
4243        assert_eq!(fills, expected_fills);
4244        assert_eq!(
4245            routed
4246                .iter()
4247                .filter(|msg| matches!(msg, NautilusWsMessage::AccountStreamFirstFrame(_)))
4248                .count(),
4249            1,
4250        );
4251        assert!(
4252            !routed
4253                .iter()
4254                .any(|msg| matches!(msg, NautilusWsMessage::Raw(_)))
4255        );
4256    }
4257
4258    #[rstest]
4259    fn handle_frame_account_orders_skips_unknown_market() {
4260        // Build a handler with the execution context but no instrument
4261        // cached for the order's market_index; the handler should log and
4262        // emit no execution reports (the trailing readiness marker still
4263        // fires so `connect()` does not stall when the venue resubscribes
4264        // before instrument bootstrap completes).
4265        let signal = Arc::new(AtomicBool::new(false));
4266        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4267        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4268        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4269        let mut handler =
4270            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4271        handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
4272        // No instrument inserted for market_index=0.
4273
4274        let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
4275        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4276
4277        assert!(messages.is_empty());
4278    }
4279
4280    #[rstest]
4281    fn handle_frame_account_assets_invalid_timestamp_returns_empty() {
4282        let mut handler = make_handler_with_account();
4283        // u64::MAX millis overflows the seconds-to-nanos conversion in
4284        // `parse_millis_to_nanos`; the handler must log and emit no typed
4285        // account state rather than surface a partially constructed one.
4286        // The trailing readiness marker is still emitted so `connect()`
4287        // does not stall if the venue ever sends a malformed timestamp on
4288        // the initial frame.
4289        let frame_json = r#"{
4290            "type": "update/account_all_assets",
4291            "channel": "account_all_assets:1234",
4292            "timestamp": 18446744073709551615,
4293            "assets": {
4294                "0": {
4295                    "symbol": "USDC",
4296                    "asset_id": 0,
4297                    "balance": "100.000000",
4298                    "locked_balance": "1.000000"
4299                }
4300            }
4301        }"#;
4302        let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
4303
4304        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4305
4306        assert!(messages.is_empty());
4307    }
4308
4309    #[rstest]
4310    fn handle_frame_account_all_orders_routes_to_execution_reports() {
4311        // The `update/account_all_orders` variant lacks the per-account
4312        // top-level `account` and `nonce` fields that `account_orders`
4313        // carries, but the handler should treat them identically.
4314        let mut handler = make_handler_with_account();
4315        let frame_json = r#"{
4316            "type": "update/account_all_orders",
4317            "channel": "account_all_orders:1234",
4318            "orders": {
4319                "0": [{
4320                    "order_index": 281476929510110,
4321                    "client_order_index": 42,
4322                    "order_id": "281476929510110",
4323                    "client_order_id": "42",
4324                    "market_index": 0,
4325                    "owner_account_index": 1234,
4326                    "initial_base_amount": "0.0050",
4327                    "price": "2352.74",
4328                    "nonce": 9182390020,
4329                    "remaining_base_amount": "0.0050",
4330                    "is_ask": true,
4331                    "base_size": 50,
4332                    "base_price": 235274,
4333                    "filled_base_amount": "0.0000",
4334                    "filled_quote_amount": "0.000000",
4335                    "side": "sell",
4336                    "type": "limit",
4337                    "time_in_force": "good-till-time",
4338                    "reduce_only": false,
4339                    "trigger_price": "0.00",
4340                    "order_expiry": 1780360584479,
4341                    "status": "open",
4342                    "trigger_status": "na",
4343                    "trigger_time": 0,
4344                    "parent_order_index": 0,
4345                    "parent_order_id": "0",
4346                    "to_trigger_order_id_0": "0",
4347                    "to_trigger_order_id_1": "0",
4348                    "to_cancel_order_id_0": "0",
4349                    "integrator_fee_collector_index": "0",
4350                    "integrator_taker_fee": "0",
4351                    "integrator_maker_fee": "0",
4352                    "block_height": 227535532,
4353                    "timestamp": 1777941383576,
4354                    "created_at": 1777941383576,
4355                    "updated_at": 1777941383576,
4356                    "transaction_time": 1777941383576735
4357                }]
4358            }
4359        }"#;
4360        let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
4361
4362        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4363
4364        assert_eq!(messages.len(), 1);
4365        match &messages[0] {
4366            NautilusWsMessage::ExecutionReports(reports) => {
4367                assert_eq!(reports.len(), 1);
4368                match &reports[0] {
4369                    super::ExecutionReport::Order(order) => {
4370                        assert_eq!(order.order_id, "281476929510110");
4371                    }
4372                    other => panic!("expected order report, was {other:?}"),
4373                }
4374            }
4375            other => panic!("expected execution reports, was {other:?}"),
4376        }
4377    }
4378
4379    fn snapshot_trade_frame_json() -> &'static str {
4380        r#"{
4381            "type": "subscribed/account_all_trades",
4382            "channel": "account_all_trades:1234",
4383            "trades": [{
4384                "trade_id": 19209006902,
4385                "trade_id_str": "19209006902",
4386                "tx_hash": "000000128b1ee814",
4387                "type": "trade",
4388                "market_id": 0,
4389                "size": "0.1336",
4390                "price": "2352.73",
4391                "usd_amount": "314.324728",
4392                "ask_id": 281476929510102,
4393                "bid_id": 562947905631053,
4394                "ask_client_id": 0,
4395                "bid_client_id": 7001011966,
4396                "ask_account_id": 91249,
4397                "bid_account_id": 1234,
4398                "is_maker_ask": true,
4399                "block_height": 227535535,
4400                "timestamp": 1777941384181,
4401                "transaction_time": 1777941384181586
4402            }],
4403            "total_volume": "100.0",
4404            "monthly_volume": "100.0",
4405            "weekly_volume": "100.0",
4406            "daily_volume": "100.0"
4407        }"#
4408    }
4409
4410    #[rstest]
4411    fn handle_frame_account_all_trades_snapshot_is_dropped_with_context() {
4412        let mut handler = make_handler_with_account();
4413        // With execution context active the snapshot must be dropped to
4414        // avoid replaying historical fills as live FillReports on reconnect.
4415        // The trailing readiness marker is still emitted so `connect()`
4416        // observes the trades stream as having delivered.
4417        let frame: super::LighterWsFrame =
4418            serde_json::from_str(snapshot_trade_frame_json()).unwrap();
4419
4420        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4421
4422        assert!(messages.is_empty());
4423    }
4424
4425    #[rstest]
4426    fn handle_frame_market_stats_emits_mark_index_and_funding_updates() {
4427        let mut handler = make_handler_with_account();
4428        let frame: super::LighterWsFrame =
4429            serde_json::from_str(WS_MARKET_STATS_UPDATE_SINGLE).unwrap();
4430
4431        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4432
4433        assert_eq!(messages.len(), 3);
4434        match &messages[0] {
4435            NautilusWsMessage::MarkPrice(update) => {
4436                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4437                assert_eq!(update.value, Price::from("2064.47"));
4438                assert_eq!(update.ts_event, UnixNanos::from(1_774_883_844_933_000_000));
4439            }
4440            event => panic!("expected mark price update, was {event:?}"),
4441        }
4442
4443        match &messages[1] {
4444            NautilusWsMessage::IndexPrice(update) => {
4445                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4446                assert_eq!(update.value, Price::from("2064.48"));
4447            }
4448            event => panic!("expected index price update, was {event:?}"),
4449        }
4450
4451        match &messages[2] {
4452            NautilusWsMessage::FundingRate(update) => {
4453                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4454                assert_eq!(update.rate, Decimal::new(1, 8));
4455                assert_eq!(update.next_funding_ns, None);
4456            }
4457            event => panic!("expected funding rate update, was {event:?}"),
4458        }
4459    }
4460
4461    #[rstest]
4462    fn handle_frame_market_stats_all_emits_mark_index_and_funding_updates() {
4463        let mut handler = make_handler_with_account();
4464        let frame: super::LighterWsFrame =
4465            serde_json::from_str(WS_MARKET_STATS_UPDATE_ALL).unwrap();
4466
4467        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4468
4469        assert_eq!(messages.len(), 3);
4470        assert!(matches!(&messages[0], NautilusWsMessage::MarkPrice(_)));
4471        assert!(matches!(&messages[1], NautilusWsMessage::IndexPrice(_)));
4472        assert!(matches!(&messages[2], NautilusWsMessage::FundingRate(_)));
4473
4474        match &messages[0] {
4475            NautilusWsMessage::MarkPrice(update) => {
4476                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4477                assert_eq!(update.value, Price::from("2064.47"));
4478            }
4479            event => panic!("expected mark price update, was {event:?}"),
4480        }
4481
4482        match &messages[2] {
4483            NautilusWsMessage::FundingRate(update) => {
4484                assert_eq!(update.rate, Decimal::new(1, 8));
4485                assert_eq!(update.next_funding_ns, None);
4486            }
4487            event => panic!("expected funding rate update, was {event:?}"),
4488        }
4489    }
4490
4491    #[rstest]
4492    fn handle_frame_spot_market_stats_emits_index_update() {
4493        let signal = Arc::new(AtomicBool::new(false));
4494        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4495        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4496        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4497        let mut handler =
4498            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4499        handler.instruments.insert(2048, stub_eth_spot_instrument());
4500        let frame: super::LighterWsFrame =
4501            serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_SINGLE).unwrap();
4502
4503        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4504
4505        assert_eq!(messages.len(), 1);
4506        match &messages[0] {
4507            NautilusWsMessage::IndexPrice(update) => {
4508                assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
4509                assert_eq!(update.value, Price::from("1.00"));
4510            }
4511            event => panic!("expected spot index price update, was {event:?}"),
4512        }
4513    }
4514
4515    #[rstest]
4516    fn handle_frame_spot_market_stats_all_emits_index_update() {
4517        let signal = Arc::new(AtomicBool::new(false));
4518        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4519        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4520        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4521        let mut handler =
4522            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4523        handler.instruments.insert(2048, stub_eth_spot_instrument());
4524        let frame: super::LighterWsFrame =
4525            serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_ALL).unwrap();
4526
4527        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4528
4529        assert_eq!(messages.len(), 1);
4530        match &messages[0] {
4531            NautilusWsMessage::IndexPrice(update) => {
4532                assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
4533                assert_eq!(update.value, Price::from("1.00"));
4534            }
4535            event => panic!("expected spot index price update, was {event:?}"),
4536        }
4537    }
4538
4539    #[rstest]
4540    #[case(LighterWsChannel::OrderBook(0), "order_book:0", "order_book/0")]
4541    #[case(LighterWsChannel::Trade(7), "trade:7", "trade/7")]
4542    #[case(LighterWsChannel::Ticker(2), "ticker:2", "ticker/2")]
4543    #[case(LighterWsChannel::Height, "height", "height")]
4544    #[case(
4545        LighterWsChannel::MarketStats(LighterMarketSelection::All),
4546        "market_stats:all",
4547        "market_stats/all"
4548    )]
4549    #[case(
4550        LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(2048)),
4551        "spot_market_stats:2048",
4552        "spot_market_stats/2048"
4553    )]
4554    #[case(
4555        LighterWsChannel::OrderBook(4095),
4556        "order_book:4095",
4557        "order_book/4095"
4558    )]
4559    #[case(
4560        LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(50_000)),
4561        "spot_market_stats:50000",
4562        "spot_market_stats/50000"
4563    )]
4564    #[case(
4565        LighterWsChannel::Candle {
4566            market_index: 40_000,
4567            resolution: LighterCandleResolution::OneHour,
4568        },
4569        "candle:40000:1h",
4570        "candle/40000/1h"
4571    )]
4572    #[case(
4573        LighterWsChannel::AccountOrders { market_index: 0, account_index: 1234 },
4574        "account_orders:0:1234",
4575        "account_orders/0/1234",
4576    )]
4577    fn topic_and_subscription_round_trip(
4578        #[case] channel: LighterWsChannel,
4579        #[case] expected_topic: &str,
4580        #[case] expected_subscription: &str,
4581    ) {
4582        assert_eq!(channel.topic_key(), expected_topic);
4583        assert_eq!(channel.subscription_channel(), expected_subscription);
4584    }
4585
4586    #[rstest]
4587    #[case("order_book:0", Some(0))]
4588    #[case("trade:42", Some(42))]
4589    #[case("order_book:4095", Some(4095))]
4590    #[case("trade:40000", Some(40_000))]
4591    #[case("market_stats:50000", Some(50_000))]
4592    #[case("height", None)]
4593    #[case("malformed", None)]
4594    fn market_index_extraction(#[case] topic: &str, #[case] expected: Option<i64>) {
4595        assert_eq!(market_index_from_topic(topic), expected);
4596    }
4597
4598    #[rstest]
4599    #[case("order_book:0", Some(0))]
4600    #[case("order_book:42", Some(42))]
4601    #[case("order_book:40000", Some(40_000))]
4602    #[case("trade:42", None)]
4603    #[case("ticker:2", None)]
4604    #[case("market_stats:0", None)]
4605    #[case("height", None)]
4606    #[case("order_book:not-an-int", None)]
4607    fn order_book_market_index_only_matches_order_book_channel(
4608        #[case] topic: &str,
4609        #[case] expected: Option<i64>,
4610    ) {
4611        assert_eq!(order_book_market_index_from_topic(topic), expected);
4612    }
4613
4614    #[rstest]
4615    #[case(LighterWsChannel::AccountAll(1234), true)]
4616    #[case(LighterWsChannel::OrderBook(0), false)]
4617    #[case(LighterWsChannel::AccountAllPositions(1), true)]
4618    #[case(LighterWsChannel::Trade(0), false)]
4619    fn requires_auth_classification(#[case] channel: LighterWsChannel, #[case] expected: bool) {
4620        assert_eq!(channel.requires_auth(), expected);
4621    }
4622
4623    #[rstest]
4624    fn handler_command_subscribe_debug_redacts_auth_token() {
4625        let token = "schnorr-signature-bytes-do-not-leak";
4626        let cmd = HandlerCommand::Subscribe {
4627            channel: LighterWsChannel::AccountAll(1234),
4628            auth: Some(SecretString::from(token)),
4629            response_tx: None,
4630        };
4631
4632        let dbg = format!("{cmd:?}");
4633
4634        assert!(
4635            !dbg.contains(token),
4636            "Debug output must not contain the auth token, found: {dbg}",
4637        );
4638        assert!(dbg.contains(REDACTED));
4639    }
4640
4641    #[tokio::test]
4642    async fn send_tx_command_returns_handler_send_error_without_active_client() {
4643        let signal = Arc::new(AtomicBool::new(false));
4644        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4645        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4646        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4647        let mut handler =
4648            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4649        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
4650
4651        cmd_tx
4652            .send(HandlerCommand::SendTx {
4653                tx_type: LighterTxType::CreateOrder as u8,
4654                tx_info: serde_json::value::RawValue::from_string(
4655                    r#"{"AccountIndex":12345,"Nonce":42}"#.to_string(),
4656                )
4657                .unwrap(),
4658                connection_epoch: 0,
4659                response_tx,
4660            })
4661            .unwrap();
4662        drop(cmd_tx);
4663        drop(raw_tx);
4664
4665        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4666            .await
4667            .expect("timed out waiting for handler to drain command");
4668        let result = response_rx.await.expect("sendTx response channel closed");
4669
4670        assert!(next.is_none());
4671        let Err(LighterWsError::Client(message)) = result else {
4672            panic!("expected client send error, was {result:?}");
4673        };
4674        assert!(message.contains("no active WebSocket client"));
4675    }
4676
4677    #[tokio::test]
4678    async fn resubscribe_order_book_command_skips_when_reference_removed() {
4679        let signal = Arc::new(AtomicBool::new(false));
4680        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4681        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4682        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4683        let subscriptions = SubscriptionState::new(':');
4684        let topic = LighterWsChannel::OrderBook(0).topic_key();
4685        assert!(subscriptions.add_reference(&topic));
4686        assert!(subscriptions.remove_reference(&topic));
4687
4688        let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions.clone());
4689        handler.book.delta_subs.insert(0);
4690
4691        let (completion, result) = tokio::sync::oneshot::channel();
4692        cmd_tx
4693            .send(HandlerCommand::RecoverBook {
4694                market_index: 0,
4695                cancel: CancellationToken::new(),
4696                gate: SnapshotGate::default(),
4697                completion,
4698            })
4699            .expect("queue resync");
4700        drop(cmd_tx);
4701        drop(raw_tx);
4702
4703        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4704            .await
4705            .expect("timed out waiting for handler to drain command");
4706
4707        assert!(next.is_none());
4708        assert!(subscriptions.pending_subscribe_topics().is_empty());
4709        assert!(subscriptions.pending_unsubscribe_topics().is_empty());
4710        assert!(result.await.is_err());
4711    }
4712
4713    #[tokio::test]
4714    async fn resubscribe_order_book_queues_when_inflight_is_at_cap() {
4715        let signal = Arc::new(AtomicBool::new(false));
4716        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4717        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4718        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4719        let subscriptions = SubscriptionState::new(':');
4720        let topic = LighterWsChannel::OrderBook(0).topic_key();
4721        assert!(subscriptions.add_reference(&topic));
4722
4723        let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions);
4724        handler.book.delta_subs.insert(0);
4725        saturate_subscription_gate(&mut handler);
4726
4727        let (completion, _rx) = tokio::sync::oneshot::channel();
4728        handler.queue_book_replacement(
4729            0,
4730            BookWrite {
4731                cancel: CancellationToken::new(),
4732                gate: SnapshotGate::default(),
4733                completion,
4734            },
4735        );
4736
4737        handler.pump_pending_subscribes().await;
4738
4739        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4740        assert!(
4741            handler
4742                .pending_subs
4743                .iter()
4744                .any(|(topic, _)| *topic == Ustr::from("order_book:0")),
4745        );
4746    }
4747
4748    fn stub_candle(
4749        t: i64,
4750        open: i64,
4751        high: i64,
4752        low: i64,
4753        close: i64,
4754        volume_ticks: i64,
4755    ) -> LighterWsCandle {
4756        LighterWsCandle {
4757            t,
4758            o: Decimal::new(open, 2),
4759            h: Decimal::new(high, 2),
4760            l: Decimal::new(low, 2),
4761            c: Decimal::new(close, 2),
4762            v: Decimal::new(volume_ticks, 4),
4763            quote_volume: Decimal::ZERO,
4764            i: 0,
4765        }
4766    }
4767
4768    fn candle_frame(channel: &str, candle: LighterWsCandle, is_snapshot: bool) -> LighterWsFrame {
4769        if is_snapshot {
4770            LighterWsFrame::CandleSnapshot {
4771                channel: Ustr::from(channel),
4772                candles: vec![candle],
4773                timestamp: 0,
4774            }
4775        } else {
4776            LighterWsFrame::Candle {
4777                channel: Ustr::from(channel),
4778                candles: vec![candle],
4779                timestamp: 0,
4780            }
4781        }
4782    }
4783
4784    #[rstest]
4785    fn handle_candles_first_observation_caches_without_emit() {
4786        let mut handler = make_handler_with_account();
4787        let frame = candle_frame(
4788            "candle:0:1m",
4789            stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
4790            true,
4791        );
4792
4793        let messages = handler.handle_frame(frame, UnixNanos::from(99));
4794
4795        assert!(messages.is_empty(), "first observation must not emit");
4796        let key = (0_i64, LighterCandleResolution::OneMinute);
4797        assert_eq!(handler.last_candles.get(&key).map(|c| c.t), Some(1_000_000));
4798    }
4799
4800    #[rstest]
4801    fn handle_candles_t_advance_emits_bar_for_previous_candle() {
4802        let mut handler = make_handler_with_account();
4803        let prev = stub_candle(1_000_000, 10_000, 11_000, 9_900, 10_500, 10_000);
4804        let next = stub_candle(1_060_000, 10_500, 10_600, 10_450, 10_550, 20_000);
4805        let next_t = next.t;
4806        handler.handle_frame(candle_frame("candle:0:1m", prev, true), UnixNanos::from(1));
4807
4808        let messages =
4809            handler.handle_frame(candle_frame("candle:0:1m", next, false), UnixNanos::from(2));
4810
4811        assert_eq!(messages.len(), 1);
4812        match &messages[0] {
4813            NautilusWsMessage::Bar(bar) => {
4814                // The emitted bar reflects the previous (closed) candle, not the new one.
4815                assert_eq!(bar.open, Price::from("100.00"));
4816                assert_eq!(bar.high, Price::from("110.00"));
4817                assert_eq!(bar.low, Price::from("99.00"));
4818                assert_eq!(bar.close, Price::from("105.00"));
4819                assert_eq!(bar.volume, Quantity::from("1.0000"));
4820                assert_eq!(bar.ts_event, UnixNanos::from(1_000_000 * 1_000_000));
4821            }
4822            other => panic!("expected Bar message, was {other:?}"),
4823        }
4824        let cached = handler
4825            .last_candles
4826            .get(&(0_i64, LighterCandleResolution::OneMinute))
4827            .expect("cache populated");
4828        assert_eq!(cached.t, next_t);
4829    }
4830
4831    #[rstest]
4832    fn handle_candles_same_t_updates_cache_without_emit() {
4833        let mut handler = make_handler_with_account();
4834        let initial = stub_candle(1_000_000, 10_000, 10_050, 9_950, 10_025, 5_000);
4835        let same_t_updated = stub_candle(1_000_000, 10_000, 10_100, 9_950, 10_075, 7_500);
4836        let same_t_h = same_t_updated.h;
4837        let same_t_c = same_t_updated.c;
4838        handler.handle_frame(
4839            candle_frame("candle:0:1m", initial, true),
4840            UnixNanos::from(1),
4841        );
4842
4843        let messages = handler.handle_frame(
4844            candle_frame("candle:0:1m", same_t_updated, false),
4845            UnixNanos::from(2),
4846        );
4847
4848        assert!(messages.is_empty(), "same-`t` update must not emit");
4849        let cached = handler
4850            .last_candles
4851            .get(&(0_i64, LighterCandleResolution::OneMinute))
4852            .expect("cache populated");
4853        assert_eq!(cached.h, same_t_h);
4854        assert_eq!(cached.c, same_t_c);
4855    }
4856
4857    #[rstest]
4858    fn handle_candles_regressed_t_is_skipped() {
4859        let mut handler = make_handler_with_account();
4860        let initial = stub_candle(2_000_000, 10_000, 10_000, 10_000, 10_000, 5_000);
4861        let regressed = stub_candle(1_000_000, 9_000, 9_000, 9_000, 9_000, 5_000);
4862        let initial_t = initial.t;
4863        handler.handle_frame(
4864            candle_frame("candle:0:1m", initial, true),
4865            UnixNanos::from(1),
4866        );
4867
4868        let messages = handler.handle_frame(
4869            candle_frame("candle:0:1m", regressed, false),
4870            UnixNanos::from(2),
4871        );
4872
4873        assert!(messages.is_empty(), "regressed `t` must not emit");
4874        let cached = handler
4875            .last_candles
4876            .get(&(0_i64, LighterCandleResolution::OneMinute))
4877            .expect("cache populated");
4878        // Regressed frame is skipped entirely; cache stays on the original entry.
4879        assert_eq!(cached.t, initial_t);
4880    }
4881
4882    #[rstest]
4883    fn handle_candles_unknown_market_returns_empty() {
4884        let mut handler = make_handler_with_account();
4885        let frame = candle_frame(
4886            "candle:99:1m",
4887            stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 5_000),
4888            true,
4889        );
4890
4891        let messages = handler.handle_frame(frame, UnixNanos::from(1));
4892
4893        assert!(messages.is_empty());
4894    }
4895
4896    #[rstest]
4897    fn handle_unsubscribe_ack_clears_only_matching_candle_key() {
4898        let mut handler = make_handler_with_account();
4899        handler.last_candles.insert(
4900            (0, LighterCandleResolution::OneMinute),
4901            stub_candle(1, 0, 0, 0, 0, 0),
4902        );
4903        handler.last_candles.insert(
4904            (0, LighterCandleResolution::FiveMinute),
4905            stub_candle(2, 0, 0, 0, 0, 0),
4906        );
4907        handler.subscriptions.mark_unsubscribe("candle:0:1m");
4908
4909        let payload = json!({"type": "unsubscribed", "channel": "candle:0:1m"});
4910        let (matched, _) = handle_control_text(&mut handler, &payload.to_string());
4911
4912        assert!(matched);
4913        assert!(
4914            handler
4915                .last_candles
4916                .get(&(0, LighterCandleResolution::OneMinute))
4917                .is_none(),
4918        );
4919        assert!(
4920            handler
4921                .last_candles
4922                .get(&(0, LighterCandleResolution::FiveMinute))
4923                .is_some(),
4924        );
4925    }
4926
4927    #[rstest]
4928    #[case::well_formed("candle:0:1m", Some((0, LighterCandleResolution::OneMinute)))]
4929    #[case::weekly("candle:3:1w", Some((3, LighterCandleResolution::OneWeek)))]
4930    #[case::widened_id("candle:40000:1h", Some((40_000, LighterCandleResolution::OneHour)))]
4931    #[case::other_kind("order_book:0", None)]
4932    #[case::missing_resolution("candle:0", None)]
4933    #[case::bad_market("candle:notanint:1m", None)]
4934    #[case::bad_resolution("candle:0:bogus", None)]
4935    fn test_candle_market_and_resolution_from_topic(
4936        #[case] topic: &str,
4937        #[case] expected: Option<(i64, LighterCandleResolution)>,
4938    ) {
4939        assert_eq!(candle_market_and_resolution_from_topic(topic), expected);
4940    }
4941
4942    #[rstest]
4943    #[case::network_retries(LighterWsError::Network("disconnected".into()), true)]
4944    #[case::auth_does_not_retry(LighterWsError::Authentication("bad token".into()), false)]
4945    #[case::parse_does_not_retry(LighterWsError::Parse("bad json".into()), false)]
4946    #[case::client_does_not_retry(LighterWsError::Client("no active WebSocket client".into()), false)]
4947    #[case::transport_closed_does_not_retry(LighterWsError::Transport(SendError::Closed), false)]
4948    #[case::transport_invalid_input_does_not_retry(
4949        LighterWsError::Transport(SendError::InvalidInput("pong payload too large".into())),
4950        false
4951    )]
4952    #[case::transport_connection_changed_does_not_retry(
4953        LighterWsError::Transport(SendError::ConnectionChanged),
4954        false
4955    )]
4956    #[case::transport_timeout_retries(LighterWsError::Transport(SendError::Timeout), true)]
4957    #[case::transport_write_timeout_does_not_retry(
4958        LighterWsError::Transport(SendError::WriteTimeout),
4959        false
4960    )]
4961    #[case::transport_broken_pipe_does_not_retry(
4962        LighterWsError::Transport(SendError::BrokenPipe(
4963            "writer closed".into(),
4964        )),
4965        false,
4966    )]
4967    #[case::buffer_full(LighterWsError::Transport(SendError::BufferFull), false)]
4968    fn test_should_retry_lighter_ws_error(#[case] error: LighterWsError, #[case] expected: bool) {
4969        assert_eq!(should_retry_lighter_ws_error(&error), expected);
4970    }
4971
4972    // Pins the `#[from] SendError` derive that `.map_err(LighterWsError::Transport)` relies on.
4973    #[rstest]
4974    #[case::closed(SendError::Closed)]
4975    #[case::invalid_input(SendError::InvalidInput("pong payload too large".into()))]
4976    #[case::timeout(SendError::Timeout)]
4977    #[case::write_timeout(SendError::WriteTimeout)]
4978    #[case::connection_changed(SendError::ConnectionChanged)]
4979    #[case::broken_pipe(SendError::BrokenPipe("writer dropped".into()))]
4980    fn send_error_converts_into_transport_variant(#[case] send_error: SendError) {
4981        let err: LighterWsError = send_error.into();
4982        assert!(
4983            matches!(err, LighterWsError::Transport(_)),
4984            "expected Transport variant, was {err:?}",
4985        );
4986    }
4987
4988    #[tokio::test]
4989    async fn subscribe_command_parks_in_pending_subs_when_inflight_at_cap() {
4990        let signal = Arc::new(AtomicBool::new(false));
4991        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4992        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4993        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4994        let mut handler =
4995            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4996
4997        // Saturate the gate so the pump cannot dispatch the queued subscribe
4998        saturate_subscription_gate(&mut handler);
4999
5000        cmd_tx
5001            .send(HandlerCommand::Subscribe {
5002                channel: LighterWsChannel::Candle {
5003                    market_index: 0,
5004                    resolution: LighterCandleResolution::OneMinute,
5005                },
5006                auth: None,
5007                response_tx: None,
5008            })
5009            .expect("queue subscribe");
5010        drop(cmd_tx);
5011        drop(raw_tx);
5012
5013        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5014            .await
5015            .expect("timed out waiting for handler to drain command");
5016
5017        assert!(next.is_none());
5018        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
5019        assert_eq!(handler.pending_subs.len(), 1);
5020        assert_eq!(handler.subscription_attempts.len(), 1);
5021        assert_eq!(handler.pending_subs[0].0, Ustr::from("candle:0:1m"));
5022    }
5023
5024    #[tokio::test]
5025    async fn unsubscribe_drops_queued_subscribe_while_gate_full() {
5026        let signal = Arc::new(AtomicBool::new(false));
5027        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5028        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5029        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5030        let mut handler =
5031            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5032
5033        // Saturate the gate so the queued subscribe cannot dispatch before the unsubscribe
5034        saturate_subscription_gate(&mut handler);
5035
5036        cmd_tx
5037            .send(HandlerCommand::Subscribe {
5038                channel: LighterWsChannel::Trade(0),
5039                auth: None,
5040                response_tx: None,
5041            })
5042            .expect("queue subscribe");
5043        cmd_tx
5044            .send(HandlerCommand::Unsubscribe {
5045                channel: LighterWsChannel::Trade(0),
5046            })
5047            .expect("queue unsubscribe");
5048        drop(cmd_tx);
5049        drop(raw_tx);
5050
5051        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5052            .await
5053            .expect("timed out waiting for handler to drain commands");
5054
5055        assert!(next.is_none());
5056        assert!(handler.pending_subs.is_empty());
5057        assert!(handler.subscription_attempts.is_empty());
5058    }
5059
5060    #[tokio::test]
5061    async fn reconnect_requeues_attempt_with_new_generation() {
5062        let signal = Arc::new(AtomicBool::new(false));
5063        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5064        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5065        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5066        let mut handler =
5067            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5068
5069        saturate_subscription_gate(&mut handler);
5070        handler.queue_subscribe(LighterWsChannel::Trade(0), None, None);
5071        let old_generation = handler.pending_subs[0].1;
5072
5073        raw_tx
5074            .send((7, Message::Text(RECONNECTED.to_string().into())))
5075            .expect("queue reconnect sentinel");
5076
5077        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5078            .await
5079            .expect("timed out waiting for reconnect");
5080
5081        assert!(matches!(next, Some(NautilusWsMessage::Reconnected { .. })));
5082        assert!(handler.inflight_subs.is_empty());
5083        assert_eq!(handler.pending_subs.len(), 1);
5084        assert_eq!(handler.pending_subs[0].0, Ustr::from("trade:0"));
5085        assert_ne!(handler.pending_subs[0].1, old_generation);
5086        assert_eq!(
5087            handler.subscription_attempts[&Ustr::from("trade:0")].generation,
5088            handler.pending_subs[0].1,
5089        );
5090    }
5091
5092    #[rstest]
5093    #[tokio::test]
5094    async fn pump_releases_inflight_slot_and_schedules_send_failure_retry() {
5095        let signal = Arc::new(AtomicBool::new(false));
5096        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5097        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5098        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5099        let mut handler =
5100            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5101
5102        // No active client, so every dispatch fails; the gate must not leak slots
5103        for market_index in 0..3 {
5104            handler.queue_subscribe(LighterWsChannel::Trade(market_index), None, None);
5105        }
5106
5107        handler.pump_pending_subscribes().await;
5108
5109        assert!(handler.pending_subs.is_empty());
5110        assert!(handler.inflight_subs.is_empty());
5111        assert_eq!(handler.subscription_attempts.len(), 3);
5112        assert!(
5113            handler
5114                .subscription_attempts
5115                .values()
5116                .all(|attempt| attempt.retries == 1),
5117        );
5118        assert_eq!(handler.subscription_retries.len(), 3);
5119    }
5120
5121    #[rstest]
5122    fn subscribed_control_frame_releases_inflight_slot() {
5123        let mut handler = make_handler_with_account();
5124        mark_subscription_inflight(
5125            &mut handler,
5126            LighterWsChannel::Candle {
5127                market_index: 0,
5128                resolution: LighterCandleResolution::OneMinute,
5129            },
5130            None,
5131        );
5132
5133        let (matched, msg) = handle_control_text(
5134            &mut handler,
5135            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5136        );
5137
5138        assert!(matched);
5139        assert!(msg.is_none());
5140        assert!(
5141            !handler
5142                .inflight_subs
5143                .contains_key(&Ustr::from("candle:0:1m"))
5144        );
5145        assert!(handler.subscription_attempts.is_empty());
5146    }
5147
5148    #[rstest]
5149    #[case::top_level(
5150        r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_orders:12345"}"#
5151    )]
5152    #[case::nested(
5153        r#"{"error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
5154    )]
5155    #[case::nested_typed(
5156        r#"{"type":"error","error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
5157    )]
5158    fn already_subscribed_confirms_matching_inflight_topic(#[case] payload: &str) {
5159        let mut handler = make_handler_with_account();
5160        mark_subscription_inflight(
5161            &mut handler,
5162            LighterWsChannel::AccountAllOrders(12345),
5163            None,
5164        );
5165
5166        let (matched, msg) = handle_control_text(&mut handler, payload);
5167
5168        assert!(matched);
5169        assert!(msg.is_none());
5170        assert!(handler.inflight_subs.is_empty());
5171        assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
5172        assert_eq!(handler.subscriptions.len(), 1);
5173    }
5174
5175    #[rstest]
5176    fn already_subscribed_does_not_confirm_unmatched_topic() {
5177        let mut handler = make_handler_with_account();
5178        let inflight = "account_all_orders:12345";
5179        mark_subscription_inflight(
5180            &mut handler,
5181            LighterWsChannel::AccountAllOrders(12345),
5182            None,
5183        );
5184
5185        let (matched, msg) = handle_control_text(
5186            &mut handler,
5187            r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_trades:12345"}"#,
5188        );
5189
5190        assert!(matched);
5191        assert!(msg.is_none());
5192        assert!(handler.inflight_subs.contains_key(&Ustr::from(inflight)));
5193        assert_eq!(
5194            handler.subscriptions.pending_subscribe_topics(),
5195            vec![inflight]
5196        );
5197        assert_eq!(handler.subscriptions.len(), 0);
5198    }
5199
5200    #[tokio::test]
5201    async fn duplicate_queued_and_inflight_topics_share_one_generation() {
5202        let mut handler = make_handler_with_account();
5203        let (response_tx_1, mut response_rx_1) = tokio::sync::oneshot::channel();
5204        let (response_tx_2, mut response_rx_2) = tokio::sync::oneshot::channel();
5205        let (response_tx_3, mut response_rx_3) = tokio::sync::oneshot::channel();
5206        let channel = LighterWsChannel::Trade(7);
5207
5208        handler.queue_subscribe(channel.clone(), None, Some(response_tx_1));
5209        let generation = handler.pending_subs[0].1;
5210        handler.queue_subscribe(channel.clone(), None, Some(response_tx_2));
5211
5212        assert_eq!(handler.pending_subs.len(), 1);
5213        assert_eq!(handler.subscription_attempts.len(), 1);
5214        assert_eq!(
5215            handler.subscription_attempts[&Ustr::from("trade:7")]
5216                .response_txs
5217                .len(),
5218            2,
5219        );
5220
5221        let (topic, queued_generation) = handler.pending_subs.pop_front().unwrap();
5222        handler.inflight_subs.insert(topic, queued_generation);
5223        handler.queue_subscribe(channel, None, Some(response_tx_3));
5224
5225        assert_eq!(queued_generation, generation);
5226        assert!(handler.pending_subs.is_empty());
5227        assert_eq!(handler.inflight_subs.len(), 1);
5228        assert_eq!(
5229            handler.subscription_attempts[&Ustr::from("trade:7")]
5230                .response_txs
5231                .len(),
5232            3,
5233        );
5234        assert!(matches!(
5235            response_rx_1.try_recv(),
5236            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5237        ));
5238        assert!(matches!(
5239            response_rx_2.try_recv(),
5240            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5241        ));
5242        assert!(matches!(
5243            response_rx_3.try_recv(),
5244            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5245        ));
5246
5247        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5248
5249        assert_eq!(response_rx_1.await.unwrap(), Ok(()));
5250        assert_eq!(response_rx_2.await.unwrap(), Ok(()));
5251        assert_eq!(response_rx_3.await.unwrap(), Ok(()));
5252        assert!(handler.inflight_subs.is_empty());
5253        assert!(handler.subscription_attempts.is_empty());
5254    }
5255
5256    #[tokio::test]
5257    async fn changed_inflight_auth_waits_for_serialized_successor() {
5258        let mut handler = make_handler_with_account();
5259        let channel = LighterWsChannel::AccountAllOrders(12345);
5260        let topic = Ustr::from(channel.topic_key().as_str());
5261        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5262        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5263
5264        handler.queue_subscribe(
5265            channel.clone(),
5266            Some(SecretString::from("old-token")),
5267            Some(old_tx),
5268        );
5269        let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
5270        handler.inflight_subs.insert(queued_topic, old_generation);
5271        handler.queue_subscribe(
5272            channel,
5273            Some(SecretString::from("fresh-token")),
5274            Some(fresh_tx),
5275        );
5276
5277        let attempt = &handler.subscription_attempts[&topic];
5278        assert_eq!(
5279            attempt.auth.as_ref().map(SecretString::expose_secret),
5280            Some("old-token")
5281        );
5282        assert_eq!(
5283            attempt
5284                .pending_auth
5285                .as_ref()
5286                .map(SecretString::expose_secret),
5287            Some("fresh-token")
5288        );
5289        assert_eq!(attempt.response_txs.len(), 1);
5290        assert_eq!(attempt.pending_response_txs.len(), 1);
5291
5292        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5293        assert_eq!(old_rx.await.unwrap(), Ok(()));
5294        assert!(matches!(
5295            fresh_rx.try_recv(),
5296            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5297        ));
5298        let attempt = &handler.subscription_attempts[&topic];
5299        assert_ne!(attempt.generation, old_generation);
5300        assert_eq!(
5301            attempt.auth.as_ref().map(SecretString::expose_secret),
5302            Some("fresh-token")
5303        );
5304        assert!(attempt.pending_auth.is_none());
5305        assert_eq!(
5306            handler.pending_subs.front(),
5307            Some(&(topic, attempt.generation)),
5308        );
5309
5310        let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
5311        handler.inflight_subs.insert(topic, fresh_generation);
5312
5313        // The predecessor's trailing typed snapshot must not complete the
5314        // dispatched successor.
5315        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
5316        assert!(matches!(
5317            fresh_rx.try_recv(),
5318            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5319        ));
5320        assert_eq!(
5321            handler.subscription_attempts[&topic].generation,
5322            fresh_generation,
5323        );
5324
5325        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5326        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5327        assert!(handler.subscription_attempts.is_empty());
5328    }
5329
5330    #[tokio::test]
5331    async fn stale_typed_frame_does_not_complete_fresh_attempt_after_predecessor_removed() {
5332        let mut handler = make_handler_with_account();
5333        let channel = LighterWsChannel::AccountAllOrders(12345);
5334        let topic = Ustr::from(channel.topic_key().as_str());
5335        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5336        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5337
5338        // Predecessor completes on its control ack and is removed before any
5339        // pending auth exists.
5340        handler.queue_subscribe(
5341            channel.clone(),
5342            Some(SecretString::from("old-token")),
5343            Some(old_tx),
5344        );
5345        let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
5346        handler.inflight_subs.insert(queued_topic, old_generation);
5347        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5348        assert_eq!(old_rx.await.unwrap(), Ok(()));
5349        assert!(handler.subscription_attempts.is_empty());
5350
5351        // The fresh command then creates and dispatches a brand-new attempt.
5352        handler.queue_subscribe(
5353            channel,
5354            Some(SecretString::from("fresh-token")),
5355            Some(fresh_tx),
5356        );
5357        let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
5358        handler.inflight_subs.insert(topic, fresh_generation);
5359
5360        // The predecessor's trailing typed snapshot must not complete it.
5361        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
5362        assert!(matches!(
5363            fresh_rx.try_recv(),
5364            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5365        ));
5366        assert_eq!(
5367            handler.subscription_attempts[&topic].generation,
5368            fresh_generation,
5369        );
5370
5371        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5372        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5373        assert!(handler.subscription_attempts.is_empty());
5374    }
5375
5376    #[tokio::test]
5377    async fn changed_queued_auth_updates_existing_generation_before_dispatch() {
5378        let mut handler = make_handler_with_account();
5379        let channel = LighterWsChannel::AccountAllOrders(12345);
5380        let topic = Ustr::from(channel.topic_key().as_str());
5381        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5382        let (fresh_tx, fresh_rx) = tokio::sync::oneshot::channel();
5383
5384        handler.queue_subscribe(
5385            channel.clone(),
5386            Some(SecretString::from("old-token")),
5387            Some(old_tx),
5388        );
5389        let generation = handler.pending_subs[0].1;
5390        handler.queue_subscribe(
5391            channel,
5392            Some(SecretString::from("fresh-token")),
5393            Some(fresh_tx),
5394        );
5395
5396        assert_eq!(handler.pending_subs.len(), 1);
5397        assert_eq!(handler.pending_subs[0], (topic, generation));
5398        let attempt = &handler.subscription_attempts[&topic];
5399        assert_eq!(attempt.generation, generation);
5400        assert_eq!(
5401            attempt.auth.as_ref().map(SecretString::expose_secret),
5402            Some("fresh-token")
5403        );
5404        assert!(attempt.pending_auth.is_none());
5405        assert_eq!(attempt.response_txs.len(), 2);
5406        assert!(attempt.pending_response_txs.is_empty());
5407
5408        handler.pending_subs.pop_front();
5409        handler.inflight_subs.insert(topic, generation);
5410        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5411        assert_eq!(old_rx.await.unwrap(), Ok(()));
5412        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5413    }
5414
5415    #[rstest]
5416    fn second_reconnect_folds_pending_auth_into_requeued_generation() {
5417        let mut handler = make_handler_with_account();
5418        let channel = LighterWsChannel::AccountAllOrders(12345);
5419        let topic = Ustr::from(channel.topic_key().as_str());
5420        let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
5421        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5422
5423        handler.queue_subscribe(
5424            channel.clone(),
5425            Some(SecretString::from("old-token")),
5426            Some(old_tx),
5427        );
5428        handler.reset_subscription_attempts_after_reconnect();
5429        let (_, replay_generation) = handler.pending_subs.pop_front().unwrap();
5430        handler.inflight_subs.insert(topic, replay_generation);
5431        handler.queue_subscribe(
5432            channel,
5433            Some(SecretString::from("fresh-token")),
5434            Some(fresh_tx),
5435        );
5436
5437        let attempt = &handler.subscription_attempts[&topic];
5438        assert_eq!(
5439            attempt.auth.as_ref().map(SecretString::expose_secret),
5440            Some("old-token")
5441        );
5442        assert_eq!(
5443            attempt
5444                .pending_auth
5445                .as_ref()
5446                .map(SecretString::expose_secret),
5447            Some("fresh-token")
5448        );
5449        assert_eq!(attempt.response_txs.len(), 1);
5450        assert_eq!(attempt.pending_response_txs.len(), 1);
5451
5452        handler.reset_subscription_attempts_after_reconnect();
5453
5454        let attempt = &handler.subscription_attempts[&topic];
5455        assert_ne!(attempt.generation, replay_generation);
5456        assert_eq!(
5457            attempt.auth.as_ref().map(SecretString::expose_secret),
5458            Some("fresh-token")
5459        );
5460        assert!(attempt.pending_auth.is_none());
5461        assert_eq!(attempt.response_txs.len(), 2);
5462        assert!(attempt.pending_response_txs.is_empty());
5463        assert_eq!(handler.pending_subs.len(), 1);
5464        assert_eq!(handler.pending_subs[0], (topic, attempt.generation));
5465        assert!(handler.inflight_subs.is_empty());
5466        assert!(matches!(
5467            old_rx.try_recv(),
5468            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5469        ));
5470        assert!(matches!(
5471            fresh_rx.try_recv(),
5472            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5473        ));
5474
5475        let (_, generation) = handler.pending_subs.pop_front().unwrap();
5476        handler.inflight_subs.insert(topic, generation);
5477        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5478
5479        assert_eq!(old_rx.try_recv(), Ok(Ok(())));
5480        assert_eq!(fresh_rx.try_recv(), Ok(Ok(())));
5481        assert!(handler.subscription_attempts.is_empty());
5482    }
5483
5484    #[tokio::test]
5485    async fn retry_folds_pending_auth_without_resetting_retry_budget() {
5486        let mut handler = make_handler_with_account();
5487        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5488        handler.set_command_sender(cmd_tx);
5489        let channel = LighterWsChannel::AccountAllOrders(12345);
5490        let topic = Ustr::from(channel.topic_key().as_str());
5491        let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
5492        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5493
5494        handler.queue_subscribe(
5495            channel.clone(),
5496            Some(SecretString::from("old-token")),
5497            Some(old_tx),
5498        );
5499        let (_, generation) = handler.pending_subs.pop_front().unwrap();
5500        handler.inflight_subs.insert(topic, generation);
5501        handler
5502            .subscription_attempts
5503            .get_mut(&topic)
5504            .unwrap()
5505            .retries = 2;
5506        handler.queue_subscribe(
5507            channel,
5508            Some(SecretString::from("fresh-token")),
5509            Some(fresh_tx),
5510        );
5511
5512        handler.schedule_subscription_retry(topic, generation, "retry");
5513
5514        let attempt = &handler.subscription_attempts[&topic];
5515        assert_eq!(attempt.retries, 3);
5516        assert_ne!(attempt.generation, generation);
5517        assert_eq!(
5518            attempt.auth.as_ref().map(SecretString::expose_secret),
5519            Some("fresh-token")
5520        );
5521        assert!(attempt.pending_auth.is_none());
5522        assert_eq!(attempt.response_txs.len(), 2);
5523        assert!(attempt.pending_response_txs.is_empty());
5524        assert!(handler.pending_subs.is_empty());
5525        assert!(handler.inflight_subs.is_empty());
5526        assert_eq!(handler.subscription_retries.len(), 1);
5527        assert!(matches!(
5528            old_rx.try_recv(),
5529            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5530        ));
5531        assert!(matches!(
5532            fresh_rx.try_recv(),
5533            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5534        ));
5535    }
5536
5537    #[tokio::test]
5538    async fn confirmed_authenticated_topic_opens_new_generation() {
5539        let mut handler = make_handler_with_account();
5540        let channel = LighterWsChannel::AccountAllOrders(12345);
5541        let topic = channel.topic_key();
5542        handler.subscriptions.mark_subscribe(&topic);
5543        handler.subscriptions.confirm_subscribe(&topic);
5544        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5545
5546        handler.queue_subscribe(
5547            channel,
5548            Some(SecretString::from("rotated-auth-token")),
5549            Some(response_tx),
5550        );
5551
5552        assert_eq!(handler.pending_subs.len(), 1);
5553        assert_eq!(handler.subscription_attempts.len(), 1);
5554        assert_eq!(handler.subscriptions.len(), 1);
5555        assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
5556
5557        let (topic, generation) = handler.pending_subs.pop_front().unwrap();
5558        handler.inflight_subs.insert(topic, generation);
5559        handle_control_text(
5560            &mut handler,
5561            r#"{"type":"subscribed","channel":"account_all_orders:12345"}"#,
5562        );
5563
5564        assert_eq!(response_rx.await.unwrap(), Ok(()));
5565        assert!(handler.subscription_attempts.is_empty());
5566        assert_eq!(handler.subscriptions.len(), 1);
5567    }
5568
5569    #[tokio::test]
5570    async fn typed_update_before_ack_does_not_complete_subscription_generation() {
5571        let mut handler = make_handler_with_account();
5572        let (response_tx, mut response_rx) = tokio::sync::oneshot::channel();
5573        let (topic, generation) = mark_subscription_inflight(
5574            &mut handler,
5575            LighterWsChannel::Candle {
5576                market_index: 0,
5577                resolution: LighterCandleResolution::OneMinute,
5578            },
5579            Some(response_tx),
5580        );
5581
5582        handler.handle_frame(
5583            candle_frame(
5584                "candle:0:1m",
5585                stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
5586                false,
5587            ),
5588            UnixNanos::from(1),
5589        );
5590
5591        assert_eq!(
5592            handler
5593                .last_candles
5594                .get(&(0, LighterCandleResolution::OneMinute))
5595                .map(|candle| candle.t),
5596            Some(1_000_000),
5597        );
5598        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5599        assert_eq!(
5600            handler.subscriptions.pending_subscribe_topics(),
5601            vec!["candle:0:1m"],
5602        );
5603        assert_eq!(handler.subscriptions.len(), 0);
5604        assert!(matches!(
5605            response_rx.try_recv(),
5606            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5607        ));
5608
5609        handle_control_text(
5610            &mut handler,
5611            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5612        );
5613
5614        assert_eq!(response_rx.await.unwrap(), Ok(()));
5615        assert_eq!(handler.subscriptions.len(), 1);
5616    }
5617
5618    #[tokio::test]
5619    async fn typed_subscribed_frame_completes_subscription_generation() {
5620        let signal = Arc::new(AtomicBool::new(false));
5621        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5622        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5623        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5624        let mut handler =
5625            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5626        handler.instruments.insert(0, stub_eth_perp_instrument());
5627        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5628        mark_subscription_inflight(
5629            &mut handler,
5630            LighterWsChannel::Candle {
5631                market_index: 0,
5632                resolution: LighterCandleResolution::OneMinute,
5633            },
5634            Some(response_tx),
5635        );
5636
5637        raw_tx
5638            .send((7, Message::Text(WS_CANDLE_SUBSCRIBED.into())))
5639            .expect("typed subscribed frame");
5640        drop(raw_tx);
5641        drop(cmd_tx);
5642
5643        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5644            .await
5645            .expect("handler did not process typed subscribed frame");
5646
5647        assert!(next.is_none());
5648        assert_eq!(response_rx.await.unwrap(), Ok(()));
5649        assert!(handler.inflight_subs.is_empty());
5650        assert!(handler.subscription_attempts.is_empty());
5651        assert_eq!(handler.subscriptions.len(), 1);
5652    }
5653
5654    #[tokio::test]
5655    async fn unparsable_subscribed_frame_still_completes_subscription() {
5656        let signal = Arc::new(AtomicBool::new(false));
5657        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5658        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5659        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5660        let mut handler =
5661            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5662        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5663        mark_subscription_inflight(
5664            &mut handler,
5665            LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(4098)),
5666            Some(response_tx),
5667        );
5668
5669        raw_tx
5670            .send((7, Message::Text(WS_SPOT_STATS_SUBSCRIBED_BAD_BODY.into())))
5671            .expect("unparsable subscribed frame");
5672        drop(raw_tx);
5673        drop(cmd_tx);
5674
5675        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5676            .await
5677            .expect("handler did not process unparsable subscribed frame");
5678
5679        assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5680        let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5681            .await
5682            .expect("subscription was not completed");
5683        assert_eq!(response.unwrap(), Ok(()));
5684        assert!(handler.inflight_subs.is_empty());
5685        assert!(handler.subscription_attempts.is_empty());
5686        assert_eq!(handler.subscriptions.len(), 1);
5687    }
5688
5689    #[tokio::test]
5690    async fn unparsable_book_subscribed_frame_still_completes_subscription() {
5691        let signal = Arc::new(AtomicBool::new(false));
5692        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5693        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5694        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5695        let mut handler =
5696            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5697        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5698        mark_subscription_inflight(
5699            &mut handler,
5700            LighterWsChannel::OrderBook(4095),
5701            Some(response_tx),
5702        );
5703
5704        // Epoch 0 matches the generation entry seeded by the helper, so the
5705        // book-specific snapshot matching accepts this confirmation.
5706        raw_tx
5707            .send((0, Message::Text(WS_BOOK_SUBSCRIBED_BAD_BODY.into())))
5708            .expect("unparsable book subscribed frame");
5709        drop(raw_tx);
5710        drop(cmd_tx);
5711
5712        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5713            .await
5714            .expect("handler did not process unparsable book subscribed frame");
5715
5716        assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5717        let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5718            .await
5719            .expect("book subscription was not completed");
5720        assert_eq!(response.unwrap(), Ok(()));
5721        assert!(handler.inflight_subs.is_empty());
5722        assert!(handler.subscription_attempts.is_empty());
5723        assert_eq!(handler.subscriptions.len(), 1);
5724    }
5725
5726    #[rstest]
5727    #[case::nonbook_top_level(false, false, "market_stats:0")]
5728    #[case::nonbook_wrapped(false, true, "market_stats/0")]
5729    #[case::book_top_level(true, false, "order_book/0")]
5730    #[case::book_wrapped(true, true, "order_book:0")]
5731    #[tokio::test]
5732    async fn subscribe_failure_preserves_distinct_inflight_subscription(
5733        #[case] book: bool,
5734        #[case] wrapped: bool,
5735        #[case] channel: &str,
5736    ) {
5737        let mut handler = make_handler_with_account();
5738        let (failed_tx, mut failed_rx) = tokio::sync::oneshot::channel();
5739        let (valid_tx, mut valid_rx) = tokio::sync::oneshot::channel();
5740
5741        let failed_channel = if book {
5742            LighterWsChannel::OrderBook(0)
5743        } else {
5744            LighterWsChannel::MarketStats(LighterMarketSelection::Market(0))
5745        };
5746
5747        let (failed_topic, _) =
5748            mark_subscription_inflight(&mut handler, failed_channel, Some(failed_tx));
5749        let (valid_topic, generation) =
5750            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(valid_tx));
5751        let error = serde_json::json!({"code": 30012, "channel": channel});
5752
5753        let value = if wrapped {
5754            serde_json::json!({"error": error})
5755        } else {
5756            error
5757        };
5758
5759        let (matched, message) = handler.handle_control_value(&value);
5760        let pending = valid_rx.try_recv();
5761        let inflight = handler.inflight_subs.get(&valid_topic).copied();
5762        let retries = handler.subscription_attempts[&valid_topic].retries;
5763        let rejected = failed_rx.try_recv();
5764        let failure_retained = handler.subscription_attempts.contains_key(&failed_topic);
5765        handler.handle_control_value(&serde_json::json!({
5766            "type": "subscribed", "channel": "trade:7"
5767        }));
5768
5769        let expected_error = if book {
5770            BOOK_SUBSCRIBE_REJECTION.to_string()
5771        } else {
5772            "subscription market_stats:0 failed after 1 attempts: venue rejected the WebSocket \
5773             subscribe with code 30012"
5774                .to_string()
5775        };
5776
5777        assert!(matched);
5778        assert!(message.is_none());
5779        assert_eq!(
5780            pending,
5781            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
5782        );
5783        assert_eq!(inflight, Some(generation));
5784        assert_eq!(retries, 0);
5785        assert_eq!(rejected, Ok(Err(expected_error)));
5786        assert!(!failure_retained);
5787        assert_eq!(valid_rx.try_recv(), Ok(Ok(())));
5788        assert!(handler.inflight_subs.is_empty());
5789        assert!(handler.subscription_attempts.is_empty());
5790        assert_eq!(handler.subscriptions.len(), 1);
5791    }
5792
5793    #[rstest]
5794    #[case::unknown("trade:999")]
5795    #[case::confirmed("trade:7")]
5796    #[case::queued("market_stats:0")]
5797    #[tokio::test]
5798    async fn subscribe_failure_ignores_channel_without_inflight_attempt(#[case] channel: &str) {
5799        let mut handler = make_handler_with_account();
5800        let (response, mut result) = tokio::sync::oneshot::channel();
5801        let (topic, generation) =
5802            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(3), Some(response));
5803        handler.subscriptions.mark_subscribe("trade:7");
5804        handler.subscriptions.confirm_subscribe("trade:7");
5805        handler.queue_subscribe(
5806            LighterWsChannel::MarketStats(LighterMarketSelection::Market(0)),
5807            None,
5808            None,
5809        );
5810        let (queued_topic, queued_generation) = *handler.pending_subs.front().unwrap();
5811
5812        handler.handle_control_value(&serde_json::json!({"code": 30012, "channel": channel}));
5813
5814        assert_eq!(
5815            result.try_recv(),
5816            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
5817        );
5818        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5819        assert_eq!(
5820            handler.subscription_attempts[&queued_topic].generation,
5821            queued_generation
5822        );
5823        assert_eq!(handler.subscription_attempts[&queued_topic].retries, 0);
5824        assert_eq!(
5825            handler.pending_subs.iter().copied().collect::<Vec<_>>(),
5826            [(queued_topic, queued_generation)]
5827        );
5828        assert!(handler.subscription_retries.is_empty());
5829        assert_eq!(handler.subscriptions.len(), 1);
5830    }
5831
5832    #[tokio::test]
5833    async fn failed_subscribe_error_fails_every_waiter_for_the_generation() {
5834        let mut handler = make_handler_with_account();
5835        let (response_tx_1, response_rx_1) = tokio::sync::oneshot::channel();
5836        let (response_tx_2, response_rx_2) = tokio::sync::oneshot::channel();
5837        let channel = LighterWsChannel::MarketStats(LighterMarketSelection::Market(0));
5838        mark_subscription_inflight(&mut handler, channel.clone(), Some(response_tx_1));
5839        handler.queue_subscribe(channel, None, Some(response_tx_2));
5840
5841        let (matched, msg) = handle_control_text(
5842            &mut handler,
5843            &serde_json::json!({
5844                "type": "error", "code": 30012, "channel": "market_stats:0"
5845            })
5846            .to_string(),
5847        );
5848
5849        assert!(matched);
5850        assert!(msg.is_none());
5851        let error_1 = response_rx_1.await.unwrap().unwrap_err();
5852        let error_2 = response_rx_2.await.unwrap().unwrap_err();
5853        assert!(error_1.contains("market_stats:0"));
5854        assert!(error_1.contains("30012"));
5855        assert_eq!(error_1, error_2);
5856        assert!(handler.inflight_subs.is_empty());
5857        assert!(handler.subscription_attempts.is_empty());
5858        assert!(handler.subscriptions.is_empty());
5859    }
5860
5861    #[rstest]
5862    #[case::rate_limit(30009)]
5863    #[case::unattributed_failure(30012)]
5864    #[tokio::test]
5865    async fn subscription_retry_exhaustion_fails_waiter_and_clears_intent(#[case] code: u64) {
5866        let mut handler = make_handler_with_account();
5867        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5868        let (topic, _) =
5869            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(response_tx));
5870        handler
5871            .subscription_attempts
5872            .get_mut(&topic)
5873            .expect("subscription attempt")
5874            .retries = SUBSCRIBE_RETRY_MAX;
5875
5876        let (matched, msg) =
5877            handle_control_text(&mut handler, &serde_json::json!({"code": code}).to_string());
5878
5879        assert!(matched);
5880        assert!(msg.is_none());
5881        assert_eq!(
5882            response_rx.await.unwrap(),
5883            Err(format!(
5884                "subscription trade:7 failed after 6 attempts: venue rejected the WebSocket \
5885                 subscribe with code {code}"
5886            )),
5887        );
5888        assert!(handler.inflight_subs.is_empty());
5889        assert!(handler.subscription_attempts.is_empty());
5890        assert!(handler.subscriptions.is_empty());
5891    }
5892
5893    #[rstest]
5894    #[case::rate_limit(serde_json::json!({"code": 30009}))]
5895    #[case::unattributed_failure(serde_json::json!({"code": 30012}))]
5896    #[case::wrapped_failure(serde_json::json!({"error": {"code": 30012}}))]
5897    #[case::invalid_channel(serde_json::json!({"code": 30012, "channel": 7}))]
5898    #[case::empty_channel(serde_json::json!({"code": 30012, "channel": ""}))]
5899    #[tokio::test]
5900    async fn subscription_error_retries_each_inflight_topic_with_new_generation(
5901        #[case] value: serde_json::Value,
5902    ) {
5903        let mut handler = make_handler_with_account();
5904        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5905        handler.set_command_sender(cmd_tx);
5906        let (trade_tx, mut trade_rx) = tokio::sync::oneshot::channel();
5907        let (candle_tx, mut candle_rx) = tokio::sync::oneshot::channel();
5908        let (trade_topic, trade_generation) =
5909            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(trade_tx));
5910        let (candle_topic, candle_generation) = mark_subscription_inflight(
5911            &mut handler,
5912            LighterWsChannel::Candle {
5913                market_index: 0,
5914                resolution: LighterCandleResolution::OneMinute,
5915            },
5916            Some(candle_tx),
5917        );
5918
5919        let (matched, msg) = handler.handle_control_value(&value);
5920
5921        assert!(matched);
5922        assert!(msg.is_none());
5923        assert!(handler.inflight_subs.is_empty());
5924        assert!(handler.pending_subs.is_empty());
5925        let trade_retry = &handler.subscription_attempts[&trade_topic];
5926        let candle_retry = &handler.subscription_attempts[&candle_topic];
5927        assert_eq!(trade_retry.retries, 1);
5928        assert_eq!(candle_retry.retries, 1);
5929        assert_ne!(trade_retry.generation, trade_generation);
5930        assert_ne!(candle_retry.generation, candle_generation);
5931        assert_eq!(handler.subscription_retries.len(), 2);
5932        assert!(matches!(
5933            trade_rx.try_recv(),
5934            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5935        ));
5936        assert!(matches!(
5937            candle_rx.try_recv(),
5938            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5939        ));
5940
5941        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5942        assert_eq!(handler.subscription_attempts.len(), 2);
5943        assert_eq!(handler.subscriptions.len(), 0);
5944
5945        let retry_generations = [
5946            (
5947                trade_topic,
5948                handler.subscription_attempts[&trade_topic].generation,
5949            ),
5950            (
5951                candle_topic,
5952                handler.subscription_attempts[&candle_topic].generation,
5953            ),
5954        ];
5955
5956        for (topic, generation) in retry_generations {
5957            handler.queue_subscription_retry(topic, generation);
5958            let queued = handler.pending_subs.pop_front().unwrap();
5959            assert_eq!(queued, (topic, generation));
5960            handler.inflight_subs.insert(topic, generation);
5961        }
5962
5963        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5964        handle_control_text(
5965            &mut handler,
5966            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5967        );
5968
5969        assert_eq!(trade_rx.await.unwrap(), Ok(()));
5970        assert_eq!(candle_rx.await.unwrap(), Ok(()));
5971        assert!(handler.subscription_attempts.is_empty());
5972        assert_eq!(handler.subscriptions.len(), 2);
5973    }
5974}