Skip to main content

nautilus_lighter/websocket/
handler.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Inner WebSocket feed handler running on a dedicated tokio task.
17
18use std::{
19    collections::VecDeque,
20    fmt::Debug,
21    future::Future,
22    pin::Pin,
23    sync::{
24        Arc,
25        atomic::{AtomicBool, Ordering},
26    },
27    time::Duration,
28};
29
30use ahash::AHashMap;
31use futures_util::{StreamExt, stream::FuturesUnordered};
32use nautilus_core::{
33    AtomicTime,
34    nanos::UnixNanos,
35    string::secret::{REDACTED, SecretString},
36    time::get_atomic_clock_realtime,
37};
38#[cfg(test)]
39use nautilus_live::book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS;
40use nautilus_live::book::{
41    BookSequenceOutcome,
42    recovery::{BookRecoveryOutcome, BookRecoveryState},
43    snapshot::{PendingSnapshot, SnapshotGate},
44};
45use nautilus_model::{identifiers::AccountId, instruments::InstrumentAny, types::Currency};
46use nautilus_network::{
47    RECONNECTED,
48    error::SendError,
49    retry::{RetryManager, create_websocket_retry_manager},
50    websocket::{SubscriptionState, WebSocketClient},
51};
52use tokio_tungstenite::tungstenite::Message;
53use tokio_util::sync::CancellationToken;
54use ustr::Ustr;
55use zeroize::Zeroize;
56
57use super::{
58    account_state::LighterAccountStateReconciler,
59    error::LighterWsError,
60    messages::{
61        AccountStream, ExecutionReport, LighterAsset, LighterPosition, LighterUserStats,
62        LighterWsCandle, LighterWsChannel, LighterWsChannelKind, LighterWsFrame,
63        LighterWsOrderBook, LighterWsRequest, NautilusWsMessage, SendTxRejectionSource,
64        TX_BATCH_ID_PREFIX,
65    },
66    parse::{
67        parse_ws_bar, parse_ws_funding_rate_update, parse_ws_index_price_update,
68        parse_ws_mark_price_update, parse_ws_position_status_report, parse_ws_quote_tick,
69        parse_ws_spot_index_price_update, parse_ws_trade_tick,
70    },
71};
72use crate::{
73    book::{
74        recovery::{self, BookWorkResult, BookWrite},
75        sync::BookSyncTracker,
76    },
77    common::{
78        consts::{
79            LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED, LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED,
80            LIGHTER_ERROR_CODE_TX_RANGE, LIGHTER_ERROR_CODE_WS_RATE_LIMITED,
81            LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED, LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL,
82            SUBSCRIBE_INFLIGHT_MAX, SUBSCRIBE_RETRY_BASE_BACKOFF, SUBSCRIBE_RETRY_MAX,
83        },
84        enums::LighterCandleResolution,
85        rate_limit::LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY,
86    },
87    http::models::{LighterOrder, LighterTrade},
88};
89
90// Lighter control-frame `type` field values that fall outside the typed
91// `LighterWsFrame` variants. These are protocol-level frames the handler
92// inspects via the typed-frame parse fallback in `handle_control_value`.
93const CTRL_TYPE_CONNECTED: &str = "connected";
94const CTRL_TYPE_SUBSCRIBED: &str = "subscribed";
95const CTRL_TYPE_UNSUBSCRIBED: &str = "unsubscribed";
96const CTRL_TYPE_PING: &str = "ping";
97const CTRL_TYPE_PONG: &str = "pong";
98const CTRL_TYPE_ERROR: &str = "error";
99const CTRL_TYPE_SEND_TX: &str = "jsonapi/sendtx";
100
101#[derive(serde::Deserialize)]
102struct LighterWsFrameHeader<'a> {
103    #[serde(rename = "type", borrow)]
104    kind: &'a str,
105    #[serde(borrow)]
106    channel: Option<&'a str>,
107}
108
109/// Commands sent from the outer client to the inner feed handler.
110#[expect(
111    clippy::large_enum_variant,
112    reason = "commands are ephemeral and immediately consumed"
113)]
114pub enum HandlerCommand {
115    /// Hand the live `WebSocketClient` to the handler after the outer client
116    /// completes the network connect.
117    SetClient(WebSocketClient),
118    /// Drain the queue and shut the handler down.
119    Disconnect,
120    /// Subscribe to a channel. `auth` carries an optional auth token for the
121    /// account-scoped channels.
122    Subscribe {
123        channel: LighterWsChannel,
124        auth: Option<SecretString>,
125        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
126    },
127    /// Unsubscribe from a channel.
128    Unsubscribe { channel: LighterWsChannel },
129    /// Replace a book subscription for the current recovery owner.
130    RecoverBook {
131        market_index: i64,
132        cancel: CancellationToken,
133        gate: SnapshotGate,
134        completion: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
135    },
136    /// Replace the handler's instrument cache (used on initial connect).
137    InitializeInstruments(Vec<(i64, InstrumentAny)>),
138    /// Insert or replace a single instrument by `market_index`.
139    UpdateInstrument {
140        market_index: i64,
141        instrument: InstrumentAny,
142    },
143    /// Toggle whether `update/order_book` frames for `market_index` should
144    /// be emitted as [`NautilusWsMessage::Deltas`].
145    SetBookDeltasSub { market_index: i64, subscribed: bool },
146    /// Toggle whether `update/order_book` frames for `market_index` should
147    /// also be emitted as a [`NautilusWsMessage::Depth`] snapshot.
148    SetDepthSub { market_index: i64, subscribed: bool },
149    /// Provide the execution context (`AccountId` and venue `account_index`)
150    /// the handler stamps onto reports parsed from `account_*` frames.
151    /// Without this context the handler cannot construct typed reports and
152    /// account frames are forwarded as raw JSON instead.
153    SetExecutionContext {
154        account_id: AccountId,
155        account_index: i64,
156    },
157    /// Dispatch a signed L2 transaction over the WebSocket.
158    ///
159    /// The handler serializes a [`LighterWsRequest::SendTx`] frame; the venue
160    /// confirms acceptance via the `account_*` streams (and reports any
161    /// rejection via a control-plane `error` frame).
162    SendTx {
163        tx_type: u8,
164        tx_info: Box<serde_json::value::RawValue>,
165        connection_epoch: u64,
166        response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
167    },
168    SendTxBatch {
169        data: super::messages::LighterWsSendTxBatch,
170        connection_epoch: u64,
171        response_tx: tokio::sync::oneshot::Sender<Result<(), LighterWsError>>,
172    },
173}
174
175impl Debug for HandlerCommand {
176    /// Custom `Debug` that redacts the `auth` field of `Subscribe`. The
177    /// command is routed in-process via mpsc and may be printed by error
178    /// paths or trace logging; deriving `Debug` would otherwise leak the
179    /// venue bearer token verbatim.
180    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181        match self {
182            Self::SetClient(_) => f.write_str("SetClient(<WebSocketClient>)"),
183            Self::Disconnect => f.write_str("Disconnect"),
184            Self::Subscribe { channel, auth, .. } => f
185                .debug_struct(stringify!(Subscribe))
186                .field("channel", channel)
187                .field("auth", auth)
188                .finish(),
189            Self::Unsubscribe { channel } => f
190                .debug_struct(stringify!(Unsubscribe))
191                .field("channel", channel)
192                .finish(),
193            Self::RecoverBook { market_index, .. } => f
194                .debug_struct(stringify!(RecoverBook))
195                .field("market_index", market_index)
196                .finish(),
197            Self::InitializeInstruments(instruments) => f
198                .debug_tuple(stringify!(InitializeInstruments))
199                .field(&instruments.len())
200                .finish(),
201            Self::UpdateInstrument { market_index, .. } => f
202                .debug_struct(stringify!(UpdateInstrument))
203                .field("market_index", market_index)
204                .finish(),
205            Self::SetBookDeltasSub {
206                market_index,
207                subscribed,
208            } => f
209                .debug_struct(stringify!(SetBookDeltasSub))
210                .field("market_index", market_index)
211                .field("subscribed", subscribed)
212                .finish(),
213            Self::SetDepthSub {
214                market_index,
215                subscribed,
216            } => f
217                .debug_struct(stringify!(SetDepthSub))
218                .field("market_index", market_index)
219                .field("subscribed", subscribed)
220                .finish(),
221            Self::SetExecutionContext {
222                account_id,
223                account_index,
224            } => f
225                .debug_struct(stringify!(SetExecutionContext))
226                .field("account_id", account_id)
227                .field("account_index", account_index)
228                .finish(),
229            Self::SendTxBatch { .. } => f.write_str("SendTxBatch(<redacted>)"),
230            Self::SendTx { tx_type, .. } => f
231                .debug_struct(stringify!(SendTx))
232                .field("tx_type", tx_type)
233                .field("tx_info", &REDACTED)
234                .finish(),
235        }
236    }
237}
238
239/// Inner feed handler. Owns the [`WebSocketClient`] exclusively and routes
240/// raw frames into the venue-message channel.
241///
242/// Subscribe dispatch is gated by a closed-loop inflight count: the venue caps
243/// unacknowledged client messages per connection, and acknowledgement latency
244/// is multi-second, so queued subscribes are held back until acks free a slot.
245pub(super) struct FeedHandler {
246    clock: &'static AtomicTime,
247    signal: Arc<AtomicBool>,
248    inner: Option<Arc<WebSocketClient>>,
249    cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
250    cmd_tx: Option<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>,
251    raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
252    out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
253    subscriptions: SubscriptionState,
254    retry_manager: RetryManager<LighterWsError>,
255    pending_messages: VecDeque<NautilusWsMessage>,
256    pending_subs: VecDeque<(Ustr, u64)>,
257    inflight_subs: AHashMap<Ustr, u64>,
258    subscription_attempts: AHashMap<Ustr, SubscriptionAttempt>,
259    subscription_retries: FuturesUnordered<SubscriptionRetry>,
260    ignored_completions: AHashMap<Ustr, CompletionKind>,
261    next_subscription_generation: u64,
262    instruments: AHashMap<i64, InstrumentAny>,
263    book: BookSyncTracker,
264    book_snapshot_timeout: Duration,
265    last_candles: AHashMap<(i64, LighterCandleResolution), LighterWsCandle>,
266    exec_account: Option<(AccountId, i64)>,
267    account_state_reconciler: LighterAccountStateReconciler,
268}
269
270type SubscriptionRetry = Pin<Box<dyn Future<Output = (Ustr, u64)> + Send + Sync + 'static>>;
271
272struct SubscriptionAttempt {
273    channel: LighterWsChannel,
274    auth: Option<SecretString>,
275    pending_auth: Option<SecretString>,
276    generation: u64,
277    retries: u8,
278    response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
279    pending_response_txs: Vec<tokio::sync::oneshot::Sender<Result<(), String>>>,
280}
281
282impl SubscriptionAttempt {
283    fn fold_pending_auth(&mut self) {
284        if let Some(auth) = self.pending_auth.take() {
285            self.auth = Some(auth);
286            self.response_txs.append(&mut self.pending_response_txs);
287        } else {
288            debug_assert!(self.pending_response_txs.is_empty());
289        }
290    }
291}
292
293/// Which venue frame produced a subscription completion.
294///
295/// The venue answers a subscribe with a generic control ack followed by a
296/// typed `subscribed/<channel>` snapshot, and both complete by topic. The kind
297/// lets `complete_subscription` swallow the predecessor's trailing frame so a
298/// same-topic generation dispatched between the two cannot be completed by a
299/// response that belongs to the earlier request.
300#[derive(Clone, Copy, Debug, PartialEq, Eq)]
301enum CompletionKind {
302    ControlAck,
303    Typed,
304    AlreadySubscribed,
305}
306
307impl FeedHandler {
308    #[cfg(test)]
309    pub(super) fn new(
310        signal: Arc<AtomicBool>,
311        cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
312        raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
313        out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
314        subscriptions: SubscriptionState,
315    ) -> Self {
316        Self::new_with_settlement_currency(
317            signal,
318            cmd_rx,
319            raw_rx,
320            out_tx,
321            subscriptions,
322            Currency::get_or_create_crypto("USDC"),
323            Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
324        )
325    }
326
327    pub(super) fn new_with_settlement_currency(
328        signal: Arc<AtomicBool>,
329        cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
330        raw_rx: tokio::sync::mpsc::UnboundedReceiver<(u64, Message)>,
331        out_tx: tokio::sync::mpsc::UnboundedSender<NautilusWsMessage>,
332        subscriptions: SubscriptionState,
333        settlement_currency: Currency,
334        book_snapshot_timeout: Duration,
335    ) -> Self {
336        Self {
337            clock: get_atomic_clock_realtime(),
338            signal,
339            inner: None,
340            cmd_rx,
341            cmd_tx: None,
342            raw_rx,
343            out_tx,
344            subscriptions,
345            retry_manager: create_websocket_retry_manager(),
346            pending_messages: VecDeque::new(),
347            pending_subs: VecDeque::new(),
348            inflight_subs: AHashMap::new(),
349            subscription_attempts: AHashMap::new(),
350            subscription_retries: FuturesUnordered::new(),
351            ignored_completions: AHashMap::new(),
352            next_subscription_generation: 1,
353            instruments: AHashMap::new(),
354            book: BookSyncTracker::default(),
355            book_snapshot_timeout,
356            last_candles: AHashMap::new(),
357            exec_account: None,
358            account_state_reconciler: LighterAccountStateReconciler::new_with_settlement_currency(
359                settlement_currency,
360            ),
361        }
362    }
363
364    pub(super) fn send(&self, msg: NautilusWsMessage) -> Result<(), String> {
365        self.out_tx
366            .send(msg)
367            .map_err(|e| format!("Failed to send message: {e}"))
368    }
369
370    pub(super) fn set_command_sender(
371        &mut self,
372        cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
373    ) {
374        self.cmd_tx = Some(cmd_tx);
375    }
376
377    pub(super) fn is_stopped(&self) -> bool {
378        self.signal.load(Ordering::Relaxed)
379    }
380
381    async fn send_with_retry(&self, payload: String) -> Result<(), LighterWsError> {
382        self.send_secret_with_retry(SecretString::from(payload))
383            .await
384    }
385
386    async fn send_secret_with_retry(&self, payload: SecretString) -> Result<(), LighterWsError> {
387        if let Some(client) = &self.inner {
388            self.retry_manager
389                .invocation(
390                    "websocket_send",
391                    || {
392                        let payload = payload.clone();
393                        async move {
394                            client
395                                .send_text(
396                                    payload.expose_secret().to_owned(),
397                                    Some(LIGHTER_WS_MESSAGE_RATE_LIMIT_KEY.as_slice()),
398                                )
399                                .await
400                                .map_err(LighterWsError::Transport)
401                        }
402                    },
403                    should_retry_lighter_ws_error,
404                    |e| create_lighter_ws_timeout_error(e.to_string()),
405                )
406                .execute()
407                .await
408        } else {
409            Err(LighterWsError::Client(
410                "no active WebSocket client".to_string(),
411            ))
412        }
413    }
414
415    // Single-shot: sendTx payloads carry a signed nonce; transport-layer
416    // retry could double-submit if the original landed and only the ack was lost.
417    async fn send_once(
418        &self,
419        payload: String,
420        connection_epoch: u64,
421    ) -> Result<(), LighterWsError> {
422        if let Some(client) = &self.inner {
423            match client
424                .send_text_on_connection(payload, None, connection_epoch)
425                .await
426            {
427                Err(SendError::BrokenPipe(message)) => {
428                    Err(LighterWsError::SendTxOutcomeUnknown(message))
429                }
430                result => result.map_err(LighterWsError::Transport),
431            }
432        } else {
433            Err(LighterWsError::Client(
434                "no active WebSocket client".to_string(),
435            ))
436        }
437    }
438
439    async fn dispatch_subscribe(
440        &self,
441        channel: LighterWsChannel,
442        auth: Option<SecretString>,
443    ) -> Result<(), String> {
444        let topic = channel.topic_key();
445
446        let authed = auth.is_some();
447        let mut request = match auth {
448            Some(token) => LighterWsRequest::subscribe_auth(
449                channel.subscription_channel(),
450                token.expose_secret(),
451            ),
452            None => LighterWsRequest::subscribe(channel.subscription_channel()),
453        };
454
455        let payload = serde_json::to_string(&request);
456        request.zeroize();
457
458        match payload {
459            Ok(payload) => {
460                // Avoid logging the serialized payload for authenticated channels;
461                // it embeds a live Lighter L2 bearer token.
462                log::debug!("Sending Lighter subscribe: topic={topic} authed={authed}");
463                let result = if authed {
464                    self.send_secret_with_retry(SecretString::from(payload))
465                        .await
466                } else {
467                    self.send_with_retry(payload).await
468                };
469
470                if let Err(e) = result {
471                    log::error!("Error subscribing to {topic}: {e}");
472                    Err(e.to_string())
473                } else {
474                    Ok(())
475                }
476            }
477            Err(e) => {
478                log::error!("Error serializing subscription for {topic}: {e}");
479                Err(format!("failed to serialize subscription for {topic}: {e}"))
480            }
481        }
482    }
483
484    async fn dispatch_send_tx(
485        &self,
486        tx_type: u8,
487        tx_info: Box<serde_json::value::RawValue>,
488        connection_epoch: u64,
489    ) -> Result<(), LighterWsError> {
490        let request = LighterWsRequest::SendTx {
491            data: super::messages::LighterWsSendTx { tx_type, tx_info },
492        };
493
494        match serde_json::to_string(&request) {
495            Ok(payload) => {
496                log::debug!(
497                    "Sending Lighter sendTx: tx_type={tx_type} ({} bytes)",
498                    payload.len(),
499                );
500
501                match self.send_once(payload, connection_epoch).await {
502                    Ok(()) => Ok(()),
503                    Err(e) => {
504                        log::debug!("Error dispatching Lighter sendTx (tx_type={tx_type}): {e}");
505                        Err(e)
506                    }
507                }
508            }
509            Err(e) => {
510                log::error!("Error serializing Lighter sendTx (tx_type={tx_type}): {e}");
511                Err(LighterWsError::Client(format!(
512                    "failed to serialize Lighter sendTx: {e}"
513                )))
514            }
515        }
516    }
517
518    async fn dispatch_unsubscribe(&self, channel: LighterWsChannel) {
519        let topic = channel.topic_key();
520        self.subscriptions.mark_unsubscribe(&topic);
521
522        let request = LighterWsRequest::unsubscribe(channel.subscription_channel());
523        match serde_json::to_string(&request) {
524            Ok(payload) => {
525                log::debug!("Sending Lighter unsubscribe ({} bytes)", payload.len());
526                if let Err(e) = self.send_with_retry(payload).await {
527                    log::error!("Error unsubscribing from {topic}: {e}");
528                }
529            }
530            Err(e) => {
531                log::error!("Error serializing unsubscription for {topic}: {e}");
532            }
533        }
534    }
535
536    pub(super) async fn next(&mut self) -> Option<NautilusWsMessage> {
537        if let Some(msg) = self.pending_messages.pop_front() {
538            return Some(msg);
539        }
540
541        loop {
542            // Pump each iteration: the select below drains acks that free slots
543            self.pump_pending_subscribes().await;
544
545            tokio::select! {
546                biased;
547                Some(work) = self.book.work.next(), if !self.book.work.is_empty() => {
548                    match work {
549                        BookWorkResult::Sent { market_index, generation, cancel, write, result } => {
550                            self.complete_book_send(market_index, generation, cancel, write, result);
551                        }
552                        BookWorkResult::Initial { market_index, cancel } => {
553                            if !cancel.is_cancelled() {
554                                self.start_book_recovery(market_index);
555                            }
556                        }
557                        // Recovery ends once a snapshot is accepted or the episode is cancelled
558                        BookWorkResult::Recovery => {}
559                    }
560                }
561                Some(cmd) = self.cmd_rx.recv() => {
562                    match cmd {
563                        HandlerCommand::SetClient(client) => {
564                            log::debug!("Setting WebSocket client in Lighter handler");
565                            self.inner = Some(Arc::new(client));
566                        }
567                        HandlerCommand::Disconnect => {
568                            log::debug!("Lighter handler received disconnect");
569                            if let Some(ref client) = self.inner {
570                                client.disconnect().await;
571                            }
572                            self.signal.store(true, Ordering::SeqCst);
573                            return None;
574                        }
575                        HandlerCommand::Subscribe {
576                            channel,
577                            auth,
578                            response_tx,
579                        } => {
580                            self.queue_subscribe(channel, auth, response_tx);
581                        }
582                        HandlerCommand::Unsubscribe { channel } => {
583                            // Drop a queued-but-unsent subscribe for this channel so a freed
584                            // slot cannot resubscribe after the caller unsubscribed.
585                            let topic = channel.topic_key();
586                            self.cancel_subscription_attempt(
587                                &topic,
588                                "subscription cancelled before venue acknowledgement",
589                            );
590
591                            if let LighterWsChannel::OrderBook(market_index) = &channel {
592                                self.book.cancel(*market_index);
593                                self.book.clear_cached_order_book(*market_index);
594                            }
595                            self.dispatch_unsubscribe(channel).await;
596                        }
597                        HandlerCommand::RecoverBook { market_index, cancel, gate, completion } => {
598                            self.queue_book_replacement(market_index, BookWrite { cancel, gate, completion });
599                        }
600                        HandlerCommand::InitializeInstruments(instruments) => {
601                            self.instruments.clear();
602                            for (market_index, inst) in instruments {
603                                self.instruments.insert(market_index, inst);
604                            }
605                        }
606                        HandlerCommand::UpdateInstrument { market_index, instrument } => {
607                            self.instruments.insert(market_index, instrument);
608                        }
609                        HandlerCommand::SetBookDeltasSub { market_index, subscribed } => {
610                            if subscribed {
611                                let inserted = self.book.delta_subs.insert(market_index);
612                                if inserted
613                                    && let Some(first) = self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_deltas_snapshot(market_index, instrument, self.clock.get_time_ns()))
614                                {
615                                    return Some(first);
616                                }
617                            } else {
618                                self.book.delta_subs.remove(&market_index);
619                                self.cancel_orphaned_book_recovery(market_index);
620                            }
621                        }
622                        HandlerCommand::SetDepthSub { market_index, subscribed } => {
623                            if subscribed {
624                                let inserted = self.book.depth_subs.insert(market_index);
625                                if inserted
626                                    && let Some(first) =
627                                        self.instruments.get(&market_index).and_then(|instrument| self.book.emit_cached_order_book_depth_snapshot(market_index, instrument, self.clock.get_time_ns()))
628                                {
629                                    return Some(first);
630                                }
631                            } else {
632                                self.book.depth_subs.remove(&market_index);
633                                self.cancel_orphaned_book_recovery(market_index);
634                            }
635                        }
636                        HandlerCommand::SetExecutionContext { account_id, account_index } => {
637                            self.exec_account = Some((account_id, account_index));
638                        }
639                        HandlerCommand::SendTxBatch { data, connection_epoch, response_tx } => {
640                            let result = match serde_json::to_string(&LighterWsRequest::SendTxBatch { data }) {
641                                Ok(payload) => self.send_once(payload, connection_epoch).await,
642                                Err(e) => Err(LighterWsError::Client(format!("failed to serialize Lighter sendTxBatch: {e}"))),
643                            };
644                            let _ = response_tx.send(result);
645                        }
646                        HandlerCommand::SendTx {
647                            tx_type,
648                            tx_info,
649                            connection_epoch,
650                            response_tx,
651                        } => {
652                            let result = self
653                                .dispatch_send_tx(tx_type, tx_info, connection_epoch)
654                                .await;
655
656                            if response_tx.send(result).is_err() {
657                                log::debug!("Lighter sendTx result receiver dropped");
658                            }
659                        }
660                    }
661                }
662                Some((topic, generation)) = self.subscription_retries.next(),
663                    if !self.subscription_retries.is_empty() =>
664                {
665                    self.queue_subscription_retry(topic, generation);
666                }
667                Some((connection_epoch, raw_msg)) = self.raw_rx.recv() => {
668                    match raw_msg {
669                        Message::Text(text) => {
670                            if text == RECONNECTED {
671                                log::debug!("Received Lighter WebSocket RECONNECTED sentinel");
672                                self.book.reset_on_reconnect();
673                                // Resubscribe replays a fresh `subscribed/candle`; pre-disconnect cache is stale.
674                                self.last_candles.clear();
675                                // The new socket starts with zero inflight and restore_subscriptions
676                                // re-queues every topic. Retire stale generations before accepting
677                                // replay commands so late work cannot complete a replacement attempt.
678                                self.reset_subscription_attempts_after_reconnect();
679                                self.account_state_reconciler.reset();
680                                return Some(NautilusWsMessage::Reconnected {
681                                    connection_epoch,
682                                });
683                            }
684
685                            let ts_init = self.clock.get_time_ns();
686                            let subscribed_topic = typed_subscribe_topic(&text);
687
688                            if let Ok(frame) = serde_json::from_str::<LighterWsFrame>(&text) {
689                                if let LighterWsFrame::Shutdown { close_in_ms } = frame {
690                                    if let Some(ws_client) = &self.inner
691                                        && ws_client.connection_epoch() == connection_epoch
692                                        && ws_client.request_reconnect()
693                                    {
694                                        log::info!("Lighter server shutdown in {close_in_ms} ms, reconnect requested");
695                                    }
696                                    continue;
697                                }
698
699                                if let Some(topic) = subscribed_topic
700                                    && !self.complete_typed_subscription(topic, connection_epoch)
701                                {
702                                    continue;
703                                }
704                                let messages = self
705                                    .handle_frame(frame, ts_init)
706                                    .into_iter()
707                                    .map(|msg| msg.with_connection_epoch(connection_epoch))
708                                    .collect();
709
710                                if let Some(first) = self.dispatch_results(messages) {
711                                    return Some(first);
712                                }
713                            } else if let Ok(value) =
714                                serde_json::from_str::<serde_json::Value>(&text)
715                            {
716                                // The venue acknowledged the subscription even
717                                // though the snapshot body is unparsable.
718                                // Complete from the header so subscribers never
719                                // hang on a malformed confirmation frame.
720                                if let Some(topic) = subscribed_topic {
721                                    self.complete_typed_subscription(topic, connection_epoch);
722                                }
723
724                                let (matched, msg) = self.handle_control_value(&value);
725                                if let Some(first) = msg {
726                                    return Some(first.with_connection_epoch(connection_epoch));
727                                }
728
729                                if !matched {
730                                    // Neither a typed frame nor a recognized
731                                    // control type. Surface raw so venue
732                                    // errors (bad signature, margin rejection,
733                                    // ...) don't drop silently.
734                                    log::warn!("Lighter WS unparsed frame: {value}");
735                                    return Some(NautilusWsMessage::Raw(value));
736                                }
737                            } else {
738                                log::warn!("Lighter WS non-JSON text: {text}");
739                            }
740                        }
741                        Message::Ping(data) => {
742                            if let Some(ref client) = self.inner
743                                && let Err(e) = client.send_pong(data.to_vec()).await {
744                                log::error!("Error sending Lighter pong: {e}");
745                            }
746                        }
747                        Message::Close(frame) => {
748                            log::debug!("Received Lighter WebSocket close frame: {frame:?}");
749                            return None;
750                        }
751                        _ => {}
752                    }
753                }
754                else => {
755                    log::debug!("Lighter handler shutting down: stream ended or command channel closed");
756                    return None;
757                }
758            }
759        }
760    }
761
762    fn dispatch_results(
763        &mut self,
764        mut messages: Vec<NautilusWsMessage>,
765    ) -> Option<NautilusWsMessage> {
766        if messages.is_empty() {
767            return None;
768        }
769        let first = messages.remove(0);
770        for extra in messages {
771            self.pending_messages.push_back(extra);
772        }
773        Some(first)
774    }
775
776    // Non-blocking count check, not a permit await: this task also drains the
777    // acks that free slots, so awaiting a permit here would deadlock.
778    async fn pump_pending_subscribes(&mut self) {
779        while self.inflight_subs.len() < SUBSCRIBE_INFLIGHT_MAX {
780            let Some((topic, generation)) = self.pending_subs.pop_front() else {
781                break;
782            };
783
784            let Some(attempt) = self.subscription_attempts.get(&topic) else {
785                continue;
786            };
787
788            if attempt.generation != generation {
789                continue;
790            }
791
792            let channel = attempt.channel.clone();
793            let auth = attempt.auth.clone();
794            self.inflight_subs.insert(topic, generation);
795            if let LighterWsChannel::OrderBook(market_index) = channel {
796                self.send_book_subscribe(market_index, generation);
797            } else if let Err(message) = self.dispatch_subscribe(channel, auth).await {
798                self.schedule_subscription_retry(topic, generation, &message);
799            }
800        }
801    }
802
803    fn queue_subscribe(
804        &mut self,
805        channel: LighterWsChannel,
806        auth: Option<SecretString>,
807        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
808    ) {
809        let topic = Ustr::from(channel.topic_key().as_str());
810
811        if let LighterWsChannel::OrderBook(market_index) = channel
812            && !self.book.writes.contains_key(&market_index)
813            && self
814                .book
815                .recovery
816                .get(&market_index)
817                .and_then(BookRecoveryState::current)
818                .is_some_and(|recovery| !recovery.is_accepted())
819        {
820            return;
821        }
822
823        if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
824            attempt.channel = channel;
825            if matches!(attempt.channel, LighterWsChannel::OrderBook(_))
826                && !self.inflight_subs.contains_key(&topic)
827                && !self
828                    .pending_subs
829                    .iter()
830                    .any(|(pending, _)| *pending == topic)
831            {
832                self.pending_subs.push_back((topic, attempt.generation));
833            }
834
835            let effective_auth = attempt
836                .pending_auth
837                .as_ref()
838                .or(attempt.auth.as_ref())
839                .map(SecretString::expose_secret);
840            let auth_changed = auth
841                .as_ref()
842                .map(SecretString::expose_secret)
843                .is_some_and(|auth| Some(auth) != effective_auth);
844
845            if auth_changed {
846                let is_inflight = self.inflight_subs.get(&topic) == Some(&attempt.generation);
847                if is_inflight {
848                    attempt.pending_auth = auth;
849                    if let Some(response_tx) = response_tx {
850                        attempt.pending_response_txs.push(response_tx);
851                    }
852                } else {
853                    debug_assert!(attempt.pending_auth.is_none());
854                    debug_assert!(attempt.pending_response_txs.is_empty());
855                    attempt.auth = auth;
856                    if let Some(response_tx) = response_tx {
857                        attempt.response_txs.push(response_tx);
858                    }
859                }
860            } else if let Some(response_tx) = response_tx {
861                if auth.is_some() && attempt.pending_auth.is_some() {
862                    attempt.pending_response_txs.push(response_tx);
863                } else {
864                    attempt.response_txs.push(response_tx);
865                }
866            }
867            return;
868        }
869
870        let newly_pending = self.subscriptions.try_mark_subscribe(topic.as_str());
871        if !newly_pending
872            && auth.is_none()
873            && !self
874                .subscriptions
875                .pending_subscribe_topics()
876                .iter()
877                .any(|pending| pending == topic.as_str())
878        {
879            if let Some(response_tx) = response_tx {
880                let _ = response_tx.send(Ok(()));
881            }
882            return;
883        }
884
885        let generation = self.take_subscription_generation();
886        self.subscription_attempts.insert(
887            topic,
888            SubscriptionAttempt {
889                channel,
890                auth,
891                pending_auth: None,
892                generation,
893                retries: 0,
894                response_txs: response_tx.into_iter().collect(),
895                pending_response_txs: Vec::new(),
896            },
897        );
898        self.pending_subs.push_back((topic, generation));
899    }
900
901    fn queue_subscription_retry(&mut self, topic: Ustr, generation: u64) {
902        if self
903            .subscription_attempts
904            .get(&topic)
905            .is_some_and(|attempt| attempt.generation == generation)
906            && self.inflight_subs.get(&topic) != Some(&generation)
907            && !self
908                .pending_subs
909                .iter()
910                .any(|pending| *pending == (topic, generation))
911        {
912            self.pending_subs.push_back((topic, generation));
913        }
914    }
915
916    fn complete_subscription(&mut self, topic: &str, kind: CompletionKind) -> bool {
917        let topic = Ustr::from(topic);
918
919        // Swallow the predecessor's trailing frame before the inflight guard,
920        // so it cannot complete a same-topic generation dispatched between the
921        // venue's control ack and its typed snapshot - whether that generation
922        // is a serialized successor, a fresh attempt, queued, or in backoff.
923        if self.ignored_completions.get(&topic) == Some(&kind) {
924            self.ignored_completions.remove(&topic);
925            return false;
926        }
927
928        let Some(generation) = self.inflight_subs.get(&topic).copied() else {
929            return false;
930        };
931
932        if !self
933            .subscription_attempts
934            .get(&topic)
935            .is_some_and(|attempt| attempt.generation == generation)
936        {
937            return false;
938        }
939
940        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
941            && !self
942                .book
943                .expected
944                .get(&market_index)
945                .is_some_and(|expected| expected.0 == generation)
946        {
947            return false;
948        }
949
950        self.inflight_subs.remove(&topic);
951        self.subscriptions.confirm_subscribe(topic.as_str());
952
953        // The venue contract is control ack first, typed snapshot second, so a
954        // control-ack completion owes one trailing typed frame. Typed and
955        // already-subscribed completions have no established trailing frame.
956        if kind == CompletionKind::ControlAck {
957            if let Some(market_index) = order_book_market_index_from_topic(topic.as_str())
958                && let Some(expected) = self.book.expected.get(&market_index)
959                && expected.0 == generation
960            {
961                self.book.trailing.entry(market_index).or_insert(*expected);
962            }
963
964            self.ignored_completions
965                .insert(topic, CompletionKind::Typed);
966        }
967        let mut attempt = self
968            .subscription_attempts
969            .remove(&topic)
970            .expect("matching subscription attempt disappeared");
971        for response_tx in std::mem::take(&mut attempt.response_txs) {
972            let _ = response_tx.send(Ok(()));
973        }
974
975        if let Some(auth) = attempt.pending_auth.take() {
976            let generation = self.take_subscription_generation();
977            attempt.auth = Some(auth);
978            attempt.generation = generation;
979            attempt.retries = 0;
980            attempt.response_txs = std::mem::take(&mut attempt.pending_response_txs);
981            self.subscription_attempts.insert(topic, attempt);
982            self.pending_subs.push_back((topic, generation));
983        } else {
984            debug_assert!(attempt.pending_response_txs.is_empty());
985        }
986        true
987    }
988
989    fn retry_inflight_subscriptions(&mut self, message: &str) {
990        let mut inflight: Vec<(Ustr, u64)> = self
991            .inflight_subs
992            .iter()
993            .map(|(topic, generation)| (*topic, *generation))
994            .collect();
995        inflight.sort_unstable_by_key(|(topic, _)| *topic);
996
997        for (topic, generation) in inflight {
998            self.schedule_subscription_retry(topic, generation, message);
999        }
1000    }
1001
1002    fn schedule_subscription_retry(&mut self, topic: Ustr, generation: u64, message: &str) {
1003        if self.inflight_subs.get(&topic) != Some(&generation) {
1004            return;
1005        }
1006
1007        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1008            self.reject_book_subscription(
1009                market_index,
1010                generation,
1011                LighterWsError::Network(message.into()),
1012            );
1013            return;
1014        }
1015
1016        self.inflight_subs.remove(&topic);
1017        self.subscriptions.mark_failure(topic.as_str());
1018
1019        let Some(attempt) = self.subscription_attempts.get_mut(&topic) else {
1020            return;
1021        };
1022
1023        if attempt.generation != generation {
1024            return;
1025        }
1026
1027        attempt.fold_pending_auth();
1028        if attempt.retries >= SUBSCRIBE_RETRY_MAX {
1029            self.fail_subscription_attempt(topic, message);
1030            return;
1031        }
1032
1033        attempt.retries += 1;
1034        let retry = attempt.retries;
1035        let next_generation = self.take_subscription_generation();
1036        let attempt = self
1037            .subscription_attempts
1038            .get_mut(&topic)
1039            .expect("subscription attempt disappeared before retry");
1040        attempt.generation = next_generation;
1041
1042        let delay = SUBSCRIBE_RETRY_BASE_BACKOFF.saturating_mul(1_u32 << (retry - 1));
1043        self.subscription_retries.push(Box::pin(async move {
1044            tokio::time::sleep(delay).await;
1045            (topic, next_generation)
1046        }));
1047    }
1048
1049    fn fail_subscription_attempt(&mut self, topic: Ustr, message: &str) {
1050        self.inflight_subs.remove(&topic);
1051        self.pending_subs.retain(|(pending, _)| *pending != topic);
1052        self.subscriptions.mark_unsubscribe(topic.as_str());
1053        self.subscriptions.confirm_unsubscribe(topic.as_str());
1054
1055        if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1056            let attempts = attempt.retries + 1;
1057            for response_tx in attempt
1058                .response_txs
1059                .into_iter()
1060                .chain(attempt.pending_response_txs)
1061            {
1062                let _ = response_tx.send(Err(format!(
1063                    "subscription {topic} failed after {attempts} attempts: {message}",
1064                )));
1065            }
1066        }
1067    }
1068
1069    fn cancel_subscription_attempt(&mut self, topic: &str, message: &str) {
1070        let topic = Ustr::from(topic);
1071        self.inflight_subs.remove(&topic);
1072        self.pending_subs.retain(|(pending, _)| *pending != topic);
1073        if let Some(attempt) = self.subscription_attempts.remove(&topic) {
1074            for response_tx in attempt
1075                .response_txs
1076                .into_iter()
1077                .chain(attempt.pending_response_txs)
1078            {
1079                let _ = response_tx.send(Err(format!("{message}: {topic}")));
1080            }
1081        }
1082    }
1083
1084    fn reset_subscription_attempts_after_reconnect(&mut self) {
1085        for topic in self.subscriptions.all_topics() {
1086            self.subscriptions.mark_failure(&topic);
1087        }
1088        self.pending_subs.clear();
1089        self.inflight_subs.clear();
1090        // Completion frames from the old connection must not be applied to -
1091        // or swallowed from - the new connection.
1092        self.ignored_completions.clear();
1093
1094        let mut topics: Vec<Ustr> = self.subscription_attempts.keys().copied().collect();
1095        topics.sort_unstable();
1096        for topic in topics {
1097            let generation = self.take_subscription_generation();
1098            let attempt = self
1099                .subscription_attempts
1100                .get_mut(&topic)
1101                .expect("subscription attempt disappeared during reconnect");
1102            attempt.fold_pending_auth();
1103            attempt.generation = generation;
1104            attempt.retries = 0;
1105
1106            if order_book_market_index_from_topic(topic.as_str()).is_none() {
1107                self.pending_subs.push_back((topic, generation));
1108            }
1109        }
1110    }
1111
1112    fn take_subscription_generation(&mut self) -> u64 {
1113        let generation = self.next_subscription_generation;
1114        self.next_subscription_generation =
1115            self.next_subscription_generation.wrapping_add(1).max(1);
1116        generation
1117    }
1118
1119    /// Returns `(matched, msg)` where `matched=true` means the frame's
1120    /// `type` was recognized as a known control type (whether or not a
1121    /// message is emitted); `matched=false` lets the caller surface the
1122    /// raw frame so venue errors with unknown shapes aren't lost.
1123    fn handle_control_value(
1124        &mut self,
1125        value: &serde_json::Value,
1126    ) -> (bool, Option<NautilusWsMessage>) {
1127        if value.get("type").and_then(|v| v.as_str()) == Some("jsonapi/sendtxbatch")
1128            || value
1129                .get("id")
1130                .and_then(|v| v.as_str())
1131                .is_some_and(|id| id.starts_with(TX_BATCH_ID_PREFIX))
1132        {
1133            let body = value.get("error").unwrap_or(value);
1134            let id = value.get("id").and_then(|v| v.as_str());
1135            let code = body.get("code").and_then(|v| v.as_i64());
1136
1137            let (Some(id), Some(code)) = (id, code) else {
1138                log::warn!("Ignoring malformed Lighter sendTxBatch response");
1139                return (true, None);
1140            };
1141
1142            return (
1143                true,
1144                Some(NautilusWsMessage::SendTxBatchResult {
1145                    connection_epoch: 0,
1146                    id: id.to_string(),
1147                    code,
1148                    message: body
1149                        .get("message")
1150                        .and_then(|v| v.as_str())
1151                        .unwrap_or("batch rejected")
1152                        .to_string(),
1153                    tx_hashes: value
1154                        .get("tx_hash")
1155                        .and_then(|v| serde_json::from_value(v.clone()).ok())
1156                        .unwrap_or_default(),
1157                }),
1158            );
1159        }
1160
1161        if let Some(error) = already_subscribed_error(value) {
1162            self.confirm_already_subscribed(error);
1163            return (true, None);
1164        }
1165        let subscription_code = subscription_error_code(value);
1166
1167        if subscription_code == Some(LIGHTER_ERROR_CODE_WS_RATE_LIMITED) {
1168            self.retry_inflight_subscriptions(&format!(
1169                "venue rejected the WebSocket subscribe with code \
1170                 {LIGHTER_ERROR_CODE_WS_RATE_LIMITED}",
1171            ));
1172            return (true, None);
1173        }
1174
1175        if subscription_code == Some(LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED) {
1176            self.handle_subscribe_failure(value);
1177            return (true, None);
1178        }
1179
1180        let kind = value.get("type").and_then(|v| v.as_str()).unwrap_or("");
1181
1182        match kind {
1183            CTRL_TYPE_CONNECTED => {
1184                log::debug!("Lighter WebSocket handshake complete");
1185                (true, None)
1186            }
1187            CTRL_TYPE_PING | CTRL_TYPE_PONG => (true, None),
1188            CTRL_TYPE_SEND_TX => {
1189                let raw_code = value.get("code").and_then(|v| v.as_u64());
1190                match raw_code {
1191                    Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) => {
1192                        log_integrator_not_approved();
1193                        (
1194                            true,
1195                            Some(send_tx_rejected_from_value(
1196                                value,
1197                                SendTxRejectionSource::Ack,
1198                            )),
1199                        )
1200                    }
1201                    Some(200) => {
1202                        log::debug!("Lighter WebSocket sendTx ack: {value}");
1203                        let tx_hash = value
1204                            .get("tx_hash")
1205                            .and_then(|v| v.as_str())
1206                            .map(str::to_string);
1207                        (
1208                            true,
1209                            Some(NautilusWsMessage::SendTxAck {
1210                                connection_epoch: 0,
1211                                tx_hash,
1212                                code: 200,
1213                            }),
1214                        )
1215                    }
1216                    Some(_) => (
1217                        true,
1218                        Some(send_tx_rejected_from_value(
1219                            value,
1220                            SendTxRejectionSource::Ack,
1221                        )),
1222                    ),
1223                    None => {
1224                        log::warn!(
1225                            "Ignoring malformed Lighter sendTx response without numeric code: {value}",
1226                        );
1227                        (true, None)
1228                    }
1229                }
1230            }
1231            CTRL_TYPE_SUBSCRIBED | CTRL_TYPE_UNSUBSCRIBED => {
1232                if let Some(topic) = value.get("channel").and_then(|v| v.as_str()) {
1233                    if kind == CTRL_TYPE_SUBSCRIBED {
1234                        self.complete_subscription(topic, CompletionKind::ControlAck);
1235                    } else {
1236                        let was_pending_unsubscribe = self
1237                            .subscriptions
1238                            .pending_unsubscribe_topics()
1239                            .iter()
1240                            .any(|pending| pending == topic);
1241                        self.subscriptions.confirm_unsubscribe(topic);
1242
1243                        if was_pending_unsubscribe {
1244                            // Only matched unsubscribe ACKs should reset stream state
1245                            if let Some(market_index) = order_book_market_index_from_topic(topic) {
1246                                self.book.clear_cached_order_book(market_index);
1247                            }
1248
1249                            if let Some(key) = candle_market_and_resolution_from_topic(topic) {
1250                                self.last_candles.remove(&key);
1251                            }
1252                        }
1253                    }
1254                }
1255                (true, None)
1256            }
1257            CTRL_TYPE_ERROR => {
1258                let code = value.get("code").and_then(|v| v.as_u64());
1259                if code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1260                    log_integrator_not_approved();
1261                } else {
1262                    log::warn!("Lighter WebSocket error frame: {value}");
1263                }
1264
1265                if is_sendtx_error_code(code) {
1266                    (
1267                        true,
1268                        Some(send_tx_rejected_from_value(
1269                            value,
1270                            SendTxRejectionSource::BareError,
1271                        )),
1272                    )
1273                } else {
1274                    (true, None)
1275                }
1276            }
1277            _ => {
1278                if let Some(error) = value.get("error") {
1279                    let nested_code = error.get("code").and_then(|v| v.as_u64());
1280                    if nested_code == Some(LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED) {
1281                        log_integrator_not_approved();
1282                    } else {
1283                        log::warn!("Lighter WebSocket error frame: {value}");
1284                    }
1285                    let rejected = is_sendtx_error_code(nested_code).then(|| {
1286                        send_tx_rejected_from_nested_error(error, SendTxRejectionSource::BareError)
1287                    });
1288                    return (true, rejected);
1289                }
1290                (false, None)
1291            }
1292        }
1293    }
1294
1295    fn handle_subscribe_failure(&mut self, value: &serde_json::Value) {
1296        let message = format!(
1297            "venue rejected the WebSocket subscribe with code \
1298             {LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED}",
1299        );
1300
1301        let channel = value
1302            .get("channel")
1303            .and_then(serde_json::Value::as_str)
1304            .or_else(|| {
1305                value
1306                    .get("error")
1307                    .and_then(|e| e.get("channel"))
1308                    .and_then(serde_json::Value::as_str)
1309            })
1310            .filter(|channel| !channel.is_empty());
1311
1312        let Some(channel) = channel else {
1313            log::warn!(
1314                "Lighter subscribe error code={LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED} channel=none: retrying inflight subscriptions"
1315            );
1316            self.retry_inflight_subscriptions(&message);
1317            return;
1318        };
1319
1320        let topic = Ustr::from(channel.replace('/', ":").as_str());
1321
1322        let Some(generation) = self.inflight_subs.get(&topic).copied() else {
1323            log::warn!(
1324                "Lighter subscribe error code={LIGHTER_ERROR_CODE_WS_SUBSCRIBE_FAILED} channel={channel}: no matching inflight subscription"
1325            );
1326            return;
1327        };
1328
1329        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
1330            self.reject_book_subscription(
1331                market_index,
1332                generation,
1333                LighterWsError::Client(message),
1334            );
1335        } else {
1336            self.fail_subscription_attempt(topic, &message);
1337        }
1338    }
1339
1340    fn confirm_already_subscribed(&mut self, error: &serde_json::Value) {
1341        let Some(topic) = error
1342            .get("message")
1343            .and_then(|value| value.as_str())
1344            .and_then(|message| message.strip_prefix("Already Subscribed to : "))
1345        else {
1346            log::debug!(
1347                "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1348            );
1349            return;
1350        };
1351
1352        if !self.complete_subscription(topic, CompletionKind::AlreadySubscribed) {
1353            log::debug!(
1354                "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}",
1355            );
1356            return;
1357        }
1358
1359        log::debug!(
1360            "Lighter WebSocket subscription response: code={LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED}, topic={topic}",
1361        );
1362    }
1363
1364    fn handle_frame(
1365        &mut self,
1366        frame: LighterWsFrame,
1367        ts_init: UnixNanos,
1368    ) -> Vec<NautilusWsMessage> {
1369        match frame {
1370            LighterWsFrame::OrderBookSnapshot {
1371                channel,
1372                order_book,
1373                timestamp,
1374                ..
1375            } => self.handle_order_book(channel, &order_book, timestamp, true, ts_init),
1376            LighterWsFrame::OrderBook {
1377                channel,
1378                order_book,
1379                timestamp,
1380                ..
1381            } => self.handle_order_book(channel, &order_book, timestamp, false, ts_init),
1382            LighterWsFrame::TickerSnapshot {
1383                channel,
1384                ticker,
1385                timestamp,
1386                ..
1387            }
1388            | LighterWsFrame::Ticker {
1389                channel,
1390                ticker,
1391                timestamp,
1392                ..
1393            } => self.handle_ticker(channel, &ticker, timestamp, ts_init),
1394            LighterWsFrame::TradeSnapshot {
1395                trades,
1396                liquidation_trades,
1397                ..
1398            }
1399            | LighterWsFrame::Trade {
1400                trades,
1401                liquidation_trades,
1402                ..
1403            } => self.handle_trades(&trades, &liquidation_trades, ts_init),
1404            LighterWsFrame::AccountOrders { ref orders, .. }
1405            | LighterWsFrame::AccountAllOrders { ref orders, .. } => {
1406                if self.exec_account.is_none() {
1407                    return raw_message(&frame);
1408                }
1409                let mut msgs = self.handle_account_orders(orders, ts_init);
1410                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1411                    AccountStream::Orders,
1412                ));
1413                msgs
1414            }
1415            // Lighter publishes the historical fill replay as `subscribed/`
1416            // rather than `update/`. When typed execution routing is active
1417            // we drop the snapshot to avoid re-emitting prior executions as
1418            // fresh fill reports on every reconnect (HTTP reconciliation
1419            // owns historical state recovery). When no execution context is
1420            // set we still forward as `Raw` to preserve the prior contract
1421            // for unauthenticated subscribers.
1422            LighterWsFrame::AccountAllTradesSnapshot { .. } => {
1423                if self.exec_account.is_none() {
1424                    return raw_message(&frame);
1425                }
1426                log::debug!(
1427                    "Skipping Lighter account_all_trades snapshot frame; \
1428                     reconcile historical fills via HTTP",
1429                );
1430                vec![NautilusWsMessage::AccountStreamFirstFrame(
1431                    AccountStream::Trades,
1432                )]
1433            }
1434            LighterWsFrame::AccountAllTrades { ref trades, .. } => {
1435                if self.exec_account.is_none() {
1436                    return raw_message(&frame);
1437                }
1438                let mut msgs = self.handle_account_trades(trades.values().flatten(), ts_init);
1439                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1440                    AccountStream::Trades,
1441                ));
1442                msgs
1443            }
1444            LighterWsFrame::AccountAllPositionsSnapshot { ref positions, .. } => {
1445                if self.exec_account.is_none() {
1446                    return raw_message(&frame);
1447                }
1448                let mut msgs =
1449                    self.handle_account_positions(positions, ts_init, PositionFrameType::Snapshot);
1450                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1451                    AccountStream::Positions,
1452                ));
1453                msgs
1454            }
1455            LighterWsFrame::AccountAllPositions { ref positions, .. } => {
1456                if self.exec_account.is_none() {
1457                    return raw_message(&frame);
1458                }
1459                self.handle_account_positions(positions, ts_init, PositionFrameType::Update)
1460            }
1461            LighterWsFrame::AccountAllAssets {
1462                ref assets,
1463                timestamp,
1464                ..
1465            } => {
1466                if self.exec_account.is_none() {
1467                    return raw_message(&frame);
1468                }
1469                let mut msgs = self.handle_account_assets(assets, timestamp, ts_init);
1470                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1471                    AccountStream::Assets,
1472                ));
1473                msgs
1474            }
1475            LighterWsFrame::UserStats {
1476                ref stats,
1477                timestamp,
1478                ..
1479            } => {
1480                if self.exec_account.is_none() {
1481                    return raw_message(&frame);
1482                }
1483                let mut msgs = self.handle_user_stats(stats, timestamp, ts_init);
1484                msgs.push(NautilusWsMessage::AccountStreamFirstFrame(
1485                    AccountStream::UserStats,
1486                ));
1487                msgs
1488            }
1489            LighterWsFrame::MarketStats {
1490                ref market_stats,
1491                timestamp,
1492                ..
1493            } => self.handle_market_stats(market_stats, timestamp, ts_init),
1494            LighterWsFrame::SpotMarketStats {
1495                ref spot_market_stats,
1496                timestamp,
1497                ..
1498            } => self.handle_spot_market_stats(spot_market_stats, timestamp, ts_init),
1499            LighterWsFrame::CandleSnapshot {
1500                channel,
1501                ref candles,
1502                ..
1503            }
1504            | LighterWsFrame::Candle {
1505                channel,
1506                ref candles,
1507                ..
1508            } => self.handle_candles(channel, candles, ts_init),
1509            LighterWsFrame::Shutdown { .. } => Vec::new(),
1510            LighterWsFrame::Height { .. } => raw_message(&frame),
1511        }
1512    }
1513
1514    fn handle_order_book(
1515        &mut self,
1516        channel: Ustr,
1517        book: &LighterWsOrderBook,
1518        timestamp: u64,
1519        is_snapshot: bool,
1520        ts_init: UnixNanos,
1521    ) -> Vec<NautilusWsMessage> {
1522        let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1523            log::debug!("Lighter order_book frame missing market index in channel '{channel}'");
1524            return Vec::new();
1525        };
1526
1527        let Some(instrument) = self.instruments.get(&market_index) else {
1528            log::debug!("No instrument cached for Lighter market_index={market_index}");
1529            return Vec::new();
1530        };
1531
1532        match self.book.validate_sequence(market_index, book, is_snapshot) {
1533            BookSequenceOutcome::Accept => self.book.apply(
1534                market_index,
1535                instrument,
1536                book,
1537                timestamp,
1538                is_snapshot,
1539                ts_init,
1540            ),
1541            BookSequenceOutcome::Suppress => Vec::new(),
1542            BookSequenceOutcome::Recover => {
1543                self.start_book_recovery(market_index);
1544                Vec::new()
1545            }
1546        }
1547    }
1548
1549    fn queue_book_replacement(&mut self, market_index: i64, write: BookWrite) {
1550        if write.cancel.is_cancelled() || !self.order_book_stream_is_referenced(market_index) {
1551            return;
1552        }
1553
1554        self.book.clear_cached_order_book(market_index);
1555        self.book.expected.remove(&market_index);
1556        let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1557        self.retire_book_attempt(&topic);
1558        self.subscriptions.mark_failure(&topic);
1559        self.book.writes.insert(market_index, write);
1560        self.queue_subscribe(LighterWsChannel::OrderBook(market_index), None, None);
1561    }
1562
1563    fn retire_book_attempt(&mut self, topic: &str) {
1564        let topic = Ustr::from(topic);
1565        self.inflight_subs.remove(&topic);
1566        self.pending_subs.retain(|(pending, _)| *pending != topic);
1567        if let Some(attempt) = self.subscription_attempts.get_mut(&topic) {
1568            // Keep callers waiting for the eventual replacement acknowledgement.
1569            attempt.generation = self.next_subscription_generation;
1570            self.next_subscription_generation =
1571                self.next_subscription_generation.wrapping_add(1).max(1);
1572            self.pending_subs.push_back((topic, attempt.generation));
1573        }
1574    }
1575
1576    fn reject_book_subscription(
1577        &mut self,
1578        market_index: i64,
1579        generation: u64,
1580        error: LighterWsError,
1581    ) {
1582        if !self
1583            .book
1584            .expected
1585            .get(&market_index)
1586            .is_some_and(|expected| expected.0 == generation)
1587            || self
1588                .book
1589                .recovery
1590                .get(&market_index)
1591                .and_then(BookRecoveryState::current)
1592                .is_some_and(|recovery| recovery.gate.lock().is_closed())
1593        {
1594            return;
1595        }
1596
1597        self.reject_book(market_index, error);
1598    }
1599
1600    fn reject_book(&mut self, market_index: i64, error: LighterWsError) {
1601        self.book.expected.remove(&market_index);
1602        self.book.clear_cached_order_book(market_index);
1603        let topic = Ustr::from(
1604            LighterWsChannel::OrderBook(market_index)
1605                .topic_key()
1606                .as_str(),
1607        );
1608
1609        let waiting = matches!(error, LighterWsError::Client(_))
1610            && self
1611                .subscription_attempts
1612                .get(&topic)
1613                .is_some_and(|attempt| {
1614                    !attempt.response_txs.is_empty() || !attempt.pending_response_txs.is_empty()
1615                });
1616
1617        if waiting {
1618            // The waiting subscriber owns the failure and releases its stream reference; an expired
1619            // initial wait must not start recovery before that release.
1620            self.book.initial.remove(&market_index);
1621            self.cancel_subscription_attempt(&topic, &error.to_string());
1622            log::error!(
1623                "Lighter book subscription rejected for market_index={market_index}: {error}"
1624            );
1625        }
1626
1627        if let Some(recovery) = self
1628            .book
1629            .recovery
1630            .get(&market_index)
1631            .and_then(BookRecoveryState::current)
1632            && !recovery.is_accepted()
1633        {
1634            // Frees the slot during backoff; the next replacement takes a new generation
1635            self.inflight_subs.remove(&topic);
1636            recovery.gate.lock().close();
1637            recovery
1638                .outcome
1639                .send_replace(BookRecoveryOutcome::Rejected(error));
1640        } else if !waiting {
1641            self.start_book_recovery(market_index);
1642        }
1643    }
1644
1645    fn start_book_recovery(&mut self, market_index: i64) {
1646        if !self.order_book_stream_is_referenced(market_index) {
1647            return;
1648        }
1649
1650        let Some(cmd_tx) = self.cmd_tx.clone() else {
1651            return;
1652        };
1653
1654        let Some(recovery) = self.book.recovery.entry(market_index).or_default().claim() else {
1655            return;
1656        };
1657
1658        recovery.gate.lock().close();
1659        self.book.initial.remove(&market_index);
1660        self.book.expected.remove(&market_index);
1661        self.book.clear_cached_order_book(market_index);
1662        let topic = LighterWsChannel::OrderBook(market_index).topic_key();
1663        self.inflight_subs.remove(&Ustr::from(topic.as_str()));
1664        self.pending_subs
1665            .retain(|(pending, _)| pending.as_str() != topic);
1666        self.book.work.push(recovery::recover(
1667            market_index,
1668            recovery,
1669            cmd_tx,
1670            self.book_snapshot_timeout,
1671        ));
1672    }
1673
1674    // A rejected subscriber releases its stream reference without an unsubscribe, so the last
1675    // consumer's release must end the recovery.
1676    fn cancel_orphaned_book_recovery(&mut self, market_index: i64) {
1677        if !self.order_book_stream_is_referenced(market_index) {
1678            self.book.recovery.remove(&market_index);
1679        }
1680    }
1681
1682    fn complete_typed_subscription(&mut self, topic: &str, epoch: u64) -> bool {
1683        if let Some(market_index) = order_book_market_index_from_topic(topic)
1684            && self
1685                .book
1686                .expected
1687                .get(&market_index)
1688                .is_some_and(|expected| expected.1 != epoch)
1689        {
1690            return false;
1691        }
1692
1693        if !self.book_snapshot_matches(topic, epoch) {
1694            if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1695                self.ignored_completions.remove(&Ustr::from(topic));
1696            }
1697
1698            return false;
1699        }
1700
1701        self.complete_subscription(topic, CompletionKind::Typed);
1702        true
1703    }
1704
1705    fn book_snapshot_matches(&mut self, topic: &str, epoch: u64) -> bool {
1706        let Some(market_index) = order_book_market_index_from_topic(topic) else {
1707            return true;
1708        };
1709
1710        let source =
1711            if self.ignored_completions.get(&Ustr::from(topic)) == Some(&CompletionKind::Typed) {
1712                self.book.trailing.remove(&market_index)
1713            } else {
1714                self.inflight_subs
1715                    .get(&Ustr::from(topic))
1716                    .map(|generation| (*generation, epoch))
1717            };
1718
1719        source.is_some_and(|source| {
1720            source.1 == epoch && Some(source) == self.book.expected.get(&market_index).copied()
1721        })
1722    }
1723
1724    fn send_book_subscribe(&mut self, market_index: i64, generation: u64) {
1725        let write = self.book.writes.remove(&market_index);
1726        let cancel = write
1727            .as_ref()
1728            .map_or_else(CancellationToken::new, |write| write.cancel.clone());
1729        if write.is_none() {
1730            self.book.initial.insert(
1731                market_index,
1732                PendingSnapshot {
1733                    deadline: None,
1734                    cancel: cancel.clone(),
1735                    gate: SnapshotGate::default(),
1736                },
1737            );
1738        }
1739
1740        let client = self.inner.clone();
1741        let subscriptions = self.subscriptions.clone();
1742        let book_snapshot_timeout = self.book_snapshot_timeout;
1743        self.book.work.push(recovery::subscribe(
1744            market_index,
1745            generation,
1746            cancel,
1747            write,
1748            client,
1749            subscriptions,
1750            book_snapshot_timeout,
1751        ));
1752    }
1753
1754    fn complete_book_send(
1755        &mut self,
1756        market_index: i64,
1757        generation: u64,
1758        cancel: CancellationToken,
1759        write: Option<BookWrite>,
1760        result: Result<u64, LighterWsError>,
1761    ) {
1762        let topic = Ustr::from(
1763            LighterWsChannel::OrderBook(market_index)
1764                .topic_key()
1765                .as_str(),
1766        );
1767
1768        if cancel.is_cancelled()
1769            || self.inflight_subs.get(&topic) != Some(&generation)
1770            || !self.order_book_stream_is_referenced(market_index)
1771        {
1772            return;
1773        }
1774
1775        match result {
1776            Ok(epoch) => {
1777                self.book.expected.insert(market_index, (generation, epoch));
1778
1779                if let Some(write) = write {
1780                    write.gate.open();
1781                    let _ = write.completion.send(Ok(()));
1782                } else {
1783                    self.book.work.push(recovery::wait_for_snapshot(
1784                        market_index,
1785                        cancel,
1786                        self.book_snapshot_timeout,
1787                    ));
1788                }
1789            }
1790            Err(e) => {
1791                if let Some(write) = write {
1792                    // Frees the slot during backoff; the next replacement takes a new generation
1793                    self.inflight_subs.remove(&topic);
1794                    let _ = write.completion.send(Err(e));
1795                } else {
1796                    self.reject_book(market_index, e);
1797                }
1798            }
1799        }
1800    }
1801
1802    fn order_book_stream_is_referenced(&self, market_index: i64) -> bool {
1803        let channel = LighterWsChannel::OrderBook(market_index);
1804        self.subscriptions.get_reference_count(&channel.topic_key()) > 0
1805            && (self.book.delta_subs.contains(&market_index)
1806                || self.book.depth_subs.contains(&market_index))
1807    }
1808
1809    fn handle_ticker(
1810        &self,
1811        channel: Ustr,
1812        ticker: &super::messages::LighterTicker,
1813        timestamp: u64,
1814        ts_init: UnixNanos,
1815    ) -> Vec<NautilusWsMessage> {
1816        // Resolve the instrument from the channel's `ticker:N` market index,
1817        // not the ticker payload's `s` symbol. Lighter sends the unsuffixed
1818        // venue symbol (e.g. "ETH"), while cached `raw_symbol` carries the
1819        // -PERP / -SPOT suffix, so a symbol-name match never resolves and
1820        // would conflate spot vs perp listings of the same asset.
1821        let Some(market_index) = market_index_from_topic(channel.as_str()) else {
1822            log::debug!("Lighter ticker frame missing market index in channel '{channel}'");
1823            return Vec::new();
1824        };
1825
1826        let Some(instrument) = self.instruments.get(&market_index) else {
1827            log::debug!("No instrument cached for Lighter ticker market_index={market_index}");
1828            return Vec::new();
1829        };
1830
1831        match parse_ws_quote_tick(ticker, instrument, timestamp, ts_init) {
1832            Ok(Some(quote)) => vec![NautilusWsMessage::Quote(quote)],
1833            Ok(None) => {
1834                log::debug!(
1835                    "Skipping Lighter ticker for market_index={market_index}: one-sided book",
1836                );
1837                Vec::new()
1838            }
1839            Err(e) => {
1840                log::error!("Error parsing Lighter ticker frame: {e}");
1841                Vec::new()
1842            }
1843        }
1844    }
1845
1846    fn handle_trades(
1847        &self,
1848        trades: &[crate::http::models::LighterTrade],
1849        liquidation_trades: &[crate::http::models::LighterTrade],
1850        ts_init: UnixNanos,
1851    ) -> Vec<NautilusWsMessage> {
1852        // Lighter splits trade prints across `trades` and `liquidation_trades`
1853        // on the same `update/trade` frame; both share the LighterTrade model
1854        // and are public executions, so emit them together.
1855        let Some(market_index) = trades
1856            .first()
1857            .or_else(|| liquidation_trades.first())
1858            .map(|t| t.market_id)
1859        else {
1860            return Vec::new();
1861        };
1862
1863        let Some(instrument) = self.instruments.get(&market_index) else {
1864            log::debug!("No instrument cached for Lighter trade market_index={market_index}");
1865            return Vec::new();
1866        };
1867
1868        let mut ticks = Vec::with_capacity(trades.len() + liquidation_trades.len());
1869        for trade in trades.iter().chain(liquidation_trades.iter()) {
1870            match parse_ws_trade_tick(trade, instrument, ts_init) {
1871                Ok(tick) => ticks.push(tick),
1872                Err(e) => log::error!("Error parsing Lighter trade tick: {e}"),
1873            }
1874        }
1875
1876        if ticks.is_empty() {
1877            Vec::new()
1878        } else {
1879            vec![NautilusWsMessage::Trades(ticks)]
1880        }
1881    }
1882
1883    fn handle_market_stats(
1884        &self,
1885        payload: &super::messages::LighterMarketStatsPayload,
1886        timestamp: u64,
1887        ts_init: UnixNanos,
1888    ) -> Vec<NautilusWsMessage> {
1889        match payload {
1890            super::messages::LighterMarketStatsPayload::All(stats) => stats
1891                .values()
1892                .flat_map(|stats| self.handle_one_market_stats(stats, timestamp, ts_init))
1893                .collect(),
1894            super::messages::LighterMarketStatsPayload::One(stats) => {
1895                self.handle_one_market_stats(stats, timestamp, ts_init)
1896            }
1897        }
1898    }
1899
1900    fn handle_one_market_stats(
1901        &self,
1902        stats: &super::messages::LighterMarketStats,
1903        timestamp: u64,
1904        ts_init: UnixNanos,
1905    ) -> Vec<NautilusWsMessage> {
1906        let Some(instrument) = self.instruments.get(&stats.market_id) else {
1907            log::debug!(
1908                "No instrument cached for Lighter market_stats market_id={}",
1909                stats.market_id,
1910            );
1911            return Vec::new();
1912        };
1913
1914        let mut messages = Vec::with_capacity(3);
1915
1916        match parse_ws_mark_price_update(stats, instrument, timestamp, ts_init) {
1917            Ok(mark_price) => messages.push(NautilusWsMessage::MarkPrice(mark_price)),
1918            Err(e) => log::error!("Error parsing Lighter mark price: {e}"),
1919        }
1920
1921        match parse_ws_index_price_update(stats, instrument, timestamp, ts_init) {
1922            Ok(index_price) => messages.push(NautilusWsMessage::IndexPrice(index_price)),
1923            Err(e) => log::error!("Error parsing Lighter index price: {e}"),
1924        }
1925
1926        match parse_ws_funding_rate_update(stats, instrument, timestamp, ts_init) {
1927            Ok(funding_rate) => messages.push(NautilusWsMessage::FundingRate(funding_rate)),
1928            Err(e) => log::error!("Error parsing Lighter funding rate: {e}"),
1929        }
1930
1931        messages
1932    }
1933
1934    fn handle_spot_market_stats(
1935        &self,
1936        payload: &super::messages::LighterSpotMarketStatsPayload,
1937        timestamp: u64,
1938        ts_init: UnixNanos,
1939    ) -> Vec<NautilusWsMessage> {
1940        match payload {
1941            super::messages::LighterSpotMarketStatsPayload::All(stats) => stats
1942                .values()
1943                .filter_map(|stats| self.handle_one_spot_market_stats(stats, timestamp, ts_init))
1944                .collect(),
1945            super::messages::LighterSpotMarketStatsPayload::One(stats) => self
1946                .handle_one_spot_market_stats(stats, timestamp, ts_init)
1947                .into_iter()
1948                .collect(),
1949        }
1950    }
1951
1952    fn handle_one_spot_market_stats(
1953        &self,
1954        stats: &super::messages::LighterSpotMarketStats,
1955        timestamp: u64,
1956        ts_init: UnixNanos,
1957    ) -> Option<NautilusWsMessage> {
1958        let Some(instrument) = self.instruments.get(&stats.market_id) else {
1959            log::debug!(
1960                "No instrument cached for Lighter spot_market_stats market_id={}",
1961                stats.market_id,
1962            );
1963            return None;
1964        };
1965
1966        match parse_ws_spot_index_price_update(stats, instrument, timestamp, ts_init) {
1967            Ok(index_price) => Some(NautilusWsMessage::IndexPrice(index_price)),
1968            Err(e) => {
1969                log::error!("Error parsing Lighter spot index price: {e}");
1970                None
1971            }
1972        }
1973    }
1974
1975    fn handle_candles(
1976        &mut self,
1977        channel: Ustr,
1978        candles: &[LighterWsCandle],
1979        ts_init: UnixNanos,
1980    ) -> Vec<NautilusWsMessage> {
1981        let Some((market_index, resolution)) =
1982            candle_market_and_resolution_from_topic(channel.as_str())
1983        else {
1984            log::warn!("Lighter candle frame with unparsable channel `{channel}`");
1985            return Vec::new();
1986        };
1987
1988        let Some(instrument) = self.instruments.get(&market_index) else {
1989            log::debug!("No instrument cached for Lighter candle market_index={market_index}");
1990            return Vec::new();
1991        };
1992
1993        let key = (market_index, resolution);
1994        let mut emitted = Vec::new();
1995
1996        for candle in candles {
1997            let previous = self.last_candles.get(&key).cloned();
1998            match previous {
1999                None => {}
2000                Some(prev) if candle.t > prev.t => {
2001                    // `t` advanced; previous candle is closed.
2002                    match parse_ws_bar(instrument, &prev, resolution, ts_init) {
2003                        Ok(bar) => emitted.push(NautilusWsMessage::Bar(bar)),
2004                        Err(e) => log::error!("Error parsing Lighter candle bar: {e}"),
2005                    }
2006                }
2007                Some(prev) if candle.t < prev.t => continue,
2008                Some(_) => {}
2009            }
2010            self.last_candles.insert(key, candle.clone());
2011        }
2012
2013        emitted
2014    }
2015
2016    fn handle_account_orders(
2017        &self,
2018        orders_by_market: &AHashMap<Ustr, Vec<LighterOrder>>,
2019        _ts_init: UnixNanos,
2020    ) -> Vec<NautilusWsMessage> {
2021        if self.exec_account.is_none() {
2022            log::debug!("Lighter account_orders frame skipped: no execution context set");
2023            return Vec::new();
2024        }
2025
2026        let mut reports = Vec::new();
2027
2028        for orders in orders_by_market.values() {
2029            for order in orders {
2030                if !self.instruments.contains_key(&order.market_index) {
2031                    log::debug!(
2032                        "No instrument cached for Lighter order market_index={}",
2033                        order.market_index,
2034                    );
2035                    continue;
2036                }
2037
2038                reports.push(ExecutionReport::Order(order.clone()));
2039            }
2040        }
2041
2042        if reports.is_empty() {
2043            Vec::new()
2044        } else {
2045            vec![NautilusWsMessage::ExecutionReports(reports)]
2046        }
2047    }
2048
2049    fn handle_account_trades<'a>(
2050        &self,
2051        trades: impl IntoIterator<Item = &'a LighterTrade>,
2052        _ts_init: UnixNanos,
2053    ) -> Vec<NautilusWsMessage> {
2054        let Some((_account_id, account_index)) = self.exec_account else {
2055            log::debug!("Lighter account_trades frame skipped: no execution context set");
2056            return Vec::new();
2057        };
2058
2059        let mut reports = Vec::new();
2060
2061        for trade in trades {
2062            if !self.instruments.contains_key(&trade.market_id) {
2063                log::debug!(
2064                    "No instrument cached for Lighter account trade market_id={}",
2065                    trade.market_id,
2066                );
2067                continue;
2068            }
2069
2070            // The venue shares the channel with crossed pairs the account is
2071            // not part of; gate the cheap account check at the handler so
2072            // unrelated traffic never reaches the execution consumer.
2073            if trade.bid_account_id != account_index && trade.ask_account_id != account_index {
2074                continue;
2075            }
2076
2077            reports.push(ExecutionReport::Fill(trade.clone()));
2078        }
2079
2080        if reports.is_empty() {
2081            Vec::new()
2082        } else {
2083            vec![NautilusWsMessage::ExecutionReports(reports)]
2084        }
2085    }
2086
2087    fn handle_account_positions(
2088        &self,
2089        positions: &AHashMap<Ustr, LighterPosition>,
2090        ts_init: UnixNanos,
2091        frame_type: PositionFrameType,
2092    ) -> Vec<NautilusWsMessage> {
2093        let Some((account_id, _)) = self.exec_account else {
2094            log::debug!("Lighter account_positions frame skipped: no execution context set");
2095            return Vec::new();
2096        };
2097
2098        // The position frame carries no top-level event timestamp, so stamp
2099        // ts_event with the wall-clock time the handler observed the frame.
2100        let ts_event = ts_init;
2101
2102        let mut reports = Vec::new();
2103        let mut skipped_market_ids = Vec::new();
2104        let mut closed_market_ids = Vec::new();
2105
2106        for position in positions.values() {
2107            if position.position.is_zero() {
2108                if matches!(frame_type, PositionFrameType::Update) {
2109                    closed_market_ids.push(position.market_id);
2110                }
2111                continue;
2112            }
2113
2114            let Some(instrument) = self.instruments.get(&position.market_id) else {
2115                log::debug!(
2116                    "No instrument cached for Lighter position market_id={}",
2117                    position.market_id,
2118                );
2119
2120                skipped_market_ids.push(position.market_id);
2121                continue;
2122            };
2123
2124            match parse_ws_position_status_report(
2125                position, instrument, account_id, ts_event, ts_init,
2126            ) {
2127                Ok(report) => reports.push(report),
2128                Err(e) => {
2129                    skipped_market_ids.push(position.market_id);
2130                    log::error!("Error parsing Lighter position status report: {e}");
2131                }
2132            }
2133        }
2134
2135        match frame_type {
2136            PositionFrameType::Snapshot => {
2137                // Emit even when empty: signals the last position closed.
2138                vec![NautilusWsMessage::PositionSnapshot {
2139                    reports,
2140                    skipped_market_ids,
2141                }]
2142            }
2143            PositionFrameType::Update => vec![NautilusWsMessage::PositionUpdate {
2144                reports,
2145                closed_market_ids,
2146                skipped_market_ids,
2147            }],
2148        }
2149    }
2150
2151    fn handle_account_assets(
2152        &self,
2153        assets: &AHashMap<Ustr, LighterAsset>,
2154        timestamp_ms: u64,
2155        ts_init: UnixNanos,
2156    ) -> Vec<NautilusWsMessage> {
2157        let Some((account_id, _)) = self.exec_account else {
2158            log::debug!("Lighter account_assets frame skipped: no execution context set");
2159            return Vec::new();
2160        };
2161
2162        let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2163            Ok(ts) => ts,
2164            Err(e) => {
2165                log::error!("Invalid Lighter account_assets timestamp {timestamp_ms}: {e}");
2166                return Vec::new();
2167            }
2168        };
2169
2170        self.account_state_reconciler.update_assets(assets);
2171        self.emit_unified_account_state(account_id, ts_event, ts_init)
2172    }
2173
2174    fn handle_user_stats(
2175        &self,
2176        stats: &LighterUserStats,
2177        timestamp_ms: u64,
2178        ts_init: UnixNanos,
2179    ) -> Vec<NautilusWsMessage> {
2180        let Some((account_id, _)) = self.exec_account else {
2181            log::debug!("Lighter user_stats frame skipped: no execution context set");
2182            return Vec::new();
2183        };
2184
2185        let ts_event = match crate::common::parse::parse_millis_to_nanos(timestamp_ms) {
2186            Ok(ts) => ts,
2187            Err(e) => {
2188                log::error!("Invalid Lighter user_stats timestamp {timestamp_ms}: {e}");
2189                return Vec::new();
2190            }
2191        };
2192
2193        self.account_state_reconciler.update_user_stats(stats);
2194        self.emit_unified_account_state(account_id, ts_event, ts_init)
2195    }
2196
2197    /// Asks the reconciler for a merged [`AccountState`] and wraps it as a
2198    /// `NautilusWsMessage`. Returns an empty vec when the second stream
2199    /// hasn't arrived yet; the AccountStreamFirstFrame marker still gets
2200    /// pushed by the caller so the startup gate progresses.
2201    fn emit_unified_account_state(
2202        &self,
2203        account_id: AccountId,
2204        ts_event: UnixNanos,
2205        ts_init: UnixNanos,
2206    ) -> Vec<NautilusWsMessage> {
2207        match self
2208            .account_state_reconciler
2209            .build_state(account_id, ts_event, ts_init)
2210        {
2211            Some(Ok(state)) => vec![NautilusWsMessage::AccountState(Box::new(state))],
2212            Some(Err(e)) => {
2213                log::error!("Error building unified Lighter account state: {e}");
2214                Vec::new()
2215            }
2216            None => Vec::new(),
2217        }
2218    }
2219}
2220
2221#[derive(Clone, Copy)]
2222enum PositionFrameType {
2223    Snapshot,
2224    Update,
2225}
2226
2227fn raw_message(frame: &LighterWsFrame) -> Vec<NautilusWsMessage> {
2228    let value = serde_json::to_value(frame).unwrap_or(serde_json::Value::Null);
2229    vec![NautilusWsMessage::Raw(value)]
2230}
2231
2232// Codes outside the transaction range (e.g. 30003 "Already Subscribed")
2233// would falsely reject a live order; see `LIGHTER_ERROR_CODE_TX_RANGE`.
2234fn is_sendtx_error_code(code: Option<u64>) -> bool {
2235    code.is_some_and(|c| LIGHTER_ERROR_CODE_TX_RANGE.contains(&c))
2236}
2237
2238fn already_subscribed_error(value: &serde_json::Value) -> Option<&serde_json::Value> {
2239    if value.get("code").and_then(|code| code.as_u64())
2240        == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2241    {
2242        Some(value)
2243    } else {
2244        value.get("error").filter(|e| {
2245            e.get("code").and_then(|code| code.as_u64())
2246                == Some(LIGHTER_ERROR_CODE_ALREADY_SUBSCRIBED)
2247        })
2248    }
2249}
2250
2251fn subscription_error_code(value: &serde_json::Value) -> Option<u64> {
2252    value
2253        .get("code")
2254        .and_then(|code| code.as_u64())
2255        .or_else(|| {
2256            value
2257                .get("error")
2258                .and_then(|e| e.get("code"))
2259                .and_then(|code| code.as_u64())
2260        })
2261}
2262
2263fn typed_subscribe_topic(text: &str) -> Option<&str> {
2264    let header = serde_json::from_str::<LighterWsFrameHeader<'_>>(text).ok()?;
2265    header
2266        .kind
2267        .starts_with("subscribed/")
2268        .then_some(header.channel?)
2269}
2270
2271// SendTxRejected from a top-level `{code, message}` frame: non-200 sendTx
2272// ACK or `{"type":"error",...}`.
2273fn send_tx_rejected_from_value(
2274    value: &serde_json::Value,
2275    source: SendTxRejectionSource,
2276) -> NautilusWsMessage {
2277    let code = value.get("code").and_then(|v| v.as_i64());
2278    let message = value
2279        .get("message")
2280        .and_then(|v| v.as_str())
2281        .unwrap_or("")
2282        .to_string();
2283    let tx_hash = value
2284        .get("tx_hash")
2285        .and_then(|v| v.as_str())
2286        .map(str::to_string);
2287    NautilusWsMessage::SendTxRejected {
2288        connection_epoch: 0,
2289        source,
2290        code,
2291        message,
2292        tx_hash,
2293    }
2294}
2295
2296// SendTxRejected from a nested `{"error":{"code":N,"message":...}}` frame;
2297// these frames never carry a `tx_hash`.
2298fn send_tx_rejected_from_nested_error(
2299    error: &serde_json::Value,
2300    source: SendTxRejectionSource,
2301) -> NautilusWsMessage {
2302    let code = error.get("code").and_then(|v| v.as_i64());
2303    let message = error
2304        .get("message")
2305        .and_then(|v| v.as_str())
2306        .unwrap_or("")
2307        .to_string();
2308    NautilusWsMessage::SendTxRejected {
2309        connection_epoch: 0,
2310        source,
2311        code,
2312        message,
2313        tx_hash: None,
2314    }
2315}
2316
2317fn log_integrator_not_approved() {
2318    log::error!(
2319        "Lighter venue rejected with code {LIGHTER_ERROR_CODE_INTEGRATOR_NOT_APPROVED} \
2320         'integrator is not approved'.\n\
2321         Tagged orders require Nautilus integrator approval. \
2322         See: {LIGHTER_INTEGRATOR_APPROVAL_DOCS_URL}",
2323    );
2324}
2325
2326fn market_index_from_topic(topic: &str) -> Option<i64> {
2327    let (_, rest) = topic.split_once(':')?;
2328    rest.parse::<i64>().ok()
2329}
2330
2331fn candle_market_and_resolution_from_topic(topic: &str) -> Option<(i64, LighterCandleResolution)> {
2332    let (channel, rest) = topic.split_once(':')?;
2333    if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::Candle) {
2334        return None;
2335    }
2336    let (market, res) = rest.split_once(':')?;
2337    let market_index = market.parse::<i64>().ok()?;
2338    let resolution = res.parse::<LighterCandleResolution>().ok()?;
2339    Some((market_index, resolution))
2340}
2341
2342fn order_book_market_index_from_topic(topic: &str) -> Option<i64> {
2343    let (channel, rest) = topic.split_once(':')?;
2344    if LighterWsChannelKind::from_wire_str(channel) != Some(LighterWsChannelKind::OrderBook) {
2345        return None;
2346    }
2347
2348    rest.parse::<i64>().ok()
2349}
2350
2351pub(crate) fn should_retry_lighter_ws_error(error: &LighterWsError) -> bool {
2352    match error {
2353        LighterWsError::Network(_) => true,
2354        // Closed and BrokenPipe are terminal on this client; only Timeout
2355        // (wait_for_active) can recover if the connection comes up. WriteTimeout
2356        // is not retryable: the write was cancelled after it began, so the peer
2357        // may already have the message and a retry could duplicate it.
2358        LighterWsError::Transport(send_error) => match send_error {
2359            SendError::Timeout => true,
2360            SendError::InvalidInput(_)
2361            | SendError::BufferFull
2362            | SendError::Closed
2363            | SendError::ConnectionChanged
2364            | SendError::BrokenPipe(_)
2365            | SendError::WriteTimeout => false,
2366        },
2367        LighterWsError::Authentication(_)
2368        | LighterWsError::Parse(_)
2369        | LighterWsError::Client(_)
2370        | LighterWsError::SendTxOutcomeUnknown(_) => false,
2371    }
2372}
2373
2374pub(crate) fn create_lighter_ws_timeout_error(_msg: String) -> LighterWsError {
2375    // Structured variant so the classifier retries; the retry manager
2376    // already logs the textual timeout context.
2377    LighterWsError::Transport(SendError::Timeout)
2378}
2379
2380#[cfg(test)]
2381mod tests {
2382    use std::time::Duration;
2383
2384    use futures_util::FutureExt;
2385    use log::{Level, LevelFilter, Log, Metadata, Record};
2386    use nautilus_model::{
2387        enums::{AccountType, BookAction, RecordFlag},
2388        identifiers::{InstrumentId, Symbol, Venue},
2389        instruments::{CryptoPerpetual, CurrencyPair},
2390        types::{Currency, Money, Price, Quantity},
2391    };
2392    use parking_lot::Mutex;
2393    use rstest::rstest;
2394    use rust_decimal::Decimal;
2395    use serde_json::json;
2396
2397    use super::*;
2398    use crate::{
2399        common::enums::{LighterCandleResolution, LighterTxType},
2400        websocket::messages::{LighterMarketSelection, LighterWsCandle, LighterWsChannel},
2401    };
2402
2403    const SECRET_MARKER: &str = "426426426";
2404
2405    struct OutboundLogCapture {
2406        messages: Mutex<Vec<String>>,
2407    }
2408
2409    static OUTBOUND_LOG_CAPTURE: OutboundLogCapture = OutboundLogCapture {
2410        messages: Mutex::new(Vec::new()),
2411    };
2412
2413    impl OutboundLogCapture {
2414        fn clear(&self) {
2415            self.messages.lock().clear();
2416        }
2417
2418        fn messages(&self) -> Vec<String> {
2419            self.messages.lock().clone()
2420        }
2421    }
2422
2423    impl Log for OutboundLogCapture {
2424        fn enabled(&self, metadata: &Metadata<'_>) -> bool {
2425            matches!(metadata.level(), Level::Debug | Level::Warn)
2426                && metadata.target() == "nautilus_lighter::websocket::handler"
2427        }
2428
2429        fn log(&self, record: &Record<'_>) {
2430            if self.enabled(record.metadata()) {
2431                let message = record.args().to_string();
2432                if (record.level() == Level::Debug
2433                    && message.starts_with("Sending Lighter unsubscribe"))
2434                    || (record.level() == Level::Warn
2435                        && message.starts_with("Lighter subscribe error"))
2436                {
2437                    self.messages.lock().push(message);
2438                }
2439            }
2440        }
2441
2442        fn flush(&self) {}
2443    }
2444
2445    const WS_ACCOUNT_ORDERS_UPDATE: &str =
2446        include_str!("../../test_data/ws_account_orders_update.json");
2447    const WS_ACCOUNT_ALL_TRADES_UPDATE: &str =
2448        include_str!("../../test_data/ws_account_all_trades_update.json");
2449    const WS_ACCOUNT_ALL_POSITIONS_UPDATE: &str =
2450        include_str!("../../test_data/ws_account_all_positions_update.json");
2451    const WS_ACCOUNT_ALL_ASSETS_UPDATE: &str =
2452        include_str!("../../test_data/ws_account_all_assets_update.json");
2453    const WS_USER_STATS_UPDATE: &str = include_str!("../../test_data/ws_user_stats_update.json");
2454    const WS_ACCOUNT_ALL_ASSETS_WITH_POSITION: &str =
2455        include_str!("../../test_data/ws_account_all_assets_with_position.json");
2456    const WS_USER_STATS_WITH_POSITION: &str =
2457        include_str!("../../test_data/ws_user_stats_with_position.json");
2458    const WS_MARKET_STATS_UPDATE_SINGLE: &str =
2459        include_str!("../../test_data/ws_market_stats_update_single.json");
2460    const WS_MARKET_STATS_UPDATE_ALL: &str =
2461        include_str!("../../test_data/ws_market_stats_update_all.json");
2462    const WS_SPOT_MARKET_STATS_UPDATE_SINGLE: &str =
2463        include_str!("../../test_data/ws_spot_market_stats_update_single.json");
2464    const WS_SPOT_MARKET_STATS_UPDATE_ALL: &str =
2465        include_str!("../../test_data/ws_spot_market_stats_update_all.json");
2466    const WS_CANDLE_SUBSCRIBED: &str = include_str!("../../test_data/ws_candle_subscribed.json");
2467    const WS_SPOT_STATS_SUBSCRIBED_BAD_BODY: &str =
2468        include_str!("../../test_data/ws_spot_market_stats_subscribed_single_bad_body.json");
2469    const WS_BOOK_SUBSCRIBED_BAD_BODY: &str =
2470        include_str!("../../test_data/ws_order_book_subscribed_bad_body.json");
2471    const WS_SUBSCRIBE_FAILED: &str =
2472        include_str!("../../test_data/ws_order_book_subscribe_failed.json");
2473    const BOOK_SUBSCRIBE_REJECTION: &str =
2474        "client error: venue rejected the WebSocket subscribe with code 30012: order_book:0";
2475
2476    fn handle_control_text(
2477        handler: &mut FeedHandler,
2478        text: &str,
2479    ) -> (bool, Option<NautilusWsMessage>) {
2480        let Ok(value) = serde_json::from_str::<serde_json::Value>(text) else {
2481            return (false, None);
2482        };
2483        handler.handle_control_value(&value)
2484    }
2485
2486    fn stub_eth_perp_instrument() -> InstrumentAny {
2487        let instrument_id = InstrumentId::new(Symbol::new("ETH-PERP"), Venue::new("LIGHTER"));
2488        InstrumentAny::CryptoPerpetual(
2489            CryptoPerpetual::builder()
2490                .instrument_id(instrument_id)
2491                .raw_symbol(Symbol::new("ETH-PERP"))
2492                .base_currency(Currency::from("ETH"))
2493                .quote_currency(Currency::from("USDC"))
2494                .settlement_currency(Currency::from("USDC"))
2495                .is_inverse(false)
2496                .price_precision(2)
2497                .size_precision(4)
2498                .price_increment(Price::from("0.01"))
2499                .size_increment(Quantity::from("0.0001"))
2500                .ts_event(UnixNanos::default())
2501                .ts_init(UnixNanos::default())
2502                .build()
2503                .unwrap(),
2504        )
2505    }
2506
2507    fn stub_eth_spot_instrument() -> InstrumentAny {
2508        let instrument_id = InstrumentId::new(Symbol::new("ETH-SPOT"), Venue::new("LIGHTER"));
2509        InstrumentAny::CurrencyPair(
2510            CurrencyPair::builder()
2511                .instrument_id(instrument_id)
2512                .raw_symbol(Symbol::new("ETH-SPOT"))
2513                .base_currency(Currency::from("ETH"))
2514                .quote_currency(Currency::from("USDC"))
2515                .price_precision(2)
2516                .size_precision(4)
2517                .price_increment(Price::from("0.01"))
2518                .size_increment(Quantity::from("0.0001"))
2519                .ts_event(UnixNanos::default())
2520                .ts_init(UnixNanos::default())
2521                .build()
2522                .unwrap(),
2523        )
2524    }
2525
2526    fn make_handler_with_account() -> FeedHandler {
2527        let signal = Arc::new(AtomicBool::new(false));
2528        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
2529        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
2530        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
2531        let mut handler =
2532            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2533        handler.instruments.insert(0, stub_eth_perp_instrument());
2534        handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
2535        handler
2536    }
2537
2538    // A recovery past its retry budget keeps running, so a late snapshot still restores the book
2539    #[rstest]
2540    fn book_late_snapshot_completes_running_recovery() {
2541        let mut handler = make_handler_with_account();
2542        handler.book.delta_subs.insert(0);
2543        let frame: LighterWsFrame = serde_json::from_str(include_str!(
2544            "../../test_data/ws_order_book_subscribed.json"
2545        ))
2546        .unwrap();
2547        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2548        handler.subscriptions.add_reference("order_book:0");
2549        assert!(recovery.begin_replacement());
2550
2551        let during_write = handler.handle_frame(frame.clone(), UnixNanos::from(1));
2552        recovery.gate.open();
2553        let after_write = handler.handle_frame(frame, UnixNanos::from(2));
2554
2555        assert!(during_write.is_empty());
2556        assert_eq!(after_write.len(), 1);
2557        assert!(recovery.is_accepted());
2558        assert!(handler.book.snapshots_seen.contains(&0));
2559    }
2560
2561    #[rstest]
2562    #[case::unconfirmed_permanent(false, false, false)]
2563    #[case::closed_gate_permanent(true, false, false)]
2564    #[case::confirmed_permanent(true, true, false)]
2565    #[case::unconfirmed_retry(false, false, true)]
2566    #[case::closed_gate_retry(true, false, true)]
2567    #[case::confirmed_retry(true, true, true)]
2568    fn book_venue_rejection_requires_confirmed_write(
2569        #[case] confirmed: bool,
2570        #[case] gate_open: bool,
2571        #[case] retry: bool,
2572    ) {
2573        let mut handler = make_handler_with_account();
2574        let (topic, generation) =
2575            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2576        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2577        assert!(recovery.begin_replacement());
2578
2579        if gate_open {
2580            recovery.gate.open();
2581        }
2582
2583        if !confirmed {
2584            handler.book.expected.remove(&0);
2585        }
2586
2587        if retry {
2588            handler.retry_inflight_subscriptions("venue rejection");
2589        } else {
2590            let value = serde_json::json!({"code": 30012, "channel": "order_book:0"});
2591            handler.handle_control_value(&value);
2592        }
2593
2594        let rejected = confirmed && gate_open;
2595        assert_eq!(
2596            handler.inflight_subs.get(&topic),
2597            (!rejected).then_some(&generation)
2598        );
2599        assert!(!recovery.cancellation.is_cancelled());
2600        assert!(recovery.gate.lock().is_closed());
2601
2602        match &*recovery.outcome.borrow() {
2603            BookRecoveryOutcome::Pending => assert!(!confirmed || !gate_open),
2604            BookRecoveryOutcome::Rejected(LighterWsError::Network(message)) => {
2605                assert!(confirmed && gate_open && retry);
2606                assert_eq!(message, "venue rejection");
2607            }
2608            BookRecoveryOutcome::Rejected(LighterWsError::Client(message)) => {
2609                assert!(confirmed && gate_open && !retry);
2610                assert_eq!(
2611                    message,
2612                    "venue rejected the WebSocket subscribe with code 30012"
2613                );
2614            }
2615            outcome => panic!("unexpected recovery outcome: {outcome:?}"),
2616        }
2617    }
2618
2619    // A waiting subscriber owns a rejection and releases the stream; a replayed subscription
2620    // recovers instead of failing the book.
2621    #[rstest]
2622    #[case::waiting_subscriber(true)]
2623    #[case::replayed(false)]
2624    fn book_client_rejection_recovers_only_without_waiting_subscriber(#[case] waiting: bool) {
2625        let mut handler = make_handler_with_account();
2626        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2627        handler.cmd_tx = Some(cmd_tx);
2628        handler.book.delta_subs.insert(0);
2629        handler.subscriptions.add_reference("order_book:0");
2630        let (response, mut result) = tokio::sync::oneshot::channel();
2631        let (topic, generation) = mark_subscription_inflight(
2632            &mut handler,
2633            LighterWsChannel::OrderBook(0),
2634            waiting.then_some(response),
2635        );
2636        let initial = CancellationToken::new();
2637        handler.book.initial.insert(
2638            0,
2639            PendingSnapshot {
2640                deadline: None,
2641                cancel: initial.clone(),
2642                gate: SnapshotGate::default(),
2643            },
2644        );
2645
2646        handler.reject_book_subscription(
2647            0,
2648            generation,
2649            LighterWsError::Client("invalid market".into()),
2650        );
2651
2652        let recovering = handler
2653            .book
2654            .recovery
2655            .get(&0)
2656            .is_some_and(BookRecoveryState::is_running);
2657
2658        if waiting {
2659            assert!(
2660                matches!(result.try_recv(), Ok(Err(message)) if message.contains("invalid market"))
2661            );
2662        } else {
2663            assert_eq!(
2664                result.try_recv(),
2665                Err(tokio::sync::oneshot::error::TryRecvError::Closed)
2666            );
2667        }
2668
2669        assert_eq!(recovering, !waiting);
2670        assert_eq!(handler.book.work.len(), usize::from(!waiting));
2671        assert_eq!(handler.subscription_attempts.contains_key(&topic), !waiting);
2672        assert!(!handler.inflight_subs.contains_key(&topic));
2673        assert!(!handler.book.initial.contains_key(&0));
2674        assert!(initial.is_cancelled());
2675    }
2676
2677    // A rejected replacement answers the subscriber still waiting on the initial subscribe, and
2678    // the recovery keeps its gate closed until the subscriber releases the book.
2679    #[rstest]
2680    #[case::response_txs(false)]
2681    #[case::pending_response_txs(true)]
2682    #[tokio::test]
2683    async fn book_client_rejection_during_recovery_answers_waiting_subscriber(
2684        #[case] pending: bool,
2685    ) {
2686        let (mut handler, _cmd_tx, mut result) =
2687            recovering_book_with_waiting_subscriber(false).await;
2688        let topic = Ustr::from("order_book:0");
2689
2690        if pending {
2691            // Book subscribes carry no auth, so seed a waiter behind a pending auth change directly
2692            let attempt = handler.subscription_attempts.get_mut(&topic).unwrap();
2693            attempt.pending_auth = Some(SecretString::from("token"));
2694            attempt.pending_response_txs = std::mem::take(&mut attempt.response_txs);
2695        }
2696
2697        let recovery = Arc::clone(handler.book.recovery[&0].current().unwrap());
2698        let gate_opened = !recovery.gate.lock().is_closed();
2699        let pending = result.try_recv();
2700
2701        handle_control_text(&mut handler, WS_SUBSCRIBE_FAILED);
2702
2703        assert!(gate_opened);
2704        assert_eq!(
2705            pending,
2706            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2707        );
2708        assert_eq!(
2709            result.try_recv(),
2710            Ok(Err(BOOK_SUBSCRIBE_REJECTION.to_string()))
2711        );
2712        assert!(!handler.subscription_attempts.contains_key(&topic));
2713        assert!(!handler.inflight_subs.contains_key(&topic));
2714        assert!(!handler.book.expected.contains_key(&0));
2715        assert!(recovery.is_running());
2716        assert!(recovery.gate.lock().is_closed());
2717        assert!(matches!(
2718            &*recovery.outcome.borrow(),
2719            BookRecoveryOutcome::Rejected(LighterWsError::Client(message))
2720                if message == "venue rejected the WebSocket subscribe with code 30012"
2721        ));
2722    }
2723
2724    // The rejected subscriber's release ends the recovery unless another consumer still references
2725    // the book, whose next replacement then queues a fresh attempt; either way a late snapshot
2726    // cannot reopen the book.
2727    #[rstest]
2728    #[case::deltas_last_reference(false, false)]
2729    #[case::deltas_other_consumer(false, true)]
2730    #[case::depth_last_reference(true, false)]
2731    #[case::depth_other_consumer(true, true)]
2732    #[tokio::test]
2733    async fn book_rejected_subscriber_release_cancels_orphaned_recovery(
2734        #[case] depth: bool,
2735        #[case] other_consumer: bool,
2736    ) {
2737        let (mut handler, cmd_tx, mut result) =
2738            recovering_book_with_waiting_subscriber(depth).await;
2739        let topic = Ustr::from("order_book:0");
2740
2741        if other_consumer {
2742            let other = if depth {
2743                &mut handler.book.delta_subs
2744            } else {
2745                &mut handler.book.depth_subs
2746            };
2747
2748            other.insert(0);
2749            assert!(!handler.subscriptions.add_reference("order_book:0"));
2750        }
2751
2752        let recovery = Arc::clone(handler.book.recovery[&0].current().unwrap());
2753        handle_control_text(&mut handler, WS_SUBSCRIBE_FAILED);
2754        let answer = result.try_recv();
2755
2756        // Release as the rejected client does: drop the reference, then clear the consumer flag
2757        let released = handler.subscriptions.remove_reference("order_book:0");
2758
2759        let release = if depth {
2760            HandlerCommand::SetDepthSub {
2761                market_index: 0,
2762                subscribed: false,
2763            }
2764        } else {
2765            HandlerCommand::SetBookDeltasSub {
2766                market_index: 0,
2767                subscribed: false,
2768            }
2769        };
2770
2771        cmd_tx.send(release).unwrap();
2772        cmd_tx.send(HandlerCommand::Disconnect).unwrap();
2773        assert!(handler.next().await.is_none());
2774
2775        let frame: LighterWsFrame = serde_json::from_str(include_str!(
2776            "../../test_data/ws_order_book_subscribed.json"
2777        ))
2778        .unwrap();
2779        let late = handler.handle_frame(frame, UnixNanos::from(1));
2780        let (completion, _completed) = tokio::sync::oneshot::channel();
2781        handler.queue_book_replacement(
2782            0,
2783            BookWrite {
2784                cancel: recovery.cancellation.child_token(),
2785                gate: recovery.gate.clone(),
2786                completion,
2787            },
2788        );
2789
2790        let replacement = handler.subscription_attempts.get(&topic).map(|attempt| {
2791            (
2792                attempt.response_txs.len(),
2793                handler.pending_subs.contains(&(topic, attempt.generation)),
2794            )
2795        });
2796
2797        assert_eq!(answer, Ok(Err(BOOK_SUBSCRIBE_REJECTION.to_string())));
2798        assert_eq!(released, !other_consumer);
2799        assert_eq!(recovery.is_running(), other_consumer);
2800        assert_eq!(handler.book.recovery.contains_key(&0), other_consumer);
2801        assert_eq!(handler.book.work.len(), usize::from(other_consumer));
2802        assert!(!recovery.is_accepted());
2803        assert!(recovery.gate.lock().is_closed());
2804        assert!(late.is_empty());
2805        assert!(!handler.book.snapshots_seen.contains(&0));
2806        assert_eq!(handler.book.writes.contains_key(&0), other_consumer);
2807        assert_eq!(replacement, other_consumer.then_some((0, true)));
2808    }
2809
2810    // Rate limits the initial book subscribe, which starts a recovery, then confirms the
2811    // recovery's replacement write while the initial subscriber still waits.
2812    async fn recovering_book_with_waiting_subscriber(
2813        depth: bool,
2814    ) -> (
2815        FeedHandler,
2816        tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2817        tokio::sync::oneshot::Receiver<Result<(), String>>,
2818    ) {
2819        let signal = Arc::new(AtomicBool::new(false));
2820        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2821        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
2822        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
2823        let mut handler =
2824            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
2825        handler.set_command_sender(cmd_tx.clone());
2826        handler.instruments.insert(0, stub_eth_perp_instrument());
2827
2828        if depth {
2829            handler.book.depth_subs.insert(0);
2830        } else {
2831            handler.book.delta_subs.insert(0);
2832        }
2833
2834        handler.subscriptions.add_reference("order_book:0");
2835        let (response, result) = tokio::sync::oneshot::channel();
2836        mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(response));
2837        handle_control_text(
2838            &mut handler,
2839            r#"{"type":"error","code":30009,"message":"rate limit exceeded"}"#,
2840        );
2841
2842        // The first attempt requests its replacement on the first poll
2843        assert!(handler.book.work.next().now_or_never().is_none());
2844
2845        let Ok(HandlerCommand::RecoverBook {
2846            market_index,
2847            cancel,
2848            gate,
2849            completion,
2850        }) = handler.cmd_rx.try_recv()
2851        else {
2852            panic!("expected book replacement request");
2853        };
2854
2855        handler.queue_book_replacement(
2856            market_index,
2857            BookWrite {
2858                cancel,
2859                gate,
2860                completion,
2861            },
2862        );
2863
2864        handler.pump_pending_subscribes().await;
2865
2866        // The send fails without a socket, so confirm the write as the connection would
2867        let Some(BookWorkResult::Sent {
2868            market_index,
2869            generation,
2870            cancel,
2871            write,
2872            ..
2873        }) = handler.book.work.next().await
2874        else {
2875            panic!("expected replacement send");
2876        };
2877
2878        handler.complete_book_send(market_index, generation, cancel, write, Ok(0));
2879
2880        (handler, cmd_tx, result)
2881    }
2882
2883    // A failed replacement frees its slot during backoff so other subscriptions can proceed
2884    #[rstest]
2885    #[case::rejected(true)]
2886    #[case::send_failed(false)]
2887    #[tokio::test]
2888    async fn book_failed_replacement_releases_subscription_slot(#[case] rejected: bool) {
2889        let mut handler = make_handler_with_account();
2890        handler.book.delta_subs.insert(0);
2891        handler.subscriptions.add_reference("order_book:0");
2892        let (topic, generation) =
2893            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2894
2895        for i in 0..SUBSCRIBE_INFLIGHT_MAX - 1 {
2896            handler
2897                .inflight_subs
2898                .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
2899        }
2900
2901        handler.queue_subscribe(LighterWsChannel::OrderBook(1), None, None);
2902        let other = Ustr::from("order_book:1");
2903        handler.pump_pending_subscribes().await;
2904        let blocked = !handler.inflight_subs.contains_key(&other);
2905        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
2906        assert!(recovery.begin_replacement());
2907        recovery.gate.open();
2908        let (completion, completed) = tokio::sync::oneshot::channel();
2909
2910        if rejected {
2911            handler.reject_book_subscription(
2912                0,
2913                generation,
2914                LighterWsError::Client("venue rejection".into()),
2915            );
2916        } else {
2917            handler.complete_book_send(
2918                0,
2919                generation,
2920                recovery.cancellation.child_token(),
2921                Some(BookWrite {
2922                    cancel: recovery.cancellation.child_token(),
2923                    gate: recovery.gate.clone(),
2924                    completion,
2925                }),
2926                Err(LighterWsError::Network("send failed".into())),
2927            );
2928        }
2929
2930        handler.pump_pending_subscribes().await;
2931
2932        assert!(blocked);
2933        assert!(!handler.inflight_subs.contains_key(&topic));
2934        assert!(handler.inflight_subs.contains_key(&other));
2935        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
2936        assert!(recovery.is_running());
2937
2938        if rejected {
2939            assert!(matches!(
2940                &*recovery.outcome.borrow(),
2941                BookRecoveryOutcome::Rejected(LighterWsError::Client(message))
2942                    if message == "venue rejection"
2943            ));
2944        } else {
2945            assert!(
2946                matches!(completed.await, Ok(Err(LighterWsError::Network(message))) if message == "send failed")
2947            );
2948        }
2949    }
2950
2951    #[tokio::test]
2952    async fn book_queued_write_skips_after_last_reference_disappears() {
2953        let mut handler = make_handler_with_account();
2954        handler.book.delta_subs.insert(0);
2955        let channel = LighterWsChannel::OrderBook(0);
2956        handler.subscriptions.add_reference(&channel.topic_key());
2957        let (_, generation) = mark_subscription_inflight(&mut handler, channel.clone(), None);
2958        handler.send_book_subscribe(0, generation);
2959        assert!(handler.subscriptions.remove_reference(&channel.topic_key()));
2960
2961        let BookWorkResult::Sent { result, .. } = handler.book.work.next().await.unwrap() else {
2962            panic!("expected send completion");
2963        };
2964
2965        assert_eq!(
2966            result.unwrap_err().to_string(),
2967            "client error: book subscription cancelled"
2968        );
2969        assert!(handler.inner.is_none());
2970    }
2971
2972    #[rstest]
2973    fn book_wrong_epoch_snapshot_preserves_current_completion() {
2974        let mut handler = make_handler_with_account();
2975        let (tx, mut rx) = tokio::sync::oneshot::channel();
2976        let (topic, generation) =
2977            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
2978        handler.book.expected.insert(0, (generation, 12));
2979
2980        assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
2981        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
2982        assert_eq!(
2983            rx.try_recv(),
2984            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
2985        );
2986        assert!(handler.complete_typed_subscription(topic.as_str(), 12));
2987        assert_eq!(rx.try_recv(), Ok(Ok(())));
2988        assert!(!handler.inflight_subs.contains_key(&topic));
2989    }
2990
2991    #[rstest]
2992    fn book_control_ack_preserves_snapshot_but_not_successor_completion() {
2993        let mut handler = make_handler_with_account();
2994        let (topic, old_generation) =
2995            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
2996        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
2997        assert!(handler.complete_typed_subscription(topic.as_str(), 0));
2998        assert_eq!(handler.book.expected[&0], (old_generation, 0));
2999
3000        handler.subscriptions.mark_failure(topic.as_str());
3001        let (_, generation) =
3002            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3003        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3004        handler.subscriptions.mark_failure(topic.as_str());
3005        let (tx, mut rx) = tokio::sync::oneshot::channel();
3006        let (_, successor) =
3007            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), Some(tx));
3008
3009        assert_ne!(generation, successor);
3010        assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
3011        assert_eq!(handler.inflight_subs.get(&topic), Some(&successor));
3012        assert_eq!(
3013            rx.try_recv(),
3014            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
3015        );
3016        assert!(handler.complete_typed_subscription(topic.as_str(), 0));
3017        assert_eq!(rx.try_recv(), Ok(Ok(())));
3018    }
3019
3020    #[rstest]
3021    fn book_old_epoch_does_not_consume_current_trailing_snapshot() {
3022        let mut handler = make_handler_with_account();
3023        let (topic, generation) =
3024            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3025        handler.book.expected.insert(0, (generation, 12));
3026        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3027
3028        assert!(!handler.complete_typed_subscription(topic.as_str(), 11));
3029        assert_eq!(handler.book.trailing.get(&0), Some(&(generation, 12)));
3030        assert_eq!(
3031            handler.ignored_completions.get(&topic),
3032            Some(&CompletionKind::Typed)
3033        );
3034        assert!(handler.complete_typed_subscription(topic.as_str(), 12));
3035        assert!(!handler.book.trailing.contains_key(&0));
3036        assert!(!handler.ignored_completions.contains_key(&topic));
3037    }
3038
3039    #[rstest]
3040    fn book_ack_before_write_cannot_complete_subscription() {
3041        let mut handler = make_handler_with_account();
3042        let (topic, generation) =
3043            mark_subscription_inflight(&mut handler, LighterWsChannel::OrderBook(0), None);
3044        handler.book.expected.clear();
3045
3046        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
3047        assert!(!handler.complete_typed_subscription(topic.as_str(), 0));
3048        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
3049    }
3050
3051    #[rstest]
3052    fn book_empty_snapshot_replaces_cached_levels_for_both_consumers() {
3053        let mut handler = make_handler_with_account();
3054        handler.book.delta_subs.insert(0);
3055        handler.book.depth_subs.insert(0);
3056        let frame: LighterWsFrame = serde_json::from_str(include_str!(
3057            "../../test_data/ws_order_book_subscribed.json"
3058        ))
3059        .unwrap();
3060        assert_eq!(
3061            handler
3062                .handle_frame(frame.clone(), UnixNanos::from(1))
3063                .len(),
3064            2
3065        );
3066
3067        let LighterWsFrame::OrderBookSnapshot { order_book, .. } = &frame else {
3068            panic!("expected book snapshot");
3069        };
3070
3071        let mut empty = order_book.clone();
3072        empty.bids.clear();
3073        empty.asks.clear();
3074        empty.nonce += 1;
3075        let messages = handler.handle_order_book(
3076            Ustr::from("order_book:0"),
3077            &empty,
3078            123,
3079            true,
3080            UnixNanos::from(2),
3081        );
3082
3083        assert_eq!(messages.len(), 2);
3084
3085        let NautilusWsMessage::Deltas(deltas) = &messages[0] else {
3086            panic!("expected deltas");
3087        };
3088
3089        assert_eq!(deltas.deltas.len(), 1);
3090        assert_eq!(deltas.deltas[0].action, BookAction::Clear);
3091        assert_eq!(deltas.deltas[0].sequence, empty.nonce as u64);
3092        assert_eq!(
3093            deltas.deltas[0].flags,
3094            RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8
3095        );
3096
3097        let NautilusWsMessage::Depth(depth) = &messages[1] else {
3098            panic!("expected depth");
3099        };
3100
3101        assert!(depth.bids.is_empty());
3102        assert!(depth.asks.is_empty());
3103        assert!(depth.bid_counts.is_empty());
3104        assert!(depth.ask_counts.is_empty());
3105        assert!(handler.book.states[&0].book.bids.is_empty());
3106        assert!(handler.book.states[&0].book.asks.is_empty());
3107    }
3108
3109    #[tokio::test]
3110    async fn book_initial_deadline_is_cancelled_by_replacement_or_unsubscribe() {
3111        let mut handler = make_handler_with_account();
3112        handler.book.delta_subs.insert(0);
3113        let channel = LighterWsChannel::OrderBook(0);
3114        handler.subscriptions.add_reference(&channel.topic_key());
3115        let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
3116        let cancel = CancellationToken::new();
3117        handler.book.initial.insert(
3118            0,
3119            PendingSnapshot {
3120                deadline: None,
3121                cancel: cancel.clone(),
3122                gate: SnapshotGate::default(),
3123            },
3124        );
3125
3126        handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
3127        handler.book.cancel(0);
3128
3129        let BookWorkResult::Initial {
3130            cancel: completed,
3131            market_index,
3132        } = handler.book.work.next().await.unwrap()
3133        else {
3134            panic!("expected deadline completion");
3135        };
3136
3137        assert_eq!(market_index, 0);
3138        assert!(completed.is_cancelled());
3139        assert!(cancel.is_cancelled());
3140        assert!(handler.book.recovery.is_empty());
3141        assert!(handler.book.expected.is_empty());
3142    }
3143
3144    #[tokio::test]
3145    async fn book_initial_deadline_expires_without_cancellation() {
3146        let mut handler = make_handler_with_account();
3147        handler.book_snapshot_timeout = Duration::from_millis(50);
3148        handler.book.delta_subs.insert(0);
3149        let channel = LighterWsChannel::OrderBook(0);
3150        handler.subscriptions.add_reference(&channel.topic_key());
3151        let (_, generation) = mark_subscription_inflight(&mut handler, channel, None);
3152        let cancel = CancellationToken::new();
3153        handler.book.initial.insert(
3154            0,
3155            PendingSnapshot {
3156                deadline: None,
3157                cancel: cancel.clone(),
3158                gate: SnapshotGate::default(),
3159            },
3160        );
3161
3162        handler.complete_book_send(0, generation, cancel.clone(), None, Ok(7));
3163
3164        let BookWorkResult::Initial {
3165            market_index,
3166            cancel: completed,
3167        } = handler.book.work.next().await.unwrap()
3168        else {
3169            panic!("expected deadline completion");
3170        };
3171
3172        assert_eq!(market_index, 0);
3173        assert!(!completed.is_cancelled());
3174        assert!(!cancel.is_cancelled());
3175    }
3176
3177    #[tokio::test]
3178    async fn book_reconnect_retains_recovery_owner_and_retires_old_writes() {
3179        let signal = Arc::new(AtomicBool::new(false));
3180        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
3181        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel();
3182        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel();
3183        let mut handler =
3184            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3185        let recovery = handler.book.recovery.entry(0).or_default().claim().unwrap();
3186        handler.book.expected.insert(0, (3, 9));
3187        raw_tx
3188            .send((10, Message::Text(RECONNECTED.into())))
3189            .unwrap();
3190        assert!(matches!(
3191            handler.next().await,
3192            Some(NautilusWsMessage::Reconnected {
3193                connection_epoch: 10
3194            })
3195        ));
3196
3197        assert!(Arc::ptr_eq(
3198            handler.book.recovery[&0].current().unwrap(),
3199            &recovery
3200        ));
3201        assert!(!recovery.cancellation.is_cancelled());
3202        assert!(recovery.gate.lock().is_closed());
3203        assert!(matches!(
3204            *recovery.outcome.borrow(),
3205            BookRecoveryOutcome::Rejected(LighterWsError::Network(_))
3206        ));
3207        assert!(handler.book.expected.is_empty());
3208        assert!(handler.book.writes.is_empty());
3209    }
3210
3211    #[rstest]
3212    fn book_recovery_is_scoped_to_handler_and_market() {
3213        let mut first = make_handler_with_account();
3214        let mut second = make_handler_with_account();
3215        let recovery = first.book.recovery.entry(0).or_default().claim().unwrap();
3216        let other_market = first.book.recovery.entry(1).or_default().claim().unwrap();
3217        let other_socket = second.book.recovery.entry(0).or_default().claim().unwrap();
3218        first.book.cancel(0);
3219
3220        assert!(recovery.cancellation.is_cancelled());
3221        assert!(!other_market.cancellation.is_cancelled());
3222        assert!(!other_socket.cancellation.is_cancelled());
3223    }
3224
3225    #[rstest]
3226    #[tokio::test]
3227    async fn test_subscription_logs_omit_payload_body() {
3228        log::set_logger(&OUTBOUND_LOG_CAPTURE).expect("test logger already installed");
3229        log::set_max_level(LevelFilter::Debug);
3230
3231        let mut handler = make_handler_with_account();
3232        let account_index = SECRET_MARKER.parse::<i64>().unwrap();
3233        let channel = LighterWsChannel::AccountAll(account_index);
3234        let payload_len = serde_json::to_string(&LighterWsRequest::unsubscribe(
3235            channel.subscription_channel(),
3236        ))
3237        .unwrap()
3238        .len();
3239        OUTBOUND_LOG_CAPTURE.clear();
3240
3241        handler.dispatch_unsubscribe(channel).await;
3242        handler.handle_control_value(&json!({"code": 30012, "auth": SECRET_MARKER}));
3243        handler.handle_control_value(&json!({
3244            "code": 30012, "channel": "trade/999", "auth": SECRET_MARKER,
3245        }));
3246
3247        let messages = OUTBOUND_LOG_CAPTURE.messages();
3248
3249        assert!(
3250            messages
3251                .iter()
3252                .all(|message| !message.contains(SECRET_MARKER)),
3253            "outbound logs exposed the secret marker: {messages:?}"
3254        );
3255        assert!(
3256            messages.iter().any(|message| {
3257                message == &format!("Sending Lighter unsubscribe ({payload_len} bytes)")
3258            }),
3259            "unsubscribe metadata missing or inaccurate: {messages:?}"
3260        );
3261        assert!(messages.iter().any(|message| {
3262            message == "Lighter subscribe error code=30012 channel=none: retrying inflight subscriptions"
3263        }));
3264        assert!(messages.iter().any(|message| {
3265            message == "Lighter subscribe error code=30012 channel=trade/999: no matching inflight subscription"
3266        }));
3267    }
3268
3269    fn mark_subscription_inflight(
3270        handler: &mut FeedHandler,
3271        channel: LighterWsChannel,
3272        response_tx: Option<tokio::sync::oneshot::Sender<Result<(), String>>>,
3273    ) -> (Ustr, u64) {
3274        handler.queue_subscribe(channel, None, response_tx);
3275        let (topic, generation) = handler
3276            .pending_subs
3277            .pop_front()
3278            .expect("subscription was not queued");
3279        handler.inflight_subs.insert(topic, generation);
3280        if let Some(market_index) = order_book_market_index_from_topic(topic.as_str()) {
3281            handler.book.expected.insert(market_index, (generation, 0));
3282        }
3283
3284        (topic, generation)
3285    }
3286
3287    fn saturate_subscription_gate(handler: &mut FeedHandler) {
3288        for i in 0..SUBSCRIBE_INFLIGHT_MAX {
3289            handler
3290                .inflight_subs
3291                .insert(Ustr::from(format!("dummy:{i}").as_str()), i as u64 + 1);
3292        }
3293    }
3294
3295    /// Strip the trailing `AccountStreamFirstFrame` marker from a handler
3296    /// emission so account-stream tests can assert on the typed payload
3297    /// alone. The marker contract is pinned in
3298    /// `handle_frame_emits_account_stream_first_frame_marker_per_variant`.
3299    fn strip_account_marker(mut msgs: Vec<NautilusWsMessage>) -> Vec<NautilusWsMessage> {
3300        if matches!(
3301            msgs.last(),
3302            Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
3303        ) {
3304            msgs.pop();
3305        }
3306        msgs
3307    }
3308
3309    #[rstest]
3310    fn handle_frame_routes_account_orders_to_execution_reports() {
3311        let mut handler = make_handler_with_account();
3312        let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3313
3314        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3315
3316        assert_eq!(messages.len(), 1);
3317        match &messages[0] {
3318            NautilusWsMessage::ExecutionReports(reports) => {
3319                assert_eq!(reports.len(), 1);
3320                match &reports[0] {
3321                    super::ExecutionReport::Order(order) => {
3322                        assert_eq!(order.order_id, "281476929510110");
3323                        assert_eq!(order.client_order_id, "42");
3324                    }
3325                    other => panic!("expected order report, was {other:?}"),
3326                }
3327            }
3328            other => panic!("expected execution reports, was {other:?}"),
3329        }
3330    }
3331
3332    #[rstest]
3333    fn handle_frame_routes_account_trades_to_execution_reports() {
3334        let mut handler = make_handler_with_account();
3335        let frame: super::LighterWsFrame =
3336            serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
3337
3338        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3339
3340        assert_eq!(messages.len(), 1);
3341        match &messages[0] {
3342            NautilusWsMessage::ExecutionReports(reports) => {
3343                assert_eq!(reports.len(), 1);
3344                match &reports[0] {
3345                    super::ExecutionReport::Fill(fill) => {
3346                        // Bid side is the user's account in this fixture
3347                        // (see handle_frame_routes_account_trades_to_execution_reports
3348                        // fixture: bid_account_id == 1234). The raw payload
3349                        // carries the bid order id as the venue id; the
3350                        // execution loop derives `venue_order_id` from it.
3351                        assert_eq!(fill.bid_id_str.as_deref(), Some("562947905631053"),);
3352                    }
3353                    other => panic!("expected fill report, was {other:?}"),
3354                }
3355            }
3356            other => panic!("expected execution reports, was {other:?}"),
3357        }
3358    }
3359
3360    #[rstest]
3361    fn handle_frame_routes_account_positions_to_update_without_readiness_marker() {
3362        let mut handler = make_handler_with_account();
3363        let frame: super::LighterWsFrame =
3364            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3365
3366        let messages = handler.handle_frame(frame, UnixNanos::from(11));
3367
3368        assert_eq!(messages.len(), 1);
3369        match &messages[0] {
3370            NautilusWsMessage::PositionUpdate {
3371                reports,
3372                closed_market_ids,
3373                skipped_market_ids,
3374            } => {
3375                assert!(closed_market_ids.is_empty());
3376                assert!(skipped_market_ids.is_empty());
3377                assert_eq!(reports.len(), 1);
3378                assert_eq!(reports[0].quantity, Quantity::from("1.5000"));
3379            }
3380            other => panic!("expected position update, was {other:?}"),
3381        }
3382    }
3383
3384    #[rstest]
3385    fn handle_frame_tolerates_unknown_position_margin_mode() {
3386        let mut handler = make_handler_with_account();
3387        let mut frame_json: serde_json::Value =
3388            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3389        frame_json["positions"]["0"]["margin_mode"] = json!(99);
3390        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3391
3392        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3393
3394        assert_eq!(messages.len(), 1);
3395        match &messages[0] {
3396            NautilusWsMessage::PositionUpdate {
3397                reports,
3398                closed_market_ids,
3399                skipped_market_ids,
3400            } => {
3401                assert!(closed_market_ids.is_empty());
3402                assert!(skipped_market_ids.is_empty());
3403                assert_eq!(reports.len(), 1);
3404            }
3405            other => panic!("expected position update, was {other:?}"),
3406        }
3407    }
3408
3409    #[rstest]
3410    fn handle_frame_routes_empty_account_positions_to_empty_update() {
3411        let mut handler = make_handler_with_account();
3412        let frame_json = serde_json::json!({
3413            "type": "update/account_all_positions",
3414            "channel": "account_all_positions:1234",
3415            "positions": {},
3416            "shares": [],
3417            "last_funding_round": null,
3418            "last_funding_discount": null,
3419        });
3420        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3421
3422        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3423
3424        assert_eq!(messages.len(), 1);
3425        match &messages[0] {
3426            NautilusWsMessage::PositionUpdate {
3427                reports,
3428                closed_market_ids,
3429                skipped_market_ids,
3430            } => {
3431                assert!(closed_market_ids.is_empty());
3432                assert!(skipped_market_ids.is_empty());
3433                assert!(reports.is_empty());
3434            }
3435            other => panic!("expected empty position update, was {other:?}"),
3436        }
3437    }
3438
3439    #[rstest]
3440    fn handle_frame_routes_zero_account_position_to_closed_update() {
3441        let mut handler = make_handler_with_account();
3442        let mut frame_json: serde_json::Value =
3443            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3444        frame_json["positions"]["0"]["position"] = json!("0.0000");
3445        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3446
3447        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3448
3449        assert_eq!(messages.len(), 1);
3450        match &messages[0] {
3451            NautilusWsMessage::PositionUpdate {
3452                reports,
3453                closed_market_ids,
3454                skipped_market_ids,
3455            } => {
3456                assert!(reports.is_empty());
3457                assert!(skipped_market_ids.is_empty());
3458                assert_eq!(closed_market_ids, &[0]);
3459            }
3460            other => panic!("expected closed position update, was {other:?}"),
3461        }
3462    }
3463
3464    #[rstest]
3465    fn handle_frame_marks_account_positions_incomplete_when_position_instrument_uncached() {
3466        let mut handler = make_handler_with_account();
3467        let mut frame_json: serde_json::Value =
3468            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3469        frame_json["type"] = json!("subscribed/account_all_positions");
3470        frame_json["positions"]["0"]["market_id"] = json!(999);
3471        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3472
3473        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3474
3475        assert_eq!(messages.len(), 1);
3476        match &messages[0] {
3477            NautilusWsMessage::PositionSnapshot {
3478                reports,
3479                skipped_market_ids,
3480            } => {
3481                assert_eq!(skipped_market_ids, &[999]);
3482                assert!(reports.is_empty());
3483            }
3484            other => panic!("expected incomplete position snapshot, was {other:?}"),
3485        }
3486    }
3487
3488    #[rstest]
3489    fn handle_frame_marks_account_positions_incomplete_when_position_parse_fails() {
3490        let mut handler = make_handler_with_account();
3491        let mut frame_json: serde_json::Value =
3492            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3493        frame_json["type"] = json!("subscribed/account_all_positions");
3494        frame_json["positions"]["0"]["position"] = json!("-1.5000");
3495        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3496
3497        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3498
3499        assert_eq!(messages.len(), 1);
3500        match &messages[0] {
3501            NautilusWsMessage::PositionSnapshot {
3502                reports,
3503                skipped_market_ids,
3504            } => {
3505                assert_eq!(skipped_market_ids, &[0]);
3506                assert!(reports.is_empty());
3507            }
3508            other => panic!("expected incomplete position snapshot, was {other:?}"),
3509        }
3510    }
3511
3512    #[rstest]
3513    fn handle_frame_marks_position_update_incomplete_when_position_parse_fails() {
3514        let mut handler = make_handler_with_account();
3515        let mut frame_json: serde_json::Value =
3516            serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3517        frame_json["positions"]["0"]["position"] = json!("-1.5000");
3518        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3519
3520        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3521
3522        assert_eq!(messages.len(), 1);
3523        match &messages[0] {
3524            NautilusWsMessage::PositionUpdate {
3525                reports,
3526                closed_market_ids,
3527                skipped_market_ids,
3528            } => {
3529                assert!(reports.is_empty());
3530                assert!(closed_market_ids.is_empty());
3531                assert_eq!(skipped_market_ids, &[0]);
3532            }
3533            other => panic!("expected incomplete position update, was {other:?}"),
3534        }
3535    }
3536
3537    #[rstest]
3538    fn handle_frame_routes_subscribed_account_all_positions_snapshot() {
3539        let mut handler = make_handler_with_account();
3540        let frame_json = serde_json::json!({
3541            "type": "subscribed/account_all_positions",
3542            "channel": "account_all_positions:1234",
3543            "positions": {
3544                "0": {
3545                    "allocated_margin": "0.000000",
3546                    "avg_entry_price": "0.111230",
3547                    "initial_margin_fraction": "10.00",
3548                    "liquidation_price": "0.100598",
3549                    "margin_mode": 0,
3550                    "market_id": 0,
3551                    "open_order_count": 0,
3552                    "pending_order_count": 0,
3553                    "position": "100",
3554                    "position_tied_order_count": 0,
3555                    "position_value": "11.123000",
3556                    "realized_pnl": "0.000000",
3557                    "sign": 1,
3558                    "symbol": "ETH",
3559                    "total_discount": "0.000000",
3560                    "total_funding_paid_out": "0.000000",
3561                    "unrealized_pnl": "0.000000"
3562                }
3563            },
3564        });
3565        let frame: super::LighterWsFrame = serde_json::from_value(frame_json).unwrap();
3566
3567        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
3568
3569        assert_eq!(messages.len(), 1);
3570        match &messages[0] {
3571            NautilusWsMessage::PositionSnapshot {
3572                reports,
3573                skipped_market_ids,
3574            } => {
3575                assert!(skipped_market_ids.is_empty());
3576                assert_eq!(reports.len(), 1);
3577                assert_eq!(reports[0].quantity, Quantity::from("100"));
3578            }
3579            other => panic!("expected position snapshot, was {other:?}"),
3580        }
3581    }
3582
3583    #[rstest]
3584    #[case::connected(serde_json::json!({"type": "connected", "session_id": "x"}), true, false)]
3585    #[case::ping(serde_json::json!({"type": "ping"}), true, false)]
3586    #[case::pong(serde_json::json!({"type": "pong"}), true, false)]
3587    #[case::send_tx_ack(
3588        serde_json::json!({"type": "jsonapi/sendtx", "code": 200, "tx_hash": "abc"}),
3589        true,
3590        true,
3591    )]
3592    #[case::send_tx_without_code(
3593        serde_json::json!({"type": "jsonapi/sendtx", "tx_hash": "abc"}),
3594        true,
3595        false,
3596    )]
3597    #[case::send_tx_with_nonnumeric_code(
3598        serde_json::json!({"type": "jsonapi/sendtx", "code": "200", "tx_hash": "abc"}),
3599        true,
3600        false,
3601    )]
3602    #[case::error_frame(
3603        serde_json::json!({"type": "error", "code": 21727, "message": "invalid client order index"}),
3604        true,
3605        true,
3606    )]
3607    #[case::error_frame_integrator_not_approved(
3608        serde_json::json!({"type": "error", "code": 21149, "message": "integrator is not approved"}),
3609        true,
3610        true,
3611    )]
3612    #[case::send_tx_ack_integrator_not_approved(
3613        serde_json::json!({"type": "jsonapi/sendtx", "code": 21149, "message": "integrator is not approved"}),
3614        true,
3615        true,
3616    )]
3617    #[case::wrapped_error_integrator_not_approved(
3618        serde_json::json!({"error": {"code": 21149, "message": "integrator is not approved"}}),
3619        true,
3620        true,
3621    )]
3622    #[case::subscription_error_frame(
3623        serde_json::json!({"type": "error", "code": 30003, "message": "Already Subscribed to : ticker:3"}),
3624        true,
3625        false,
3626    )]
3627    #[case::wrapped_subscription_error(
3628        serde_json::json!({"error": {"code": 30003, "message": "Already Subscribed to : ticker:3"}}),
3629        true,
3630        false,
3631    )]
3632    #[case::codeless_error_frame(
3633        serde_json::json!({"type": "error", "message": "unclassifiable"}),
3634        true,
3635        false,
3636    )]
3637    #[case::unknown_type(
3638        serde_json::json!({"type": "something_unexpected", "payload": "x"}),
3639        false,
3640        false,
3641    )]
3642    #[case::no_type_field(
3643        serde_json::json!({"error": {"code": 21702, "message": "invalid price"}}),
3644        true,
3645        true,
3646    )]
3647    fn handle_control_text_tri_state(
3648        #[case] payload: serde_json::Value,
3649        #[case] expected_matched: bool,
3650        #[case] expected_has_msg: bool,
3651    ) {
3652        // Pins the (matched, msg) contract that lets the WS receive loop
3653        // distinguish "known control type, consume silently" from
3654        // "surface a typed/Raw message to the consumer". sendTx success ACKs
3655        // and venue-error frames now produce typed SendTxAck/SendTxRejected
3656        // variants instead of Raw, so the execution loop can correlate them
3657        // back to in-flight cloids.
3658        let mut handler = make_handler_with_account();
3659        let text = payload.to_string();
3660        let (matched, msg) = handle_control_text(&mut handler, &text);
3661        assert_eq!(matched, expected_matched, "matched flag");
3662        assert_eq!(msg.is_some(), expected_has_msg, "msg presence");
3663    }
3664
3665    #[rstest]
3666    #[case::ack(serde_json::json!({"type":"jsonapi/sendtxbatch","id":"tx-batch:abc","code":200,"tx_hash":["abc","def"]}), 200, "batch rejected", vec!["abc", "def"])]
3667    #[case::error(serde_json::json!({"id":"tx-batch:abc","error":{"code":21104,"message":"invalid nonce"}}), 21104, "invalid nonce", vec![])]
3668    fn batch_response_preserves_correlation_and_epoch(
3669        #[case] value: serde_json::Value,
3670        #[case] expected_code: i64,
3671        #[case] expected_message: &str,
3672        #[case] expected_hashes: Vec<&str>,
3673    ) {
3674        let mut handler = make_handler_with_account();
3675        let (matched, message) = handler.handle_control_value(&value);
3676        assert!(matched);
3677
3678        match message.unwrap().with_connection_epoch(7) {
3679            NautilusWsMessage::SendTxBatchResult {
3680                connection_epoch,
3681                id,
3682                code,
3683                message,
3684                tx_hashes,
3685            } => {
3686                assert_eq!(connection_epoch, 7);
3687                assert_eq!(id, "tx-batch:abc");
3688                assert_eq!(code, expected_code);
3689                assert_eq!(message, expected_message);
3690                assert_eq!(tx_hashes, expected_hashes);
3691            }
3692            message => panic!("expected batch response, was {message:?}"),
3693        }
3694    }
3695
3696    #[rstest]
3697    fn handle_control_text_sendtx_success_emits_typed_ack() {
3698        let mut handler = make_handler_with_account();
3699        let payload = serde_json::json!({
3700            "type": "jsonapi/sendtx",
3701            "code": 200,
3702            "tx_hash": "0000abcd",
3703        })
3704        .to_string();
3705
3706        let (_, msg) = handle_control_text(&mut handler, &payload);
3707
3708        match msg.expect("SendTxAck emitted") {
3709            NautilusWsMessage::SendTxAck { tx_hash, code, .. } => {
3710                assert_eq!(code, 200);
3711                assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3712            }
3713            other => panic!("expected SendTxAck, was {other:?}"),
3714        }
3715    }
3716
3717    #[rstest]
3718    fn handle_control_text_sendtx_failure_emits_ack_sourced_rejection() {
3719        let mut handler = make_handler_with_account();
3720        let payload = serde_json::json!({
3721            "type": "jsonapi/sendtx",
3722            "code": 21727,
3723            "message": "invalid client order index",
3724        })
3725        .to_string();
3726
3727        let (_, msg) = handle_control_text(&mut handler, &payload);
3728
3729        match msg.expect("SendTxRejected emitted") {
3730            NautilusWsMessage::SendTxRejected {
3731                source,
3732                code,
3733                message,
3734                tx_hash,
3735                ..
3736            } => {
3737                assert_eq!(source, SendTxRejectionSource::Ack);
3738                assert_eq!(code, Some(21727));
3739                assert_eq!(message, "invalid client order index");
3740                assert_eq!(tx_hash, None);
3741            }
3742            other => panic!("expected SendTxRejected, was {other:?}"),
3743        }
3744    }
3745
3746    #[rstest]
3747    fn handle_control_text_sendtx_failure_carries_echoed_tx_hash() {
3748        let mut handler = make_handler_with_account();
3749        let payload = serde_json::json!({
3750            "type": "jsonapi/sendtx",
3751            "code": 21727,
3752            "message": "invalid client order index",
3753            "tx_hash": "0000abcd",
3754        })
3755        .to_string();
3756
3757        let (_, msg) = handle_control_text(&mut handler, &payload);
3758
3759        match msg.expect("SendTxRejected emitted") {
3760            NautilusWsMessage::SendTxRejected { tx_hash, .. } => {
3761                assert_eq!(tx_hash.as_deref(), Some("0000abcd"));
3762            }
3763            other => panic!("expected SendTxRejected, was {other:?}"),
3764        }
3765    }
3766
3767    #[rstest]
3768    fn handle_control_text_bare_error_frame_emits_bare_error_rejection() {
3769        let mut handler = make_handler_with_account();
3770        let payload = serde_json::json!({
3771            "type": "error",
3772            "code": 21702,
3773            "message": "invalid price",
3774        })
3775        .to_string();
3776
3777        let (_, msg) = handle_control_text(&mut handler, &payload);
3778
3779        match msg.expect("SendTxRejected emitted") {
3780            NautilusWsMessage::SendTxRejected {
3781                source,
3782                code,
3783                message,
3784                tx_hash,
3785                ..
3786            } => {
3787                assert_eq!(source, SendTxRejectionSource::BareError);
3788                assert_eq!(code, Some(21702));
3789                assert_eq!(message, "invalid price");
3790                assert_eq!(tx_hash, None);
3791            }
3792            other => panic!("expected SendTxRejected, was {other:?}"),
3793        }
3794    }
3795
3796    #[rstest]
3797    fn handle_control_text_wrapped_error_emits_bare_error_rejection() {
3798        // No top-level `type` field; the rejection sits under `error.{code,message}`.
3799        let mut handler = make_handler_with_account();
3800        let payload = serde_json::json!({
3801            "error": {"code": 21149, "message": "integrator is not approved"},
3802        })
3803        .to_string();
3804
3805        let (_, msg) = handle_control_text(&mut handler, &payload);
3806
3807        match msg.expect("SendTxRejected emitted") {
3808            NautilusWsMessage::SendTxRejected {
3809                source,
3810                code,
3811                message,
3812                tx_hash,
3813                ..
3814            } => {
3815                assert_eq!(source, SendTxRejectionSource::BareError);
3816                assert_eq!(code, Some(21149));
3817                assert_eq!(message, "integrator is not approved");
3818                assert_eq!(tx_hash, None);
3819            }
3820            other => panic!("expected SendTxRejected, was {other:?}"),
3821        }
3822    }
3823
3824    #[rstest]
3825    fn handle_frame_emits_no_account_state_until_both_streams_seen() {
3826        // Reconciler refuses to emit until both `account_all_assets` and
3827        // `user_stats` have delivered. An assets frame alone produces only
3828        // the AccountStreamFirstFrame marker, no AccountState payload.
3829        let mut handler = make_handler_with_account();
3830        let assets_only: super::LighterWsFrame =
3831            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3832
3833        let messages = strip_account_marker(handler.handle_frame(assets_only, UnixNanos::from(11)));
3834
3835        assert!(
3836            messages.is_empty(),
3837            "expected no AccountState before user_stats arrives, received {messages:?}"
3838        );
3839    }
3840
3841    #[rstest]
3842    fn handle_frame_routes_account_assets_and_user_stats_to_unified_state() {
3843        // Fixture is the captured production no-position payload (10 USDC
3844        // on spot, 40 USDC pledged as perp collateral, no resting orders).
3845        // Lighter runs unified margin: both legs are deployable equity, so
3846        // total = balance + margin_balance, locked = locked_balance only,
3847        // and MarginBalance.initial = 0 because user_stats.collateral ==
3848        // available_balance (no margin in use).
3849        let mut handler = make_handler_with_account();
3850        let assets_frame: super::LighterWsFrame =
3851            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3852        let user_stats_frame: super::LighterWsFrame =
3853            serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
3854
3855        let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
3856        let messages =
3857            strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
3858
3859        assert_eq!(messages.len(), 1);
3860        match &messages[0] {
3861            NautilusWsMessage::AccountState(state) => {
3862                let usdc = Currency::get_or_create_crypto("USDC");
3863                assert_eq!(state.account_type, AccountType::Margin);
3864                assert_eq!(state.base_currency, None);
3865                assert_eq!(state.balances.len(), 1);
3866                assert_eq!(state.balances[0].currency, usdc);
3867                // balance 10 + margin_balance 40 = 50 total; locked = 0
3868                // (no resting spot orders); free = 50 (all deployable).
3869                assert_eq!(state.balances[0].total, Money::from("50.000000 USDC"));
3870                assert_eq!(state.balances[0].locked, Money::from("0 USDC"));
3871                assert_eq!(state.balances[0].free, Money::from("50.000000 USDC"));
3872                assert_eq!(state.margins.len(), 1);
3873                assert_eq!(state.margins[0].currency, usdc);
3874                assert_eq!(state.margins[0].initial, Money::from("0 USDC"));
3875                assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
3876                assert!(state.margins[0].instrument_id.is_none());
3877                assert!(state.is_reported);
3878            }
3879            other => panic!("expected account state, was {other:?}"),
3880        }
3881    }
3882
3883    #[rstest]
3884    fn handle_frame_unified_state_reflects_open_position() {
3885        // Fixture pair captured live with one open ETH-LONG position:
3886        //   account_all_assets[USDC] = {balance:10, locked:0, margin_balance:39.995369556}
3887        //   user_stats              = {collateral:39.995369, available:39.168314, margin_usage:2.07}
3888        //
3889        // The 5 mUSDC haircut on margin_balance is the entry fee. The
3890        // 0.827055 USDC gap between collateral and available_balance is
3891        // the initial margin pledged to the open position. AccountBalance
3892        // is unchanged in shape; perp-margin-in-use lives on
3893        // MarginBalance, not in `locked`. Maintenance is zero because
3894        // Lighter doesn't publish a maintenance value on user_stats.
3895        let mut handler = make_handler_with_account();
3896        let assets_frame: super::LighterWsFrame =
3897            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_WITH_POSITION).unwrap();
3898        let user_stats_frame: super::LighterWsFrame =
3899            serde_json::from_str(WS_USER_STATS_WITH_POSITION).unwrap();
3900
3901        let _ = handler.handle_frame(assets_frame, UnixNanos::from(11));
3902        let messages =
3903            strip_account_marker(handler.handle_frame(user_stats_frame, UnixNanos::from(12)));
3904
3905        assert_eq!(messages.len(), 1);
3906        match &messages[0] {
3907            NautilusWsMessage::AccountState(state) => {
3908                assert_eq!(state.account_type, AccountType::Margin);
3909                assert_eq!(state.base_currency, None);
3910                assert_eq!(state.balances.len(), 1);
3911                // total = 10 + 39.995369556 = 49.99536956 (Money truncates
3912                // the trailing digit to 8 decimals, matching the
3913                // production log).
3914                assert_eq!(state.balances[0].total, Money::from("49.99536956 USDC"));
3915                assert_eq!(state.balances[0].locked, Money::from("0 USDC"));
3916                assert_eq!(state.balances[0].free, Money::from("49.99536956 USDC"));
3917                assert_eq!(state.margins.len(), 1);
3918                // initial = collateral 39.995369 - available 39.168314 = 0.827055
3919                assert_eq!(state.margins[0].initial, Money::from("0.82705500 USDC"));
3920                assert_eq!(state.margins[0].maintenance, Money::from("0 USDC"));
3921                assert!(state.margins[0].instrument_id.is_none());
3922            }
3923            other => panic!("expected account state, was {other:?}"),
3924        }
3925    }
3926
3927    #[rstest]
3928    fn handle_frame_emits_account_stream_first_frame_marker_per_variant() {
3929        // Pins the strict-await wiring: the handler appends an
3930        // `AccountStreamFirstFrame` marker after any typed reports so the
3931        // execution consumption loop can mark readiness only after the
3932        // accompanying state has been applied. The marker fires even for
3933        // empty content frames that produce no typed reports.
3934        let mut handler = make_handler_with_account();
3935        let orders_frame: super::LighterWsFrame =
3936            serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3937        let trades_frame: super::LighterWsFrame =
3938            serde_json::from_str(WS_ACCOUNT_ALL_TRADES_UPDATE).unwrap();
3939        let positions_frame: super::LighterWsFrame = serde_json::from_value({
3940            let mut value: serde_json::Value =
3941                serde_json::from_str(WS_ACCOUNT_ALL_POSITIONS_UPDATE).unwrap();
3942            value["type"] = json!("subscribed/account_all_positions");
3943            value
3944        })
3945        .unwrap();
3946        let assets_frame: super::LighterWsFrame =
3947            serde_json::from_str(WS_ACCOUNT_ALL_ASSETS_UPDATE).unwrap();
3948        let user_stats_frame: super::LighterWsFrame =
3949            serde_json::from_str(WS_USER_STATS_UPDATE).unwrap();
3950
3951        let cases = [
3952            (orders_frame, AccountStream::Orders),
3953            (trades_frame, AccountStream::Trades),
3954            (positions_frame, AccountStream::Positions),
3955            (assets_frame, AccountStream::Assets),
3956            (user_stats_frame, AccountStream::UserStats),
3957        ];
3958
3959        for (frame, expected) in cases {
3960            let msgs = handler.handle_frame(frame, UnixNanos::from(11));
3961            let marker = msgs
3962                .iter()
3963                .find(|m| matches!(m, NautilusWsMessage::AccountStreamFirstFrame(_)))
3964                .unwrap_or_else(|| panic!("missing marker for {expected:?}"));
3965            match marker {
3966                NautilusWsMessage::AccountStreamFirstFrame(stream) => {
3967                    assert_eq!(*stream, expected);
3968                }
3969                other => panic!("expected AccountStreamFirstFrame, was {other:?}"),
3970            }
3971            // The marker must trail any typed reports the handler emitted
3972            // so the consumption loop applies state first.
3973            assert!(
3974                matches!(
3975                    msgs.last(),
3976                    Some(NautilusWsMessage::AccountStreamFirstFrame(_)),
3977                ),
3978                "marker must trail typed reports for {expected:?}",
3979            );
3980        }
3981    }
3982
3983    #[rstest]
3984    fn handle_frame_account_orders_without_context_falls_back_to_raw() {
3985        let signal = Arc::new(AtomicBool::new(false));
3986        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3987        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
3988        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
3989        let mut handler =
3990            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
3991        handler.instruments.insert(0, stub_eth_perp_instrument());
3992        // exec_account intentionally left unset; the handler must preserve
3993        // the prior `Raw` forwarding contract for unauthenticated subscribers.
3994
3995        let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
3996        let messages = handler.handle_frame(frame, UnixNanos::from(11));
3997
3998        assert_eq!(messages.len(), 1);
3999        match &messages[0] {
4000            NautilusWsMessage::Raw(value) => {
4001                assert_eq!(value["type"], "update/account_orders");
4002            }
4003            other => panic!("expected raw fallback, was {other:?}"),
4004        }
4005    }
4006
4007    #[rstest]
4008    fn handle_frame_account_orders_skips_unknown_market() {
4009        // Build a handler with the execution context but no instrument
4010        // cached for the order's market_index; the handler should log and
4011        // emit no execution reports (the trailing readiness marker still
4012        // fires so `connect()` does not stall when the venue resubscribes
4013        // before instrument bootstrap completes).
4014        let signal = Arc::new(AtomicBool::new(false));
4015        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4016        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4017        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4018        let mut handler =
4019            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4020        handler.exec_account = Some((AccountId::from("LIGHTER-1234"), 1234));
4021        // No instrument inserted for market_index=0.
4022
4023        let frame: super::LighterWsFrame = serde_json::from_str(WS_ACCOUNT_ORDERS_UPDATE).unwrap();
4024        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4025
4026        assert!(messages.is_empty());
4027    }
4028
4029    #[rstest]
4030    fn handle_frame_account_assets_invalid_timestamp_returns_empty() {
4031        let mut handler = make_handler_with_account();
4032        // u64::MAX millis overflows the seconds-to-nanos conversion in
4033        // `parse_millis_to_nanos`; the handler must log and emit no typed
4034        // account state rather than surface a partially constructed one.
4035        // The trailing readiness marker is still emitted so `connect()`
4036        // does not stall if the venue ever sends a malformed timestamp on
4037        // the initial frame.
4038        let frame_json = r#"{
4039            "type": "update/account_all_assets",
4040            "channel": "account_all_assets:1234",
4041            "timestamp": 18446744073709551615,
4042            "assets": {
4043                "0": {
4044                    "symbol": "USDC",
4045                    "asset_id": 0,
4046                    "balance": "100.000000",
4047                    "locked_balance": "1.000000"
4048                }
4049            }
4050        }"#;
4051        let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
4052
4053        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4054
4055        assert!(messages.is_empty());
4056    }
4057
4058    #[rstest]
4059    fn handle_frame_account_all_orders_routes_to_execution_reports() {
4060        // The `update/account_all_orders` variant lacks the per-account
4061        // top-level `account` and `nonce` fields that `account_orders`
4062        // carries, but the handler should treat them identically.
4063        let mut handler = make_handler_with_account();
4064        let frame_json = r#"{
4065            "type": "update/account_all_orders",
4066            "channel": "account_all_orders:1234",
4067            "orders": {
4068                "0": [{
4069                    "order_index": 281476929510110,
4070                    "client_order_index": 42,
4071                    "order_id": "281476929510110",
4072                    "client_order_id": "42",
4073                    "market_index": 0,
4074                    "owner_account_index": 1234,
4075                    "initial_base_amount": "0.0050",
4076                    "price": "2352.74",
4077                    "nonce": 9182390020,
4078                    "remaining_base_amount": "0.0050",
4079                    "is_ask": true,
4080                    "base_size": 50,
4081                    "base_price": 235274,
4082                    "filled_base_amount": "0.0000",
4083                    "filled_quote_amount": "0.000000",
4084                    "side": "sell",
4085                    "type": "limit",
4086                    "time_in_force": "good-till-time",
4087                    "reduce_only": false,
4088                    "trigger_price": "0.00",
4089                    "order_expiry": 1780360584479,
4090                    "status": "open",
4091                    "trigger_status": "na",
4092                    "trigger_time": 0,
4093                    "parent_order_index": 0,
4094                    "parent_order_id": "0",
4095                    "to_trigger_order_id_0": "0",
4096                    "to_trigger_order_id_1": "0",
4097                    "to_cancel_order_id_0": "0",
4098                    "integrator_fee_collector_index": "0",
4099                    "integrator_taker_fee": "0",
4100                    "integrator_maker_fee": "0",
4101                    "block_height": 227535532,
4102                    "timestamp": 1777941383576,
4103                    "created_at": 1777941383576,
4104                    "updated_at": 1777941383576,
4105                    "transaction_time": 1777941383576735
4106                }]
4107            }
4108        }"#;
4109        let frame: super::LighterWsFrame = serde_json::from_str(frame_json).unwrap();
4110
4111        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4112
4113        assert_eq!(messages.len(), 1);
4114        match &messages[0] {
4115            NautilusWsMessage::ExecutionReports(reports) => {
4116                assert_eq!(reports.len(), 1);
4117                match &reports[0] {
4118                    super::ExecutionReport::Order(order) => {
4119                        assert_eq!(order.order_id, "281476929510110");
4120                    }
4121                    other => panic!("expected order report, was {other:?}"),
4122                }
4123            }
4124            other => panic!("expected execution reports, was {other:?}"),
4125        }
4126    }
4127
4128    fn snapshot_trade_frame_json() -> &'static str {
4129        r#"{
4130            "type": "subscribed/account_all_trades",
4131            "channel": "account_all_trades:1234",
4132            "trades": [{
4133                "trade_id": 19209006902,
4134                "trade_id_str": "19209006902",
4135                "tx_hash": "000000128b1ee814",
4136                "type": "trade",
4137                "market_id": 0,
4138                "size": "0.1336",
4139                "price": "2352.73",
4140                "usd_amount": "314.324728",
4141                "ask_id": 281476929510102,
4142                "bid_id": 562947905631053,
4143                "ask_client_id": 0,
4144                "bid_client_id": 7001011966,
4145                "ask_account_id": 91249,
4146                "bid_account_id": 1234,
4147                "is_maker_ask": true,
4148                "block_height": 227535535,
4149                "timestamp": 1777941384181,
4150                "transaction_time": 1777941384181586
4151            }],
4152            "total_volume": "100.0",
4153            "monthly_volume": "100.0",
4154            "weekly_volume": "100.0",
4155            "daily_volume": "100.0"
4156        }"#
4157    }
4158
4159    #[rstest]
4160    fn handle_frame_account_all_trades_snapshot_is_dropped_with_context() {
4161        let mut handler = make_handler_with_account();
4162        // With execution context active the snapshot must be dropped to
4163        // avoid replaying historical fills as live FillReports on reconnect.
4164        // The trailing readiness marker is still emitted so `connect()`
4165        // observes the trades stream as having delivered.
4166        let frame: super::LighterWsFrame =
4167            serde_json::from_str(snapshot_trade_frame_json()).unwrap();
4168
4169        let messages = strip_account_marker(handler.handle_frame(frame, UnixNanos::from(11)));
4170
4171        assert!(messages.is_empty());
4172    }
4173
4174    #[rstest]
4175    fn handle_frame_account_all_trades_snapshot_falls_back_to_raw_without_context() {
4176        let signal = Arc::new(AtomicBool::new(false));
4177        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4178        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4179        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4180        let mut handler =
4181            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4182        handler.instruments.insert(0, stub_eth_perp_instrument());
4183        // No exec_account: preserve prior Raw forwarding so unauthenticated
4184        // subscribers still see the snapshot frame.
4185
4186        let frame: super::LighterWsFrame =
4187            serde_json::from_str(snapshot_trade_frame_json()).unwrap();
4188        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4189
4190        assert_eq!(messages.len(), 1);
4191        match &messages[0] {
4192            NautilusWsMessage::Raw(value) => {
4193                assert_eq!(value["type"], "subscribed/account_all_trades");
4194            }
4195            other => panic!("expected raw fallback, was {other:?}"),
4196        }
4197    }
4198
4199    #[rstest]
4200    fn handle_frame_market_stats_emits_mark_index_and_funding_updates() {
4201        let mut handler = make_handler_with_account();
4202        let frame: super::LighterWsFrame =
4203            serde_json::from_str(WS_MARKET_STATS_UPDATE_SINGLE).unwrap();
4204
4205        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4206
4207        assert_eq!(messages.len(), 3);
4208        match &messages[0] {
4209            NautilusWsMessage::MarkPrice(update) => {
4210                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4211                assert_eq!(update.value, Price::from("2064.47"));
4212                assert_eq!(update.ts_event, UnixNanos::from(1_774_883_844_933_000_000));
4213            }
4214            event => panic!("expected mark price update, was {event:?}"),
4215        }
4216
4217        match &messages[1] {
4218            NautilusWsMessage::IndexPrice(update) => {
4219                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4220                assert_eq!(update.value, Price::from("2064.48"));
4221            }
4222            event => panic!("expected index price update, was {event:?}"),
4223        }
4224
4225        match &messages[2] {
4226            NautilusWsMessage::FundingRate(update) => {
4227                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4228                assert_eq!(update.rate, Decimal::new(1, 8));
4229                assert_eq!(update.next_funding_ns, None);
4230            }
4231            event => panic!("expected funding rate update, was {event:?}"),
4232        }
4233    }
4234
4235    #[rstest]
4236    fn handle_frame_market_stats_all_emits_mark_index_and_funding_updates() {
4237        let mut handler = make_handler_with_account();
4238        let frame: super::LighterWsFrame =
4239            serde_json::from_str(WS_MARKET_STATS_UPDATE_ALL).unwrap();
4240
4241        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4242
4243        assert_eq!(messages.len(), 3);
4244        assert!(matches!(&messages[0], NautilusWsMessage::MarkPrice(_)));
4245        assert!(matches!(&messages[1], NautilusWsMessage::IndexPrice(_)));
4246        assert!(matches!(&messages[2], NautilusWsMessage::FundingRate(_)));
4247
4248        match &messages[0] {
4249            NautilusWsMessage::MarkPrice(update) => {
4250                assert_eq!(update.instrument_id.to_string(), "ETH-PERP.LIGHTER");
4251                assert_eq!(update.value, Price::from("2064.47"));
4252            }
4253            event => panic!("expected mark price update, was {event:?}"),
4254        }
4255
4256        match &messages[2] {
4257            NautilusWsMessage::FundingRate(update) => {
4258                assert_eq!(update.rate, Decimal::new(1, 8));
4259                assert_eq!(update.next_funding_ns, None);
4260            }
4261            event => panic!("expected funding rate update, was {event:?}"),
4262        }
4263    }
4264
4265    #[rstest]
4266    fn handle_frame_spot_market_stats_emits_index_update() {
4267        let signal = Arc::new(AtomicBool::new(false));
4268        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4269        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4270        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4271        let mut handler =
4272            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4273        handler.instruments.insert(2048, stub_eth_spot_instrument());
4274        let frame: super::LighterWsFrame =
4275            serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_SINGLE).unwrap();
4276
4277        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4278
4279        assert_eq!(messages.len(), 1);
4280        match &messages[0] {
4281            NautilusWsMessage::IndexPrice(update) => {
4282                assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
4283                assert_eq!(update.value, Price::from("1.00"));
4284            }
4285            event => panic!("expected spot index price update, was {event:?}"),
4286        }
4287    }
4288
4289    #[rstest]
4290    fn handle_frame_spot_market_stats_all_emits_index_update() {
4291        let signal = Arc::new(AtomicBool::new(false));
4292        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4293        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4294        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4295        let mut handler =
4296            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4297        handler.instruments.insert(2048, stub_eth_spot_instrument());
4298        let frame: super::LighterWsFrame =
4299            serde_json::from_str(WS_SPOT_MARKET_STATS_UPDATE_ALL).unwrap();
4300
4301        let messages = handler.handle_frame(frame, UnixNanos::from(11));
4302
4303        assert_eq!(messages.len(), 1);
4304        match &messages[0] {
4305            NautilusWsMessage::IndexPrice(update) => {
4306                assert_eq!(update.instrument_id.to_string(), "ETH-SPOT.LIGHTER");
4307                assert_eq!(update.value, Price::from("1.00"));
4308            }
4309            event => panic!("expected spot index price update, was {event:?}"),
4310        }
4311    }
4312
4313    #[rstest]
4314    #[case(LighterWsChannel::OrderBook(0), "order_book:0", "order_book/0")]
4315    #[case(LighterWsChannel::Trade(7), "trade:7", "trade/7")]
4316    #[case(LighterWsChannel::Ticker(2), "ticker:2", "ticker/2")]
4317    #[case(LighterWsChannel::Height, "height", "height")]
4318    #[case(
4319        LighterWsChannel::MarketStats(LighterMarketSelection::All),
4320        "market_stats:all",
4321        "market_stats/all"
4322    )]
4323    #[case(
4324        LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(2048)),
4325        "spot_market_stats:2048",
4326        "spot_market_stats/2048"
4327    )]
4328    #[case(
4329        LighterWsChannel::OrderBook(4095),
4330        "order_book:4095",
4331        "order_book/4095"
4332    )]
4333    #[case(
4334        LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(50_000)),
4335        "spot_market_stats:50000",
4336        "spot_market_stats/50000"
4337    )]
4338    #[case(
4339        LighterWsChannel::Candle {
4340            market_index: 40_000,
4341            resolution: LighterCandleResolution::OneHour,
4342        },
4343        "candle:40000:1h",
4344        "candle/40000/1h"
4345    )]
4346    #[case(
4347        LighterWsChannel::AccountOrders { market_index: 0, account_index: 1234 },
4348        "account_orders:0:1234",
4349        "account_orders/0/1234",
4350    )]
4351    fn topic_and_subscription_round_trip(
4352        #[case] channel: LighterWsChannel,
4353        #[case] expected_topic: &str,
4354        #[case] expected_subscription: &str,
4355    ) {
4356        assert_eq!(channel.topic_key(), expected_topic);
4357        assert_eq!(channel.subscription_channel(), expected_subscription);
4358    }
4359
4360    #[rstest]
4361    #[case("order_book:0", Some(0))]
4362    #[case("trade:42", Some(42))]
4363    #[case("order_book:4095", Some(4095))]
4364    #[case("trade:40000", Some(40_000))]
4365    #[case("market_stats:50000", Some(50_000))]
4366    #[case("height", None)]
4367    #[case("malformed", None)]
4368    fn market_index_extraction(#[case] topic: &str, #[case] expected: Option<i64>) {
4369        assert_eq!(market_index_from_topic(topic), expected);
4370    }
4371
4372    #[rstest]
4373    #[case("order_book:0", Some(0))]
4374    #[case("order_book:42", Some(42))]
4375    #[case("order_book:40000", Some(40_000))]
4376    #[case("trade:42", None)]
4377    #[case("ticker:2", None)]
4378    #[case("market_stats:0", None)]
4379    #[case("height", None)]
4380    #[case("order_book:not-an-int", None)]
4381    fn order_book_market_index_only_matches_order_book_channel(
4382        #[case] topic: &str,
4383        #[case] expected: Option<i64>,
4384    ) {
4385        assert_eq!(order_book_market_index_from_topic(topic), expected);
4386    }
4387
4388    #[rstest]
4389    #[case(LighterWsChannel::AccountAll(1234), true)]
4390    #[case(LighterWsChannel::OrderBook(0), false)]
4391    #[case(LighterWsChannel::AccountAllPositions(1), true)]
4392    #[case(LighterWsChannel::Trade(0), false)]
4393    fn requires_auth_classification(#[case] channel: LighterWsChannel, #[case] expected: bool) {
4394        assert_eq!(channel.requires_auth(), expected);
4395    }
4396
4397    #[rstest]
4398    fn handler_command_subscribe_debug_redacts_auth_token() {
4399        let token = "schnorr-signature-bytes-do-not-leak";
4400        let cmd = HandlerCommand::Subscribe {
4401            channel: LighterWsChannel::AccountAll(1234),
4402            auth: Some(SecretString::from(token)),
4403            response_tx: None,
4404        };
4405
4406        let dbg = format!("{cmd:?}");
4407
4408        assert!(
4409            !dbg.contains(token),
4410            "Debug output must not contain the auth token, found: {dbg}",
4411        );
4412        assert!(dbg.contains(REDACTED));
4413    }
4414
4415    #[tokio::test]
4416    async fn send_tx_command_returns_handler_send_error_without_active_client() {
4417        let signal = Arc::new(AtomicBool::new(false));
4418        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4419        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4420        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4421        let mut handler =
4422            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4423        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
4424
4425        cmd_tx
4426            .send(HandlerCommand::SendTx {
4427                tx_type: LighterTxType::CreateOrder as u8,
4428                tx_info: serde_json::value::RawValue::from_string(
4429                    r#"{"AccountIndex":12345,"Nonce":42}"#.to_string(),
4430                )
4431                .unwrap(),
4432                connection_epoch: 0,
4433                response_tx,
4434            })
4435            .unwrap();
4436        drop(cmd_tx);
4437        drop(raw_tx);
4438
4439        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4440            .await
4441            .expect("timed out waiting for handler to drain command");
4442        let result = response_rx.await.expect("sendTx response channel closed");
4443
4444        assert!(next.is_none());
4445        let Err(LighterWsError::Client(message)) = result else {
4446            panic!("expected client send error, was {result:?}");
4447        };
4448        assert!(message.contains("no active WebSocket client"));
4449    }
4450
4451    #[tokio::test]
4452    async fn resubscribe_order_book_command_skips_when_reference_removed() {
4453        let signal = Arc::new(AtomicBool::new(false));
4454        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4455        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4456        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4457        let subscriptions = SubscriptionState::new(':');
4458        let topic = LighterWsChannel::OrderBook(0).topic_key();
4459        assert!(subscriptions.add_reference(&topic));
4460        assert!(subscriptions.remove_reference(&topic));
4461
4462        let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions.clone());
4463        handler.book.delta_subs.insert(0);
4464
4465        let (completion, result) = tokio::sync::oneshot::channel();
4466        cmd_tx
4467            .send(HandlerCommand::RecoverBook {
4468                market_index: 0,
4469                cancel: CancellationToken::new(),
4470                gate: SnapshotGate::default(),
4471                completion,
4472            })
4473            .expect("queue resync");
4474        drop(cmd_tx);
4475        drop(raw_tx);
4476
4477        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4478            .await
4479            .expect("timed out waiting for handler to drain command");
4480
4481        assert!(next.is_none());
4482        assert!(subscriptions.pending_subscribe_topics().is_empty());
4483        assert!(subscriptions.pending_unsubscribe_topics().is_empty());
4484        assert!(result.await.is_err());
4485    }
4486
4487    #[tokio::test]
4488    async fn resubscribe_order_book_queues_when_inflight_is_at_cap() {
4489        let signal = Arc::new(AtomicBool::new(false));
4490        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4491        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4492        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4493        let subscriptions = SubscriptionState::new(':');
4494        let topic = LighterWsChannel::OrderBook(0).topic_key();
4495        assert!(subscriptions.add_reference(&topic));
4496
4497        let mut handler = FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, subscriptions);
4498        handler.book.delta_subs.insert(0);
4499        saturate_subscription_gate(&mut handler);
4500
4501        let (completion, _rx) = tokio::sync::oneshot::channel();
4502        handler.queue_book_replacement(
4503            0,
4504            BookWrite {
4505                cancel: CancellationToken::new(),
4506                gate: SnapshotGate::default(),
4507                completion,
4508            },
4509        );
4510
4511        handler.pump_pending_subscribes().await;
4512
4513        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4514        assert!(
4515            handler
4516                .pending_subs
4517                .iter()
4518                .any(|(topic, _)| *topic == Ustr::from("order_book:0")),
4519        );
4520    }
4521
4522    fn stub_candle(
4523        t: i64,
4524        open: i64,
4525        high: i64,
4526        low: i64,
4527        close: i64,
4528        volume_ticks: i64,
4529    ) -> LighterWsCandle {
4530        LighterWsCandle {
4531            t,
4532            o: Decimal::new(open, 2),
4533            h: Decimal::new(high, 2),
4534            l: Decimal::new(low, 2),
4535            c: Decimal::new(close, 2),
4536            v: Decimal::new(volume_ticks, 4),
4537            quote_volume: Decimal::ZERO,
4538            i: 0,
4539        }
4540    }
4541
4542    fn candle_frame(channel: &str, candle: LighterWsCandle, is_snapshot: bool) -> LighterWsFrame {
4543        if is_snapshot {
4544            LighterWsFrame::CandleSnapshot {
4545                channel: Ustr::from(channel),
4546                candles: vec![candle],
4547                timestamp: 0,
4548            }
4549        } else {
4550            LighterWsFrame::Candle {
4551                channel: Ustr::from(channel),
4552                candles: vec![candle],
4553                timestamp: 0,
4554            }
4555        }
4556    }
4557
4558    #[rstest]
4559    fn handle_candles_first_observation_caches_without_emit() {
4560        let mut handler = make_handler_with_account();
4561        let frame = candle_frame(
4562            "candle:0:1m",
4563            stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
4564            true,
4565        );
4566
4567        let messages = handler.handle_frame(frame, UnixNanos::from(99));
4568
4569        assert!(messages.is_empty(), "first observation must not emit");
4570        let key = (0_i64, LighterCandleResolution::OneMinute);
4571        assert_eq!(handler.last_candles.get(&key).map(|c| c.t), Some(1_000_000));
4572    }
4573
4574    #[rstest]
4575    fn handle_candles_t_advance_emits_bar_for_previous_candle() {
4576        let mut handler = make_handler_with_account();
4577        let prev = stub_candle(1_000_000, 10_000, 11_000, 9_900, 10_500, 10_000);
4578        let next = stub_candle(1_060_000, 10_500, 10_600, 10_450, 10_550, 20_000);
4579        let next_t = next.t;
4580        handler.handle_frame(candle_frame("candle:0:1m", prev, true), UnixNanos::from(1));
4581
4582        let messages =
4583            handler.handle_frame(candle_frame("candle:0:1m", next, false), UnixNanos::from(2));
4584
4585        assert_eq!(messages.len(), 1);
4586        match &messages[0] {
4587            NautilusWsMessage::Bar(bar) => {
4588                // The emitted bar reflects the previous (closed) candle, not the new one.
4589                assert_eq!(bar.open, Price::from("100.00"));
4590                assert_eq!(bar.high, Price::from("110.00"));
4591                assert_eq!(bar.low, Price::from("99.00"));
4592                assert_eq!(bar.close, Price::from("105.00"));
4593                assert_eq!(bar.volume, Quantity::from("1.0000"));
4594                assert_eq!(bar.ts_event, UnixNanos::from(1_000_000 * 1_000_000));
4595            }
4596            other => panic!("expected Bar message, was {other:?}"),
4597        }
4598        let cached = handler
4599            .last_candles
4600            .get(&(0_i64, LighterCandleResolution::OneMinute))
4601            .expect("cache populated");
4602        assert_eq!(cached.t, next_t);
4603    }
4604
4605    #[rstest]
4606    fn handle_candles_same_t_updates_cache_without_emit() {
4607        let mut handler = make_handler_with_account();
4608        let initial = stub_candle(1_000_000, 10_000, 10_050, 9_950, 10_025, 5_000);
4609        let same_t_updated = stub_candle(1_000_000, 10_000, 10_100, 9_950, 10_075, 7_500);
4610        let same_t_h = same_t_updated.h;
4611        let same_t_c = same_t_updated.c;
4612        handler.handle_frame(
4613            candle_frame("candle:0:1m", initial, true),
4614            UnixNanos::from(1),
4615        );
4616
4617        let messages = handler.handle_frame(
4618            candle_frame("candle:0:1m", same_t_updated, false),
4619            UnixNanos::from(2),
4620        );
4621
4622        assert!(messages.is_empty(), "same-`t` update must not emit");
4623        let cached = handler
4624            .last_candles
4625            .get(&(0_i64, LighterCandleResolution::OneMinute))
4626            .expect("cache populated");
4627        assert_eq!(cached.h, same_t_h);
4628        assert_eq!(cached.c, same_t_c);
4629    }
4630
4631    #[rstest]
4632    fn handle_candles_regressed_t_is_skipped() {
4633        let mut handler = make_handler_with_account();
4634        let initial = stub_candle(2_000_000, 10_000, 10_000, 10_000, 10_000, 5_000);
4635        let regressed = stub_candle(1_000_000, 9_000, 9_000, 9_000, 9_000, 5_000);
4636        let initial_t = initial.t;
4637        handler.handle_frame(
4638            candle_frame("candle:0:1m", initial, true),
4639            UnixNanos::from(1),
4640        );
4641
4642        let messages = handler.handle_frame(
4643            candle_frame("candle:0:1m", regressed, false),
4644            UnixNanos::from(2),
4645        );
4646
4647        assert!(messages.is_empty(), "regressed `t` must not emit");
4648        let cached = handler
4649            .last_candles
4650            .get(&(0_i64, LighterCandleResolution::OneMinute))
4651            .expect("cache populated");
4652        // Regressed frame is skipped entirely; cache stays on the original entry.
4653        assert_eq!(cached.t, initial_t);
4654    }
4655
4656    #[rstest]
4657    fn handle_candles_unknown_market_returns_empty() {
4658        let mut handler = make_handler_with_account();
4659        let frame = candle_frame(
4660            "candle:99:1m",
4661            stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 5_000),
4662            true,
4663        );
4664
4665        let messages = handler.handle_frame(frame, UnixNanos::from(1));
4666
4667        assert!(messages.is_empty());
4668    }
4669
4670    #[rstest]
4671    fn handle_unsubscribe_ack_clears_only_matching_candle_key() {
4672        let mut handler = make_handler_with_account();
4673        handler.last_candles.insert(
4674            (0, LighterCandleResolution::OneMinute),
4675            stub_candle(1, 0, 0, 0, 0, 0),
4676        );
4677        handler.last_candles.insert(
4678            (0, LighterCandleResolution::FiveMinute),
4679            stub_candle(2, 0, 0, 0, 0, 0),
4680        );
4681        handler.subscriptions.mark_unsubscribe("candle:0:1m");
4682
4683        let payload = json!({"type": "unsubscribed", "channel": "candle:0:1m"});
4684        let (matched, _) = handle_control_text(&mut handler, &payload.to_string());
4685
4686        assert!(matched);
4687        assert!(
4688            handler
4689                .last_candles
4690                .get(&(0, LighterCandleResolution::OneMinute))
4691                .is_none(),
4692        );
4693        assert!(
4694            handler
4695                .last_candles
4696                .get(&(0, LighterCandleResolution::FiveMinute))
4697                .is_some(),
4698        );
4699    }
4700
4701    #[rstest]
4702    #[case::well_formed("candle:0:1m", Some((0, LighterCandleResolution::OneMinute)))]
4703    #[case::weekly("candle:3:1w", Some((3, LighterCandleResolution::OneWeek)))]
4704    #[case::widened_id("candle:40000:1h", Some((40_000, LighterCandleResolution::OneHour)))]
4705    #[case::other_kind("order_book:0", None)]
4706    #[case::missing_resolution("candle:0", None)]
4707    #[case::bad_market("candle:notanint:1m", None)]
4708    #[case::bad_resolution("candle:0:bogus", None)]
4709    fn test_candle_market_and_resolution_from_topic(
4710        #[case] topic: &str,
4711        #[case] expected: Option<(i64, LighterCandleResolution)>,
4712    ) {
4713        assert_eq!(candle_market_and_resolution_from_topic(topic), expected);
4714    }
4715
4716    #[rstest]
4717    #[case::network_retries(LighterWsError::Network("disconnected".into()), true)]
4718    #[case::auth_does_not_retry(LighterWsError::Authentication("bad token".into()), false)]
4719    #[case::parse_does_not_retry(LighterWsError::Parse("bad json".into()), false)]
4720    #[case::client_does_not_retry(LighterWsError::Client("no active WebSocket client".into()), false)]
4721    #[case::transport_closed_does_not_retry(LighterWsError::Transport(SendError::Closed), false)]
4722    #[case::transport_invalid_input_does_not_retry(
4723        LighterWsError::Transport(SendError::InvalidInput("pong payload too large".into())),
4724        false
4725    )]
4726    #[case::transport_connection_changed_does_not_retry(
4727        LighterWsError::Transport(SendError::ConnectionChanged),
4728        false
4729    )]
4730    #[case::transport_timeout_retries(LighterWsError::Transport(SendError::Timeout), true)]
4731    #[case::transport_write_timeout_does_not_retry(
4732        LighterWsError::Transport(SendError::WriteTimeout),
4733        false
4734    )]
4735    #[case::transport_broken_pipe_does_not_retry(
4736        LighterWsError::Transport(SendError::BrokenPipe(
4737            "writer closed".into(),
4738        )),
4739        false,
4740    )]
4741    #[case::buffer_full(LighterWsError::Transport(SendError::BufferFull), false)]
4742    fn test_should_retry_lighter_ws_error(#[case] error: LighterWsError, #[case] expected: bool) {
4743        assert_eq!(should_retry_lighter_ws_error(&error), expected);
4744    }
4745
4746    // Pins the `#[from] SendError` derive that `.map_err(LighterWsError::Transport)` relies on.
4747    #[rstest]
4748    #[case::closed(SendError::Closed)]
4749    #[case::invalid_input(SendError::InvalidInput("pong payload too large".into()))]
4750    #[case::timeout(SendError::Timeout)]
4751    #[case::write_timeout(SendError::WriteTimeout)]
4752    #[case::connection_changed(SendError::ConnectionChanged)]
4753    #[case::broken_pipe(SendError::BrokenPipe("writer dropped".into()))]
4754    fn send_error_converts_into_transport_variant(#[case] send_error: SendError) {
4755        let err: LighterWsError = send_error.into();
4756        assert!(
4757            matches!(err, LighterWsError::Transport(_)),
4758            "expected Transport variant, was {err:?}",
4759        );
4760    }
4761
4762    #[tokio::test]
4763    async fn subscribe_command_parks_in_pending_subs_when_inflight_at_cap() {
4764        let signal = Arc::new(AtomicBool::new(false));
4765        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4766        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4767        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4768        let mut handler =
4769            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4770
4771        // Saturate the gate so the pump cannot dispatch the queued subscribe
4772        saturate_subscription_gate(&mut handler);
4773
4774        cmd_tx
4775            .send(HandlerCommand::Subscribe {
4776                channel: LighterWsChannel::Candle {
4777                    market_index: 0,
4778                    resolution: LighterCandleResolution::OneMinute,
4779                },
4780                auth: None,
4781                response_tx: None,
4782            })
4783            .expect("queue subscribe");
4784        drop(cmd_tx);
4785        drop(raw_tx);
4786
4787        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4788            .await
4789            .expect("timed out waiting for handler to drain command");
4790
4791        assert!(next.is_none());
4792        assert_eq!(handler.inflight_subs.len(), SUBSCRIBE_INFLIGHT_MAX);
4793        assert_eq!(handler.pending_subs.len(), 1);
4794        assert_eq!(handler.subscription_attempts.len(), 1);
4795        assert_eq!(handler.pending_subs[0].0, Ustr::from("candle:0:1m"));
4796    }
4797
4798    #[tokio::test]
4799    async fn unsubscribe_drops_queued_subscribe_while_gate_full() {
4800        let signal = Arc::new(AtomicBool::new(false));
4801        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4802        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4803        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4804        let mut handler =
4805            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4806
4807        // Saturate the gate so the queued subscribe cannot dispatch before the unsubscribe
4808        saturate_subscription_gate(&mut handler);
4809
4810        cmd_tx
4811            .send(HandlerCommand::Subscribe {
4812                channel: LighterWsChannel::Trade(0),
4813                auth: None,
4814                response_tx: None,
4815            })
4816            .expect("queue subscribe");
4817        cmd_tx
4818            .send(HandlerCommand::Unsubscribe {
4819                channel: LighterWsChannel::Trade(0),
4820            })
4821            .expect("queue unsubscribe");
4822        drop(cmd_tx);
4823        drop(raw_tx);
4824
4825        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4826            .await
4827            .expect("timed out waiting for handler to drain commands");
4828
4829        assert!(next.is_none());
4830        assert!(handler.pending_subs.is_empty());
4831        assert!(handler.subscription_attempts.is_empty());
4832    }
4833
4834    #[tokio::test]
4835    async fn reconnect_requeues_attempt_with_new_generation() {
4836        let signal = Arc::new(AtomicBool::new(false));
4837        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4838        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4839        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4840        let mut handler =
4841            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4842
4843        saturate_subscription_gate(&mut handler);
4844        handler.queue_subscribe(LighterWsChannel::Trade(0), None, None);
4845        let old_generation = handler.pending_subs[0].1;
4846
4847        raw_tx
4848            .send((7, Message::Text(RECONNECTED.to_string().into())))
4849            .expect("queue reconnect sentinel");
4850
4851        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
4852            .await
4853            .expect("timed out waiting for reconnect");
4854
4855        assert!(matches!(next, Some(NautilusWsMessage::Reconnected { .. })));
4856        assert!(handler.inflight_subs.is_empty());
4857        assert_eq!(handler.pending_subs.len(), 1);
4858        assert_eq!(handler.pending_subs[0].0, Ustr::from("trade:0"));
4859        assert_ne!(handler.pending_subs[0].1, old_generation);
4860        assert_eq!(
4861            handler.subscription_attempts[&Ustr::from("trade:0")].generation,
4862            handler.pending_subs[0].1,
4863        );
4864    }
4865
4866    #[rstest]
4867    #[tokio::test]
4868    async fn pump_releases_inflight_slot_and_schedules_send_failure_retry() {
4869        let signal = Arc::new(AtomicBool::new(false));
4870        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
4871        let (_raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
4872        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
4873        let mut handler =
4874            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
4875
4876        // No active client, so every dispatch fails; the gate must not leak slots
4877        for market_index in 0..3 {
4878            handler.queue_subscribe(LighterWsChannel::Trade(market_index), None, None);
4879        }
4880
4881        handler.pump_pending_subscribes().await;
4882
4883        assert!(handler.pending_subs.is_empty());
4884        assert!(handler.inflight_subs.is_empty());
4885        assert_eq!(handler.subscription_attempts.len(), 3);
4886        assert!(
4887            handler
4888                .subscription_attempts
4889                .values()
4890                .all(|attempt| attempt.retries == 1),
4891        );
4892        assert_eq!(handler.subscription_retries.len(), 3);
4893    }
4894
4895    #[rstest]
4896    fn subscribed_control_frame_releases_inflight_slot() {
4897        let mut handler = make_handler_with_account();
4898        mark_subscription_inflight(
4899            &mut handler,
4900            LighterWsChannel::Candle {
4901                market_index: 0,
4902                resolution: LighterCandleResolution::OneMinute,
4903            },
4904            None,
4905        );
4906
4907        let (matched, msg) = handle_control_text(
4908            &mut handler,
4909            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
4910        );
4911
4912        assert!(matched);
4913        assert!(msg.is_none());
4914        assert!(
4915            !handler
4916                .inflight_subs
4917                .contains_key(&Ustr::from("candle:0:1m"))
4918        );
4919        assert!(handler.subscription_attempts.is_empty());
4920    }
4921
4922    #[rstest]
4923    #[case::top_level(
4924        r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_orders:12345"}"#
4925    )]
4926    #[case::nested(
4927        r#"{"error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
4928    )]
4929    #[case::nested_typed(
4930        r#"{"type":"error","error":{"code":30003,"message":"Already Subscribed to : account_all_orders:12345"}}"#
4931    )]
4932    fn already_subscribed_confirms_matching_inflight_topic(#[case] payload: &str) {
4933        let mut handler = make_handler_with_account();
4934        mark_subscription_inflight(
4935            &mut handler,
4936            LighterWsChannel::AccountAllOrders(12345),
4937            None,
4938        );
4939
4940        let (matched, msg) = handle_control_text(&mut handler, payload);
4941
4942        assert!(matched);
4943        assert!(msg.is_none());
4944        assert!(handler.inflight_subs.is_empty());
4945        assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
4946        assert_eq!(handler.subscriptions.len(), 1);
4947    }
4948
4949    #[rstest]
4950    fn already_subscribed_does_not_confirm_unmatched_topic() {
4951        let mut handler = make_handler_with_account();
4952        let inflight = "account_all_orders:12345";
4953        mark_subscription_inflight(
4954            &mut handler,
4955            LighterWsChannel::AccountAllOrders(12345),
4956            None,
4957        );
4958
4959        let (matched, msg) = handle_control_text(
4960            &mut handler,
4961            r#"{"type":"error","code":30003,"message":"Already Subscribed to : account_all_trades:12345"}"#,
4962        );
4963
4964        assert!(matched);
4965        assert!(msg.is_none());
4966        assert!(handler.inflight_subs.contains_key(&Ustr::from(inflight)));
4967        assert_eq!(
4968            handler.subscriptions.pending_subscribe_topics(),
4969            vec![inflight]
4970        );
4971        assert_eq!(handler.subscriptions.len(), 0);
4972    }
4973
4974    #[tokio::test]
4975    async fn duplicate_queued_and_inflight_topics_share_one_generation() {
4976        let mut handler = make_handler_with_account();
4977        let (response_tx_1, mut response_rx_1) = tokio::sync::oneshot::channel();
4978        let (response_tx_2, mut response_rx_2) = tokio::sync::oneshot::channel();
4979        let (response_tx_3, mut response_rx_3) = tokio::sync::oneshot::channel();
4980        let channel = LighterWsChannel::Trade(7);
4981
4982        handler.queue_subscribe(channel.clone(), None, Some(response_tx_1));
4983        let generation = handler.pending_subs[0].1;
4984        handler.queue_subscribe(channel.clone(), None, Some(response_tx_2));
4985
4986        assert_eq!(handler.pending_subs.len(), 1);
4987        assert_eq!(handler.subscription_attempts.len(), 1);
4988        assert_eq!(
4989            handler.subscription_attempts[&Ustr::from("trade:7")]
4990                .response_txs
4991                .len(),
4992            2,
4993        );
4994
4995        let (topic, queued_generation) = handler.pending_subs.pop_front().unwrap();
4996        handler.inflight_subs.insert(topic, queued_generation);
4997        handler.queue_subscribe(channel, None, Some(response_tx_3));
4998
4999        assert_eq!(queued_generation, generation);
5000        assert!(handler.pending_subs.is_empty());
5001        assert_eq!(handler.inflight_subs.len(), 1);
5002        assert_eq!(
5003            handler.subscription_attempts[&Ustr::from("trade:7")]
5004                .response_txs
5005                .len(),
5006            3,
5007        );
5008        assert!(matches!(
5009            response_rx_1.try_recv(),
5010            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5011        ));
5012        assert!(matches!(
5013            response_rx_2.try_recv(),
5014            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5015        ));
5016        assert!(matches!(
5017            response_rx_3.try_recv(),
5018            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5019        ));
5020
5021        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5022
5023        assert_eq!(response_rx_1.await.unwrap(), Ok(()));
5024        assert_eq!(response_rx_2.await.unwrap(), Ok(()));
5025        assert_eq!(response_rx_3.await.unwrap(), Ok(()));
5026        assert!(handler.inflight_subs.is_empty());
5027        assert!(handler.subscription_attempts.is_empty());
5028    }
5029
5030    #[tokio::test]
5031    async fn changed_inflight_auth_waits_for_serialized_successor() {
5032        let mut handler = make_handler_with_account();
5033        let channel = LighterWsChannel::AccountAllOrders(12345);
5034        let topic = Ustr::from(channel.topic_key().as_str());
5035        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5036        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5037
5038        handler.queue_subscribe(
5039            channel.clone(),
5040            Some(SecretString::from("old-token")),
5041            Some(old_tx),
5042        );
5043        let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
5044        handler.inflight_subs.insert(queued_topic, old_generation);
5045        handler.queue_subscribe(
5046            channel,
5047            Some(SecretString::from("fresh-token")),
5048            Some(fresh_tx),
5049        );
5050
5051        let attempt = &handler.subscription_attempts[&topic];
5052        assert_eq!(
5053            attempt.auth.as_ref().map(SecretString::expose_secret),
5054            Some("old-token")
5055        );
5056        assert_eq!(
5057            attempt
5058                .pending_auth
5059                .as_ref()
5060                .map(SecretString::expose_secret),
5061            Some("fresh-token")
5062        );
5063        assert_eq!(attempt.response_txs.len(), 1);
5064        assert_eq!(attempt.pending_response_txs.len(), 1);
5065
5066        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5067        assert_eq!(old_rx.await.unwrap(), Ok(()));
5068        assert!(matches!(
5069            fresh_rx.try_recv(),
5070            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5071        ));
5072        let attempt = &handler.subscription_attempts[&topic];
5073        assert_ne!(attempt.generation, old_generation);
5074        assert_eq!(
5075            attempt.auth.as_ref().map(SecretString::expose_secret),
5076            Some("fresh-token")
5077        );
5078        assert!(attempt.pending_auth.is_none());
5079        assert_eq!(
5080            handler.pending_subs.front(),
5081            Some(&(topic, attempt.generation)),
5082        );
5083
5084        let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
5085        handler.inflight_subs.insert(topic, fresh_generation);
5086
5087        // The predecessor's trailing typed snapshot must not complete the
5088        // dispatched successor.
5089        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
5090        assert!(matches!(
5091            fresh_rx.try_recv(),
5092            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5093        ));
5094        assert_eq!(
5095            handler.subscription_attempts[&topic].generation,
5096            fresh_generation,
5097        );
5098
5099        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5100        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5101        assert!(handler.subscription_attempts.is_empty());
5102    }
5103
5104    #[tokio::test]
5105    async fn stale_typed_frame_does_not_complete_fresh_attempt_after_predecessor_removed() {
5106        let mut handler = make_handler_with_account();
5107        let channel = LighterWsChannel::AccountAllOrders(12345);
5108        let topic = Ustr::from(channel.topic_key().as_str());
5109        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5110        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5111
5112        // Predecessor completes on its control ack and is removed before any
5113        // pending auth exists.
5114        handler.queue_subscribe(
5115            channel.clone(),
5116            Some(SecretString::from("old-token")),
5117            Some(old_tx),
5118        );
5119        let (queued_topic, old_generation) = handler.pending_subs.pop_front().unwrap();
5120        handler.inflight_subs.insert(queued_topic, old_generation);
5121        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5122        assert_eq!(old_rx.await.unwrap(), Ok(()));
5123        assert!(handler.subscription_attempts.is_empty());
5124
5125        // The fresh command then creates and dispatches a brand-new attempt.
5126        handler.queue_subscribe(
5127            channel,
5128            Some(SecretString::from("fresh-token")),
5129            Some(fresh_tx),
5130        );
5131        let (_, fresh_generation) = handler.pending_subs.pop_front().unwrap();
5132        handler.inflight_subs.insert(topic, fresh_generation);
5133
5134        // The predecessor's trailing typed snapshot must not complete it.
5135        assert!(!handler.complete_subscription(topic.as_str(), CompletionKind::Typed));
5136        assert!(matches!(
5137            fresh_rx.try_recv(),
5138            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5139        ));
5140        assert_eq!(
5141            handler.subscription_attempts[&topic].generation,
5142            fresh_generation,
5143        );
5144
5145        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5146        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5147        assert!(handler.subscription_attempts.is_empty());
5148    }
5149
5150    #[tokio::test]
5151    async fn changed_queued_auth_updates_existing_generation_before_dispatch() {
5152        let mut handler = make_handler_with_account();
5153        let channel = LighterWsChannel::AccountAllOrders(12345);
5154        let topic = Ustr::from(channel.topic_key().as_str());
5155        let (old_tx, old_rx) = tokio::sync::oneshot::channel();
5156        let (fresh_tx, fresh_rx) = tokio::sync::oneshot::channel();
5157
5158        handler.queue_subscribe(
5159            channel.clone(),
5160            Some(SecretString::from("old-token")),
5161            Some(old_tx),
5162        );
5163        let generation = handler.pending_subs[0].1;
5164        handler.queue_subscribe(
5165            channel,
5166            Some(SecretString::from("fresh-token")),
5167            Some(fresh_tx),
5168        );
5169
5170        assert_eq!(handler.pending_subs.len(), 1);
5171        assert_eq!(handler.pending_subs[0], (topic, generation));
5172        let attempt = &handler.subscription_attempts[&topic];
5173        assert_eq!(attempt.generation, generation);
5174        assert_eq!(
5175            attempt.auth.as_ref().map(SecretString::expose_secret),
5176            Some("fresh-token")
5177        );
5178        assert!(attempt.pending_auth.is_none());
5179        assert_eq!(attempt.response_txs.len(), 2);
5180        assert!(attempt.pending_response_txs.is_empty());
5181
5182        handler.pending_subs.pop_front();
5183        handler.inflight_subs.insert(topic, generation);
5184        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5185        assert_eq!(old_rx.await.unwrap(), Ok(()));
5186        assert_eq!(fresh_rx.await.unwrap(), Ok(()));
5187    }
5188
5189    #[rstest]
5190    fn second_reconnect_folds_pending_auth_into_requeued_generation() {
5191        let mut handler = make_handler_with_account();
5192        let channel = LighterWsChannel::AccountAllOrders(12345);
5193        let topic = Ustr::from(channel.topic_key().as_str());
5194        let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
5195        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5196
5197        handler.queue_subscribe(
5198            channel.clone(),
5199            Some(SecretString::from("old-token")),
5200            Some(old_tx),
5201        );
5202        handler.reset_subscription_attempts_after_reconnect();
5203        let (_, replay_generation) = handler.pending_subs.pop_front().unwrap();
5204        handler.inflight_subs.insert(topic, replay_generation);
5205        handler.queue_subscribe(
5206            channel,
5207            Some(SecretString::from("fresh-token")),
5208            Some(fresh_tx),
5209        );
5210
5211        let attempt = &handler.subscription_attempts[&topic];
5212        assert_eq!(
5213            attempt.auth.as_ref().map(SecretString::expose_secret),
5214            Some("old-token")
5215        );
5216        assert_eq!(
5217            attempt
5218                .pending_auth
5219                .as_ref()
5220                .map(SecretString::expose_secret),
5221            Some("fresh-token")
5222        );
5223        assert_eq!(attempt.response_txs.len(), 1);
5224        assert_eq!(attempt.pending_response_txs.len(), 1);
5225
5226        handler.reset_subscription_attempts_after_reconnect();
5227
5228        let attempt = &handler.subscription_attempts[&topic];
5229        assert_ne!(attempt.generation, replay_generation);
5230        assert_eq!(
5231            attempt.auth.as_ref().map(SecretString::expose_secret),
5232            Some("fresh-token")
5233        );
5234        assert!(attempt.pending_auth.is_none());
5235        assert_eq!(attempt.response_txs.len(), 2);
5236        assert!(attempt.pending_response_txs.is_empty());
5237        assert_eq!(handler.pending_subs.len(), 1);
5238        assert_eq!(handler.pending_subs[0], (topic, attempt.generation));
5239        assert!(handler.inflight_subs.is_empty());
5240        assert!(matches!(
5241            old_rx.try_recv(),
5242            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5243        ));
5244        assert!(matches!(
5245            fresh_rx.try_recv(),
5246            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5247        ));
5248
5249        let (_, generation) = handler.pending_subs.pop_front().unwrap();
5250        handler.inflight_subs.insert(topic, generation);
5251        assert!(handler.complete_subscription(topic.as_str(), CompletionKind::ControlAck));
5252
5253        assert_eq!(old_rx.try_recv(), Ok(Ok(())));
5254        assert_eq!(fresh_rx.try_recv(), Ok(Ok(())));
5255        assert!(handler.subscription_attempts.is_empty());
5256    }
5257
5258    #[tokio::test]
5259    async fn retry_folds_pending_auth_without_resetting_retry_budget() {
5260        let mut handler = make_handler_with_account();
5261        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5262        handler.set_command_sender(cmd_tx);
5263        let channel = LighterWsChannel::AccountAllOrders(12345);
5264        let topic = Ustr::from(channel.topic_key().as_str());
5265        let (old_tx, mut old_rx) = tokio::sync::oneshot::channel();
5266        let (fresh_tx, mut fresh_rx) = tokio::sync::oneshot::channel();
5267
5268        handler.queue_subscribe(
5269            channel.clone(),
5270            Some(SecretString::from("old-token")),
5271            Some(old_tx),
5272        );
5273        let (_, generation) = handler.pending_subs.pop_front().unwrap();
5274        handler.inflight_subs.insert(topic, generation);
5275        handler
5276            .subscription_attempts
5277            .get_mut(&topic)
5278            .unwrap()
5279            .retries = 2;
5280        handler.queue_subscribe(
5281            channel,
5282            Some(SecretString::from("fresh-token")),
5283            Some(fresh_tx),
5284        );
5285
5286        handler.schedule_subscription_retry(topic, generation, "retry");
5287
5288        let attempt = &handler.subscription_attempts[&topic];
5289        assert_eq!(attempt.retries, 3);
5290        assert_ne!(attempt.generation, generation);
5291        assert_eq!(
5292            attempt.auth.as_ref().map(SecretString::expose_secret),
5293            Some("fresh-token")
5294        );
5295        assert!(attempt.pending_auth.is_none());
5296        assert_eq!(attempt.response_txs.len(), 2);
5297        assert!(attempt.pending_response_txs.is_empty());
5298        assert!(handler.pending_subs.is_empty());
5299        assert!(handler.inflight_subs.is_empty());
5300        assert_eq!(handler.subscription_retries.len(), 1);
5301        assert!(matches!(
5302            old_rx.try_recv(),
5303            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5304        ));
5305        assert!(matches!(
5306            fresh_rx.try_recv(),
5307            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5308        ));
5309    }
5310
5311    #[tokio::test]
5312    async fn confirmed_authenticated_topic_opens_new_generation() {
5313        let mut handler = make_handler_with_account();
5314        let channel = LighterWsChannel::AccountAllOrders(12345);
5315        let topic = channel.topic_key();
5316        handler.subscriptions.mark_subscribe(&topic);
5317        handler.subscriptions.confirm_subscribe(&topic);
5318        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5319
5320        handler.queue_subscribe(
5321            channel,
5322            Some(SecretString::from("rotated-auth-token")),
5323            Some(response_tx),
5324        );
5325
5326        assert_eq!(handler.pending_subs.len(), 1);
5327        assert_eq!(handler.subscription_attempts.len(), 1);
5328        assert_eq!(handler.subscriptions.len(), 1);
5329        assert!(handler.subscriptions.pending_subscribe_topics().is_empty());
5330
5331        let (topic, generation) = handler.pending_subs.pop_front().unwrap();
5332        handler.inflight_subs.insert(topic, generation);
5333        handle_control_text(
5334            &mut handler,
5335            r#"{"type":"subscribed","channel":"account_all_orders:12345"}"#,
5336        );
5337
5338        assert_eq!(response_rx.await.unwrap(), Ok(()));
5339        assert!(handler.subscription_attempts.is_empty());
5340        assert_eq!(handler.subscriptions.len(), 1);
5341    }
5342
5343    #[tokio::test]
5344    async fn typed_update_before_ack_does_not_complete_subscription_generation() {
5345        let mut handler = make_handler_with_account();
5346        let (response_tx, mut response_rx) = tokio::sync::oneshot::channel();
5347        let (topic, generation) = mark_subscription_inflight(
5348            &mut handler,
5349            LighterWsChannel::Candle {
5350                market_index: 0,
5351                resolution: LighterCandleResolution::OneMinute,
5352            },
5353            Some(response_tx),
5354        );
5355
5356        handler.handle_frame(
5357            candle_frame(
5358                "candle:0:1m",
5359                stub_candle(1_000_000, 10_000, 10_000, 10_000, 10_000, 10_000),
5360                false,
5361            ),
5362            UnixNanos::from(1),
5363        );
5364
5365        assert_eq!(
5366            handler
5367                .last_candles
5368                .get(&(0, LighterCandleResolution::OneMinute))
5369                .map(|candle| candle.t),
5370            Some(1_000_000),
5371        );
5372        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5373        assert_eq!(
5374            handler.subscriptions.pending_subscribe_topics(),
5375            vec!["candle:0:1m"],
5376        );
5377        assert_eq!(handler.subscriptions.len(), 0);
5378        assert!(matches!(
5379            response_rx.try_recv(),
5380            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5381        ));
5382
5383        handle_control_text(
5384            &mut handler,
5385            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5386        );
5387
5388        assert_eq!(response_rx.await.unwrap(), Ok(()));
5389        assert_eq!(handler.subscriptions.len(), 1);
5390    }
5391
5392    #[tokio::test]
5393    async fn typed_subscribed_frame_completes_subscription_generation() {
5394        let signal = Arc::new(AtomicBool::new(false));
5395        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5396        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5397        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5398        let mut handler =
5399            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5400        handler.instruments.insert(0, stub_eth_perp_instrument());
5401        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5402        mark_subscription_inflight(
5403            &mut handler,
5404            LighterWsChannel::Candle {
5405                market_index: 0,
5406                resolution: LighterCandleResolution::OneMinute,
5407            },
5408            Some(response_tx),
5409        );
5410
5411        raw_tx
5412            .send((7, Message::Text(WS_CANDLE_SUBSCRIBED.into())))
5413            .expect("typed subscribed frame");
5414        drop(raw_tx);
5415        drop(cmd_tx);
5416
5417        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5418            .await
5419            .expect("handler did not process typed subscribed frame");
5420
5421        assert!(next.is_none());
5422        assert_eq!(response_rx.await.unwrap(), Ok(()));
5423        assert!(handler.inflight_subs.is_empty());
5424        assert!(handler.subscription_attempts.is_empty());
5425        assert_eq!(handler.subscriptions.len(), 1);
5426    }
5427
5428    #[tokio::test]
5429    async fn unparsable_subscribed_frame_still_completes_subscription() {
5430        let signal = Arc::new(AtomicBool::new(false));
5431        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5432        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5433        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5434        let mut handler =
5435            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5436        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5437        mark_subscription_inflight(
5438            &mut handler,
5439            LighterWsChannel::SpotMarketStats(LighterMarketSelection::Market(4098)),
5440            Some(response_tx),
5441        );
5442
5443        raw_tx
5444            .send((7, Message::Text(WS_SPOT_STATS_SUBSCRIBED_BAD_BODY.into())))
5445            .expect("unparsable subscribed frame");
5446        drop(raw_tx);
5447        drop(cmd_tx);
5448
5449        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5450            .await
5451            .expect("handler did not process unparsable subscribed frame");
5452
5453        assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5454        let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5455            .await
5456            .expect("subscription was not completed");
5457        assert_eq!(response.unwrap(), Ok(()));
5458        assert!(handler.inflight_subs.is_empty());
5459        assert!(handler.subscription_attempts.is_empty());
5460        assert_eq!(handler.subscriptions.len(), 1);
5461    }
5462
5463    #[tokio::test]
5464    async fn unparsable_book_subscribed_frame_still_completes_subscription() {
5465        let signal = Arc::new(AtomicBool::new(false));
5466        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
5467        let (raw_tx, raw_rx) = tokio::sync::mpsc::unbounded_channel::<(u64, Message)>();
5468        let (out_tx, _out_rx) = tokio::sync::mpsc::unbounded_channel::<NautilusWsMessage>();
5469        let mut handler =
5470            FeedHandler::new(signal, cmd_rx, raw_rx, out_tx, SubscriptionState::new(':'));
5471        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5472        mark_subscription_inflight(
5473            &mut handler,
5474            LighterWsChannel::OrderBook(4095),
5475            Some(response_tx),
5476        );
5477
5478        // Epoch 0 matches the generation entry seeded by the helper, so the
5479        // book-specific snapshot matching accepts this confirmation.
5480        raw_tx
5481            .send((0, Message::Text(WS_BOOK_SUBSCRIBED_BAD_BODY.into())))
5482            .expect("unparsable book subscribed frame");
5483        drop(raw_tx);
5484        drop(cmd_tx);
5485
5486        let next = tokio::time::timeout(Duration::from_secs(2), handler.next())
5487            .await
5488            .expect("handler did not process unparsable book subscribed frame");
5489
5490        assert!(matches!(next, Some(NautilusWsMessage::Raw(_))));
5491        let response = tokio::time::timeout(Duration::from_secs(2), response_rx)
5492            .await
5493            .expect("book subscription was not completed");
5494        assert_eq!(response.unwrap(), Ok(()));
5495        assert!(handler.inflight_subs.is_empty());
5496        assert!(handler.subscription_attempts.is_empty());
5497        assert_eq!(handler.subscriptions.len(), 1);
5498    }
5499
5500    #[rstest]
5501    #[case::nonbook_top_level(false, false, "market_stats:0")]
5502    #[case::nonbook_wrapped(false, true, "market_stats/0")]
5503    #[case::book_top_level(true, false, "order_book/0")]
5504    #[case::book_wrapped(true, true, "order_book:0")]
5505    #[tokio::test]
5506    async fn subscribe_failure_preserves_distinct_inflight_subscription(
5507        #[case] book: bool,
5508        #[case] wrapped: bool,
5509        #[case] channel: &str,
5510    ) {
5511        let mut handler = make_handler_with_account();
5512        let (failed_tx, mut failed_rx) = tokio::sync::oneshot::channel();
5513        let (valid_tx, mut valid_rx) = tokio::sync::oneshot::channel();
5514
5515        let failed_channel = if book {
5516            LighterWsChannel::OrderBook(0)
5517        } else {
5518            LighterWsChannel::MarketStats(LighterMarketSelection::Market(0))
5519        };
5520
5521        let (failed_topic, _) =
5522            mark_subscription_inflight(&mut handler, failed_channel, Some(failed_tx));
5523        let (valid_topic, generation) =
5524            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(valid_tx));
5525        let error = serde_json::json!({"code": 30012, "channel": channel});
5526
5527        let value = if wrapped {
5528            serde_json::json!({"error": error})
5529        } else {
5530            error
5531        };
5532
5533        let (matched, message) = handler.handle_control_value(&value);
5534        let pending = valid_rx.try_recv();
5535        let inflight = handler.inflight_subs.get(&valid_topic).copied();
5536        let retries = handler.subscription_attempts[&valid_topic].retries;
5537        let rejected = failed_rx.try_recv();
5538        let failure_retained = handler.subscription_attempts.contains_key(&failed_topic);
5539        handler.handle_control_value(&serde_json::json!({
5540            "type": "subscribed", "channel": "trade:7"
5541        }));
5542
5543        let expected_error = if book {
5544            BOOK_SUBSCRIBE_REJECTION.to_string()
5545        } else {
5546            "subscription market_stats:0 failed after 1 attempts: venue rejected the WebSocket \
5547             subscribe with code 30012"
5548                .to_string()
5549        };
5550
5551        assert!(matched);
5552        assert!(message.is_none());
5553        assert_eq!(
5554            pending,
5555            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
5556        );
5557        assert_eq!(inflight, Some(generation));
5558        assert_eq!(retries, 0);
5559        assert_eq!(rejected, Ok(Err(expected_error)));
5560        assert!(!failure_retained);
5561        assert_eq!(valid_rx.try_recv(), Ok(Ok(())));
5562        assert!(handler.inflight_subs.is_empty());
5563        assert!(handler.subscription_attempts.is_empty());
5564        assert_eq!(handler.subscriptions.len(), 1);
5565    }
5566
5567    #[rstest]
5568    #[case::unknown("trade:999")]
5569    #[case::confirmed("trade:7")]
5570    #[case::queued("market_stats:0")]
5571    #[tokio::test]
5572    async fn subscribe_failure_ignores_channel_without_inflight_attempt(#[case] channel: &str) {
5573        let mut handler = make_handler_with_account();
5574        let (response, mut result) = tokio::sync::oneshot::channel();
5575        let (topic, generation) =
5576            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(3), Some(response));
5577        handler.subscriptions.mark_subscribe("trade:7");
5578        handler.subscriptions.confirm_subscribe("trade:7");
5579        handler.queue_subscribe(
5580            LighterWsChannel::MarketStats(LighterMarketSelection::Market(0)),
5581            None,
5582            None,
5583        );
5584        let (queued_topic, queued_generation) = *handler.pending_subs.front().unwrap();
5585
5586        handler.handle_control_value(&serde_json::json!({"code": 30012, "channel": channel}));
5587
5588        assert_eq!(
5589            result.try_recv(),
5590            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
5591        );
5592        assert_eq!(handler.inflight_subs.get(&topic), Some(&generation));
5593        assert_eq!(
5594            handler.subscription_attempts[&queued_topic].generation,
5595            queued_generation
5596        );
5597        assert_eq!(handler.subscription_attempts[&queued_topic].retries, 0);
5598        assert_eq!(
5599            handler.pending_subs.iter().copied().collect::<Vec<_>>(),
5600            [(queued_topic, queued_generation)]
5601        );
5602        assert!(handler.subscription_retries.is_empty());
5603        assert_eq!(handler.subscriptions.len(), 1);
5604    }
5605
5606    #[tokio::test]
5607    async fn failed_subscribe_error_fails_every_waiter_for_the_generation() {
5608        let mut handler = make_handler_with_account();
5609        let (response_tx_1, response_rx_1) = tokio::sync::oneshot::channel();
5610        let (response_tx_2, response_rx_2) = tokio::sync::oneshot::channel();
5611        let channel = LighterWsChannel::MarketStats(LighterMarketSelection::Market(0));
5612        mark_subscription_inflight(&mut handler, channel.clone(), Some(response_tx_1));
5613        handler.queue_subscribe(channel, None, Some(response_tx_2));
5614
5615        let (matched, msg) = handle_control_text(
5616            &mut handler,
5617            &serde_json::json!({
5618                "type": "error", "code": 30012, "channel": "market_stats:0"
5619            })
5620            .to_string(),
5621        );
5622
5623        assert!(matched);
5624        assert!(msg.is_none());
5625        let error_1 = response_rx_1.await.unwrap().unwrap_err();
5626        let error_2 = response_rx_2.await.unwrap().unwrap_err();
5627        assert!(error_1.contains("market_stats:0"));
5628        assert!(error_1.contains("30012"));
5629        assert_eq!(error_1, error_2);
5630        assert!(handler.inflight_subs.is_empty());
5631        assert!(handler.subscription_attempts.is_empty());
5632        assert!(handler.subscriptions.is_empty());
5633    }
5634
5635    #[rstest]
5636    #[case::rate_limit(30009)]
5637    #[case::unattributed_failure(30012)]
5638    #[tokio::test]
5639    async fn subscription_retry_exhaustion_fails_waiter_and_clears_intent(#[case] code: u64) {
5640        let mut handler = make_handler_with_account();
5641        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
5642        let (topic, _) =
5643            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(response_tx));
5644        handler
5645            .subscription_attempts
5646            .get_mut(&topic)
5647            .expect("subscription attempt")
5648            .retries = SUBSCRIBE_RETRY_MAX;
5649
5650        let (matched, msg) =
5651            handle_control_text(&mut handler, &serde_json::json!({"code": code}).to_string());
5652
5653        assert!(matched);
5654        assert!(msg.is_none());
5655        assert_eq!(
5656            response_rx.await.unwrap(),
5657            Err(format!(
5658                "subscription trade:7 failed after 6 attempts: venue rejected the WebSocket \
5659                 subscribe with code {code}"
5660            )),
5661        );
5662        assert!(handler.inflight_subs.is_empty());
5663        assert!(handler.subscription_attempts.is_empty());
5664        assert!(handler.subscriptions.is_empty());
5665    }
5666
5667    #[rstest]
5668    #[case::rate_limit(serde_json::json!({"code": 30009}))]
5669    #[case::unattributed_failure(serde_json::json!({"code": 30012}))]
5670    #[case::wrapped_failure(serde_json::json!({"error": {"code": 30012}}))]
5671    #[case::invalid_channel(serde_json::json!({"code": 30012, "channel": 7}))]
5672    #[case::empty_channel(serde_json::json!({"code": 30012, "channel": ""}))]
5673    #[tokio::test]
5674    async fn subscription_error_retries_each_inflight_topic_with_new_generation(
5675        #[case] value: serde_json::Value,
5676    ) {
5677        let mut handler = make_handler_with_account();
5678        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
5679        handler.set_command_sender(cmd_tx);
5680        let (trade_tx, mut trade_rx) = tokio::sync::oneshot::channel();
5681        let (candle_tx, mut candle_rx) = tokio::sync::oneshot::channel();
5682        let (trade_topic, trade_generation) =
5683            mark_subscription_inflight(&mut handler, LighterWsChannel::Trade(7), Some(trade_tx));
5684        let (candle_topic, candle_generation) = mark_subscription_inflight(
5685            &mut handler,
5686            LighterWsChannel::Candle {
5687                market_index: 0,
5688                resolution: LighterCandleResolution::OneMinute,
5689            },
5690            Some(candle_tx),
5691        );
5692
5693        let (matched, msg) = handler.handle_control_value(&value);
5694
5695        assert!(matched);
5696        assert!(msg.is_none());
5697        assert!(handler.inflight_subs.is_empty());
5698        assert!(handler.pending_subs.is_empty());
5699        let trade_retry = &handler.subscription_attempts[&trade_topic];
5700        let candle_retry = &handler.subscription_attempts[&candle_topic];
5701        assert_eq!(trade_retry.retries, 1);
5702        assert_eq!(candle_retry.retries, 1);
5703        assert_ne!(trade_retry.generation, trade_generation);
5704        assert_ne!(candle_retry.generation, candle_generation);
5705        assert_eq!(handler.subscription_retries.len(), 2);
5706        assert!(matches!(
5707            trade_rx.try_recv(),
5708            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5709        ));
5710        assert!(matches!(
5711            candle_rx.try_recv(),
5712            Err(tokio::sync::oneshot::error::TryRecvError::Empty),
5713        ));
5714
5715        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5716        assert_eq!(handler.subscription_attempts.len(), 2);
5717        assert_eq!(handler.subscriptions.len(), 0);
5718
5719        let retry_generations = [
5720            (
5721                trade_topic,
5722                handler.subscription_attempts[&trade_topic].generation,
5723            ),
5724            (
5725                candle_topic,
5726                handler.subscription_attempts[&candle_topic].generation,
5727            ),
5728        ];
5729
5730        for (topic, generation) in retry_generations {
5731            handler.queue_subscription_retry(topic, generation);
5732            let queued = handler.pending_subs.pop_front().unwrap();
5733            assert_eq!(queued, (topic, generation));
5734            handler.inflight_subs.insert(topic, generation);
5735        }
5736
5737        handle_control_text(&mut handler, r#"{"type":"subscribed","channel":"trade:7"}"#);
5738        handle_control_text(
5739            &mut handler,
5740            r#"{"type":"subscribed","channel":"candle:0:1m"}"#,
5741        );
5742
5743        assert_eq!(trade_rx.await.unwrap(), Ok(()));
5744        assert_eq!(candle_rx.await.unwrap(), Ok(()));
5745        assert!(handler.subscription_attempts.is_empty());
5746        assert_eq!(handler.subscriptions.len(), 2);
5747    }
5748}