Skip to main content

nautilus_derive/websocket/
client.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! WebSocket client for the Derive JSON-RPC stream.
17//!
18//! [`DeriveWebSocketClient`] orchestrates the connection lifecycle and exposes
19//! a typed surface for `public/login` + the initial `ticker` channel. The
20//! actual I/O runs in `super::handler::FeedHandler`; the client communicates
21//! with it through an unbounded command channel and consumes
22//! [`DeriveWsMessage`] events.
23
24use std::{
25    fmt::Debug,
26    sync::{
27        Arc,
28        atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
29    },
30    time::Duration,
31};
32
33use alloy::signers::local::PrivateKeySigner;
34use arc_swap::ArcSwap;
35use nautilus_common::live::dst::time;
36#[cfg(test)]
37use nautilus_common::live::get_runtime;
38use nautilus_core::{
39    UUID4,
40    string::secret::{REDACTED, SecretString},
41};
42use nautilus_live::{
43    SocketControl,
44    book::snapshot::SnapshotGate,
45    task::{SharedTaskSlot, TaskJoinOutcome, TaskSlot, finish_task},
46};
47use nautilus_network::{
48    ConnectionUrl,
49    http::create_standard_nautilus_headers,
50    mode::ConnectionMode,
51    ratelimiter::clock::MonotonicClock,
52    websocket::{
53        AuthTracker, SubscriptionState, TransportBackend, WebSocketClient, WebSocketConfig,
54        channel_message_handler,
55    },
56};
57use serde::{Deserialize, Serialize, de::DeserializeOwned};
58use serde_json::value::RawValue;
59use tokio_util::sync::CancellationToken;
60use ustr::Ustr;
61use zeroize::Zeroizing;
62
63use super::{
64    error::{DeriveWsError, Result},
65    handler::{
66        DeriveWsMessage, FeedHandler, HandlerCommand, orderbook_subscribe_params,
67        ticker_subscribe_params, trades_subscribe_params,
68    },
69    messages::{
70        DeriveWsChannel, WsLoginParams, WsLoginResult, WsSetCancelOnDisconnectParams,
71        WsSetCancelOnDisconnectResult, WsSubscribeParams, WsSubscribeResult, WsUnsubscribeParams,
72        WsUnsubscribeResult, methods, orderbook_channel, ticker_channel, trades_channel,
73    },
74};
75use crate::{
76    book::{recovery::BookRecovery, sync::BookSyncTracker},
77    common::{
78        consts::{
79            RECONNECT_BACKOFF_FACTOR, RECONNECT_BASE_BACKOFF, RECONNECT_JITTER_MS,
80            RECONNECT_MAX_BACKOFF, RECONNECT_TIMEOUT, WS_HEARTBEAT_SECS, WS_HEARTBEAT_TIMEOUT,
81            WS_REQUEST_TIMEOUT,
82        },
83        enums::DeriveEnvironment,
84        parse::salvage_rows,
85        rate_limit::{DeriveRateLimiter, RateClass, RateReservation, rate_class_for_method},
86        retry::should_retry_ws_error,
87        urls,
88    },
89    http::{
90        models::{
91            DeriveCancelByInstrumentResult, DeriveCancelByLabelResult, DeriveEmptyResult,
92            DeriveOpenOrdersResult, DeriveOrder, DeriveOrderResult, DeriveReplaceOutcome,
93            DeriveReplaceResult, DeriveTrade,
94        },
95        query::{
96            DeriveCancelAllParams, DeriveCancelByInstrumentParams, DeriveCancelByLabelParams,
97            DeriveCancelParams, DeriveCancelTriggerOrderParams, DeriveGetTriggerOrdersParams,
98            DeriveOrderParams, DeriveReplaceParams, DeriveTriggerOrderParams,
99        },
100    },
101    signing::auth::build_ws_login,
102};
103
104/// Credentials for `public/login`. The session-key signer never escapes the
105/// client; only the wallet address is exposed via [`Debug`].
106#[derive(Clone)]
107pub struct DeriveWsCredentials {
108    /// Owner's EOA or multisig address (`0x`-prefixed, 42 chars).
109    pub wallet_address: String,
110    /// secp256k1 session-key signer.
111    pub signer: PrivateKeySigner,
112}
113
114impl DeriveWsCredentials {
115    /// Constructs credentials by parsing `session_key_hex` into a signer.
116    ///
117    /// # Errors
118    ///
119    /// Returns an error if the session-key hex cannot be parsed.
120    pub fn new(wallet_address: impl Into<String>, session_key_hex: &str) -> Result<Self> {
121        let signer: PrivateKeySigner = session_key_hex
122            .parse()
123            .map_err(|e| DeriveWsError::transport(format!("invalid session key: {e}")))?;
124        Ok(Self {
125            wallet_address: wallet_address.into(),
126            signer,
127        })
128    }
129}
130
131impl Debug for DeriveWsCredentials {
132    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
133        f.debug_struct(stringify!(DeriveWsCredentials))
134            .field("wallet_address", &self.wallet_address)
135            .field("signer", &REDACTED)
136            .finish()
137    }
138}
139
140// Wallet limiter shared with the command handles so each frame is
141// paced in the caller's task before it is enqueued for the feed handler.
142type WsRateLimiter = DeriveRateLimiter;
143
144const MAX_SESSION_RECOVERY_ATTEMPTS: u32 = 3;
145const SUBSCRIPTION_ACCEPTED_STATUSES: &[&str] = &["ok"];
146const SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES: &[&str] = &["ok", "already subscribed"];
147pub(super) const UNAUTHENTICATED_CONNECTION_EPOCH: u64 = u64::MAX;
148
149/// WebSocket client for the Derive JSON-RPC stream.
150///
151/// Construct with [`Self::new`] (public-only) or [`Self::with_credentials`]
152/// when private channels and signed actions are needed. Call [`Self::connect`]
153/// before any subscribe call; [`Self::disconnect`] tears the connection down.
154#[derive(Debug)]
155pub struct DeriveWebSocketClient {
156    url: ConnectionUrl,
157    transport_backend: TransportBackend,
158    proxy_url: Option<SecretString>,
159    connection_mode: Arc<ArcSwap<AtomicU8>>,
160    connection_epoch: Arc<ArcSwap<AtomicU64>>,
161    signal: Arc<AtomicBool>,
162    auth_tracker: AuthTracker,
163    authenticated_epoch: Arc<AtomicU64>,
164    credentials: Option<DeriveWsCredentials>,
165    next_id: Arc<AtomicU64>,
166    cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
167    out_rx: Option<tokio::sync::mpsc::UnboundedReceiver<DeriveWsMessage>>,
168    subscriptions: Arc<SubscriptionState>,
169    subscription_lock: Arc<tokio::sync::Mutex<()>>,
170    book_sync: Option<(BookSyncTracker, Duration)>,
171    task_handle: TaskSlot<()>,
172    send_task: Arc<SharedTaskSlot<()>>,
173    recovery_task: Arc<SharedTaskSlot<()>>,
174    shutdown_errors: Vec<String>,
175    request_timeout: Duration,
176    conn_id: Arc<ArcSwap<String>>,
177    rate_limiter: Arc<WsRateLimiter>,
178    socket_control: Option<SocketControl>,
179}
180
181/// Cloneable command handle for Derive public market data subscriptions.
182#[derive(Debug, Clone)]
183pub struct DeriveWebSocketSubscriptionHandle {
184    cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
185    subscriptions: Arc<SubscriptionState>,
186    subscription_lock: Arc<tokio::sync::Mutex<()>>,
187    book_sync: Option<(BookSyncTracker, Duration)>,
188    connection_epoch: Arc<ArcSwap<AtomicU64>>,
189    request_timeout: Duration,
190    rate_limiter: Arc<WsRateLimiter>,
191}
192
193/// Cloneable handle for issuing signed `private/*` trading requests over the
194/// WebSocket transport.
195///
196/// Carries the same `cmd_tx` the owning [`DeriveWebSocketClient`] swaps on
197/// connect/reconnect, so a handle obtained at construction stays valid for the
198/// client's lifetime. The handle is transport-only: it sends the pre-signed
199/// body and surfaces the venue's JSON-RPC outcome. Session authorization is the
200/// client's responsibility (via `public/login`).
201#[derive(Debug, Clone)]
202pub struct DeriveWsExecutionHandle {
203    cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
204    auth_tracker: AuthTracker,
205    request_timeout: Duration,
206    conn_id: Arc<ArcSwap<String>>,
207    rate_limiter: Arc<WsRateLimiter>,
208}
209
210#[derive(Debug)]
211pub(crate) struct MatchingRateLimitReservation {
212    method: &'static str,
213    quota: RateReservation<MonotonicClock>,
214}
215
216impl DeriveWebSocketClient {
217    /// Builds a public-only client. URL falls back to the environment default
218    /// when `url` is `None`.
219    #[must_use]
220    pub fn new(
221        url: Option<String>,
222        environment: DeriveEnvironment,
223        transport_backend: TransportBackend,
224        proxy_url: Option<String>,
225    ) -> Self {
226        let url = url.unwrap_or_else(|| urls::ws_url(environment).to_string());
227        Self::build(url, transport_backend, proxy_url, None, None, None)
228    }
229
230    /// Builds a client that will issue `public/login` on connect and replay
231    /// it after each reconnect.
232    ///
233    /// `max_matching_requests_per_second` sets the shared wallet matching
234    /// allowance. `max_per_instrument_matching_requests_per_second` sets the
235    /// independent token refill rate for perpetuals, spot, and options.
236    /// Unset values preserve existing shared settings or use reference defaults.
237    /// Clients for the same wallet on the same API host share the first explicit
238    /// settings.
239    /// See [`crate::common::rate_limit`].
240    #[must_use]
241    pub fn with_credentials(
242        url: Option<String>,
243        environment: DeriveEnvironment,
244        transport_backend: TransportBackend,
245        proxy_url: Option<String>,
246        credentials: DeriveWsCredentials,
247        max_matching_requests_per_second: Option<u32>,
248        max_per_instrument_matching_requests_per_second: Option<u32>,
249    ) -> Self {
250        let url = url.unwrap_or_else(|| urls::ws_url(environment).to_string());
251        Self::build(
252            url,
253            transport_backend,
254            proxy_url,
255            Some(credentials),
256            max_matching_requests_per_second,
257            max_per_instrument_matching_requests_per_second,
258        )
259    }
260
261    fn build(
262        url: String,
263        transport_backend: TransportBackend,
264        proxy_url: Option<String>,
265        credentials: Option<DeriveWsCredentials>,
266        matching_tps: Option<u32>,
267        instrument_tps: Option<u32>,
268    ) -> Self {
269        let connection_mode = Arc::new(ArcSwap::new(Arc::new(AtomicU8::new(
270            ConnectionMode::Closed as u8,
271        ))));
272        let connection_epoch = Arc::new(ArcSwap::new(Arc::new(AtomicU64::new(0))));
273
274        // Placeholder channel; replaced by connect() before commands are issued.
275        let (placeholder_tx, _) = tokio::sync::mpsc::unbounded_channel();
276
277        let rate_limiter = Arc::new(DeriveRateLimiter::shared(
278            &url,
279            credentials
280                .as_ref()
281                .map(|credentials| credentials.wallet_address.as_str()),
282            matching_tps,
283            instrument_tps,
284        ));
285
286        Self {
287            url: url.into(),
288            transport_backend,
289            proxy_url: proxy_url.map(SecretString::from),
290            connection_mode,
291            connection_epoch,
292            signal: Arc::new(AtomicBool::new(false)),
293            auth_tracker: AuthTracker::new(),
294            authenticated_epoch: Arc::new(AtomicU64::new(UNAUTHENTICATED_CONNECTION_EPOCH)),
295            credentials,
296            next_id: Arc::new(AtomicU64::new(1)),
297            cmd_tx: Arc::new(tokio::sync::RwLock::new(placeholder_tx)),
298            out_rx: None,
299            subscriptions: Arc::new(SubscriptionState::new('.')),
300            subscription_lock: Arc::new(tokio::sync::Mutex::new(())),
301            book_sync: None,
302            task_handle: TaskSlot::new(),
303            send_task: Arc::new(SharedTaskSlot::new()),
304            recovery_task: Arc::new(SharedTaskSlot::new()),
305            shutdown_errors: Vec::new(),
306            request_timeout: WS_REQUEST_TIMEOUT,
307            conn_id: Arc::new(ArcSwap::from_pointee(UUID4::new().to_string())),
308            rate_limiter,
309            socket_control: None,
310        }
311    }
312
313    #[must_use]
314    pub(crate) fn with_book_sync(mut self, tracker: BookSyncTracker, timeout: Duration) -> Self {
315        self.book_sync = Some((tracker, timeout));
316        self
317    }
318
319    /// Configures socket state reporting and reconnect control.
320    #[must_use]
321    pub fn with_socket_control(mut self, control: SocketControl) -> Self {
322        self.socket_control = Some(control);
323        self
324    }
325
326    /// Returns the configured WebSocket URL.
327    #[must_use]
328    pub fn url(&self) -> &str {
329        self.url.expose()
330    }
331
332    /// Sets the per-operation WebSocket timeout (login, subscribe, reads, writes).
333    ///
334    /// Must be called before `connect()`. Defaults to `WS_REQUEST_TIMEOUT`.
335    pub fn set_request_timeout(&mut self, timeout: Duration) {
336        self.request_timeout = timeout;
337    }
338
339    /// Returns `true` when credentials are configured and the venue has
340    /// confirmed the latest `public/login`. Cleared on reconnect.
341    #[must_use]
342    pub fn is_authenticated(&self) -> bool {
343        self.auth_tracker.is_authenticated()
344            && self.is_active()
345            && self.authenticated_epoch.load(Ordering::Acquire)
346                == self.connection_epoch.load().load(Ordering::Acquire)
347    }
348
349    /// Returns `true` while the underlying transport is in the active state.
350    #[must_use]
351    pub fn is_active(&self) -> bool {
352        self.connection_mode.load().load(Ordering::Relaxed) == ConnectionMode::Active as u8
353    }
354
355    /// Establishes the WebSocket connection and spawns the I/O handler task.
356    ///
357    /// When credentials are configured, issues `public/login` and awaits the
358    /// venue's acknowledgement before returning.
359    ///
360    /// # Errors
361    ///
362    /// Returns an error if the WebSocket handshake or login flow fails.
363    pub async fn connect(&mut self) -> Result<()> {
364        // Fast path requires authenticated session when creds are configured;
365        // otherwise fall through and rebuild so `Ok` always implies authenticated.
366        let auth_ok = self.credentials.is_none() || self.is_authenticated();
367        if self.is_active() && auth_ok && self.task_handle.is_some() {
368            log::warn!("Derive WebSocket already connected");
369            return Ok(());
370        }
371
372        // Tear down stale state so we don't orphan the old handler task on rebuild.
373        if self.task_handle.is_some()
374            || !self.send_task.is_empty()
375            || !self.recovery_task.is_empty()
376        {
377            log::debug!("Tearing down stale Derive WebSocket state before connect");
378            self.teardown().await?;
379        }
380
381        self.signal.store(false, Ordering::Release);
382        let setup_guard = DeriveWebSocketSetupGuard::new(self.shutdown_handle());
383
384        self.authenticated_epoch
385            .store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
386
387        let (message_handler, raw_rx) = channel_message_handler();
388        let headers = create_standard_nautilus_headers();
389
390        let cfg = WebSocketConfig {
391            url: self.url.expose().to_owned(),
392            headers,
393            heartbeat_interval_secs: Some(WS_HEARTBEAT_SECS),
394            heartbeat_payload: None,
395            connect_timeout_ms: Some(RECONNECT_TIMEOUT.as_millis() as u64),
396            reconnect_delay_initial_ms: Some(RECONNECT_BASE_BACKOFF.as_millis() as u64),
397            reconnect_delay_max_ms: Some(RECONNECT_MAX_BACKOFF.as_millis() as u64),
398            reconnect_backoff_factor: Some(RECONNECT_BACKOFF_FACTOR),
399            reconnect_jitter_ms: Some(RECONNECT_JITTER_MS),
400            reconnect_max_attempts: None,
401            heartbeat_timeout_secs: Some(WS_HEARTBEAT_TIMEOUT.as_secs()),
402            idle_timeout_ms: None,
403            writer_capacity: None,
404            backend: self.transport_backend,
405            proxy_url: self
406                .proxy_url
407                .as_ref()
408                .map(|value| value.expose_secret().to_owned()),
409            max_message_size_bytes: None,
410            max_frame_size_bytes: None,
411        };
412
413        // Rate limiting runs caller-side via `self.rate_limiter` before frames
414        // are enqueued, so the network client's own limiter is left unconfigured
415        // and never sleeps inside the single feed-handler task.
416        let client = WebSocketClient::builder()
417            .config(cfg)
418            .message_handler(message_handler)
419            .maybe_state_sink(self.socket_control.as_ref().map(SocketControl::sink))
420            .connect()
421            .await
422            .map_err(|e| DeriveWsError::transport(e.to_string()))?;
423
424        // Register the tracker so the network controller clears
425        // `is_authenticated()` on dead-socket detection, not just on the
426        // later RECONNECTED sentinel.
427        client.set_auth_tracker(self.auth_tracker.clone(), false);
428
429        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
430        let (out_tx, out_rx) = tokio::sync::mpsc::unbounded_channel::<DeriveWsMessage>();
431
432        self.out_rx = Some(out_rx);
433        self.conn_id.store(Arc::new(UUID4::new().to_string()));
434
435        let connection_mode = client.connection_mode_atomic();
436        let connection_epoch = client.connection_epoch_atomic();
437        let reconnect_handle = client.reconnect_handle();
438        if let Err(e) = cmd_tx.send(HandlerCommand::SetClient(client)) {
439            return Err(DeriveWsError::transport(format!(
440                "failed to send SetClient command: {e}",
441            )));
442        }
443
444        {
445            let mut command_sender = self.cmd_tx.write().await;
446            *command_sender = cmd_tx.clone();
447            self.connection_epoch.store(Arc::clone(&connection_epoch));
448            self.connection_mode.store(Arc::clone(&connection_mode));
449        }
450
451        log::debug!("Derive WebSocket connected: {}", self.url.origin());
452
453        let recovery = SessionRecovery {
454            subscription: self.subscription_handle(),
455            cmd_tx: cmd_tx.clone(),
456            auth_tracker: self.auth_tracker.clone(),
457            authenticated_epoch: Arc::clone(&self.authenticated_epoch),
458            connection_mode: Arc::clone(&connection_mode),
459            connection_epoch: Arc::clone(&connection_epoch),
460            credentials: self.credentials.clone(),
461            signal: Arc::clone(&self.signal),
462        };
463
464        let handler = FeedHandler::new_with_send_task(
465            Arc::clone(&self.signal),
466            cmd_rx,
467            raw_rx,
468            Arc::clone(&self.next_id),
469            self.auth_tracker.clone(),
470            Arc::clone(&self.authenticated_epoch),
471            Arc::clone(&self.send_task),
472        )
473        .with_book_sync(self.book_sync.clone());
474
475        if let Err(e) = self.task_handle.spawn(run_session_handler(
476            handler,
477            recovery,
478            out_tx,
479            Arc::clone(&self.recovery_task),
480            Arc::clone(&self.conn_id),
481        )) {
482            let shutdown_result = self.teardown().await;
483            return Err(DeriveWsError::transport(match shutdown_result {
484                Ok(()) => format!("failed to start WebSocket handler task: {e}"),
485                Err(shutdown_error) => format!(
486                    "failed to start WebSocket handler task: {e}; startup rollback failed: \
487                     {shutdown_error}"
488                ),
489            }));
490        }
491
492        if let Some(control) = &self.socket_control {
493            control.register(move || reconnect_handle.request_reconnect());
494        }
495
496        if let Some(creds) = self.credentials.clone()
497            && let Err(e) = login_via_handler(
498                &self.rate_limiter,
499                &cmd_tx,
500                &self.auth_tracker,
501                &self.authenticated_epoch,
502                &connection_mode,
503                &connection_epoch,
504                &creds,
505                self.request_timeout,
506            )
507            .await
508        {
509            // Without teardown, a retry connect() would short-circuit on
510            // is_active() and return Ok without a valid session.
511            log::warn!(
512                "Derive WebSocket login failed; tearing down transport: {}",
513                e.redacted_message()
514            );
515            self.teardown().await?;
516            return Err(e);
517        }
518
519        setup_guard.disarm();
520        Ok(())
521    }
522
523    pub(crate) fn begin_shutdown(&self) {
524        self.shutdown_handle().begin_shutdown();
525    }
526
527    pub(crate) fn shutdown_handle(&self) -> DeriveWebSocketShutdownHandle {
528        DeriveWebSocketShutdownHandle {
529            signal: Arc::clone(&self.signal),
530            recovery_task: Arc::clone(&self.recovery_task),
531        }
532    }
533
534    /// Signals the handler to disconnect, aborts the spawn task, and resets
535    /// the client's transport-related state. Shared by [`Self::disconnect`]
536    /// and the login-failure branch of [`Self::connect`].
537    async fn teardown(&mut self) -> Result<()> {
538        self.begin_shutdown();
539
540        if let Err(e) = self.cmd_tx.read().await.send(HandlerCommand::Disconnect) {
541            log::debug!(
542                "Failed to enqueue Disconnect command (handler may already be shut down): {e}",
543            );
544        }
545
546        match finish_task(
547            &mut self.task_handle,
548            Duration::from_secs(2),
549            Duration::from_secs(2),
550        )
551        .await
552        {
553            None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => {}
554            Some(TaskJoinOutcome::Failed(error)) => self
555                .shutdown_errors
556                .push(format!("WebSocket handler task failed: {error}")),
557            Some(TaskJoinOutcome::Incomplete) => self
558                .shutdown_errors
559                .push("WebSocket handler task did not stop after abort".to_string()),
560        }
561
562        if self.task_handle.is_some() {
563            if let Some(control) = &self.socket_control {
564                control.deregister();
565            }
566
567            return self.take_shutdown_result();
568        }
569
570        for (name, slot) in [
571            ("recovery", Arc::clone(&self.recovery_task)),
572            ("send", Arc::clone(&self.send_task)),
573        ] {
574            match slot
575                .finish(Duration::from_secs(1), Duration::from_secs(2))
576                .await
577            {
578                None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => {}
579                Some(TaskJoinOutcome::Failed(error)) => self
580                    .shutdown_errors
581                    .push(format!("WebSocket {name} worker failed: {error}")),
582                Some(TaskJoinOutcome::Incomplete) => self
583                    .shutdown_errors
584                    .push(format!("WebSocket {name} worker did not stop after abort")),
585            }
586
587            if !slot.is_empty() {
588                if let Some(control) = &self.socket_control {
589                    control.deregister();
590                }
591
592                return self.take_shutdown_result();
593            }
594        }
595
596        // Subscriptions are also dropped: the venue session ended with the
597        // transport, so a fresh connect() must re-issue them.
598        let (placeholder_tx, _) = tokio::sync::mpsc::unbounded_channel();
599        *self.cmd_tx.write().await = placeholder_tx;
600        self.out_rx = None;
601        self.connection_mode
602            .store(Arc::new(AtomicU8::new(ConnectionMode::Closed as u8)));
603        self.connection_epoch.store(Arc::new(AtomicU64::new(0)));
604        self.auth_tracker.invalidate();
605        self.authenticated_epoch
606            .store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
607        self.subscriptions.clear();
608
609        if let Some((tracker, _)) = &self.book_sync {
610            tracker.clear();
611        }
612
613        self.signal.store(false, Ordering::Relaxed);
614
615        if let Some(control) = &self.socket_control {
616            control.deregister();
617        }
618
619        self.take_shutdown_result()
620    }
621
622    fn take_shutdown_result(&mut self) -> Result<()> {
623        if self.shutdown_errors.is_empty() {
624            Ok(())
625        } else {
626            Err(DeriveWsError::transport(
627                std::mem::take(&mut self.shutdown_errors).join("; "),
628            ))
629        }
630    }
631
632    /// Disconnects the WebSocket connection and awaits the handler task.
633    ///
634    /// # Errors
635    ///
636    /// Returns an error if the disconnect command cannot be enqueued; the handler still tears down
637    /// on signal.
638    pub async fn disconnect(&mut self) -> Result<()> {
639        log::debug!("Disconnecting Derive WebSocket");
640        self.begin_shutdown();
641        self.teardown().await
642    }
643
644    /// Subscribes to `ticker_slim.{instrument_name}.{interval}`. `interval` is the
645    /// millisecond cadence string the venue exposes (e.g. `"100"`, `"1000"`).
646    ///
647    /// # Errors
648    ///
649    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
650    pub async fn subscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
651        self.subscription_handle()
652            .subscribe_ticker(instrument_name, interval)
653            .await
654    }
655
656    /// Unsubscribes from `ticker_slim.{instrument_name}.{interval}`.
657    ///
658    /// # Errors
659    ///
660    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
661    pub async fn unsubscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
662        self.subscription_handle()
663            .unsubscribe_ticker(instrument_name, interval)
664            .await
665    }
666
667    /// Subscribes to `orderbook.{instrument_name}.{group}.{depth}`.
668    ///
669    /// # Errors
670    ///
671    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
672    pub async fn subscribe_orderbook(
673        &self,
674        instrument_name: &str,
675        group: &str,
676        depth: &str,
677    ) -> Result<()> {
678        self.subscription_handle()
679            .subscribe_orderbook(instrument_name, group, depth)
680            .await
681    }
682
683    /// Unsubscribes from `orderbook.{instrument_name}.{group}.{depth}`.
684    ///
685    /// # Errors
686    ///
687    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
688    pub async fn unsubscribe_orderbook(
689        &self,
690        instrument_name: &str,
691        group: &str,
692        depth: &str,
693    ) -> Result<()> {
694        self.subscription_handle()
695            .unsubscribe_orderbook(instrument_name, group, depth)
696            .await
697    }
698
699    /// Subscribes to `trades.{instrument_type}.{currency}`.
700    ///
701    /// # Errors
702    ///
703    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
704    pub async fn subscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
705        self.subscription_handle()
706            .subscribe_trades(instrument_type, currency)
707            .await
708    }
709
710    /// Unsubscribes from `trades.{instrument_type}.{currency}`.
711    ///
712    /// # Errors
713    ///
714    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
715    pub async fn unsubscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
716        self.subscription_handle()
717            .unsubscribe_trades(instrument_type, currency)
718            .await
719    }
720
721    /// Subscribes to a list of channel topics in a single `subscribe` frame.
722    ///
723    /// Used by the execution client to bulk-subscribe to the private
724    /// `{subaccount_id}.orders`, `{subaccount_id}.trades`, and
725    /// `{subaccount_id}.balances` channels after login.
726    ///
727    /// # Errors
728    ///
729    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
730    pub async fn subscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
731    where
732        C: Into<DeriveWsChannel>,
733    {
734        self.subscription_handle()
735            .subscribe_channels(channels)
736            .await
737    }
738
739    /// Unsubscribes from a list of channel topics in a single
740    /// `unsubscribe` frame.
741    ///
742    /// # Errors
743    ///
744    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
745    pub async fn unsubscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
746    where
747        C: Into<DeriveWsChannel>,
748    {
749        self.subscription_handle()
750            .unsubscribe_channels(channels)
751            .await
752    }
753
754    /// Returns the next event emitted by the handler.
755    pub async fn next_event(&mut self) -> Option<DeriveWsMessage> {
756        if let Some(rx) = self.out_rx.as_mut() {
757            rx.recv().await
758        } else {
759            None
760        }
761    }
762
763    /// Returns the count of channels the client currently has confirmed
764    /// subscriptions for.
765    #[must_use]
766    pub fn subscription_count(&self) -> usize {
767        self.subscriptions.len()
768    }
769
770    /// Returns a cloneable handle for issuing subscription commands.
771    #[must_use]
772    pub fn subscription_handle(&self) -> DeriveWebSocketSubscriptionHandle {
773        DeriveWebSocketSubscriptionHandle {
774            cmd_tx: Arc::clone(&self.cmd_tx),
775            subscriptions: Arc::clone(&self.subscriptions),
776            subscription_lock: Arc::clone(&self.subscription_lock),
777            book_sync: self.book_sync.clone(),
778            connection_epoch: Arc::clone(&self.connection_epoch),
779            request_timeout: self.request_timeout,
780            rate_limiter: Arc::clone(&self.rate_limiter),
781        }
782    }
783
784    /// Enables or disables Derive's persisted, wallet-wide cancel-on-disconnect setting.
785    ///
786    /// Requires `Trade(All)` scope across all subaccounts. The venue only cancels the dropped
787    /// connection's `mmp: true` orders and RFQ quotes; trigger and algo orders are excluded.
788    /// The stored setting applies to each connection at login and persists until disabled.
789    /// This client never toggles it automatically. Before relying on a change, wait for an
790    /// acknowledgement and a matching account read, then disconnect and reconnect affected
791    /// clients. Do not assume a change affects existing connections.
792    /// See <https://docs.derive.xyz/trading/cancel-on-disconnect>.
793    ///
794    /// # Errors
795    ///
796    /// Returns an error if:
797    /// - Credentials are missing.
798    /// - Authentication fails.
799    /// - The response is rejected or malformed.
800    /// - The transport fails or the request times out.
801    ///
802    /// A timeout, lost response, or malformed response leaves the setting uncertain; coordinate
803    /// other clients and reassert the intended value before relying on it. Account reads can
804    /// lag an acknowledged WebSocket change.
805    pub async fn set_cancel_on_disconnect(&self, enabled: bool) -> Result<()> {
806        let creds = self
807            .credentials
808            .as_ref()
809            .ok_or_else(|| DeriveWsError::MissingCredentials {
810                operation: methods::PRIVATE_SET_CANCEL_ON_DISCONNECT.to_string(),
811            })?;
812
813        self.execution_handle()
814            .require_authenticated(methods::PRIVATE_SET_CANCEL_ON_DISCONNECT)
815            .await?;
816
817        let params = WsSetCancelOnDisconnectParams {
818            wallet: creds.wallet_address.clone(),
819            enabled,
820        };
821
822        let cmd_tx = self.cmd_tx.read().await.clone();
823        let _: WsSetCancelOnDisconnectResult = send_request_typed(
824            &self.rate_limiter,
825            &cmd_tx,
826            methods::PRIVATE_SET_CANCEL_ON_DISCONNECT,
827            &params,
828            self.request_timeout,
829        )
830        .await?;
831        Ok(())
832    }
833
834    /// Returns a cloneable handle for issuing signed `private/*` trading
835    /// requests.
836    ///
837    /// The handle shares the client's command channel, so it stays valid across
838    /// reconnects (the channel is swapped behind a shared lock). Obtain it once
839    /// and clone it into each order-submission task.
840    #[must_use]
841    pub fn execution_handle(&self) -> DeriveWsExecutionHandle {
842        DeriveWsExecutionHandle {
843            cmd_tx: Arc::clone(&self.cmd_tx),
844            auth_tracker: self.auth_tracker.clone(),
845            request_timeout: self.request_timeout,
846            conn_id: Arc::clone(&self.conn_id),
847            rate_limiter: Arc::clone(&self.rate_limiter),
848        }
849    }
850
851    /// Takes the event receiver from the client.
852    ///
853    /// This lets the live data client own the receive loop while subscription
854    /// commands continue through [`Self::subscription_handle`].
855    pub fn take_event_receiver(
856        &mut self,
857    ) -> Option<tokio::sync::mpsc::UnboundedReceiver<DeriveWsMessage>> {
858        self.out_rx.take()
859    }
860}
861
862impl Drop for DeriveWebSocketClient {
863    fn drop(&mut self) {
864        self.begin_shutdown();
865
866        if let Some(handle) = self.task_handle.as_ref() {
867            handle.abort();
868        }
869
870        if let Some(control) = &self.socket_control {
871            control.deregister();
872        }
873    }
874}
875
876struct SessionRecovery {
877    subscription: DeriveWebSocketSubscriptionHandle,
878    cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
879    auth_tracker: AuthTracker,
880    authenticated_epoch: Arc<AtomicU64>,
881    connection_mode: Arc<AtomicU8>,
882    connection_epoch: Arc<AtomicU64>,
883    credentials: Option<DeriveWsCredentials>,
884    signal: Arc<AtomicBool>,
885}
886
887async fn run_session_handler(
888    mut handler: FeedHandler,
889    recovery: SessionRecovery,
890    out_tx: tokio::sync::mpsc::UnboundedSender<DeriveWsMessage>,
891    recovery_task: Arc<SharedTaskSlot<()>>,
892    conn_id: Arc<ArcSwap<String>>,
893) {
894    let (recovery_tx, recovery_rx) = tokio::sync::mpsc::unbounded_channel();
895    let cmd_tx = recovery.cmd_tx.clone();
896    let connection_epoch = Arc::clone(&recovery.connection_epoch);
897    if let Err(e) = recovery_task.spawn(recovery.run(recovery_rx, out_tx.clone())) {
898        let _ = out_tx.send(DeriveWsMessage::SessionRecoveryFailed(format!(
899            "failed to start recovery worker: {e}"
900        )));
901        let _ = cmd_tx.send(HandlerCommand::Disconnect);
902        return;
903    }
904
905    loop {
906        match handler.next().await {
907            Some(DeriveWsMessage::Reconnected) => {
908                log::info!("Derive WebSocket re-establishing session after reconnect");
909                conn_id.store(Arc::new(UUID4::new().to_string()));
910                let epoch = connection_epoch.load(Ordering::Acquire);
911                if recovery_tx.send(epoch).is_err() {
912                    log::error!("Derive WebSocket recovery task stopped unexpectedly");
913                    let _ = cmd_tx.send(HandlerCommand::Disconnect);
914                }
915            }
916            Some(msg) => {
917                if out_tx.send(msg).is_err() {
918                    log::debug!("Derive outer receiver dropped, exiting stream loop");
919                    break;
920                }
921            }
922            None => {
923                log::debug!("Derive handler task ended");
924                break;
925            }
926        }
927    }
928
929    recovery_task.abort();
930}
931
932impl SessionRecovery {
933    async fn run(
934        self,
935        mut rx: tokio::sync::mpsc::UnboundedReceiver<u64>,
936        out_tx: tokio::sync::mpsc::UnboundedSender<DeriveWsMessage>,
937    ) {
938        if self.signal.load(Ordering::Acquire) {
939            return;
940        }
941
942        while let Some(mut requested_epoch) = rx.recv().await {
943            take_latest_epoch(&mut rx, &mut requested_epoch);
944            match self.recover_until_current(&mut rx, requested_epoch).await {
945                Ok(()) => {
946                    if out_tx.send(DeriveWsMessage::Reconnected).is_err() {
947                        log::debug!("Derive outer receiver dropped during recovery");
948                    }
949                }
950                Err(e) => {
951                    log::error!(
952                        "Derive WebSocket session recovery failed: {}",
953                        e.redacted_message()
954                    );
955                    let _ = out_tx.send(DeriveWsMessage::SessionRecoveryFailed(e.to_string()));
956                    let _ = self.cmd_tx.send(HandlerCommand::Disconnect);
957                    return;
958                }
959            }
960        }
961    }
962
963    async fn recover_until_current(
964        &self,
965        rx: &mut tokio::sync::mpsc::UnboundedReceiver<u64>,
966        mut requested_epoch: u64,
967    ) -> Result<()> {
968        loop {
969            match recover_session(
970                &self.subscription.rate_limiter,
971                &self.cmd_tx,
972                &self.auth_tracker,
973                &self.authenticated_epoch,
974                &self.connection_mode,
975                &self.connection_epoch,
976                self.credentials.as_ref(),
977                &self.subscription.subscriptions,
978                &self.subscription.subscription_lock,
979                self.subscription.request_timeout,
980            )
981            .await
982            {
983                Ok(recovered_epoch) => {
984                    take_latest_epoch(rx, &mut requested_epoch);
985                    if requested_epoch > recovered_epoch
986                        || !session_connection_is_active(
987                            &self.connection_mode,
988                            &self.connection_epoch,
989                            recovered_epoch,
990                        )
991                    {
992                        continue;
993                    }
994
995                    return Ok(());
996                }
997                Err(e) => {
998                    if take_latest_epoch(rx, &mut requested_epoch) {
999                        continue;
1000                    }
1001
1002                    return Err(e);
1003                }
1004            }
1005        }
1006    }
1007}
1008
1009fn take_latest_epoch(
1010    rx: &mut tokio::sync::mpsc::UnboundedReceiver<u64>,
1011    requested_epoch: &mut u64,
1012) -> bool {
1013    let mut received = false;
1014
1015    while let Ok(epoch) = rx.try_recv() {
1016        *requested_epoch = (*requested_epoch).max(epoch);
1017        received = true;
1018    }
1019
1020    received
1021}
1022
1023#[derive(Clone, Debug)]
1024pub(crate) struct DeriveWebSocketShutdownHandle {
1025    signal: Arc<AtomicBool>,
1026    recovery_task: Arc<SharedTaskSlot<()>>,
1027}
1028
1029impl DeriveWebSocketShutdownHandle {
1030    pub(crate) fn begin_shutdown(&self) {
1031        self.signal.store(true, Ordering::Release);
1032        self.recovery_task.abort();
1033    }
1034}
1035
1036struct DeriveWebSocketSetupGuard {
1037    shutdown: DeriveWebSocketShutdownHandle,
1038    armed: bool,
1039}
1040
1041impl DeriveWebSocketSetupGuard {
1042    fn new(shutdown: DeriveWebSocketShutdownHandle) -> Self {
1043        Self {
1044            shutdown,
1045            armed: true,
1046        }
1047    }
1048
1049    fn disarm(mut self) {
1050        self.armed = false;
1051    }
1052}
1053
1054impl Drop for DeriveWebSocketSetupGuard {
1055    fn drop(&mut self) {
1056        if self.armed {
1057            self.shutdown.begin_shutdown();
1058        }
1059    }
1060}
1061
1062impl DeriveWebSocketSubscriptionHandle {
1063    pub(crate) async fn subscribe_book_gated(&self, channel: String) -> Result<()> {
1064        let _guard = self.subscription_lock.lock().await;
1065        self.book_request_locked(methods::PUBLIC_SUBSCRIBE, channel, None, None)
1066            .await
1067    }
1068
1069    pub(crate) async fn replace_book_gated(
1070        &self,
1071        channel: String,
1072        cancel: CancellationToken,
1073        recovery: &Arc<BookRecovery>,
1074        gate: SnapshotGate,
1075    ) -> Result<()> {
1076        let _guard = tokio::select! {
1077            biased;
1078            () = cancel.cancelled() => return Err(DeriveWsError::NotConnected),
1079            guard = self.subscription_lock.lock() => guard,
1080        };
1081        self.book_sync
1082            .as_ref()
1083            .and_then(|(tracker, _)| tracker.begin_recovery_write(&channel, recovery, gate))
1084            .ok_or(DeriveWsError::NotConnected)?;
1085        let epoch = self.connection_epoch.load().load(Ordering::Acquire);
1086
1087        // Keep replay ownership through an interrupted replacement
1088        self.remember_subscription(&channel);
1089        self.book_request_locked(
1090            methods::PUBLIC_UNSUBSCRIBE,
1091            channel.clone(),
1092            Some(cancel.clone()),
1093            Some(epoch),
1094        )
1095        .await?;
1096        self.book_request_locked(
1097            methods::PUBLIC_SUBSCRIBE,
1098            channel,
1099            Some(cancel),
1100            Some(epoch),
1101        )
1102        .await
1103    }
1104
1105    async fn book_request_locked(
1106        &self,
1107        method: &'static str,
1108        channel: String,
1109        cancel: Option<CancellationToken>,
1110        epoch: Option<u64>,
1111    ) -> Result<()> {
1112        let params = WsSubscribeParams {
1113            channels: vec![DeriveWsChannel::from(channel.clone())],
1114        };
1115
1116        self.remember_subscription(&channel);
1117        self.rate_limiter
1118            .await_class_ready(rate_class_for_method(method), None)
1119            .await;
1120
1121        let write = self
1122            .book_sync
1123            .as_ref()
1124            .and_then(|(tracker, _)| tracker.write(&channel))
1125            .ok_or(DeriveWsError::NotConnected)?;
1126        let generation = write.generation.clone();
1127        let cancel = cancel.unwrap_or_else(|| generation.clone());
1128        let epoch = epoch.unwrap_or_else(|| self.connection_epoch.load().load(Ordering::Acquire));
1129
1130        if cancel.is_cancelled() || generation.is_cancelled() {
1131            return Err(DeriveWsError::NotConnected);
1132        }
1133
1134        let cmd_tx = self.cmd_tx.read().await.clone();
1135        let (response_tx, response_rx) = tokio::sync::oneshot::channel();
1136        cmd_tx
1137            .send(HandlerCommand::Request {
1138                method,
1139                params: SecretString::from(serde_json::to_string(&params)?),
1140                connection_epoch: Some(epoch),
1141                cancellation: Some(cancel.clone()),
1142                response_tx,
1143            })
1144            .map_err(|_| DeriveWsError::NotConnected)?;
1145
1146        let result = tokio::select! {
1147            biased;
1148            () = cancel.cancelled() => return Err(DeriveWsError::NotConnected),
1149            () = generation.cancelled() => return Err(DeriveWsError::NotConnected),
1150            result = time::timeout(self.request_timeout, response_rx) => result,
1151        };
1152
1153        let value = result
1154            .map_err(|_| DeriveWsError::Timeout {
1155                method: method.to_owned(),
1156            })?
1157            .map_err(|_| DeriveWsError::RequestCancelled {
1158                method: method.to_owned(),
1159            })??;
1160
1161        if method == methods::PUBLIC_SUBSCRIBE {
1162            let result: WsSubscribeResult = decode_default_result(&value)?;
1163            subscription_outcome(
1164                &params.channels,
1165                &result,
1166                SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES,
1167                &self.subscriptions,
1168            )
1169        } else {
1170            let result: WsUnsubscribeResult = decode_default_result(&value)?;
1171
1172            // The data client's channel registry retains ownership during replacement
1173            let topic = DeriveWsChannel::from(channel.clone());
1174
1175            if result
1176                .status
1177                .get(&topic)
1178                .is_some_and(|status| status.as_str() == "ok")
1179                && !result.remaining_subscriptions.contains(&topic)
1180            {
1181                Ok(())
1182            } else {
1183                Err(DeriveWsError::Subscription {
1184                    details: format!("{channel}: unsubscribe not confirmed"),
1185                })
1186            }
1187        }
1188    }
1189
1190    pub(crate) fn has_subscription(&self, channel: &str) -> bool {
1191        let (name, symbol) = channel.split_once('.').unwrap_or((channel, ""));
1192        self.subscriptions
1193            .is_subscribed(&Ustr::from(name), &Ustr::from(symbol))
1194    }
1195
1196    pub(crate) fn forget_subscription(&self, channel: &str) {
1197        self.subscriptions.mark_unsubscribe(channel);
1198        self.subscriptions.confirm_unsubscribe(channel);
1199    }
1200
1201    pub(crate) fn remember_subscription(&self, channel: &str) {
1202        self.subscriptions.mark_subscribe(channel);
1203    }
1204
1205    /// Subscribes to `ticker_slim.{instrument_name}.{interval}`.
1206    ///
1207    /// # Errors
1208    ///
1209    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
1210    pub async fn subscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
1211        let params = ticker_subscribe_params(instrument_name, interval);
1212        self.subscribe_channels(params.channels).await
1213    }
1214
1215    /// Unsubscribes from `ticker_slim.{instrument_name}.{interval}`.
1216    ///
1217    /// # Errors
1218    ///
1219    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
1220    pub async fn unsubscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
1221        let channel = ticker_channel(instrument_name, interval);
1222        self.send_unsubscribe(channel).await
1223    }
1224
1225    /// Subscribes to `orderbook.{instrument_name}.{group}.{depth}`.
1226    ///
1227    /// # Errors
1228    ///
1229    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
1230    pub async fn subscribe_orderbook(
1231        &self,
1232        instrument_name: &str,
1233        group: &str,
1234        depth: &str,
1235    ) -> Result<()> {
1236        let params = orderbook_subscribe_params(instrument_name, group, depth);
1237        self.subscribe_channels(params.channels).await
1238    }
1239
1240    /// Unsubscribes from `orderbook.{instrument_name}.{group}.{depth}`.
1241    ///
1242    /// # Errors
1243    ///
1244    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
1245    pub async fn unsubscribe_orderbook(
1246        &self,
1247        instrument_name: &str,
1248        group: &str,
1249        depth: &str,
1250    ) -> Result<()> {
1251        let channel = orderbook_channel(instrument_name, group, depth);
1252        self.send_unsubscribe(channel).await
1253    }
1254
1255    /// Subscribes to `trades.{instrument_type}.{currency}`.
1256    ///
1257    /// # Errors
1258    ///
1259    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
1260    pub async fn subscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
1261        let params = trades_subscribe_params(instrument_type, currency);
1262        self.subscribe_channels(params.channels).await
1263    }
1264
1265    /// Unsubscribes from `trades.{instrument_type}.{currency}`.
1266    ///
1267    /// # Errors
1268    ///
1269    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
1270    pub async fn unsubscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
1271        let channel = trades_channel(instrument_type, currency);
1272        self.send_unsubscribe(channel).await
1273    }
1274
1275    /// Subscribes to multiple channel topics in a single `subscribe` frame.
1276    ///
1277    /// # Errors
1278    ///
1279    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
1280    pub async fn subscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
1281    where
1282        C: Into<DeriveWsChannel>,
1283    {
1284        let channels = channels.into_iter().map(Into::into).collect::<Vec<_>>();
1285        if channels.is_empty() {
1286            return Ok(());
1287        }
1288
1289        let _guard = self.subscription_lock.lock().await;
1290        let params = WsSubscribeParams { channels };
1291        for topic in channel_topics(&params.channels) {
1292            self.subscriptions.mark_subscribe(&topic);
1293        }
1294
1295        let cmd_tx = self.cmd_tx.read().await.clone();
1296        let result: WsSubscribeResult = send_request(
1297            &self.rate_limiter,
1298            &cmd_tx,
1299            methods::PUBLIC_SUBSCRIBE,
1300            &params,
1301            self.request_timeout,
1302        )
1303        .await?;
1304
1305        subscription_outcome(
1306            &params.channels,
1307            &result,
1308            SUBSCRIPTION_ACCEPTED_STATUSES,
1309            &self.subscriptions,
1310        )
1311    }
1312
1313    /// Unsubscribes from multiple channel topics in a single
1314    /// `unsubscribe` frame.
1315    ///
1316    /// # Errors
1317    ///
1318    /// Returns an error if the venue raises a JSON-RPC error or the transport fails.
1319    pub async fn unsubscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
1320    where
1321        C: Into<DeriveWsChannel>,
1322    {
1323        let channels = channels.into_iter().map(Into::into).collect::<Vec<_>>();
1324        if channels.is_empty() {
1325            return Ok(());
1326        }
1327
1328        let _guard = self.subscription_lock.lock().await;
1329
1330        for topic in channel_topics(&channels) {
1331            self.subscriptions.mark_unsubscribe(&topic);
1332        }
1333
1334        let params = WsUnsubscribeParams { channels };
1335        let cmd_tx = self.cmd_tx.read().await.clone();
1336        let result: WsUnsubscribeResult = send_request(
1337            &self.rate_limiter,
1338            &cmd_tx,
1339            methods::PUBLIC_UNSUBSCRIBE,
1340            &params,
1341            self.request_timeout,
1342        )
1343        .await?;
1344
1345        unsubscribe_outcome(&params.channels, &result, &self.subscriptions)
1346    }
1347
1348    async fn send_unsubscribe(&self, channel: String) -> Result<()> {
1349        self.unsubscribe_channels(vec![DeriveWsChannel::from(channel)])
1350            .await
1351    }
1352}
1353
1354impl DeriveWsExecutionHandle {
1355    /// Returns the current WebSocket connection id used by trigger orders.
1356    #[must_use]
1357    pub fn conn_id(&self) -> String {
1358        self.conn_id.load_full().as_ref().clone()
1359    }
1360
1361    /// Submits a signed order via `private/order`.
1362    ///
1363    /// `params` must be the fully-built signed body from
1364    /// [`crate::http::query::order_to_derive_payload`]. Returns the accepted
1365    /// order echoed by the venue.
1366    ///
1367    /// # Errors
1368    ///
1369    /// Returns an error if the venue rejects the request, local throttling prevents dispatch, or
1370    /// the submission outcome is ambiguous.
1371    pub async fn submit_order(&self, params: &DeriveOrderParams) -> Result<DeriveOrder> {
1372        let reservation = self
1373            .reserve_matching_request(methods::PRIVATE_ORDER, &params.instrument_name)
1374            .await?;
1375        self.submit_order_after_rate_limit(params, reservation)
1376            .await
1377            .map(|result| result.order)
1378    }
1379
1380    pub(crate) async fn submit_order_after_rate_limit(
1381        &self,
1382        params: &DeriveOrderParams,
1383        reservation: MatchingRateLimitReservation,
1384    ) -> Result<DeriveOrderResult> {
1385        self.ensure_authenticated(methods::PRIVATE_ORDER)?;
1386        debug_assert_eq!(reservation.method, methods::PRIVATE_ORDER);
1387        let cmd_tx = self.cmd_tx.read().await.clone();
1388        self.ensure_authenticated(reservation.method)?;
1389        self.commit_matching_reservation(reservation)?;
1390        let result: DeriveOrderResult = send_request_typed_after_rate_limit(
1391            &self.rate_limiter,
1392            &cmd_tx,
1393            methods::PRIVATE_ORDER,
1394            params,
1395            self.request_timeout,
1396        )
1397        .await?;
1398        Ok(result)
1399    }
1400
1401    /// Submits a signed trigger order via `private/order`.
1402    ///
1403    /// # Errors
1404    ///
1405    /// Returns an error if the venue rejects the request, local throttling prevents dispatch, or
1406    /// the submission outcome is ambiguous.
1407    pub async fn submit_trigger_order(
1408        &self,
1409        params: &DeriveTriggerOrderParams,
1410    ) -> Result<DeriveOrder> {
1411        let reservation = self
1412            .reserve_matching_request(
1413                methods::PRIVATE_TRIGGER_ORDER,
1414                &params.order.instrument_name,
1415            )
1416            .await?;
1417        self.submit_trigger_order_after_rate_limit(params, reservation)
1418            .await
1419            .map(|result| result.order)
1420    }
1421
1422    pub(crate) async fn submit_trigger_order_after_rate_limit(
1423        &self,
1424        params: &DeriveTriggerOrderParams,
1425        reservation: MatchingRateLimitReservation,
1426    ) -> Result<DeriveOrderResult> {
1427        self.submit_order_after_rate_limit(&params.order, reservation)
1428            .await
1429    }
1430
1431    /// Modifies a working order by cancelling it and submitting a replacement
1432    /// through the venue's `private/replace`.
1433    ///
1434    /// # Errors
1435    ///
1436    /// Returns an error if the venue rejects the request, local throttling prevents dispatch, or
1437    /// the submission outcome is ambiguous.
1438    pub async fn modify_order(&self, params: &DeriveReplaceParams) -> Result<DeriveReplaceOutcome> {
1439        let reservation = self
1440            .reserve_matching_request(methods::PRIVATE_REPLACE, &params.order.instrument_name)
1441            .await?;
1442        self.modify_order_after_rate_limit(params, reservation)
1443            .await
1444            .and_then(|response| {
1445                response
1446                    .result
1447                    .and_then(|result| {
1448                        result.into_outcome(
1449                            &params.order_id_to_cancel,
1450                            &params.order.label,
1451                            params.order.envelope.subaccount_id,
1452                            params.order.instrument_name.as_str(),
1453                        )
1454                    })
1455                    .map_err(|message| {
1456                        DeriveWsError::Serde(<serde_json::Error as serde::de::Error>::custom(
1457                            message,
1458                        ))
1459                    })
1460            })
1461    }
1462
1463    pub(crate) async fn modify_order_after_rate_limit(
1464        &self,
1465        params: &DeriveReplaceParams,
1466        reservation: MatchingRateLimitReservation,
1467    ) -> Result<DeriveReplaceResponse> {
1468        self.ensure_authenticated(methods::PRIVATE_REPLACE)?;
1469        debug_assert_eq!(reservation.method, methods::PRIVATE_REPLACE);
1470        let cmd_tx = self.cmd_tx.read().await.clone();
1471        self.ensure_authenticated(reservation.method)?;
1472        self.commit_matching_reservation(reservation)?;
1473        let response = send_raw_after_rate_limit(
1474            &self.rate_limiter,
1475            &cmd_tx,
1476            methods::PRIVATE_REPLACE,
1477            params,
1478            self.request_timeout,
1479        )
1480        .await?;
1481        Ok(DeriveReplaceResponse::from_raw(&response))
1482    }
1483
1484    /// Cancels a single order via `private/cancel`.
1485    ///
1486    /// # Errors
1487    ///
1488    /// Returns an error if the venue rejects the request or the submission outcome is ambiguous.
1489    pub async fn cancel_order(&self, params: &DeriveCancelParams) -> Result<()> {
1490        self.require_authenticated(methods::PRIVATE_CANCEL).await?;
1491        let cmd_tx = self.cmd_tx.read().await.clone();
1492        let _: DeriveEmptyResult = send_request_for_instrument(
1493            &self.rate_limiter,
1494            &cmd_tx,
1495            methods::PRIVATE_CANCEL,
1496            params,
1497            self.request_timeout,
1498            params.instrument_name,
1499        )
1500        .await?;
1501        Ok(())
1502    }
1503
1504    /// Cancels every open order for one instrument via `private/cancel_by_instrument`.
1505    ///
1506    /// # Errors
1507    ///
1508    /// Returns an error if the venue rejects the request or the submission outcome is ambiguous.
1509    pub async fn cancel_by_instrument(
1510        &self,
1511        params: &DeriveCancelByInstrumentParams,
1512    ) -> Result<DeriveCancelByInstrumentResult> {
1513        self.require_authenticated(methods::PRIVATE_CANCEL_BY_INSTRUMENT)
1514            .await?;
1515        let cmd_tx = self.cmd_tx.read().await.clone();
1516        send_request_typed_for_instrument(
1517            &self.rate_limiter,
1518            &cmd_tx,
1519            methods::PRIVATE_CANCEL_BY_INSTRUMENT,
1520            params,
1521            self.request_timeout,
1522            params.instrument_name,
1523        )
1524        .await
1525    }
1526
1527    /// Cancels a single trigger order via `private/cancel_trigger_order`.
1528    ///
1529    /// # Errors
1530    ///
1531    /// Returns an error if the venue rejects the request or the submission outcome is ambiguous.
1532    pub async fn cancel_trigger_order(
1533        &self,
1534        params: &DeriveCancelTriggerOrderParams,
1535    ) -> Result<DeriveOrder> {
1536        self.require_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)
1537            .await?;
1538        let cmd_tx = self.cmd_tx.read().await.clone();
1539        send_request_typed(
1540            &self.rate_limiter,
1541            &cmd_tx,
1542            methods::PRIVATE_CANCEL_TRIGGER_ORDER,
1543            params,
1544            self.request_timeout,
1545        )
1546        .await
1547    }
1548
1549    pub(crate) async fn cancel_trigger_order_current(
1550        &self,
1551        trigger: &DeriveCancelTriggerOrderParams,
1552        regular: &DeriveCancelParams,
1553        is_active: impl Fn() -> bool,
1554    ) -> Result<Option<DeriveOrder>> {
1555        self.require_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)
1556            .await?;
1557        self.rate_limiter
1558            .await_class_ready(RateClass::NonMatching, None)
1559            .await;
1560        let cmd_tx = self.cmd_tx.read().await.clone();
1561
1562        if is_active() {
1563            return self.cancel_order(regular).await.map(|()| None);
1564        }
1565
1566        self.ensure_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)?;
1567        send_request_typed_after_rate_limit(
1568            &self.rate_limiter,
1569            &cmd_tx,
1570            methods::PRIVATE_CANCEL_TRIGGER_ORDER,
1571            trigger,
1572            self.request_timeout,
1573        )
1574        .await
1575        .map(Some)
1576    }
1577
1578    /// Cancels every open order with the given label via
1579    /// `private/cancel_by_label`.
1580    ///
1581    /// # Errors
1582    ///
1583    /// Returns an error if the venue rejects the request or the submission outcome is ambiguous.
1584    pub async fn cancel_by_label(
1585        &self,
1586        params: &DeriveCancelByLabelParams,
1587    ) -> Result<DeriveCancelByLabelResult> {
1588        self.require_authenticated(methods::PRIVATE_CANCEL_BY_LABEL)
1589            .await?;
1590        let cmd_tx = self.cmd_tx.read().await.clone();
1591        send_request_typed(
1592            &self.rate_limiter,
1593            &cmd_tx,
1594            methods::PRIVATE_CANCEL_BY_LABEL,
1595            params,
1596            self.request_timeout,
1597        )
1598        .await
1599    }
1600
1601    /// Returns currently untriggered trigger orders via
1602    /// `private/get_trigger_orders`.
1603    ///
1604    /// # Errors
1605    ///
1606    /// Returns an error if the venue rejects the request or the submission outcome is ambiguous.
1607    pub async fn get_trigger_orders(
1608        &self,
1609        params: &DeriveGetTriggerOrdersParams,
1610    ) -> Result<DeriveOpenOrdersResult> {
1611        self.require_authenticated(methods::PRIVATE_GET_TRIGGER_ORDERS)
1612            .await?;
1613        let cmd_tx = self.cmd_tx.read().await.clone();
1614        send_request_typed(
1615            &self.rate_limiter,
1616            &cmd_tx,
1617            methods::PRIVATE_GET_TRIGGER_ORDERS,
1618            params,
1619            self.request_timeout,
1620        )
1621        .await
1622    }
1623
1624    /// Cancels every open order on the subaccount (the venue's
1625    /// `private/cancel_all`), optionally scoped to an instrument.
1626    ///
1627    /// # Errors
1628    ///
1629    /// Returns an error if the venue rejects the request or the submission outcome is ambiguous.
1630    pub async fn cancel_all_orders(&self, params: &DeriveCancelAllParams) -> Result<()> {
1631        self.require_authenticated(methods::PRIVATE_CANCEL_ALL)
1632            .await?;
1633        let cmd_tx = self.cmd_tx.read().await.clone();
1634        let _: DeriveEmptyResult = send_request(
1635            &self.rate_limiter,
1636            &cmd_tx,
1637            methods::PRIVATE_CANCEL_ALL,
1638            params,
1639            self.request_timeout,
1640        )
1641        .await?;
1642        Ok(())
1643    }
1644
1645    pub(crate) async fn reserve_matching_request(
1646        &self,
1647        operation: &'static str,
1648        instrument_name: &Ustr,
1649    ) -> Result<MatchingRateLimitReservation> {
1650        self.require_authenticated(operation).await?;
1651        debug_assert_eq!(rate_class_for_method(operation), RateClass::Matching);
1652        let quota = self
1653            .rate_limiter
1654            .reserve_matching_ready(instrument_name)
1655            .await;
1656        self.ensure_authenticated(operation)?;
1657        Ok(MatchingRateLimitReservation {
1658            method: operation,
1659            quota,
1660        })
1661    }
1662
1663    fn commit_matching_reservation(&self, reservation: MatchingRateLimitReservation) -> Result<()> {
1664        reservation
1665            .quota
1666            .commit()
1667            .map_err(|retry_after| DeriveWsError::RateLimited { retry_after })
1668    }
1669
1670    fn ensure_authenticated(&self, operation: &'static str) -> Result<()> {
1671        if self.auth_tracker.is_authenticated() {
1672            return Ok(());
1673        }
1674
1675        Err(DeriveWsError::Authentication {
1676            operation: operation.to_string(),
1677            reason: "WebSocket session is not authenticated".to_string(),
1678        })
1679    }
1680
1681    async fn require_authenticated(&self, operation: &'static str) -> Result<()> {
1682        if self
1683            .auth_tracker
1684            .wait_for_authenticated(self.request_timeout)
1685            .await
1686        {
1687            return Ok(());
1688        }
1689
1690        Err(DeriveWsError::Authentication {
1691            operation: operation.to_string(),
1692            reason: "WebSocket session is not authenticated".to_string(),
1693        })
1694    }
1695}
1696
1697// Awaits the venue's raw `result`, bounded by `timeout`. A dropped responder
1698// (handler torn down on reconnect) surfaces as `RequestCancelled`, a timeout as
1699// `Timeout`; both leave a state-changing write's outcome ambiguous.
1700#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1701enum RequestRateLimit {
1702    /// Pace the request now, against the class buckets plus the carried
1703    /// instrument's per-instrument bucket when present.
1704    Await(Option<Ustr>),
1705    /// A matching reservation already consumed the cells; do not pace or
1706    /// consume again.
1707    Reserved,
1708}
1709
1710async fn send_raw<P>(
1711    rate_limiter: &WsRateLimiter,
1712    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1713    method: &'static str,
1714    params: &P,
1715    timeout: Duration,
1716) -> Result<Box<RawValue>>
1717where
1718    P: Serialize + ?Sized,
1719{
1720    send_raw_with_rate_limit(
1721        rate_limiter,
1722        cmd_tx,
1723        method,
1724        params,
1725        timeout,
1726        RequestRateLimit::Await(None),
1727        None,
1728    )
1729    .await
1730}
1731
1732// Awaits the venue's raw `result` for a matching write that carries an
1733// instrument, pacing it against the account-wide and per-instrument buckets.
1734async fn send_raw_for_instrument<P>(
1735    rate_limiter: &WsRateLimiter,
1736    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1737    method: &'static str,
1738    params: &P,
1739    timeout: Duration,
1740    instrument_name: Ustr,
1741) -> Result<Box<RawValue>>
1742where
1743    P: Serialize + ?Sized,
1744{
1745    send_raw_with_rate_limit(
1746        rate_limiter,
1747        cmd_tx,
1748        method,
1749        params,
1750        timeout,
1751        RequestRateLimit::Await(Some(instrument_name)),
1752        None,
1753    )
1754    .await
1755}
1756
1757async fn send_raw_after_rate_limit<P>(
1758    rate_limiter: &WsRateLimiter,
1759    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1760    method: &'static str,
1761    params: &P,
1762    timeout: Duration,
1763) -> Result<Box<RawValue>>
1764where
1765    P: Serialize + ?Sized,
1766{
1767    send_raw_with_rate_limit(
1768        rate_limiter,
1769        cmd_tx,
1770        method,
1771        params,
1772        timeout,
1773        RequestRateLimit::Reserved,
1774        None,
1775    )
1776    .await
1777}
1778
1779async fn send_raw_with_rate_limit<P>(
1780    rate_limiter: &WsRateLimiter,
1781    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1782    method: &'static str,
1783    params: &P,
1784    timeout: Duration,
1785    rate_limit: RequestRateLimit,
1786    connection_epoch: Option<u64>,
1787) -> Result<Box<RawValue>>
1788where
1789    P: Serialize + ?Sized,
1790{
1791    let params = SecretString::from(serde_json::to_string(params)?);
1792
1793    if let RequestRateLimit::Await(instrument_name) = rate_limit {
1794        rate_limiter
1795            .await_class_ready(rate_class_for_method(method), instrument_name.as_ref())
1796            .await;
1797    }
1798
1799    let (response_tx, response_rx) = tokio::sync::oneshot::channel();
1800    cmd_tx
1801        .send(HandlerCommand::Request {
1802            method,
1803            params,
1804            connection_epoch,
1805            cancellation: None,
1806            response_tx,
1807        })
1808        .map_err(|_| DeriveWsError::NotConnected)?;
1809
1810    // On timeout the handler's `pending` entry leaks until the next reconnect's
1811    // `fail_pending` drains it; the later send to the dropped receiver is a
1812    // no-op logged at debug.
1813    match time::timeout(timeout, response_rx).await {
1814        Ok(Ok(outcome)) => outcome,
1815        Ok(Err(_)) => Err(DeriveWsError::RequestCancelled {
1816            method: method.to_owned(),
1817        }),
1818        Err(_) => Err(DeriveWsError::Timeout {
1819            method: method.to_owned(),
1820        }),
1821    }
1822}
1823
1824// Decodes the result, treating a null/absent `result` as `R::default()` (for
1825// subscribe/unsubscribe and the cancel family's `DeriveEmptyResult`).
1826async fn send_request<P, R>(
1827    rate_limiter: &WsRateLimiter,
1828    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1829    method: &'static str,
1830    params: &P,
1831    timeout: Duration,
1832) -> Result<R>
1833where
1834    P: Serialize + ?Sized,
1835    R: Default + DeserializeOwned,
1836{
1837    let value = send_raw(rate_limiter, cmd_tx, method, params, timeout).await?;
1838    decode_default_result(&value)
1839}
1840
1841// Same as `send_request` for a matching write that carries an instrument, so
1842// the venue's per-instrument allowance is paced alongside the global one.
1843async fn send_request_for_instrument<P, R>(
1844    rate_limiter: &WsRateLimiter,
1845    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1846    method: &'static str,
1847    params: &P,
1848    timeout: Duration,
1849    instrument_name: Ustr,
1850) -> Result<R>
1851where
1852    P: Serialize + ?Sized,
1853    R: Default + DeserializeOwned,
1854{
1855    let value = send_raw_for_instrument(
1856        rate_limiter,
1857        cmd_tx,
1858        method,
1859        params,
1860        timeout,
1861        instrument_name,
1862    )
1863    .await?;
1864    decode_default_result(&value)
1865}
1866
1867// Keep strict result decoding while reserving both matching buckets
1868async fn send_request_typed_for_instrument<P, R>(
1869    rate_limiter: &WsRateLimiter,
1870    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1871    method: &'static str,
1872    params: &P,
1873    timeout: Duration,
1874    instrument_name: Ustr,
1875) -> Result<R>
1876where
1877    P: Serialize + ?Sized,
1878    R: DeserializeOwned,
1879{
1880    let value = send_raw_for_instrument(
1881        rate_limiter,
1882        cmd_tx,
1883        method,
1884        params,
1885        timeout,
1886        instrument_name,
1887    )
1888    .await?;
1889    Ok(serde_json::from_str(value.get())?)
1890}
1891
1892fn decode_default_result<R>(value: &RawValue) -> Result<R>
1893where
1894    R: Default + DeserializeOwned,
1895{
1896    if value.get() == "null" {
1897        Ok(R::default())
1898    } else {
1899        Ok(serde_json::from_str(value.get())?)
1900    }
1901}
1902
1903async fn send_request_on_connection<P, R>(
1904    rate_limiter: &WsRateLimiter,
1905    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1906    method: &'static str,
1907    params: &P,
1908    timeout: Duration,
1909    connection_epoch: u64,
1910) -> Result<R>
1911where
1912    P: Serialize + ?Sized,
1913    R: Default + DeserializeOwned,
1914{
1915    let value = send_raw_with_rate_limit(
1916        rate_limiter,
1917        cmd_tx,
1918        method,
1919        params,
1920        timeout,
1921        RequestRateLimit::Await(None),
1922        Some(connection_epoch),
1923    )
1924    .await?;
1925
1926    decode_default_result(&value)
1927}
1928
1929// Decodes the result with no `Default` fallback, for `private/order` and
1930// `private/replace` whose success result is always a populated object.
1931async fn send_request_typed<P, R>(
1932    rate_limiter: &WsRateLimiter,
1933    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1934    method: &'static str,
1935    params: &P,
1936    timeout: Duration,
1937) -> Result<R>
1938where
1939    P: Serialize + ?Sized,
1940    R: DeserializeOwned,
1941{
1942    let value = send_raw(rate_limiter, cmd_tx, method, params, timeout).await?;
1943    Ok(serde_json::from_str(value.get())?)
1944}
1945
1946async fn send_request_typed_after_rate_limit<P, R>(
1947    rate_limiter: &WsRateLimiter,
1948    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1949    method: &'static str,
1950    params: &P,
1951    timeout: Duration,
1952) -> Result<R>
1953where
1954    P: Serialize + ?Sized,
1955    R: DeserializeOwned,
1956{
1957    let value = send_raw_after_rate_limit(rate_limiter, cmd_tx, method, params, timeout).await?;
1958    Ok(serde_json::from_str(value.get())?)
1959}
1960
1961fn channel_topics(channels: &[DeriveWsChannel]) -> Vec<String> {
1962    channels.iter().map(ToString::to_string).collect()
1963}
1964
1965#[expect(
1966    clippy::too_many_arguments,
1967    reason = "authentication state is passed explicitly for epoch fencing"
1968)]
1969async fn login_via_handler(
1970    rate_limiter: &WsRateLimiter,
1971    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1972    auth_tracker: &AuthTracker,
1973    authenticated_epoch: &AtomicU64,
1974    connection_mode: &AtomicU8,
1975    connection_epoch: &AtomicU64,
1976    creds: &DeriveWsCredentials,
1977    timeout: Duration,
1978) -> Result<()> {
1979    let _receiver = auth_tracker.begin();
1980    let expected_epoch = connection_epoch.load(Ordering::Acquire);
1981
1982    match send_login_request(rate_limiter, cmd_tx, creds, timeout, expected_epoch).await {
1983        Ok(())
1984            if complete_session_authentication(
1985                auth_tracker,
1986                authenticated_epoch,
1987                connection_mode,
1988                connection_epoch,
1989                expected_epoch,
1990            ) =>
1991        {
1992            log::debug!("Derive WebSocket authenticated");
1993
1994            Ok(())
1995        }
1996        Ok(()) => {
1997            let e = DeriveWsError::transport(
1998                "connection changed while completing WebSocket authentication",
1999            );
2000            authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2001            auth_tracker.fail(e.to_string());
2002            Err(e)
2003        }
2004        Err(e) => {
2005            authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2006            auth_tracker.fail(e.to_string());
2007            Err(e)
2008        }
2009    }
2010}
2011
2012async fn send_login_request(
2013    rate_limiter: &WsRateLimiter,
2014    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2015    creds: &DeriveWsCredentials,
2016    timeout: Duration,
2017    connection_epoch: u64,
2018) -> Result<()> {
2019    rate_limiter.await_public_ready().await;
2020    let login = build_ws_login(&creds.wallet_address, &creds.signer)?;
2021
2022    let params = Zeroizing::new(WsLoginParams {
2023        wallet: login.wallet,
2024        timestamp: login.timestamp,
2025        signature: login.signature,
2026    });
2027
2028    let value = send_raw_with_rate_limit(
2029        rate_limiter,
2030        cmd_tx,
2031        methods::PUBLIC_LOGIN,
2032        &*params,
2033        timeout,
2034        RequestRateLimit::Reserved,
2035        Some(connection_epoch),
2036    )
2037    .await?;
2038
2039    let _: WsLoginResult =
2040        serde_json::from_str(value.get()).map_err(|e| DeriveWsError::Authentication {
2041            operation: methods::PUBLIC_LOGIN.to_string(),
2042            reason: format!("invalid login result: {e}"),
2043        })?;
2044
2045    Ok(())
2046}
2047
2048#[expect(
2049    clippy::too_many_arguments,
2050    reason = "recovery state is passed explicitly for epoch fencing"
2051)]
2052async fn recover_session(
2053    rate_limiter: &WsRateLimiter,
2054    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2055    auth_tracker: &AuthTracker,
2056    authenticated_epoch: &AtomicU64,
2057    connection_mode: &AtomicU8,
2058    connection_epoch: &AtomicU64,
2059    creds: Option<&DeriveWsCredentials>,
2060    subscriptions: &SubscriptionState,
2061    subscription_lock: &tokio::sync::Mutex<()>,
2062    timeout: Duration,
2063) -> Result<u64> {
2064    let _guard = subscription_lock.lock().await;
2065    let mut channels = subscriptions.reset_after_reconnect();
2066    channels.sort();
2067    let _receiver = creds.map(|_| auth_tracker.begin());
2068
2069    for attempt in 1..=MAX_SESSION_RECOVERY_ATTEMPTS {
2070        let expected_epoch = wait_for_session_connection(connection_mode, connection_epoch).await?;
2071
2072        let result = recover_session_attempt(
2073            rate_limiter,
2074            cmd_tx,
2075            connection_mode,
2076            connection_epoch,
2077            creds,
2078            subscriptions,
2079            channels.clone(),
2080            timeout,
2081            expected_epoch,
2082        )
2083        .await;
2084
2085        match result {
2086            Ok(()) => {
2087                if creds.is_some()
2088                    && !complete_session_authentication(
2089                        auth_tracker,
2090                        authenticated_epoch,
2091                        connection_mode,
2092                        connection_epoch,
2093                        expected_epoch,
2094                    )
2095                {
2096                    continue;
2097                }
2098
2099                if creds.is_some() {
2100                    log::info!("Derive WebSocket session re-authenticated");
2101                }
2102
2103                return Ok(expected_epoch);
2104            }
2105            Err(e) if attempt < MAX_SESSION_RECOVERY_ATTEMPTS && should_retry_ws_error(&e) => {
2106                let multiplier = 1_u32 << (attempt - 1);
2107                let delay = RECONNECT_BASE_BACKOFF
2108                    .saturating_mul(multiplier)
2109                    .min(RECONNECT_MAX_BACKOFF);
2110                log::warn!(
2111                    "Derive WebSocket session recovery attempt {attempt}/{MAX_SESSION_RECOVERY_ATTEMPTS} failed: {}; retrying in {delay:?}",
2112                    e.redacted_message(),
2113                );
2114                time::sleep(delay).await;
2115            }
2116            Err(e) => {
2117                authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2118
2119                if creds.is_some() {
2120                    auth_tracker.fail(e.to_string());
2121                }
2122
2123                return Err(e);
2124            }
2125        }
2126    }
2127
2128    let e = DeriveWsError::transport("WebSocket session changed while recovery completed");
2129    authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2130
2131    if creds.is_some() {
2132        auth_tracker.fail(e.to_string());
2133    }
2134
2135    Err(e)
2136}
2137
2138#[expect(
2139    clippy::too_many_arguments,
2140    reason = "a session attempt retains its fixed connection epoch and native replay context"
2141)]
2142async fn recover_session_attempt(
2143    rate_limiter: &WsRateLimiter,
2144    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2145    connection_mode: &AtomicU8,
2146    connection_epoch: &AtomicU64,
2147    creds: Option<&DeriveWsCredentials>,
2148    subscriptions: &SubscriptionState,
2149    channels: Vec<String>,
2150    timeout: Duration,
2151    expected_epoch: u64,
2152) -> Result<()> {
2153    if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2154        return Err(DeriveWsError::transport(
2155            "connection changed before WebSocket session recovery",
2156        ));
2157    }
2158
2159    if let Some(creds) = creds {
2160        send_login_request(rate_limiter, cmd_tx, creds, timeout, expected_epoch).await?;
2161    }
2162
2163    subscribe_via_handler(
2164        rate_limiter,
2165        cmd_tx,
2166        channels,
2167        timeout,
2168        expected_epoch,
2169        subscriptions,
2170    )
2171    .await?;
2172
2173    if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2174        return Err(DeriveWsError::transport(
2175            "connection changed during WebSocket session recovery",
2176        ));
2177    }
2178
2179    Ok(())
2180}
2181
2182async fn wait_for_session_connection(
2183    connection_mode: &AtomicU8,
2184    connection_epoch: &AtomicU64,
2185) -> Result<u64> {
2186    loop {
2187        match ConnectionMode::from_atomic(connection_mode) {
2188            ConnectionMode::Active => return Ok(connection_epoch.load(Ordering::Acquire)),
2189            ConnectionMode::Reconnect => time::sleep(RECONNECT_BASE_BACKOFF).await,
2190            ConnectionMode::Disconnect | ConnectionMode::Closed => {
2191                return Err(DeriveWsError::transport(
2192                    "WebSocket closed during session recovery",
2193                ));
2194            }
2195        }
2196    }
2197}
2198
2199fn complete_session_authentication(
2200    auth_tracker: &AuthTracker,
2201    authenticated_epoch: &AtomicU64,
2202    connection_mode: &AtomicU8,
2203    connection_epoch: &AtomicU64,
2204    expected_epoch: u64,
2205) -> bool {
2206    if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2207        return false;
2208    }
2209
2210    authenticated_epoch.store(expected_epoch, Ordering::Release);
2211    auth_tracker.succeed();
2212
2213    if session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2214        true
2215    } else {
2216        authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2217        auth_tracker.invalidate();
2218        false
2219    }
2220}
2221
2222fn session_connection_is_active(
2223    connection_mode: &AtomicU8,
2224    connection_epoch: &AtomicU64,
2225    expected_epoch: u64,
2226) -> bool {
2227    ConnectionMode::from_atomic(connection_mode).is_active()
2228        && connection_epoch.load(Ordering::Acquire) == expected_epoch
2229}
2230
2231async fn subscribe_via_handler(
2232    rate_limiter: &WsRateLimiter,
2233    cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2234    channels: Vec<String>,
2235    timeout: Duration,
2236    connection_epoch: u64,
2237    subscriptions: &SubscriptionState,
2238) -> Result<()> {
2239    if channels.is_empty() {
2240        return Ok(());
2241    }
2242
2243    let params = WsSubscribeParams {
2244        channels: channels.into_iter().map(DeriveWsChannel::from).collect(),
2245    };
2246
2247    let result: WsSubscribeResult = send_request_on_connection(
2248        rate_limiter,
2249        cmd_tx,
2250        methods::PUBLIC_SUBSCRIBE,
2251        &params,
2252        timeout,
2253        connection_epoch,
2254    )
2255    .await?;
2256
2257    subscription_outcome(
2258        &params.channels,
2259        &result,
2260        SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES,
2261        subscriptions,
2262    )
2263}
2264
2265fn subscription_outcome(
2266    requested: &[DeriveWsChannel],
2267    result: &WsSubscribeResult,
2268    accepted_statuses: &[&str],
2269    subscriptions: &SubscriptionState,
2270) -> Result<()> {
2271    let mut failures = Vec::new();
2272
2273    for channel in requested {
2274        let topic = channel.to_string();
2275        match result.status.get(channel) {
2276            Some(status)
2277                if accepted_statuses.contains(&status.as_str())
2278                    && result.channels.contains(channel) =>
2279            {
2280                subscriptions.confirm_subscribe(&topic);
2281            }
2282            Some(status) if accepted_statuses.contains(&status.as_str()) => {
2283                subscriptions.mark_failure(&topic);
2284                failures.push(format!("{topic}: not subscribed"));
2285            }
2286            other => {
2287                subscriptions.mark_failure(&topic);
2288                let reason = other.map_or("missing channel status", |status| status.as_str());
2289                failures.push(format!("{topic}: {reason}"));
2290            }
2291        }
2292    }
2293
2294    if failures.is_empty() {
2295        Ok(())
2296    } else {
2297        Err(DeriveWsError::Subscription {
2298            details: failures.join(", "),
2299        })
2300    }
2301}
2302
2303fn unsubscribe_outcome(
2304    requested: &[DeriveWsChannel],
2305    result: &WsUnsubscribeResult,
2306    subscriptions: &SubscriptionState,
2307) -> Result<()> {
2308    let mut failures = Vec::new();
2309
2310    for channel in requested {
2311        let topic = channel.to_string();
2312        match result.status.get(channel) {
2313            Some(status) if status == "ok" && !result.remaining_subscriptions.contains(channel) => {
2314                subscriptions.confirm_unsubscribe(&topic);
2315            }
2316            Some(status) if status == "ok" => failures.push(format!("{topic}: still subscribed")),
2317            Some(status) => failures.push(format!("{topic}: {status}")),
2318            None => failures.push(format!("{topic}: missing channel status")),
2319        }
2320    }
2321
2322    if failures.is_empty() {
2323        Ok(())
2324    } else {
2325        Err(DeriveWsError::Subscription {
2326            details: failures.join(", "),
2327        })
2328    }
2329}
2330
2331pub(crate) struct DeriveReplaceResponse {
2332    pub result: std::result::Result<DeriveReplaceResult, String>,
2333    pub trades: Vec<DeriveTrade>,
2334}
2335
2336impl DeriveReplaceResponse {
2337    fn from_raw(response: &RawValue) -> Self {
2338        #[derive(Deserialize)]
2339        struct Trades {
2340            #[serde(default)]
2341            trades: Vec<Box<RawValue>>,
2342        }
2343        let trades = serde_json::from_str::<Trades>(response.get())
2344            .map(|response| salvage_rows(response.trades))
2345            .unwrap_or_default();
2346        let result = serde_json::from_str(response.get()).map_err(|e| e.to_string());
2347        Self { result, trades }
2348    }
2349}
2350
2351#[cfg(test)]
2352mod tests {
2353    use std::{env, time::SystemTime};
2354
2355    use axum::{Json, Router, routing::post};
2356    use futures_util::StreamExt;
2357    use nautilus_common::live::dst::time::Instant;
2358    use nautilus_model::identifiers::InstrumentId;
2359    use nautilus_testkit::connection::race_command_with_publication;
2360    use rstest::rstest;
2361    use serde_json::Value;
2362
2363    use super::*;
2364    use crate::{
2365        common::{
2366            consts::REST_URL_TESTNET,
2367            rate_limit::{DERIVE_RATE_WINDOW_SECS, RateLimits, RequestLimiter},
2368            retry::{is_fatal_ws_error, is_write_outcome_ambiguous_ws},
2369        },
2370        http::{DeriveHttpClient, client::DeriveCredentials},
2371    };
2372
2373    #[tokio::test]
2374    async fn command_during_sender_publication_reaches_initialized_handler() {
2375        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2376        let addr = listener.local_addr().unwrap();
2377        let (received_tx, mut received_rx) = tokio::sync::mpsc::unbounded_channel();
2378
2379        let server = tokio::spawn(async move {
2380            let (socket, _) = listener.accept().await.unwrap();
2381            let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap();
2382            while let Some(Ok(message)) = socket.next().await {
2383                if let tokio_tungstenite::tungstenite::Message::Text(text) = message {
2384                    let payload: serde_json::Value = serde_json::from_str(text.as_str()).unwrap();
2385                    let _ = received_tx.send(payload);
2386                }
2387            }
2388        });
2389
2390        let url = format!("ws://{addr}");
2391
2392        let mut ws_client = DeriveWebSocketClient::new(
2393            Some(url),
2394            DeriveEnvironment::Testnet,
2395            TransportBackend::default(),
2396            None,
2397        );
2398        let mode = Arc::clone(&ws_client.connection_mode);
2399        let (response_tx, _response_rx) = tokio::sync::oneshot::channel();
2400
2401        let command = HandlerCommand::Request {
2402            method: "public/subscribe",
2403            params: SecretString::from(
2404                serde_json::json!({
2405                    "channels": [
2406                        "trades.ETH-USDC"
2407                    ]
2408                })
2409                .to_string(),
2410            ),
2411            connection_epoch: None,
2412            cancellation: None,
2413            response_tx,
2414        };
2415
2416        race_command_with_publication(
2417            Arc::clone(&ws_client.cmd_tx),
2418            ws_client.connect(),
2419            move || command,
2420            || ConnectionMode::from_atomic(&mode.load()).is_active(),
2421        )
2422        .await
2423        .expect("connect failed");
2424        let received = tokio::time::timeout(Duration::from_secs(2), received_rx.recv()).await;
2425        ws_client.disconnect().await.expect("disconnect failed");
2426        server.abort();
2427
2428        let payload = received
2429            .expect("startup command must reach server")
2430            .unwrap();
2431        assert_eq!(
2432            payload,
2433            serde_json::json!({
2434                "jsonrpc": "2.0",
2435                "id": 1,
2436                "method": "public/subscribe",
2437                "params": {
2438                    "channels": [
2439                        "trades.ETH-USDC"
2440                    ]
2441                }
2442            })
2443        );
2444    }
2445
2446    #[rstest]
2447    #[case("wss://stream.example.com/feed", "wss://stream.example.com")]
2448    #[case(
2449        "wss://SYN-user:SYN-password@stream.example.com:9443/SYN-path?token=SYN-query#SYN-fragment",
2450        "wss://stream.example.com:9443"
2451    )]
2452    fn test_debug_preserves_url_origin(#[case] url: &str, #[case] origin: &str) {
2453        let client = DeriveWebSocketClient::new(
2454            Some(url.to_string()),
2455            DeriveEnvironment::Testnet,
2456            TransportBackend::default(),
2457            None,
2458        );
2459        let debug = format!("{client:?}");
2460        assert!(debug.contains(&format!("url: {origin:?}")));
2461        assert!(!debug.contains("SYN-"));
2462        assert_eq!(client.url.expose(), url);
2463    }
2464
2465    #[rstest]
2466    #[case::subscription_lock(false)]
2467    #[case::rate_limit(true)]
2468    #[tokio::test]
2469    async fn test_queued_book_subscribe_uses_reconnected_generation(#[case] paced: bool) {
2470        let tracker = BookSyncTracker::default();
2471        let channel = "orderbook.ETH-USDC.1.10";
2472        tracker.record_subscription(channel, InstrumentId::from("ETH-USDC.DERIVE"));
2473        let client = DeriveWebSocketClient::new(
2474            None,
2475            DeriveEnvironment::Testnet,
2476            TransportBackend::default(),
2477            None,
2478        )
2479        .with_book_sync(tracker.clone(), Duration::from_secs(10));
2480        let mut handle = client.subscription_handle();
2481
2482        handle.rate_limiter = Arc::new(RequestLimiter::new(
2483            RateLimits::new(None, None),
2484            MonotonicClock,
2485        ));
2486        handle.connection_epoch.load().store(1, Ordering::Release);
2487        let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2488        *handle.cmd_tx.write().await = cmd_tx;
2489        if paced {
2490            while handle
2491                .rate_limiter
2492                .try_class_ready(RateClass::NonMatching, None)
2493                .is_ok()
2494            {}
2495        }
2496
2497        let lock = Arc::clone(&handle.subscription_lock);
2498
2499        let guard = if paced { None } else { Some(lock.lock().await) };
2500        let request = handle.subscribe_book_gated(channel.to_owned());
2501        tokio::pin!(request);
2502        assert!(futures_util::poll!(&mut request).is_pending());
2503
2504        tracker.reset_on_reconnect(Duration::from_secs(10));
2505        handle.connection_epoch.load().store(2, Ordering::Release);
2506        drop(guard);
2507        let ack: Value =
2508            serde_json::from_str(include_str!("../../test_data/spot/ws_subscribe_ack.json"))
2509                .unwrap();
2510        tokio::time::timeout(Duration::from_secs(8), async {
2511            tokio::join!(async { request.await.unwrap() }, async {
2512                let HandlerCommand::Request {
2513                    method,
2514                    params,
2515                    connection_epoch,
2516                    cancellation,
2517                    response_tx,
2518                } = cmd_rx.recv().await.unwrap()
2519                else {
2520                    panic!("expected book subscribe request");
2521                };
2522                let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
2523                response_tx
2524                    .send(Ok(serde_json::value::to_raw_value(&ack["result"]).unwrap()))
2525                    .unwrap();
2526
2527                assert_eq!(method, methods::PUBLIC_SUBSCRIBE);
2528                assert_eq!(params, serde_json::json!({"channels": [channel]}));
2529                assert_eq!(connection_epoch, Some(2));
2530                assert!(!cancellation.unwrap().is_cancelled());
2531            },);
2532        })
2533        .await
2534        .unwrap();
2535    }
2536
2537    #[rstest]
2538    #[tokio::test]
2539    async fn test_replay_ack_cannot_open_queued_book_replacement() {
2540        let tracker = BookSyncTracker::default();
2541        let channel = "orderbook.ETH-USDC.1.10";
2542        let now = Instant::now();
2543        tracker.record_subscription(channel, InstrumentId::from("ETH-USDC.DERIVE"));
2544        tracker.write(channel).unwrap().gate.open();
2545        assert!(tracker.record_snapshot(channel, 100, None, now));
2546        let recovery = tracker
2547            .recoveries(Duration::from_secs(1), now + Duration::from_secs(2))
2548            .pop()
2549            .unwrap()
2550            .2;
2551        assert!(recovery.begin_replacement());
2552        tracker.reset_on_reconnect(Duration::from_secs(10));
2553        let client = DeriveWebSocketClient::new(
2554            None,
2555            DeriveEnvironment::Testnet,
2556            TransportBackend::default(),
2557            None,
2558        )
2559        .with_book_sync(tracker.clone(), Duration::from_secs(10));
2560        let mut handle = client.subscription_handle();
2561
2562        handle.rate_limiter = Arc::new(RequestLimiter::new(
2563            RateLimits::new(None, None),
2564            MonotonicClock,
2565        ));
2566        handle.connection_epoch.load().store(2, Ordering::Release);
2567        let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2568        *handle.cmd_tx.write().await = cmd_tx;
2569        let lock = Arc::clone(&handle.subscription_lock);
2570        let guard = lock.lock().await;
2571        let request = handle.replace_book_gated(
2572            channel.to_owned(),
2573            CancellationToken::new(),
2574            &recovery,
2575            recovery.gate.clone(),
2576        );
2577        tokio::pin!(request);
2578        assert!(futures_util::poll!(&mut request).is_pending());
2579        let replay = tracker.write(channel).unwrap();
2580        replay.gate.open();
2581        drop(guard);
2582        let ack: Value =
2583            serde_json::from_str(include_str!("../../test_data/spot/ws_subscribe_ack.json"))
2584                .unwrap();
2585        tokio::time::timeout(Duration::from_secs(1), async {
2586            tokio::join!(async { request.await.unwrap() }, async {
2587                for method in [methods::PUBLIC_UNSUBSCRIBE, methods::PUBLIC_SUBSCRIBE] {
2588                    let HandlerCommand::Request {
2589                        method: actual_method,
2590                        params,
2591                        connection_epoch,
2592                        cancellation,
2593                        response_tx,
2594                    } = cmd_rx.recv().await.unwrap()
2595                    else {
2596                        panic!("expected book replacement request");
2597                    };
2598                    let current = tracker.write(channel).unwrap();
2599                    assert!(current.gate.lock().is_closed());
2600                    assert!(!tracker.record_snapshot(channel, 101, Some(&current.generation), now));
2601                    assert!(replay.generation.is_cancelled());
2602                    assert_eq!(actual_method, method);
2603                    assert_eq!(
2604                        serde_json::from_str::<Value>(params.expose_secret()).unwrap(),
2605                        serde_json::json!({"channels": [channel]}),
2606                    );
2607                    assert_eq!(connection_epoch, Some(2));
2608                    assert!(!cancellation.unwrap().is_cancelled());
2609                    let result = if method == methods::PUBLIC_SUBSCRIBE {
2610                        current.gate.open();
2611                        ack["result"].clone()
2612                    } else {
2613                        serde_json::json!({"status": {channel: "ok"}, "remaining_subscriptions": []})
2614                    };
2615                    response_tx
2616                        .send(Ok(serde_json::value::to_raw_value(&result).unwrap()))
2617                        .unwrap();
2618                }
2619            });
2620        })
2621        .await
2622        .unwrap();
2623
2624        assert!(tracker.record_snapshot(channel, 101, None, now));
2625        assert!(recovery.is_accepted());
2626    }
2627
2628    #[tokio::test]
2629    async fn test_expired_matching_reservation_has_local_failure_evidence() {
2630        let client = DeriveWebSocketClient::new(
2631            None,
2632            DeriveEnvironment::Testnet,
2633            TransportBackend::default(),
2634            None,
2635        );
2636        let handle = client.execution_handle();
2637        let quota = handle
2638            .rate_limiter
2639            .reserve_matching_ready(&Ustr::from("ETH-PERP"))
2640            .await;
2641        time::sleep(Duration::from_secs(DERIVE_RATE_WINDOW_SECS)).await;
2642        for _ in 0..DERIVE_RATE_WINDOW_SECS {
2643            handle
2644                .rate_limiter
2645                .await_class_ready(RateClass::Matching, None)
2646                .await;
2647        }
2648
2649        let reservation = MatchingRateLimitReservation {
2650            method: methods::PRIVATE_ORDER,
2651            quota,
2652        };
2653
2654        let error = handle.commit_matching_reservation(reservation).unwrap_err();
2655
2656        assert!(matches!(error, DeriveWsError::RateLimited { .. }));
2657        assert!(should_retry_ws_error(&error));
2658        assert!(!is_fatal_ws_error(&error));
2659        assert!(!is_write_outcome_ambiguous_ws(&error));
2660    }
2661
2662    #[rstest]
2663    fn test_replace_response_preserves_order_with_undecodable_trade() {
2664        let order: Value = serde_json::from_str(include_str!(
2665            "../../test_data/perps/http_order_eth_partially_filled.json"
2666        ))
2667        .unwrap();
2668        let trade: Value = serde_json::from_str(include_str!(
2669            "../../test_data/perps/http_private_trade_eth.json"
2670        ))
2671        .unwrap();
2672        let response = DeriveReplaceResponse::from_raw(
2673            &serde_json::value::to_raw_value(&serde_json::json!({
2674                "order": order,
2675                "trades": [{"trade_id": "bad-row"}, trade],
2676            }))
2677            .unwrap(),
2678        );
2679        let outcome = response
2680            .result
2681            .unwrap()
2682            .into_outcome("old-order", "alpha-strategy", 42, "ETH-PERP")
2683            .unwrap();
2684
2685        let DeriveReplaceOutcome::Replaced(order) = outcome else {
2686            panic!("expected replaced outcome");
2687        };
2688
2689        let trades = response.trades;
2690        assert_eq!(order.order_id, "abc-123");
2691        assert_eq!(order.label, "alpha-strategy");
2692        assert_eq!(trades.len(), 1);
2693        assert_eq!(trades[0].trade_id, "trade-xyz");
2694        assert_eq!(trades[0].order_id, "order-abc");
2695    }
2696
2697    const SESSION_KEY_HEX: &str =
2698        "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd";
2699
2700    #[rstest]
2701    #[case(Value::Null)]
2702    #[case(serde_json::json!("bad-nonce"))]
2703    #[case(serde_json::json!("18446744073709551616"))]
2704    fn test_replace_response_preserves_financial_rows_with_invalid_order_nonce(
2705        #[case] nonce: Value,
2706    ) {
2707        let mut order: Value = serde_json::from_str(include_str!(
2708            "../../test_data/perps/http_order_eth_partially_filled.json"
2709        ))
2710        .unwrap();
2711        let trade: DeriveTrade = serde_json::from_str(include_str!(
2712            "../../test_data/perps/http_private_trade_eth.json"
2713        ))
2714        .unwrap();
2715        order["nonce"] = nonce;
2716        let response = DeriveReplaceResponse::from_raw(
2717            &serde_json::value::to_raw_value(
2718                &serde_json::json!({"order": order, "trades": [trade]}),
2719            )
2720            .unwrap(),
2721        );
2722        assert!(response.result.is_err());
2723        assert_eq!(
2724            serde_json::to_value(response.trades).unwrap(),
2725            serde_json::to_value(vec![trade]).unwrap()
2726        );
2727    }
2728
2729    #[tokio::test]
2730    async fn test_http_and_websocket_share_matching_consumption() {
2731        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2732        let address = listener.local_addr().unwrap();
2733
2734        let server = tokio::spawn(async move {
2735            axum::serve(
2736                listener,
2737                Router::new().route(
2738                    "/private/cancel",
2739                    post(|| async { Json(serde_json::json!({"result": true})) }),
2740                ),
2741            )
2742            .await
2743            .unwrap();
2744        });
2745
2746        let wallet = "0x000000000000000000000000000000000000ab01";
2747        let http = DeriveHttpClient::with_credentials(
2748            format!("http://{address}"),
2749            DeriveCredentials::new(wallet, SESSION_KEY_HEX).unwrap(),
2750            None,
2751            None,
2752            None,
2753        )
2754        .unwrap();
2755        let ws = DeriveWebSocketClient::with_credentials(
2756            Some(format!("ws://{address}/ws")),
2757            DeriveEnvironment::Testnet,
2758            TransportBackend::default(),
2759            None,
2760            DeriveWsCredentials::new(wallet.to_ascii_uppercase(), SESSION_KEY_HEX).unwrap(),
2761            None,
2762            None,
2763        );
2764
2765        for _ in 0..5 {
2766            let result: bool = http
2767                .send_private_once("private/cancel", &serde_json::json!({}))
2768                .await
2769                .unwrap();
2770            assert!(result);
2771        }
2772
2773        server.abort();
2774        let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2775        let params = serde_json::json!({});
2776        let request = send_raw(
2777            &ws.rate_limiter,
2778            &cmd_tx,
2779            methods::PRIVATE_CANCEL,
2780            &params,
2781            Duration::from_secs(1),
2782        );
2783        tokio::pin!(request);
2784        assert!(futures_util::poll!(&mut request).is_pending());
2785        assert!(matches!(
2786            cmd_rx.try_recv(),
2787            Err(tokio::sync::mpsc::error::TryRecvError::Empty)
2788        ));
2789    }
2790
2791    #[tokio::test]
2792    async fn test_login_builds_auth_timestamp_after_public_quota_wait() {
2793        let rate_limiter = RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
2794        for _ in 0..25 {
2795            rate_limiter.await_public_ready().await;
2796        }
2797
2798        let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2799
2800        let captured = tokio::spawn(async move {
2801            let Some(HandlerCommand::Request {
2802                method,
2803                params,
2804                response_tx,
2805                ..
2806            }) = cmd_rx.recv().await
2807            else {
2808                panic!("login request must enqueue");
2809            };
2810
2811            assert_eq!(method, methods::PUBLIC_LOGIN);
2812            let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
2813            let timestamp: u64 = params["timestamp"].as_str().unwrap().parse().unwrap();
2814            let now = SystemTime::now()
2815                .duration_since(SystemTime::UNIX_EPOCH)
2816                .unwrap()
2817                .as_millis();
2818            response_tx
2819                .send(Ok(serde_json::value::to_raw_value(&serde_json::json!([
2820                    77
2821                ]))
2822                .unwrap()))
2823                .unwrap();
2824            now.saturating_sub(u128::from(timestamp))
2825        });
2826
2827        let started = tokio::time::Instant::now();
2828        send_login_request(
2829            &rate_limiter,
2830            &cmd_tx,
2831            &DeriveWsCredentials::new(
2832                "0x000000000000000000000000000000000000ab02",
2833                SESSION_KEY_HEX,
2834            )
2835            .unwrap(),
2836            Duration::from_secs(1),
2837            0,
2838        )
2839        .await
2840        .unwrap();
2841        assert!(started.elapsed() >= Duration::from_secs(4));
2842        assert!(
2843            captured.await.unwrap() < 1_000,
2844            "login timestamp must remain fresh after the pacing wait"
2845        );
2846    }
2847
2848    #[tokio::test]
2849    #[ignore = "requires Derive public testnet network access"]
2850    async fn test_live_testnet_unsubscribe_acknowledgement() {
2851        let mut ws = DeriveWebSocketClient::new(
2852            None,
2853            DeriveEnvironment::Testnet,
2854            TransportBackend::default(),
2855            None,
2856        );
2857        ws.connect().await.unwrap();
2858        let cmd_tx = ws.cmd_tx.read().await.clone();
2859        let params = serde_json::json!({"channels": ["ticker_slim.ETH-PERP.1000"]});
2860        let subscribe = send_raw(
2861            &ws.rate_limiter,
2862            &cmd_tx,
2863            methods::PUBLIC_SUBSCRIBE,
2864            &params,
2865            Duration::from_secs(15),
2866        )
2867        .await;
2868        let unsubscribe = send_raw(
2869            &ws.rate_limiter,
2870            &cmd_tx,
2871            methods::PUBLIC_UNSUBSCRIBE,
2872            &params,
2873            Duration::from_secs(15),
2874        )
2875        .await;
2876        ws.disconnect().await.unwrap();
2877        let subscribe: Value = serde_json::from_str(subscribe.unwrap().get()).unwrap();
2878        let unsubscribe: Value = serde_json::from_str(unsubscribe.unwrap().get()).unwrap();
2879        println!("Public testnet subscribe acknowledgement: {subscribe}");
2880        println!("Public testnet unsubscribe acknowledgement: {unsubscribe}");
2881        assert_eq!(subscribe["status"]["ticker_slim.ETH-PERP.1000"], "ok");
2882        assert_eq!(unsubscribe["status"]["ticker_slim.ETH-PERP.1000"], "ok");
2883        assert_eq!(
2884            unsubscribe["remaining_subscriptions"],
2885            serde_json::json!([])
2886        );
2887    }
2888
2889    #[tokio::test]
2890    #[ignore = "requires Derive testnet credentials and network access"]
2891    async fn test_live_testnet_request_budget_scopes() {
2892        let wallet = env::var("DERIVE_TESTNET_WALLET_ADDRESS").expect("testnet wallet configured");
2893        let key = env::var("DERIVE_TESTNET_SESSION_PRIVATE_KEY").expect("testnet key configured");
2894        let subaccount: u64 = env::var("DERIVE_TESTNET_SUBACCOUNT_ID")
2895            .expect("testnet subaccount configured")
2896            .parse()
2897            .unwrap();
2898        let http = DeriveHttpClient::with_credentials(
2899            REST_URL_TESTNET,
2900            DeriveCredentials::new(&wallet, &key).unwrap(),
2901            Some(15),
2902            None,
2903            None,
2904        )
2905        .unwrap();
2906        let mut ws = DeriveWebSocketClient::with_credentials(
2907            None,
2908            DeriveEnvironment::Testnet,
2909            TransportBackend::default(),
2910            None,
2911            DeriveWsCredentials::new(&wallet, &key).unwrap(),
2912            None,
2913            None,
2914        );
2915        ws.connect()
2916            .await
2917            .expect("testnet WebSocket login succeeds");
2918        let cmd_tx = ws.cmd_tx.read().await.clone();
2919        let before: Value = http
2920            .send_private("public/getRateLimits", &serde_json::json!({}))
2921            .await
2922            .expect("authenticated HTTP budget query succeeds");
2923        let ws_limits = send_raw(
2924            &ws.rate_limiter,
2925            &cmd_tx,
2926            "public/getRateLimits",
2927            &serde_json::json!({}),
2928            Duration::from_secs(15),
2929        )
2930        .await
2931        .expect("authenticated WebSocket budget query succeeds");
2932        println!("Testnet authenticated HTTP budgets: {before}");
2933        println!(
2934            "Testnet authenticated WebSocket budgets: {}",
2935            ws_limits.get()
2936        );
2937        let label = format!("nautilus-pacing-{}", UUID4::new());
2938
2939        for scoped in [true, false] {
2940            let mut params = serde_json::json!({"subaccount_id": subaccount, "label": label});
2941            if scoped {
2942                params["instrument_name"] = serde_json::json!("ETH-PERP");
2943            }
2944
2945            let result: Value = http
2946                .send_private_once("private/cancel_by_label", &params)
2947                .await
2948                .expect("testnet cancellation of unique empty label succeeds");
2949            let budgets: Value = http
2950                .send_private("public/getRateLimits", &serde_json::json!({}))
2951                .await
2952                .unwrap();
2953            println!("Testnet cancel-by-label scoped={scoped}: {result}; budgets={budgets}");
2954            let ws_budgets = send_raw(
2955                &ws.rate_limiter,
2956                &cmd_tx,
2957                "public/getRateLimits",
2958                &serde_json::json!({}),
2959                Duration::from_secs(15),
2960            )
2961            .await
2962            .unwrap();
2963            println!(
2964                "Testnet wallet budgets after HTTP scoped={scoped}: {}",
2965                ws_budgets.get()
2966            );
2967        }
2968
2969        let started = tokio::time::Instant::now();
2970        let name = Ustr::from("ETH-PERP");
2971        let params = serde_json::json!({"subaccount_id": subaccount, "label": label, "instrument_name": name});
2972
2973        for _ in 0..6 {
2974            let result = send_raw_for_instrument(
2975                &ws.rate_limiter,
2976                &cmd_tx,
2977                methods::PRIVATE_CANCEL_BY_LABEL,
2978                &params,
2979                Duration::from_secs(15),
2980                name,
2981            )
2982            .await
2983            .expect("paced testnet WebSocket scoped cancellation succeeds");
2984            let result: Value = serde_json::from_str(result.get()).unwrap();
2985            assert_eq!(result["cancelled_orders"], 0);
2986        }
2987
2988        let elapsed = started.elapsed();
2989        assert!(
2990            elapsed >= Duration::from_secs(1),
2991            "six same-instrument cancels must exceed the local five-token burst"
2992        );
2993        println!(
2994            "Testnet six scoped WebSocket cancellations pass with local pacing in {elapsed:?}"
2995        );
2996        let after = send_raw(
2997            &ws.rate_limiter,
2998            &cmd_tx,
2999            "public/getRateLimits",
3000            &serde_json::json!({}),
3001            Duration::from_secs(15),
3002        )
3003        .await
3004        .unwrap();
3005        println!(
3006            "Testnet wallet budgets after paced WebSocket requests: {}",
3007            after.get()
3008        );
3009        ws.disconnect().await.unwrap();
3010        ws.connect()
3011            .await
3012            .expect("testnet reconnect authenticates with preserved pacing state");
3013        ws.disconnect().await.unwrap();
3014    }
3015
3016    #[rstest]
3017    fn test_public_client_defaults_to_environment_url() {
3018        let client = DeriveWebSocketClient::new(
3019            None,
3020            DeriveEnvironment::Mainnet,
3021            TransportBackend::default(),
3022            None,
3023        );
3024        assert_eq!(client.url(), "wss://api.derive.xyz/v3/ws");
3025        assert!(!client.is_authenticated());
3026        assert!(!client.is_active());
3027        assert_eq!(client.subscription_count(), 0);
3028    }
3029
3030    #[tokio::test]
3031    async fn test_execution_auth_barrier_waits_for_authentication() {
3032        let client = DeriveWebSocketClient::with_credentials(
3033            None,
3034            DeriveEnvironment::Mainnet,
3035            TransportBackend::default(),
3036            None,
3037            DeriveWsCredentials::new(
3038                "0x000000000000000000000000000000000000aaaa",
3039                "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3040            )
3041            .unwrap(),
3042            None,
3043            None,
3044        );
3045        let execution = client.execution_handle();
3046        let auth_tracker = execution.auth_tracker.clone();
3047        let _receiver = auth_tracker.begin();
3048        let tracker_for_task = auth_tracker.clone();
3049
3050        get_runtime().spawn(async move {
3051            tokio::time::sleep(Duration::from_millis(10)).await;
3052            tracker_for_task.succeed();
3053        });
3054
3055        execution
3056            .require_authenticated(methods::PRIVATE_ORDER)
3057            .await
3058            .expect("barrier should wait for successful authentication");
3059    }
3060
3061    #[tokio::test]
3062    async fn test_execution_auth_barrier_fails_on_terminal_auth_failure() {
3063        let client = DeriveWebSocketClient::with_credentials(
3064            None,
3065            DeriveEnvironment::Mainnet,
3066            TransportBackend::default(),
3067            None,
3068            DeriveWsCredentials::new(
3069                "0x000000000000000000000000000000000000aaaa",
3070                "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3071            )
3072            .unwrap(),
3073            None,
3074            None,
3075        );
3076        let execution = client.execution_handle();
3077        let _receiver = execution.auth_tracker.begin();
3078        execution.auth_tracker.fail("bad signature");
3079
3080        let error = execution
3081            .require_authenticated(methods::PRIVATE_ORDER)
3082            .await
3083            .expect_err("terminal auth failure must reject private operations");
3084
3085        assert!(matches!(error, DeriveWsError::Authentication { .. }));
3086    }
3087
3088    #[tokio::test]
3089    async fn test_session_recovery_waits_for_reconnecting_transport() {
3090        let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect as u8));
3091        let connection_epoch = Arc::new(AtomicU64::new(1));
3092        let mode_for_task = Arc::clone(&connection_mode);
3093        let epoch_for_task = Arc::clone(&connection_epoch);
3094
3095        get_runtime().spawn(async move {
3096            tokio::time::sleep(Duration::from_millis(10)).await;
3097            epoch_for_task.store(2, Ordering::Release);
3098            mode_for_task.store(ConnectionMode::Active as u8, Ordering::Release);
3099        });
3100
3101        let epoch = tokio::time::timeout(
3102            Duration::from_secs(1),
3103            wait_for_session_connection(&connection_mode, &connection_epoch),
3104        )
3105        .await
3106        .expect("session recovery should resume after reconnect")
3107        .expect("active replacement connection should be accepted");
3108
3109        assert_eq!(epoch, 2);
3110    }
3111
3112    #[rstest]
3113    fn test_testnet_client_routes_to_v3_url() {
3114        let client = DeriveWebSocketClient::new(
3115            None,
3116            DeriveEnvironment::Testnet,
3117            TransportBackend::default(),
3118            None,
3119        );
3120        assert_eq!(client.url(), "wss://testnet.api.derive.xyz/v3/ws");
3121    }
3122
3123    #[rstest]
3124    fn test_credentials_constructor_parses_session_key() {
3125        let creds = DeriveWsCredentials::new(
3126            "0x000000000000000000000000000000000000aaaa",
3127            "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3128        )
3129        .unwrap();
3130        assert!(creds.wallet_address.starts_with("0x"));
3131        let client = DeriveWebSocketClient::with_credentials(
3132            None,
3133            DeriveEnvironment::Testnet,
3134            TransportBackend::default(),
3135            None,
3136            creds,
3137            None,
3138            None,
3139        );
3140        assert_eq!(client.url(), "wss://testnet.api.derive.xyz/v3/ws");
3141        assert!(!client.is_authenticated());
3142    }
3143
3144    #[rstest]
3145    fn test_credentials_debug_redacts_signer() {
3146        let creds = DeriveWsCredentials::new(
3147            "0xWALLET",
3148            "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3149        )
3150        .unwrap();
3151        let debug = format!("{creds:?}");
3152        assert!(debug.contains("redacted"));
3153        assert!(debug.contains("0xWALLET"));
3154        assert!(!debug.contains("2ae8be44"));
3155    }
3156
3157    #[rstest]
3158    fn test_credentials_constructor_rejects_invalid_session_key() {
3159        let err = DeriveWsCredentials::new("0xWALLET", "not-a-hex-key").unwrap_err();
3160        assert!(err.to_string().contains("invalid session key"));
3161    }
3162
3163    #[rstest]
3164    #[tokio::test]
3165    async fn test_send_raw_closed_command_queue_is_definitive() {
3166        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3167        drop(cmd_rx);
3168        let rate_limiter: WsRateLimiter =
3169            RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3170        let error = send_raw(
3171            &rate_limiter,
3172            &cmd_tx,
3173            methods::PRIVATE_REPLACE,
3174            &serde_json::json!({}),
3175            Duration::from_millis(50),
3176        )
3177        .await
3178        .unwrap_err();
3179        assert!(matches!(error, DeriveWsError::NotConnected));
3180        assert!(!crate::common::retry::is_write_outcome_ambiguous_ws(&error));
3181    }
3182
3183    #[rstest]
3184    #[tokio::test]
3185    async fn test_send_raw_times_out_when_no_response_arrives() {
3186        // Keep the receiver alive so the request enqueues, but never reply: the
3187        // bounded await must surface a Timeout rather than hang forever.
3188        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3189        let rate_limiter: WsRateLimiter =
3190            RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3191        let err = send_raw(
3192            &rate_limiter,
3193            &cmd_tx,
3194            methods::PRIVATE_ORDER,
3195            &serde_json::json!({}),
3196            Duration::from_millis(50),
3197        )
3198        .await
3199        .expect_err("must time out");
3200
3201        match err {
3202            DeriveWsError::Timeout { method } => assert_eq!(method, methods::PRIVATE_ORDER),
3203            other => panic!("expected Timeout, was {other:?}"),
3204        }
3205    }
3206
3207    #[rstest]
3208    #[tokio::test]
3209    async fn test_send_request_typed_rejects_null_result() {
3210        // `private/order` and `private/replace` always return a populated
3211        // object on success; a null result is a protocol violation that must
3212        // surface as a serde error (classified ambiguous by the exec client).
3213        let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3214        tokio::spawn(async move {
3215            if let Some(HandlerCommand::Request { response_tx, .. }) = cmd_rx.recv().await {
3216                let _ = response_tx.send(Ok(RawValue::NULL.to_owned()));
3217            }
3218        });
3219
3220        let rate_limiter: WsRateLimiter =
3221            RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3222        let result: Result<DeriveOrderResult> = send_request_typed(
3223            &rate_limiter,
3224            &cmd_tx,
3225            methods::PRIVATE_ORDER,
3226            &serde_json::json!({}),
3227            Duration::from_secs(1),
3228        )
3229        .await;
3230        assert!(matches!(result, Err(DeriveWsError::Serde(_))));
3231    }
3232
3233    #[rstest]
3234    #[case::pending(false)]
3235    #[case::fired_during_pacing(true)]
3236    #[tokio::test]
3237    async fn test_trigger_cancel_selects_endpoint_after_pacing(#[case] fires: bool) {
3238        let client = DeriveWebSocketClient::new(
3239            None,
3240            DeriveEnvironment::Testnet,
3241            TransportBackend::default(),
3242            None,
3243        );
3244        let mut handle = client.execution_handle();
3245        handle.auth_tracker.succeed();
3246
3247        handle.rate_limiter = Arc::new(RequestLimiter::new(
3248            RateLimits::new(None, None),
3249            MonotonicClock,
3250        ));
3251        let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
3252        *handle.cmd_tx.write().await = cmd_tx;
3253        for _ in 0..crate::common::rate_limit::DERIVE_WEBSOCKET_NON_MATCHING_TPS
3254            * crate::common::rate_limit::DERIVE_RATE_BURST_MULTIPLIER
3255        {
3256            handle
3257                .rate_limiter
3258                .try_class_ready(RateClass::NonMatching, None)
3259                .unwrap();
3260        }
3261
3262        assert!(
3263            handle
3264                .rate_limiter
3265                .try_class_ready(RateClass::NonMatching, None)
3266                .is_err()
3267        );
3268        let active = AtomicBool::new(false);
3269        let trigger = DeriveCancelTriggerOrderParams::new(42, "paced-trigger");
3270        let regular = DeriveCancelParams::new(42, "ETH-PERP", "paced-trigger");
3271        let cancel = handle
3272            .cancel_trigger_order_current(&trigger, &regular, || active.load(Ordering::Acquire));
3273        tokio::pin!(cancel);
3274        assert!(futures_util::poll!(cancel.as_mut()).is_pending());
3275        assert!(cmd_rx.try_recv().is_err());
3276        active.store(fires, Ordering::Release);
3277        let mut canceled: DeriveOrder = serde_json::from_str(include_str!(
3278            "../../test_data/perps/http_order_eth_partially_filled.json"
3279        ))
3280        .unwrap();
3281        canceled.order_id = "paced-trigger".into();
3282        canceled.order_status = crate::common::enums::DeriveOrderStatus::Cancelled;
3283        canceled.filled_amount = rust_decimal::Decimal::ZERO;
3284        canceled.average_price = rust_decimal::Decimal::ZERO;
3285        let expected = (!fires).then_some(canceled);
3286
3287        let response = async {
3288            let HandlerCommand::Request {
3289                method,
3290                params,
3291                response_tx,
3292                ..
3293            } = cmd_rx.recv().await.unwrap()
3294            else {
3295                panic!("expected one cancel request")
3296            };
3297
3298            let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
3299
3300            let expected_method = if fires {
3301                methods::PRIVATE_CANCEL
3302            } else {
3303                methods::PRIVATE_CANCEL_TRIGGER_ORDER
3304            };
3305
3306            assert_eq!(method, expected_method);
3307            assert_eq!(params["subaccount_id"], serde_json::json!(42));
3308            assert_eq!(params["order_id"], serde_json::json!("paced-trigger"));
3309            assert_eq!(
3310                params.get("instrument_name"),
3311                fires
3312                    .then_some(&regular.instrument_name)
3313                    .map(|name| serde_json::json!(name))
3314                    .as_ref()
3315            );
3316            response_tx
3317                .send(Ok(serde_json::value::to_raw_value(&expected).unwrap()))
3318                .unwrap();
3319        };
3320
3321        let (result, ()) = tokio::join!(cancel, response);
3322        assert_eq!(
3323            serde_json::to_value(result.unwrap()).unwrap(),
3324            serde_json::to_value(expected).unwrap()
3325        );
3326        assert!(cmd_rx.try_recv().is_err());
3327    }
3328
3329    #[rstest]
3330    #[tokio::test]
3331    async fn test_reserved_send_does_not_wait_for_or_consume_second_quota_cell() {
3332        let rate_limiter: WsRateLimiter =
3333            RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3334
3335        for _ in 0..5 {
3336            rate_limiter
3337                .try_class_ready(RateClass::Matching, None)
3338                .expect("reservation consumes the matching window");
3339        }
3340
3341        let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3342        tokio::spawn(async move {
3343            if let Some(HandlerCommand::Request { response_tx, .. }) = cmd_rx.recv().await {
3344                let _ = response_tx.send(Ok(serde_json::value::to_raw_value(
3345                    &serde_json::json!({"accepted": true}),
3346                )
3347                .unwrap()));
3348            }
3349        });
3350
3351        let response = tokio::time::timeout(
3352            Duration::from_millis(100),
3353            send_raw_after_rate_limit(
3354                &rate_limiter,
3355                &cmd_tx,
3356                methods::PRIVATE_ORDER,
3357                &serde_json::json!({}),
3358                Duration::from_secs(1),
3359            ),
3360        )
3361        .await
3362        .expect("reserved send must not wait for quota")
3363        .expect("reserved send succeeds");
3364
3365        assert_eq!(
3366            serde_json::from_str::<Value>(response.get()).unwrap(),
3367            serde_json::json!({"accepted": true}),
3368        );
3369        assert!(
3370            rate_limiter
3371                .try_class_ready(RateClass::Matching, None)
3372                .is_err(),
3373            "reserved send must not consume a second cell",
3374        );
3375    }
3376}