1use std::{
25 fmt::Debug,
26 sync::{
27 Arc,
28 atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
29 },
30 time::Duration,
31};
32
33use alloy::signers::local::PrivateKeySigner;
34use arc_swap::ArcSwap;
35use nautilus_common::live::dst::time;
36#[cfg(test)]
37use nautilus_common::live::get_runtime;
38use nautilus_core::{
39 UUID4,
40 string::secret::{REDACTED, SecretString},
41};
42use nautilus_live::{
43 SocketControl,
44 book::snapshot::SnapshotGate,
45 task::{SharedTaskSlot, TaskJoinOutcome, TaskSlot, finish_task},
46};
47use nautilus_network::{
48 ConnectionUrl,
49 http::create_standard_nautilus_headers,
50 mode::ConnectionMode,
51 ratelimiter::clock::MonotonicClock,
52 websocket::{
53 AuthTracker, SubscriptionState, TransportBackend, WebSocketClient, WebSocketConfig,
54 channel_message_handler,
55 },
56};
57use serde::{Deserialize, Serialize, de::DeserializeOwned};
58use serde_json::value::RawValue;
59use tokio_util::sync::CancellationToken;
60use ustr::Ustr;
61use zeroize::Zeroizing;
62
63use super::{
64 error::{DeriveWsError, Result},
65 handler::{
66 DeriveWsMessage, FeedHandler, HandlerCommand, orderbook_subscribe_params,
67 ticker_subscribe_params, trades_subscribe_params,
68 },
69 messages::{
70 DeriveWsChannel, WsLoginParams, WsLoginResult, WsSetCancelOnDisconnectParams,
71 WsSetCancelOnDisconnectResult, WsSubscribeParams, WsSubscribeResult, WsUnsubscribeParams,
72 WsUnsubscribeResult, methods, orderbook_channel, ticker_channel, trades_channel,
73 },
74};
75use crate::{
76 book::{recovery::BookRecovery, sync::BookSyncTracker},
77 common::{
78 consts::{
79 RECONNECT_BACKOFF_FACTOR, RECONNECT_BASE_BACKOFF, RECONNECT_JITTER_MS,
80 RECONNECT_MAX_BACKOFF, RECONNECT_TIMEOUT, WS_HEARTBEAT_SECS, WS_HEARTBEAT_TIMEOUT,
81 WS_REQUEST_TIMEOUT,
82 },
83 enums::DeriveEnvironment,
84 parse::salvage_rows,
85 rate_limit::{DeriveRateLimiter, RateClass, RateReservation, rate_class_for_method},
86 retry::should_retry_ws_error,
87 urls,
88 },
89 http::{
90 models::{
91 DeriveCancelByInstrumentResult, DeriveCancelByLabelResult, DeriveEmptyResult,
92 DeriveOpenOrdersResult, DeriveOrder, DeriveOrderResult, DeriveReplaceOutcome,
93 DeriveReplaceResult, DeriveTrade,
94 },
95 query::{
96 DeriveCancelAllParams, DeriveCancelByInstrumentParams, DeriveCancelByLabelParams,
97 DeriveCancelParams, DeriveCancelTriggerOrderParams, DeriveGetTriggerOrdersParams,
98 DeriveOrderParams, DeriveReplaceParams, DeriveTriggerOrderParams,
99 },
100 },
101 signing::auth::build_ws_login,
102};
103
104#[derive(Clone)]
107pub struct DeriveWsCredentials {
108 pub wallet_address: String,
110 pub signer: PrivateKeySigner,
112}
113
114impl DeriveWsCredentials {
115 pub fn new(wallet_address: impl Into<String>, session_key_hex: &str) -> Result<Self> {
121 let signer: PrivateKeySigner = session_key_hex
122 .parse()
123 .map_err(|e| DeriveWsError::transport(format!("invalid session key: {e}")))?;
124 Ok(Self {
125 wallet_address: wallet_address.into(),
126 signer,
127 })
128 }
129}
130
131impl Debug for DeriveWsCredentials {
132 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
133 f.debug_struct(stringify!(DeriveWsCredentials))
134 .field("wallet_address", &self.wallet_address)
135 .field("signer", &REDACTED)
136 .finish()
137 }
138}
139
140type WsRateLimiter = DeriveRateLimiter;
143
144const MAX_SESSION_RECOVERY_ATTEMPTS: u32 = 3;
145const SUBSCRIPTION_ACCEPTED_STATUSES: &[&str] = &["ok"];
146const SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES: &[&str] = &["ok", "already subscribed"];
147pub(super) const UNAUTHENTICATED_CONNECTION_EPOCH: u64 = u64::MAX;
148
149#[derive(Debug)]
155pub struct DeriveWebSocketClient {
156 url: ConnectionUrl,
157 transport_backend: TransportBackend,
158 proxy_url: Option<SecretString>,
159 connection_mode: Arc<ArcSwap<AtomicU8>>,
160 connection_epoch: Arc<ArcSwap<AtomicU64>>,
161 signal: Arc<AtomicBool>,
162 auth_tracker: AuthTracker,
163 authenticated_epoch: Arc<AtomicU64>,
164 credentials: Option<DeriveWsCredentials>,
165 next_id: Arc<AtomicU64>,
166 cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
167 out_rx: Option<tokio::sync::mpsc::UnboundedReceiver<DeriveWsMessage>>,
168 subscriptions: Arc<SubscriptionState>,
169 subscription_lock: Arc<tokio::sync::Mutex<()>>,
170 book_sync: Option<(BookSyncTracker, Duration)>,
171 task_handle: TaskSlot<()>,
172 send_task: Arc<SharedTaskSlot<()>>,
173 recovery_task: Arc<SharedTaskSlot<()>>,
174 shutdown_errors: Vec<String>,
175 request_timeout: Duration,
176 conn_id: Arc<ArcSwap<String>>,
177 rate_limiter: Arc<WsRateLimiter>,
178 socket_control: Option<SocketControl>,
179}
180
181#[derive(Debug, Clone)]
183pub struct DeriveWebSocketSubscriptionHandle {
184 cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
185 subscriptions: Arc<SubscriptionState>,
186 subscription_lock: Arc<tokio::sync::Mutex<()>>,
187 book_sync: Option<(BookSyncTracker, Duration)>,
188 connection_epoch: Arc<ArcSwap<AtomicU64>>,
189 request_timeout: Duration,
190 rate_limiter: Arc<WsRateLimiter>,
191}
192
193#[derive(Debug, Clone)]
202pub struct DeriveWsExecutionHandle {
203 cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
204 auth_tracker: AuthTracker,
205 request_timeout: Duration,
206 conn_id: Arc<ArcSwap<String>>,
207 rate_limiter: Arc<WsRateLimiter>,
208}
209
210#[derive(Debug)]
211pub(crate) struct MatchingRateLimitReservation {
212 method: &'static str,
213 quota: RateReservation<MonotonicClock>,
214}
215
216impl DeriveWebSocketClient {
217 #[must_use]
220 pub fn new(
221 url: Option<String>,
222 environment: DeriveEnvironment,
223 transport_backend: TransportBackend,
224 proxy_url: Option<String>,
225 ) -> Self {
226 let url = url.unwrap_or_else(|| urls::ws_url(environment).to_string());
227 Self::build(url, transport_backend, proxy_url, None, None, None)
228 }
229
230 #[must_use]
241 pub fn with_credentials(
242 url: Option<String>,
243 environment: DeriveEnvironment,
244 transport_backend: TransportBackend,
245 proxy_url: Option<String>,
246 credentials: DeriveWsCredentials,
247 max_matching_requests_per_second: Option<u32>,
248 max_per_instrument_matching_requests_per_second: Option<u32>,
249 ) -> Self {
250 let url = url.unwrap_or_else(|| urls::ws_url(environment).to_string());
251 Self::build(
252 url,
253 transport_backend,
254 proxy_url,
255 Some(credentials),
256 max_matching_requests_per_second,
257 max_per_instrument_matching_requests_per_second,
258 )
259 }
260
261 fn build(
262 url: String,
263 transport_backend: TransportBackend,
264 proxy_url: Option<String>,
265 credentials: Option<DeriveWsCredentials>,
266 matching_tps: Option<u32>,
267 instrument_tps: Option<u32>,
268 ) -> Self {
269 let connection_mode = Arc::new(ArcSwap::new(Arc::new(AtomicU8::new(
270 ConnectionMode::Closed as u8,
271 ))));
272 let connection_epoch = Arc::new(ArcSwap::new(Arc::new(AtomicU64::new(0))));
273
274 let (placeholder_tx, _) = tokio::sync::mpsc::unbounded_channel();
276
277 let rate_limiter = Arc::new(DeriveRateLimiter::shared(
278 &url,
279 credentials
280 .as_ref()
281 .map(|credentials| credentials.wallet_address.as_str()),
282 matching_tps,
283 instrument_tps,
284 ));
285
286 Self {
287 url: url.into(),
288 transport_backend,
289 proxy_url: proxy_url.map(SecretString::from),
290 connection_mode,
291 connection_epoch,
292 signal: Arc::new(AtomicBool::new(false)),
293 auth_tracker: AuthTracker::new(),
294 authenticated_epoch: Arc::new(AtomicU64::new(UNAUTHENTICATED_CONNECTION_EPOCH)),
295 credentials,
296 next_id: Arc::new(AtomicU64::new(1)),
297 cmd_tx: Arc::new(tokio::sync::RwLock::new(placeholder_tx)),
298 out_rx: None,
299 subscriptions: Arc::new(SubscriptionState::new('.')),
300 subscription_lock: Arc::new(tokio::sync::Mutex::new(())),
301 book_sync: None,
302 task_handle: TaskSlot::new(),
303 send_task: Arc::new(SharedTaskSlot::new()),
304 recovery_task: Arc::new(SharedTaskSlot::new()),
305 shutdown_errors: Vec::new(),
306 request_timeout: WS_REQUEST_TIMEOUT,
307 conn_id: Arc::new(ArcSwap::from_pointee(UUID4::new().to_string())),
308 rate_limiter,
309 socket_control: None,
310 }
311 }
312
313 #[must_use]
314 pub(crate) fn with_book_sync(mut self, tracker: BookSyncTracker, timeout: Duration) -> Self {
315 self.book_sync = Some((tracker, timeout));
316 self
317 }
318
319 #[must_use]
321 pub fn with_socket_control(mut self, control: SocketControl) -> Self {
322 self.socket_control = Some(control);
323 self
324 }
325
326 #[must_use]
328 pub fn url(&self) -> &str {
329 self.url.expose()
330 }
331
332 pub fn set_request_timeout(&mut self, timeout: Duration) {
336 self.request_timeout = timeout;
337 }
338
339 #[must_use]
342 pub fn is_authenticated(&self) -> bool {
343 self.auth_tracker.is_authenticated()
344 && self.is_active()
345 && self.authenticated_epoch.load(Ordering::Acquire)
346 == self.connection_epoch.load().load(Ordering::Acquire)
347 }
348
349 #[must_use]
351 pub fn is_active(&self) -> bool {
352 self.connection_mode.load().load(Ordering::Relaxed) == ConnectionMode::Active as u8
353 }
354
355 pub async fn connect(&mut self) -> Result<()> {
364 let auth_ok = self.credentials.is_none() || self.is_authenticated();
367 if self.is_active() && auth_ok && self.task_handle.is_some() {
368 log::warn!("Derive WebSocket already connected");
369 return Ok(());
370 }
371
372 if self.task_handle.is_some()
374 || !self.send_task.is_empty()
375 || !self.recovery_task.is_empty()
376 {
377 log::debug!("Tearing down stale Derive WebSocket state before connect");
378 self.teardown().await?;
379 }
380
381 self.signal.store(false, Ordering::Release);
382 let setup_guard = DeriveWebSocketSetupGuard::new(self.shutdown_handle());
383
384 self.authenticated_epoch
385 .store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
386
387 let (message_handler, raw_rx) = channel_message_handler();
388 let headers = create_standard_nautilus_headers();
389
390 let cfg = WebSocketConfig {
391 url: self.url.expose().to_owned(),
392 headers,
393 heartbeat_interval_secs: Some(WS_HEARTBEAT_SECS),
394 heartbeat_payload: None,
395 connect_timeout_ms: Some(RECONNECT_TIMEOUT.as_millis() as u64),
396 reconnect_delay_initial_ms: Some(RECONNECT_BASE_BACKOFF.as_millis() as u64),
397 reconnect_delay_max_ms: Some(RECONNECT_MAX_BACKOFF.as_millis() as u64),
398 reconnect_backoff_factor: Some(RECONNECT_BACKOFF_FACTOR),
399 reconnect_jitter_ms: Some(RECONNECT_JITTER_MS),
400 reconnect_max_attempts: None,
401 heartbeat_timeout_secs: Some(WS_HEARTBEAT_TIMEOUT.as_secs()),
402 idle_timeout_ms: None,
403 writer_capacity: None,
404 backend: self.transport_backend,
405 proxy_url: self
406 .proxy_url
407 .as_ref()
408 .map(|value| value.expose_secret().to_owned()),
409 max_message_size_bytes: None,
410 max_frame_size_bytes: None,
411 };
412
413 let client = WebSocketClient::builder()
417 .config(cfg)
418 .message_handler(message_handler)
419 .maybe_state_sink(self.socket_control.as_ref().map(SocketControl::sink))
420 .connect()
421 .await
422 .map_err(|e| DeriveWsError::transport(e.to_string()))?;
423
424 client.set_auth_tracker(self.auth_tracker.clone(), false);
428
429 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
430 let (out_tx, out_rx) = tokio::sync::mpsc::unbounded_channel::<DeriveWsMessage>();
431
432 self.out_rx = Some(out_rx);
433 self.conn_id.store(Arc::new(UUID4::new().to_string()));
434
435 let connection_mode = client.connection_mode_atomic();
436 let connection_epoch = client.connection_epoch_atomic();
437 let reconnect_handle = client.reconnect_handle();
438 if let Err(e) = cmd_tx.send(HandlerCommand::SetClient(client)) {
439 return Err(DeriveWsError::transport(format!(
440 "failed to send SetClient command: {e}",
441 )));
442 }
443
444 {
445 let mut command_sender = self.cmd_tx.write().await;
446 *command_sender = cmd_tx.clone();
447 self.connection_epoch.store(Arc::clone(&connection_epoch));
448 self.connection_mode.store(Arc::clone(&connection_mode));
449 }
450
451 log::debug!("Derive WebSocket connected: {}", self.url.origin());
452
453 let recovery = SessionRecovery {
454 subscription: self.subscription_handle(),
455 cmd_tx: cmd_tx.clone(),
456 auth_tracker: self.auth_tracker.clone(),
457 authenticated_epoch: Arc::clone(&self.authenticated_epoch),
458 connection_mode: Arc::clone(&connection_mode),
459 connection_epoch: Arc::clone(&connection_epoch),
460 credentials: self.credentials.clone(),
461 signal: Arc::clone(&self.signal),
462 };
463
464 let handler = FeedHandler::new_with_send_task(
465 Arc::clone(&self.signal),
466 cmd_rx,
467 raw_rx,
468 Arc::clone(&self.next_id),
469 self.auth_tracker.clone(),
470 Arc::clone(&self.authenticated_epoch),
471 Arc::clone(&self.send_task),
472 )
473 .with_book_sync(self.book_sync.clone());
474
475 if let Err(e) = self.task_handle.spawn(run_session_handler(
476 handler,
477 recovery,
478 out_tx,
479 Arc::clone(&self.recovery_task),
480 Arc::clone(&self.conn_id),
481 )) {
482 let shutdown_result = self.teardown().await;
483 return Err(DeriveWsError::transport(match shutdown_result {
484 Ok(()) => format!("failed to start WebSocket handler task: {e}"),
485 Err(shutdown_error) => format!(
486 "failed to start WebSocket handler task: {e}; startup rollback failed: \
487 {shutdown_error}"
488 ),
489 }));
490 }
491
492 if let Some(control) = &self.socket_control {
493 control.register(move || reconnect_handle.request_reconnect());
494 }
495
496 if let Some(creds) = self.credentials.clone()
497 && let Err(e) = login_via_handler(
498 &self.rate_limiter,
499 &cmd_tx,
500 &self.auth_tracker,
501 &self.authenticated_epoch,
502 &connection_mode,
503 &connection_epoch,
504 &creds,
505 self.request_timeout,
506 )
507 .await
508 {
509 log::warn!(
512 "Derive WebSocket login failed; tearing down transport: {}",
513 e.redacted_message()
514 );
515 self.teardown().await?;
516 return Err(e);
517 }
518
519 setup_guard.disarm();
520 Ok(())
521 }
522
523 pub(crate) fn begin_shutdown(&self) {
524 self.shutdown_handle().begin_shutdown();
525 }
526
527 pub(crate) fn shutdown_handle(&self) -> DeriveWebSocketShutdownHandle {
528 DeriveWebSocketShutdownHandle {
529 signal: Arc::clone(&self.signal),
530 recovery_task: Arc::clone(&self.recovery_task),
531 }
532 }
533
534 async fn teardown(&mut self) -> Result<()> {
538 self.begin_shutdown();
539
540 if let Err(e) = self.cmd_tx.read().await.send(HandlerCommand::Disconnect) {
541 log::debug!(
542 "Failed to enqueue Disconnect command (handler may already be shut down): {e}",
543 );
544 }
545
546 match finish_task(
547 &mut self.task_handle,
548 Duration::from_secs(2),
549 Duration::from_secs(2),
550 )
551 .await
552 {
553 None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => {}
554 Some(TaskJoinOutcome::Failed(error)) => self
555 .shutdown_errors
556 .push(format!("WebSocket handler task failed: {error}")),
557 Some(TaskJoinOutcome::Incomplete) => self
558 .shutdown_errors
559 .push("WebSocket handler task did not stop after abort".to_string()),
560 }
561
562 if self.task_handle.is_some() {
563 if let Some(control) = &self.socket_control {
564 control.deregister();
565 }
566
567 return self.take_shutdown_result();
568 }
569
570 for (name, slot) in [
571 ("recovery", Arc::clone(&self.recovery_task)),
572 ("send", Arc::clone(&self.send_task)),
573 ] {
574 match slot
575 .finish(Duration::from_secs(1), Duration::from_secs(2))
576 .await
577 {
578 None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => {}
579 Some(TaskJoinOutcome::Failed(error)) => self
580 .shutdown_errors
581 .push(format!("WebSocket {name} worker failed: {error}")),
582 Some(TaskJoinOutcome::Incomplete) => self
583 .shutdown_errors
584 .push(format!("WebSocket {name} worker did not stop after abort")),
585 }
586
587 if !slot.is_empty() {
588 if let Some(control) = &self.socket_control {
589 control.deregister();
590 }
591
592 return self.take_shutdown_result();
593 }
594 }
595
596 let (placeholder_tx, _) = tokio::sync::mpsc::unbounded_channel();
599 *self.cmd_tx.write().await = placeholder_tx;
600 self.out_rx = None;
601 self.connection_mode
602 .store(Arc::new(AtomicU8::new(ConnectionMode::Closed as u8)));
603 self.connection_epoch.store(Arc::new(AtomicU64::new(0)));
604 self.auth_tracker.invalidate();
605 self.authenticated_epoch
606 .store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
607 self.subscriptions.clear();
608
609 if let Some((tracker, _)) = &self.book_sync {
610 tracker.clear();
611 }
612
613 self.signal.store(false, Ordering::Relaxed);
614
615 if let Some(control) = &self.socket_control {
616 control.deregister();
617 }
618
619 self.take_shutdown_result()
620 }
621
622 fn take_shutdown_result(&mut self) -> Result<()> {
623 if self.shutdown_errors.is_empty() {
624 Ok(())
625 } else {
626 Err(DeriveWsError::transport(
627 std::mem::take(&mut self.shutdown_errors).join("; "),
628 ))
629 }
630 }
631
632 pub async fn disconnect(&mut self) -> Result<()> {
639 log::debug!("Disconnecting Derive WebSocket");
640 self.begin_shutdown();
641 self.teardown().await
642 }
643
644 pub async fn subscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
651 self.subscription_handle()
652 .subscribe_ticker(instrument_name, interval)
653 .await
654 }
655
656 pub async fn unsubscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
662 self.subscription_handle()
663 .unsubscribe_ticker(instrument_name, interval)
664 .await
665 }
666
667 pub async fn subscribe_orderbook(
673 &self,
674 instrument_name: &str,
675 group: &str,
676 depth: &str,
677 ) -> Result<()> {
678 self.subscription_handle()
679 .subscribe_orderbook(instrument_name, group, depth)
680 .await
681 }
682
683 pub async fn unsubscribe_orderbook(
689 &self,
690 instrument_name: &str,
691 group: &str,
692 depth: &str,
693 ) -> Result<()> {
694 self.subscription_handle()
695 .unsubscribe_orderbook(instrument_name, group, depth)
696 .await
697 }
698
699 pub async fn subscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
705 self.subscription_handle()
706 .subscribe_trades(instrument_type, currency)
707 .await
708 }
709
710 pub async fn unsubscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
716 self.subscription_handle()
717 .unsubscribe_trades(instrument_type, currency)
718 .await
719 }
720
721 pub async fn subscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
731 where
732 C: Into<DeriveWsChannel>,
733 {
734 self.subscription_handle()
735 .subscribe_channels(channels)
736 .await
737 }
738
739 pub async fn unsubscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
746 where
747 C: Into<DeriveWsChannel>,
748 {
749 self.subscription_handle()
750 .unsubscribe_channels(channels)
751 .await
752 }
753
754 pub async fn next_event(&mut self) -> Option<DeriveWsMessage> {
756 if let Some(rx) = self.out_rx.as_mut() {
757 rx.recv().await
758 } else {
759 None
760 }
761 }
762
763 #[must_use]
766 pub fn subscription_count(&self) -> usize {
767 self.subscriptions.len()
768 }
769
770 #[must_use]
772 pub fn subscription_handle(&self) -> DeriveWebSocketSubscriptionHandle {
773 DeriveWebSocketSubscriptionHandle {
774 cmd_tx: Arc::clone(&self.cmd_tx),
775 subscriptions: Arc::clone(&self.subscriptions),
776 subscription_lock: Arc::clone(&self.subscription_lock),
777 book_sync: self.book_sync.clone(),
778 connection_epoch: Arc::clone(&self.connection_epoch),
779 request_timeout: self.request_timeout,
780 rate_limiter: Arc::clone(&self.rate_limiter),
781 }
782 }
783
784 pub async fn set_cancel_on_disconnect(&self, enabled: bool) -> Result<()> {
806 let creds = self
807 .credentials
808 .as_ref()
809 .ok_or_else(|| DeriveWsError::MissingCredentials {
810 operation: methods::PRIVATE_SET_CANCEL_ON_DISCONNECT.to_string(),
811 })?;
812
813 self.execution_handle()
814 .require_authenticated(methods::PRIVATE_SET_CANCEL_ON_DISCONNECT)
815 .await?;
816
817 let params = WsSetCancelOnDisconnectParams {
818 wallet: creds.wallet_address.clone(),
819 enabled,
820 };
821
822 let cmd_tx = self.cmd_tx.read().await.clone();
823 let _: WsSetCancelOnDisconnectResult = send_request_typed(
824 &self.rate_limiter,
825 &cmd_tx,
826 methods::PRIVATE_SET_CANCEL_ON_DISCONNECT,
827 ¶ms,
828 self.request_timeout,
829 )
830 .await?;
831 Ok(())
832 }
833
834 #[must_use]
841 pub fn execution_handle(&self) -> DeriveWsExecutionHandle {
842 DeriveWsExecutionHandle {
843 cmd_tx: Arc::clone(&self.cmd_tx),
844 auth_tracker: self.auth_tracker.clone(),
845 request_timeout: self.request_timeout,
846 conn_id: Arc::clone(&self.conn_id),
847 rate_limiter: Arc::clone(&self.rate_limiter),
848 }
849 }
850
851 pub fn take_event_receiver(
856 &mut self,
857 ) -> Option<tokio::sync::mpsc::UnboundedReceiver<DeriveWsMessage>> {
858 self.out_rx.take()
859 }
860}
861
862impl Drop for DeriveWebSocketClient {
863 fn drop(&mut self) {
864 self.begin_shutdown();
865
866 if let Some(handle) = self.task_handle.as_ref() {
867 handle.abort();
868 }
869
870 if let Some(control) = &self.socket_control {
871 control.deregister();
872 }
873 }
874}
875
876struct SessionRecovery {
877 subscription: DeriveWebSocketSubscriptionHandle,
878 cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
879 auth_tracker: AuthTracker,
880 authenticated_epoch: Arc<AtomicU64>,
881 connection_mode: Arc<AtomicU8>,
882 connection_epoch: Arc<AtomicU64>,
883 credentials: Option<DeriveWsCredentials>,
884 signal: Arc<AtomicBool>,
885}
886
887async fn run_session_handler(
888 mut handler: FeedHandler,
889 recovery: SessionRecovery,
890 out_tx: tokio::sync::mpsc::UnboundedSender<DeriveWsMessage>,
891 recovery_task: Arc<SharedTaskSlot<()>>,
892 conn_id: Arc<ArcSwap<String>>,
893) {
894 let (recovery_tx, recovery_rx) = tokio::sync::mpsc::unbounded_channel();
895 let cmd_tx = recovery.cmd_tx.clone();
896 let connection_epoch = Arc::clone(&recovery.connection_epoch);
897 if let Err(e) = recovery_task.spawn(recovery.run(recovery_rx, out_tx.clone())) {
898 let _ = out_tx.send(DeriveWsMessage::SessionRecoveryFailed(format!(
899 "failed to start recovery worker: {e}"
900 )));
901 let _ = cmd_tx.send(HandlerCommand::Disconnect);
902 return;
903 }
904
905 loop {
906 match handler.next().await {
907 Some(DeriveWsMessage::Reconnected) => {
908 log::info!("Derive WebSocket re-establishing session after reconnect");
909 conn_id.store(Arc::new(UUID4::new().to_string()));
910 let epoch = connection_epoch.load(Ordering::Acquire);
911 if recovery_tx.send(epoch).is_err() {
912 log::error!("Derive WebSocket recovery task stopped unexpectedly");
913 let _ = cmd_tx.send(HandlerCommand::Disconnect);
914 }
915 }
916 Some(msg) => {
917 if out_tx.send(msg).is_err() {
918 log::debug!("Derive outer receiver dropped, exiting stream loop");
919 break;
920 }
921 }
922 None => {
923 log::debug!("Derive handler task ended");
924 break;
925 }
926 }
927 }
928
929 recovery_task.abort();
930}
931
932impl SessionRecovery {
933 async fn run(
934 self,
935 mut rx: tokio::sync::mpsc::UnboundedReceiver<u64>,
936 out_tx: tokio::sync::mpsc::UnboundedSender<DeriveWsMessage>,
937 ) {
938 if self.signal.load(Ordering::Acquire) {
939 return;
940 }
941
942 while let Some(mut requested_epoch) = rx.recv().await {
943 take_latest_epoch(&mut rx, &mut requested_epoch);
944 match self.recover_until_current(&mut rx, requested_epoch).await {
945 Ok(()) => {
946 if out_tx.send(DeriveWsMessage::Reconnected).is_err() {
947 log::debug!("Derive outer receiver dropped during recovery");
948 }
949 }
950 Err(e) => {
951 log::error!(
952 "Derive WebSocket session recovery failed: {}",
953 e.redacted_message()
954 );
955 let _ = out_tx.send(DeriveWsMessage::SessionRecoveryFailed(e.to_string()));
956 let _ = self.cmd_tx.send(HandlerCommand::Disconnect);
957 return;
958 }
959 }
960 }
961 }
962
963 async fn recover_until_current(
964 &self,
965 rx: &mut tokio::sync::mpsc::UnboundedReceiver<u64>,
966 mut requested_epoch: u64,
967 ) -> Result<()> {
968 loop {
969 match recover_session(
970 &self.subscription.rate_limiter,
971 &self.cmd_tx,
972 &self.auth_tracker,
973 &self.authenticated_epoch,
974 &self.connection_mode,
975 &self.connection_epoch,
976 self.credentials.as_ref(),
977 &self.subscription.subscriptions,
978 &self.subscription.subscription_lock,
979 self.subscription.request_timeout,
980 )
981 .await
982 {
983 Ok(recovered_epoch) => {
984 take_latest_epoch(rx, &mut requested_epoch);
985 if requested_epoch > recovered_epoch
986 || !session_connection_is_active(
987 &self.connection_mode,
988 &self.connection_epoch,
989 recovered_epoch,
990 )
991 {
992 continue;
993 }
994
995 return Ok(());
996 }
997 Err(e) => {
998 if take_latest_epoch(rx, &mut requested_epoch) {
999 continue;
1000 }
1001
1002 return Err(e);
1003 }
1004 }
1005 }
1006 }
1007}
1008
1009fn take_latest_epoch(
1010 rx: &mut tokio::sync::mpsc::UnboundedReceiver<u64>,
1011 requested_epoch: &mut u64,
1012) -> bool {
1013 let mut received = false;
1014
1015 while let Ok(epoch) = rx.try_recv() {
1016 *requested_epoch = (*requested_epoch).max(epoch);
1017 received = true;
1018 }
1019
1020 received
1021}
1022
1023#[derive(Clone, Debug)]
1024pub(crate) struct DeriveWebSocketShutdownHandle {
1025 signal: Arc<AtomicBool>,
1026 recovery_task: Arc<SharedTaskSlot<()>>,
1027}
1028
1029impl DeriveWebSocketShutdownHandle {
1030 pub(crate) fn begin_shutdown(&self) {
1031 self.signal.store(true, Ordering::Release);
1032 self.recovery_task.abort();
1033 }
1034}
1035
1036struct DeriveWebSocketSetupGuard {
1037 shutdown: DeriveWebSocketShutdownHandle,
1038 armed: bool,
1039}
1040
1041impl DeriveWebSocketSetupGuard {
1042 fn new(shutdown: DeriveWebSocketShutdownHandle) -> Self {
1043 Self {
1044 shutdown,
1045 armed: true,
1046 }
1047 }
1048
1049 fn disarm(mut self) {
1050 self.armed = false;
1051 }
1052}
1053
1054impl Drop for DeriveWebSocketSetupGuard {
1055 fn drop(&mut self) {
1056 if self.armed {
1057 self.shutdown.begin_shutdown();
1058 }
1059 }
1060}
1061
1062impl DeriveWebSocketSubscriptionHandle {
1063 pub(crate) async fn subscribe_book_gated(&self, channel: String) -> Result<()> {
1064 let _guard = self.subscription_lock.lock().await;
1065 self.book_request_locked(methods::PUBLIC_SUBSCRIBE, channel, None, None)
1066 .await
1067 }
1068
1069 pub(crate) async fn replace_book_gated(
1070 &self,
1071 channel: String,
1072 cancel: CancellationToken,
1073 recovery: &Arc<BookRecovery>,
1074 gate: SnapshotGate,
1075 ) -> Result<()> {
1076 let _guard = tokio::select! {
1077 biased;
1078 () = cancel.cancelled() => return Err(DeriveWsError::NotConnected),
1079 guard = self.subscription_lock.lock() => guard,
1080 };
1081 self.book_sync
1082 .as_ref()
1083 .and_then(|(tracker, _)| tracker.begin_recovery_write(&channel, recovery, gate))
1084 .ok_or(DeriveWsError::NotConnected)?;
1085 let epoch = self.connection_epoch.load().load(Ordering::Acquire);
1086
1087 self.remember_subscription(&channel);
1089 self.book_request_locked(
1090 methods::PUBLIC_UNSUBSCRIBE,
1091 channel.clone(),
1092 Some(cancel.clone()),
1093 Some(epoch),
1094 )
1095 .await?;
1096 self.book_request_locked(
1097 methods::PUBLIC_SUBSCRIBE,
1098 channel,
1099 Some(cancel),
1100 Some(epoch),
1101 )
1102 .await
1103 }
1104
1105 async fn book_request_locked(
1106 &self,
1107 method: &'static str,
1108 channel: String,
1109 cancel: Option<CancellationToken>,
1110 epoch: Option<u64>,
1111 ) -> Result<()> {
1112 let params = WsSubscribeParams {
1113 channels: vec![DeriveWsChannel::from(channel.clone())],
1114 };
1115
1116 self.remember_subscription(&channel);
1117 self.rate_limiter
1118 .await_class_ready(rate_class_for_method(method), None)
1119 .await;
1120
1121 let write = self
1122 .book_sync
1123 .as_ref()
1124 .and_then(|(tracker, _)| tracker.write(&channel))
1125 .ok_or(DeriveWsError::NotConnected)?;
1126 let generation = write.generation.clone();
1127 let cancel = cancel.unwrap_or_else(|| generation.clone());
1128 let epoch = epoch.unwrap_or_else(|| self.connection_epoch.load().load(Ordering::Acquire));
1129
1130 if cancel.is_cancelled() || generation.is_cancelled() {
1131 return Err(DeriveWsError::NotConnected);
1132 }
1133
1134 let cmd_tx = self.cmd_tx.read().await.clone();
1135 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
1136 cmd_tx
1137 .send(HandlerCommand::Request {
1138 method,
1139 params: SecretString::from(serde_json::to_string(¶ms)?),
1140 connection_epoch: Some(epoch),
1141 cancellation: Some(cancel.clone()),
1142 response_tx,
1143 })
1144 .map_err(|_| DeriveWsError::NotConnected)?;
1145
1146 let result = tokio::select! {
1147 biased;
1148 () = cancel.cancelled() => return Err(DeriveWsError::NotConnected),
1149 () = generation.cancelled() => return Err(DeriveWsError::NotConnected),
1150 result = time::timeout(self.request_timeout, response_rx) => result,
1151 };
1152
1153 let value = result
1154 .map_err(|_| DeriveWsError::Timeout {
1155 method: method.to_owned(),
1156 })?
1157 .map_err(|_| DeriveWsError::RequestCancelled {
1158 method: method.to_owned(),
1159 })??;
1160
1161 if method == methods::PUBLIC_SUBSCRIBE {
1162 let result: WsSubscribeResult = decode_default_result(&value)?;
1163 subscription_outcome(
1164 ¶ms.channels,
1165 &result,
1166 SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES,
1167 &self.subscriptions,
1168 )
1169 } else {
1170 let result: WsUnsubscribeResult = decode_default_result(&value)?;
1171
1172 let topic = DeriveWsChannel::from(channel.clone());
1174
1175 if result
1176 .status
1177 .get(&topic)
1178 .is_some_and(|status| status.as_str() == "ok")
1179 && !result.remaining_subscriptions.contains(&topic)
1180 {
1181 Ok(())
1182 } else {
1183 Err(DeriveWsError::Subscription {
1184 details: format!("{channel}: unsubscribe not confirmed"),
1185 })
1186 }
1187 }
1188 }
1189
1190 pub(crate) fn has_subscription(&self, channel: &str) -> bool {
1191 let (name, symbol) = channel.split_once('.').unwrap_or((channel, ""));
1192 self.subscriptions
1193 .is_subscribed(&Ustr::from(name), &Ustr::from(symbol))
1194 }
1195
1196 pub(crate) fn forget_subscription(&self, channel: &str) {
1197 self.subscriptions.mark_unsubscribe(channel);
1198 self.subscriptions.confirm_unsubscribe(channel);
1199 }
1200
1201 pub(crate) fn remember_subscription(&self, channel: &str) {
1202 self.subscriptions.mark_subscribe(channel);
1203 }
1204
1205 pub async fn subscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
1211 let params = ticker_subscribe_params(instrument_name, interval);
1212 self.subscribe_channels(params.channels).await
1213 }
1214
1215 pub async fn unsubscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
1221 let channel = ticker_channel(instrument_name, interval);
1222 self.send_unsubscribe(channel).await
1223 }
1224
1225 pub async fn subscribe_orderbook(
1231 &self,
1232 instrument_name: &str,
1233 group: &str,
1234 depth: &str,
1235 ) -> Result<()> {
1236 let params = orderbook_subscribe_params(instrument_name, group, depth);
1237 self.subscribe_channels(params.channels).await
1238 }
1239
1240 pub async fn unsubscribe_orderbook(
1246 &self,
1247 instrument_name: &str,
1248 group: &str,
1249 depth: &str,
1250 ) -> Result<()> {
1251 let channel = orderbook_channel(instrument_name, group, depth);
1252 self.send_unsubscribe(channel).await
1253 }
1254
1255 pub async fn subscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
1261 let params = trades_subscribe_params(instrument_type, currency);
1262 self.subscribe_channels(params.channels).await
1263 }
1264
1265 pub async fn unsubscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
1271 let channel = trades_channel(instrument_type, currency);
1272 self.send_unsubscribe(channel).await
1273 }
1274
1275 pub async fn subscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
1281 where
1282 C: Into<DeriveWsChannel>,
1283 {
1284 let channels = channels.into_iter().map(Into::into).collect::<Vec<_>>();
1285 if channels.is_empty() {
1286 return Ok(());
1287 }
1288
1289 let _guard = self.subscription_lock.lock().await;
1290 let params = WsSubscribeParams { channels };
1291 for topic in channel_topics(¶ms.channels) {
1292 self.subscriptions.mark_subscribe(&topic);
1293 }
1294
1295 let cmd_tx = self.cmd_tx.read().await.clone();
1296 let result: WsSubscribeResult = send_request(
1297 &self.rate_limiter,
1298 &cmd_tx,
1299 methods::PUBLIC_SUBSCRIBE,
1300 ¶ms,
1301 self.request_timeout,
1302 )
1303 .await?;
1304
1305 subscription_outcome(
1306 ¶ms.channels,
1307 &result,
1308 SUBSCRIPTION_ACCEPTED_STATUSES,
1309 &self.subscriptions,
1310 )
1311 }
1312
1313 pub async fn unsubscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
1320 where
1321 C: Into<DeriveWsChannel>,
1322 {
1323 let channels = channels.into_iter().map(Into::into).collect::<Vec<_>>();
1324 if channels.is_empty() {
1325 return Ok(());
1326 }
1327
1328 let _guard = self.subscription_lock.lock().await;
1329
1330 for topic in channel_topics(&channels) {
1331 self.subscriptions.mark_unsubscribe(&topic);
1332 }
1333
1334 let params = WsUnsubscribeParams { channels };
1335 let cmd_tx = self.cmd_tx.read().await.clone();
1336 let result: WsUnsubscribeResult = send_request(
1337 &self.rate_limiter,
1338 &cmd_tx,
1339 methods::PUBLIC_UNSUBSCRIBE,
1340 ¶ms,
1341 self.request_timeout,
1342 )
1343 .await?;
1344
1345 unsubscribe_outcome(¶ms.channels, &result, &self.subscriptions)
1346 }
1347
1348 async fn send_unsubscribe(&self, channel: String) -> Result<()> {
1349 self.unsubscribe_channels(vec![DeriveWsChannel::from(channel)])
1350 .await
1351 }
1352}
1353
1354impl DeriveWsExecutionHandle {
1355 #[must_use]
1357 pub fn conn_id(&self) -> String {
1358 self.conn_id.load_full().as_ref().clone()
1359 }
1360
1361 pub async fn submit_order(&self, params: &DeriveOrderParams) -> Result<DeriveOrder> {
1372 let reservation = self
1373 .reserve_matching_request(methods::PRIVATE_ORDER, ¶ms.instrument_name)
1374 .await?;
1375 self.submit_order_after_rate_limit(params, reservation)
1376 .await
1377 .map(|result| result.order)
1378 }
1379
1380 pub(crate) async fn submit_order_after_rate_limit(
1381 &self,
1382 params: &DeriveOrderParams,
1383 reservation: MatchingRateLimitReservation,
1384 ) -> Result<DeriveOrderResult> {
1385 self.ensure_authenticated(methods::PRIVATE_ORDER)?;
1386 debug_assert_eq!(reservation.method, methods::PRIVATE_ORDER);
1387 let cmd_tx = self.cmd_tx.read().await.clone();
1388 self.ensure_authenticated(reservation.method)?;
1389 self.commit_matching_reservation(reservation)?;
1390 let result: DeriveOrderResult = send_request_typed_after_rate_limit(
1391 &self.rate_limiter,
1392 &cmd_tx,
1393 methods::PRIVATE_ORDER,
1394 params,
1395 self.request_timeout,
1396 )
1397 .await?;
1398 Ok(result)
1399 }
1400
1401 pub async fn submit_trigger_order(
1408 &self,
1409 params: &DeriveTriggerOrderParams,
1410 ) -> Result<DeriveOrder> {
1411 let reservation = self
1412 .reserve_matching_request(
1413 methods::PRIVATE_TRIGGER_ORDER,
1414 ¶ms.order.instrument_name,
1415 )
1416 .await?;
1417 self.submit_trigger_order_after_rate_limit(params, reservation)
1418 .await
1419 .map(|result| result.order)
1420 }
1421
1422 pub(crate) async fn submit_trigger_order_after_rate_limit(
1423 &self,
1424 params: &DeriveTriggerOrderParams,
1425 reservation: MatchingRateLimitReservation,
1426 ) -> Result<DeriveOrderResult> {
1427 self.submit_order_after_rate_limit(¶ms.order, reservation)
1428 .await
1429 }
1430
1431 pub async fn modify_order(&self, params: &DeriveReplaceParams) -> Result<DeriveReplaceOutcome> {
1439 let reservation = self
1440 .reserve_matching_request(methods::PRIVATE_REPLACE, ¶ms.order.instrument_name)
1441 .await?;
1442 self.modify_order_after_rate_limit(params, reservation)
1443 .await
1444 .and_then(|response| {
1445 response
1446 .result
1447 .and_then(|result| {
1448 result.into_outcome(
1449 ¶ms.order_id_to_cancel,
1450 ¶ms.order.label,
1451 params.order.envelope.subaccount_id,
1452 params.order.instrument_name.as_str(),
1453 )
1454 })
1455 .map_err(|message| {
1456 DeriveWsError::Serde(<serde_json::Error as serde::de::Error>::custom(
1457 message,
1458 ))
1459 })
1460 })
1461 }
1462
1463 pub(crate) async fn modify_order_after_rate_limit(
1464 &self,
1465 params: &DeriveReplaceParams,
1466 reservation: MatchingRateLimitReservation,
1467 ) -> Result<DeriveReplaceResponse> {
1468 self.ensure_authenticated(methods::PRIVATE_REPLACE)?;
1469 debug_assert_eq!(reservation.method, methods::PRIVATE_REPLACE);
1470 let cmd_tx = self.cmd_tx.read().await.clone();
1471 self.ensure_authenticated(reservation.method)?;
1472 self.commit_matching_reservation(reservation)?;
1473 let response = send_raw_after_rate_limit(
1474 &self.rate_limiter,
1475 &cmd_tx,
1476 methods::PRIVATE_REPLACE,
1477 params,
1478 self.request_timeout,
1479 )
1480 .await?;
1481 Ok(DeriveReplaceResponse::from_raw(&response))
1482 }
1483
1484 pub async fn cancel_order(&self, params: &DeriveCancelParams) -> Result<()> {
1490 self.require_authenticated(methods::PRIVATE_CANCEL).await?;
1491 let cmd_tx = self.cmd_tx.read().await.clone();
1492 let _: DeriveEmptyResult = send_request_for_instrument(
1493 &self.rate_limiter,
1494 &cmd_tx,
1495 methods::PRIVATE_CANCEL,
1496 params,
1497 self.request_timeout,
1498 params.instrument_name,
1499 )
1500 .await?;
1501 Ok(())
1502 }
1503
1504 pub async fn cancel_by_instrument(
1510 &self,
1511 params: &DeriveCancelByInstrumentParams,
1512 ) -> Result<DeriveCancelByInstrumentResult> {
1513 self.require_authenticated(methods::PRIVATE_CANCEL_BY_INSTRUMENT)
1514 .await?;
1515 let cmd_tx = self.cmd_tx.read().await.clone();
1516 send_request_typed_for_instrument(
1517 &self.rate_limiter,
1518 &cmd_tx,
1519 methods::PRIVATE_CANCEL_BY_INSTRUMENT,
1520 params,
1521 self.request_timeout,
1522 params.instrument_name,
1523 )
1524 .await
1525 }
1526
1527 pub async fn cancel_trigger_order(
1533 &self,
1534 params: &DeriveCancelTriggerOrderParams,
1535 ) -> Result<DeriveOrder> {
1536 self.require_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)
1537 .await?;
1538 let cmd_tx = self.cmd_tx.read().await.clone();
1539 send_request_typed(
1540 &self.rate_limiter,
1541 &cmd_tx,
1542 methods::PRIVATE_CANCEL_TRIGGER_ORDER,
1543 params,
1544 self.request_timeout,
1545 )
1546 .await
1547 }
1548
1549 pub(crate) async fn cancel_trigger_order_current(
1550 &self,
1551 trigger: &DeriveCancelTriggerOrderParams,
1552 regular: &DeriveCancelParams,
1553 is_active: impl Fn() -> bool,
1554 ) -> Result<Option<DeriveOrder>> {
1555 self.require_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)
1556 .await?;
1557 self.rate_limiter
1558 .await_class_ready(RateClass::NonMatching, None)
1559 .await;
1560 let cmd_tx = self.cmd_tx.read().await.clone();
1561
1562 if is_active() {
1563 return self.cancel_order(regular).await.map(|()| None);
1564 }
1565
1566 self.ensure_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)?;
1567 send_request_typed_after_rate_limit(
1568 &self.rate_limiter,
1569 &cmd_tx,
1570 methods::PRIVATE_CANCEL_TRIGGER_ORDER,
1571 trigger,
1572 self.request_timeout,
1573 )
1574 .await
1575 .map(Some)
1576 }
1577
1578 pub async fn cancel_by_label(
1585 &self,
1586 params: &DeriveCancelByLabelParams,
1587 ) -> Result<DeriveCancelByLabelResult> {
1588 self.require_authenticated(methods::PRIVATE_CANCEL_BY_LABEL)
1589 .await?;
1590 let cmd_tx = self.cmd_tx.read().await.clone();
1591 send_request_typed(
1592 &self.rate_limiter,
1593 &cmd_tx,
1594 methods::PRIVATE_CANCEL_BY_LABEL,
1595 params,
1596 self.request_timeout,
1597 )
1598 .await
1599 }
1600
1601 pub async fn get_trigger_orders(
1608 &self,
1609 params: &DeriveGetTriggerOrdersParams,
1610 ) -> Result<DeriveOpenOrdersResult> {
1611 self.require_authenticated(methods::PRIVATE_GET_TRIGGER_ORDERS)
1612 .await?;
1613 let cmd_tx = self.cmd_tx.read().await.clone();
1614 send_request_typed(
1615 &self.rate_limiter,
1616 &cmd_tx,
1617 methods::PRIVATE_GET_TRIGGER_ORDERS,
1618 params,
1619 self.request_timeout,
1620 )
1621 .await
1622 }
1623
1624 pub async fn cancel_all_orders(&self, params: &DeriveCancelAllParams) -> Result<()> {
1631 self.require_authenticated(methods::PRIVATE_CANCEL_ALL)
1632 .await?;
1633 let cmd_tx = self.cmd_tx.read().await.clone();
1634 let _: DeriveEmptyResult = send_request(
1635 &self.rate_limiter,
1636 &cmd_tx,
1637 methods::PRIVATE_CANCEL_ALL,
1638 params,
1639 self.request_timeout,
1640 )
1641 .await?;
1642 Ok(())
1643 }
1644
1645 pub(crate) async fn reserve_matching_request(
1646 &self,
1647 operation: &'static str,
1648 instrument_name: &Ustr,
1649 ) -> Result<MatchingRateLimitReservation> {
1650 self.require_authenticated(operation).await?;
1651 debug_assert_eq!(rate_class_for_method(operation), RateClass::Matching);
1652 let quota = self
1653 .rate_limiter
1654 .reserve_matching_ready(instrument_name)
1655 .await;
1656 self.ensure_authenticated(operation)?;
1657 Ok(MatchingRateLimitReservation {
1658 method: operation,
1659 quota,
1660 })
1661 }
1662
1663 fn commit_matching_reservation(&self, reservation: MatchingRateLimitReservation) -> Result<()> {
1664 reservation
1665 .quota
1666 .commit()
1667 .map_err(|retry_after| DeriveWsError::RateLimited { retry_after })
1668 }
1669
1670 fn ensure_authenticated(&self, operation: &'static str) -> Result<()> {
1671 if self.auth_tracker.is_authenticated() {
1672 return Ok(());
1673 }
1674
1675 Err(DeriveWsError::Authentication {
1676 operation: operation.to_string(),
1677 reason: "WebSocket session is not authenticated".to_string(),
1678 })
1679 }
1680
1681 async fn require_authenticated(&self, operation: &'static str) -> Result<()> {
1682 if self
1683 .auth_tracker
1684 .wait_for_authenticated(self.request_timeout)
1685 .await
1686 {
1687 return Ok(());
1688 }
1689
1690 Err(DeriveWsError::Authentication {
1691 operation: operation.to_string(),
1692 reason: "WebSocket session is not authenticated".to_string(),
1693 })
1694 }
1695}
1696
1697#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1701enum RequestRateLimit {
1702 Await(Option<Ustr>),
1705 Reserved,
1708}
1709
1710async fn send_raw<P>(
1711 rate_limiter: &WsRateLimiter,
1712 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1713 method: &'static str,
1714 params: &P,
1715 timeout: Duration,
1716) -> Result<Box<RawValue>>
1717where
1718 P: Serialize + ?Sized,
1719{
1720 send_raw_with_rate_limit(
1721 rate_limiter,
1722 cmd_tx,
1723 method,
1724 params,
1725 timeout,
1726 RequestRateLimit::Await(None),
1727 None,
1728 )
1729 .await
1730}
1731
1732async fn send_raw_for_instrument<P>(
1735 rate_limiter: &WsRateLimiter,
1736 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1737 method: &'static str,
1738 params: &P,
1739 timeout: Duration,
1740 instrument_name: Ustr,
1741) -> Result<Box<RawValue>>
1742where
1743 P: Serialize + ?Sized,
1744{
1745 send_raw_with_rate_limit(
1746 rate_limiter,
1747 cmd_tx,
1748 method,
1749 params,
1750 timeout,
1751 RequestRateLimit::Await(Some(instrument_name)),
1752 None,
1753 )
1754 .await
1755}
1756
1757async fn send_raw_after_rate_limit<P>(
1758 rate_limiter: &WsRateLimiter,
1759 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1760 method: &'static str,
1761 params: &P,
1762 timeout: Duration,
1763) -> Result<Box<RawValue>>
1764where
1765 P: Serialize + ?Sized,
1766{
1767 send_raw_with_rate_limit(
1768 rate_limiter,
1769 cmd_tx,
1770 method,
1771 params,
1772 timeout,
1773 RequestRateLimit::Reserved,
1774 None,
1775 )
1776 .await
1777}
1778
1779async fn send_raw_with_rate_limit<P>(
1780 rate_limiter: &WsRateLimiter,
1781 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1782 method: &'static str,
1783 params: &P,
1784 timeout: Duration,
1785 rate_limit: RequestRateLimit,
1786 connection_epoch: Option<u64>,
1787) -> Result<Box<RawValue>>
1788where
1789 P: Serialize + ?Sized,
1790{
1791 let params = SecretString::from(serde_json::to_string(params)?);
1792
1793 if let RequestRateLimit::Await(instrument_name) = rate_limit {
1794 rate_limiter
1795 .await_class_ready(rate_class_for_method(method), instrument_name.as_ref())
1796 .await;
1797 }
1798
1799 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
1800 cmd_tx
1801 .send(HandlerCommand::Request {
1802 method,
1803 params,
1804 connection_epoch,
1805 cancellation: None,
1806 response_tx,
1807 })
1808 .map_err(|_| DeriveWsError::NotConnected)?;
1809
1810 match time::timeout(timeout, response_rx).await {
1814 Ok(Ok(outcome)) => outcome,
1815 Ok(Err(_)) => Err(DeriveWsError::RequestCancelled {
1816 method: method.to_owned(),
1817 }),
1818 Err(_) => Err(DeriveWsError::Timeout {
1819 method: method.to_owned(),
1820 }),
1821 }
1822}
1823
1824async fn send_request<P, R>(
1827 rate_limiter: &WsRateLimiter,
1828 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1829 method: &'static str,
1830 params: &P,
1831 timeout: Duration,
1832) -> Result<R>
1833where
1834 P: Serialize + ?Sized,
1835 R: Default + DeserializeOwned,
1836{
1837 let value = send_raw(rate_limiter, cmd_tx, method, params, timeout).await?;
1838 decode_default_result(&value)
1839}
1840
1841async fn send_request_for_instrument<P, R>(
1844 rate_limiter: &WsRateLimiter,
1845 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1846 method: &'static str,
1847 params: &P,
1848 timeout: Duration,
1849 instrument_name: Ustr,
1850) -> Result<R>
1851where
1852 P: Serialize + ?Sized,
1853 R: Default + DeserializeOwned,
1854{
1855 let value = send_raw_for_instrument(
1856 rate_limiter,
1857 cmd_tx,
1858 method,
1859 params,
1860 timeout,
1861 instrument_name,
1862 )
1863 .await?;
1864 decode_default_result(&value)
1865}
1866
1867async fn send_request_typed_for_instrument<P, R>(
1869 rate_limiter: &WsRateLimiter,
1870 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1871 method: &'static str,
1872 params: &P,
1873 timeout: Duration,
1874 instrument_name: Ustr,
1875) -> Result<R>
1876where
1877 P: Serialize + ?Sized,
1878 R: DeserializeOwned,
1879{
1880 let value = send_raw_for_instrument(
1881 rate_limiter,
1882 cmd_tx,
1883 method,
1884 params,
1885 timeout,
1886 instrument_name,
1887 )
1888 .await?;
1889 Ok(serde_json::from_str(value.get())?)
1890}
1891
1892fn decode_default_result<R>(value: &RawValue) -> Result<R>
1893where
1894 R: Default + DeserializeOwned,
1895{
1896 if value.get() == "null" {
1897 Ok(R::default())
1898 } else {
1899 Ok(serde_json::from_str(value.get())?)
1900 }
1901}
1902
1903async fn send_request_on_connection<P, R>(
1904 rate_limiter: &WsRateLimiter,
1905 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1906 method: &'static str,
1907 params: &P,
1908 timeout: Duration,
1909 connection_epoch: u64,
1910) -> Result<R>
1911where
1912 P: Serialize + ?Sized,
1913 R: Default + DeserializeOwned,
1914{
1915 let value = send_raw_with_rate_limit(
1916 rate_limiter,
1917 cmd_tx,
1918 method,
1919 params,
1920 timeout,
1921 RequestRateLimit::Await(None),
1922 Some(connection_epoch),
1923 )
1924 .await?;
1925
1926 decode_default_result(&value)
1927}
1928
1929async fn send_request_typed<P, R>(
1932 rate_limiter: &WsRateLimiter,
1933 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1934 method: &'static str,
1935 params: &P,
1936 timeout: Duration,
1937) -> Result<R>
1938where
1939 P: Serialize + ?Sized,
1940 R: DeserializeOwned,
1941{
1942 let value = send_raw(rate_limiter, cmd_tx, method, params, timeout).await?;
1943 Ok(serde_json::from_str(value.get())?)
1944}
1945
1946async fn send_request_typed_after_rate_limit<P, R>(
1947 rate_limiter: &WsRateLimiter,
1948 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1949 method: &'static str,
1950 params: &P,
1951 timeout: Duration,
1952) -> Result<R>
1953where
1954 P: Serialize + ?Sized,
1955 R: DeserializeOwned,
1956{
1957 let value = send_raw_after_rate_limit(rate_limiter, cmd_tx, method, params, timeout).await?;
1958 Ok(serde_json::from_str(value.get())?)
1959}
1960
1961fn channel_topics(channels: &[DeriveWsChannel]) -> Vec<String> {
1962 channels.iter().map(ToString::to_string).collect()
1963}
1964
1965#[expect(
1966 clippy::too_many_arguments,
1967 reason = "authentication state is passed explicitly for epoch fencing"
1968)]
1969async fn login_via_handler(
1970 rate_limiter: &WsRateLimiter,
1971 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1972 auth_tracker: &AuthTracker,
1973 authenticated_epoch: &AtomicU64,
1974 connection_mode: &AtomicU8,
1975 connection_epoch: &AtomicU64,
1976 creds: &DeriveWsCredentials,
1977 timeout: Duration,
1978) -> Result<()> {
1979 let _receiver = auth_tracker.begin();
1980 let expected_epoch = connection_epoch.load(Ordering::Acquire);
1981
1982 match send_login_request(rate_limiter, cmd_tx, creds, timeout, expected_epoch).await {
1983 Ok(())
1984 if complete_session_authentication(
1985 auth_tracker,
1986 authenticated_epoch,
1987 connection_mode,
1988 connection_epoch,
1989 expected_epoch,
1990 ) =>
1991 {
1992 log::debug!("Derive WebSocket authenticated");
1993
1994 Ok(())
1995 }
1996 Ok(()) => {
1997 let e = DeriveWsError::transport(
1998 "connection changed while completing WebSocket authentication",
1999 );
2000 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2001 auth_tracker.fail(e.to_string());
2002 Err(e)
2003 }
2004 Err(e) => {
2005 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2006 auth_tracker.fail(e.to_string());
2007 Err(e)
2008 }
2009 }
2010}
2011
2012async fn send_login_request(
2013 rate_limiter: &WsRateLimiter,
2014 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2015 creds: &DeriveWsCredentials,
2016 timeout: Duration,
2017 connection_epoch: u64,
2018) -> Result<()> {
2019 rate_limiter.await_public_ready().await;
2020 let login = build_ws_login(&creds.wallet_address, &creds.signer)?;
2021
2022 let params = Zeroizing::new(WsLoginParams {
2023 wallet: login.wallet,
2024 timestamp: login.timestamp,
2025 signature: login.signature,
2026 });
2027
2028 let value = send_raw_with_rate_limit(
2029 rate_limiter,
2030 cmd_tx,
2031 methods::PUBLIC_LOGIN,
2032 &*params,
2033 timeout,
2034 RequestRateLimit::Reserved,
2035 Some(connection_epoch),
2036 )
2037 .await?;
2038
2039 let _: WsLoginResult =
2040 serde_json::from_str(value.get()).map_err(|e| DeriveWsError::Authentication {
2041 operation: methods::PUBLIC_LOGIN.to_string(),
2042 reason: format!("invalid login result: {e}"),
2043 })?;
2044
2045 Ok(())
2046}
2047
2048#[expect(
2049 clippy::too_many_arguments,
2050 reason = "recovery state is passed explicitly for epoch fencing"
2051)]
2052async fn recover_session(
2053 rate_limiter: &WsRateLimiter,
2054 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2055 auth_tracker: &AuthTracker,
2056 authenticated_epoch: &AtomicU64,
2057 connection_mode: &AtomicU8,
2058 connection_epoch: &AtomicU64,
2059 creds: Option<&DeriveWsCredentials>,
2060 subscriptions: &SubscriptionState,
2061 subscription_lock: &tokio::sync::Mutex<()>,
2062 timeout: Duration,
2063) -> Result<u64> {
2064 let _guard = subscription_lock.lock().await;
2065 let mut channels = subscriptions.reset_after_reconnect();
2066 channels.sort();
2067 let _receiver = creds.map(|_| auth_tracker.begin());
2068
2069 for attempt in 1..=MAX_SESSION_RECOVERY_ATTEMPTS {
2070 let expected_epoch = wait_for_session_connection(connection_mode, connection_epoch).await?;
2071
2072 let result = recover_session_attempt(
2073 rate_limiter,
2074 cmd_tx,
2075 connection_mode,
2076 connection_epoch,
2077 creds,
2078 subscriptions,
2079 channels.clone(),
2080 timeout,
2081 expected_epoch,
2082 )
2083 .await;
2084
2085 match result {
2086 Ok(()) => {
2087 if creds.is_some()
2088 && !complete_session_authentication(
2089 auth_tracker,
2090 authenticated_epoch,
2091 connection_mode,
2092 connection_epoch,
2093 expected_epoch,
2094 )
2095 {
2096 continue;
2097 }
2098
2099 if creds.is_some() {
2100 log::info!("Derive WebSocket session re-authenticated");
2101 }
2102
2103 return Ok(expected_epoch);
2104 }
2105 Err(e) if attempt < MAX_SESSION_RECOVERY_ATTEMPTS && should_retry_ws_error(&e) => {
2106 let multiplier = 1_u32 << (attempt - 1);
2107 let delay = RECONNECT_BASE_BACKOFF
2108 .saturating_mul(multiplier)
2109 .min(RECONNECT_MAX_BACKOFF);
2110 log::warn!(
2111 "Derive WebSocket session recovery attempt {attempt}/{MAX_SESSION_RECOVERY_ATTEMPTS} failed: {}; retrying in {delay:?}",
2112 e.redacted_message(),
2113 );
2114 time::sleep(delay).await;
2115 }
2116 Err(e) => {
2117 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2118
2119 if creds.is_some() {
2120 auth_tracker.fail(e.to_string());
2121 }
2122
2123 return Err(e);
2124 }
2125 }
2126 }
2127
2128 let e = DeriveWsError::transport("WebSocket session changed while recovery completed");
2129 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2130
2131 if creds.is_some() {
2132 auth_tracker.fail(e.to_string());
2133 }
2134
2135 Err(e)
2136}
2137
2138#[expect(
2139 clippy::too_many_arguments,
2140 reason = "a session attempt retains its fixed connection epoch and native replay context"
2141)]
2142async fn recover_session_attempt(
2143 rate_limiter: &WsRateLimiter,
2144 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2145 connection_mode: &AtomicU8,
2146 connection_epoch: &AtomicU64,
2147 creds: Option<&DeriveWsCredentials>,
2148 subscriptions: &SubscriptionState,
2149 channels: Vec<String>,
2150 timeout: Duration,
2151 expected_epoch: u64,
2152) -> Result<()> {
2153 if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2154 return Err(DeriveWsError::transport(
2155 "connection changed before WebSocket session recovery",
2156 ));
2157 }
2158
2159 if let Some(creds) = creds {
2160 send_login_request(rate_limiter, cmd_tx, creds, timeout, expected_epoch).await?;
2161 }
2162
2163 subscribe_via_handler(
2164 rate_limiter,
2165 cmd_tx,
2166 channels,
2167 timeout,
2168 expected_epoch,
2169 subscriptions,
2170 )
2171 .await?;
2172
2173 if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2174 return Err(DeriveWsError::transport(
2175 "connection changed during WebSocket session recovery",
2176 ));
2177 }
2178
2179 Ok(())
2180}
2181
2182async fn wait_for_session_connection(
2183 connection_mode: &AtomicU8,
2184 connection_epoch: &AtomicU64,
2185) -> Result<u64> {
2186 loop {
2187 match ConnectionMode::from_atomic(connection_mode) {
2188 ConnectionMode::Active => return Ok(connection_epoch.load(Ordering::Acquire)),
2189 ConnectionMode::Reconnect => time::sleep(RECONNECT_BASE_BACKOFF).await,
2190 ConnectionMode::Disconnect | ConnectionMode::Closed => {
2191 return Err(DeriveWsError::transport(
2192 "WebSocket closed during session recovery",
2193 ));
2194 }
2195 }
2196 }
2197}
2198
2199fn complete_session_authentication(
2200 auth_tracker: &AuthTracker,
2201 authenticated_epoch: &AtomicU64,
2202 connection_mode: &AtomicU8,
2203 connection_epoch: &AtomicU64,
2204 expected_epoch: u64,
2205) -> bool {
2206 if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2207 return false;
2208 }
2209
2210 authenticated_epoch.store(expected_epoch, Ordering::Release);
2211 auth_tracker.succeed();
2212
2213 if session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2214 true
2215 } else {
2216 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2217 auth_tracker.invalidate();
2218 false
2219 }
2220}
2221
2222fn session_connection_is_active(
2223 connection_mode: &AtomicU8,
2224 connection_epoch: &AtomicU64,
2225 expected_epoch: u64,
2226) -> bool {
2227 ConnectionMode::from_atomic(connection_mode).is_active()
2228 && connection_epoch.load(Ordering::Acquire) == expected_epoch
2229}
2230
2231async fn subscribe_via_handler(
2232 rate_limiter: &WsRateLimiter,
2233 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2234 channels: Vec<String>,
2235 timeout: Duration,
2236 connection_epoch: u64,
2237 subscriptions: &SubscriptionState,
2238) -> Result<()> {
2239 if channels.is_empty() {
2240 return Ok(());
2241 }
2242
2243 let params = WsSubscribeParams {
2244 channels: channels.into_iter().map(DeriveWsChannel::from).collect(),
2245 };
2246
2247 let result: WsSubscribeResult = send_request_on_connection(
2248 rate_limiter,
2249 cmd_tx,
2250 methods::PUBLIC_SUBSCRIBE,
2251 ¶ms,
2252 timeout,
2253 connection_epoch,
2254 )
2255 .await?;
2256
2257 subscription_outcome(
2258 ¶ms.channels,
2259 &result,
2260 SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES,
2261 subscriptions,
2262 )
2263}
2264
2265fn subscription_outcome(
2266 requested: &[DeriveWsChannel],
2267 result: &WsSubscribeResult,
2268 accepted_statuses: &[&str],
2269 subscriptions: &SubscriptionState,
2270) -> Result<()> {
2271 let mut failures = Vec::new();
2272
2273 for channel in requested {
2274 let topic = channel.to_string();
2275 match result.status.get(channel) {
2276 Some(status)
2277 if accepted_statuses.contains(&status.as_str())
2278 && result.channels.contains(channel) =>
2279 {
2280 subscriptions.confirm_subscribe(&topic);
2281 }
2282 Some(status) if accepted_statuses.contains(&status.as_str()) => {
2283 subscriptions.mark_failure(&topic);
2284 failures.push(format!("{topic}: not subscribed"));
2285 }
2286 other => {
2287 subscriptions.mark_failure(&topic);
2288 let reason = other.map_or("missing channel status", |status| status.as_str());
2289 failures.push(format!("{topic}: {reason}"));
2290 }
2291 }
2292 }
2293
2294 if failures.is_empty() {
2295 Ok(())
2296 } else {
2297 Err(DeriveWsError::Subscription {
2298 details: failures.join(", "),
2299 })
2300 }
2301}
2302
2303fn unsubscribe_outcome(
2304 requested: &[DeriveWsChannel],
2305 result: &WsUnsubscribeResult,
2306 subscriptions: &SubscriptionState,
2307) -> Result<()> {
2308 let mut failures = Vec::new();
2309
2310 for channel in requested {
2311 let topic = channel.to_string();
2312 match result.status.get(channel) {
2313 Some(status) if status == "ok" && !result.remaining_subscriptions.contains(channel) => {
2314 subscriptions.confirm_unsubscribe(&topic);
2315 }
2316 Some(status) if status == "ok" => failures.push(format!("{topic}: still subscribed")),
2317 Some(status) => failures.push(format!("{topic}: {status}")),
2318 None => failures.push(format!("{topic}: missing channel status")),
2319 }
2320 }
2321
2322 if failures.is_empty() {
2323 Ok(())
2324 } else {
2325 Err(DeriveWsError::Subscription {
2326 details: failures.join(", "),
2327 })
2328 }
2329}
2330
2331pub(crate) struct DeriveReplaceResponse {
2332 pub result: std::result::Result<DeriveReplaceResult, String>,
2333 pub trades: Vec<DeriveTrade>,
2334}
2335
2336impl DeriveReplaceResponse {
2337 fn from_raw(response: &RawValue) -> Self {
2338 #[derive(Deserialize)]
2339 struct Trades {
2340 #[serde(default)]
2341 trades: Vec<Box<RawValue>>,
2342 }
2343 let trades = serde_json::from_str::<Trades>(response.get())
2344 .map_or_default(|response| salvage_rows(response.trades));
2345 let result = serde_json::from_str(response.get()).map_err(|e| e.to_string());
2346 Self { result, trades }
2347 }
2348}
2349
2350#[cfg(test)]
2351mod tests {
2352 use std::{env, time::SystemTime};
2353
2354 use axum::{Json, Router, routing::post};
2355 use futures_util::StreamExt;
2356 use nautilus_common::live::dst::time::Instant;
2357 use nautilus_model::identifiers::InstrumentId;
2358 use nautilus_testkit::connection::race_command_with_publication;
2359 use rstest::rstest;
2360 use serde_json::Value;
2361
2362 use super::*;
2363 use crate::{
2364 common::{
2365 consts::REST_URL_TESTNET,
2366 rate_limit::{DERIVE_RATE_WINDOW_SECS, RateLimits, RequestLimiter},
2367 retry::{is_fatal_ws_error, is_write_outcome_ambiguous_ws},
2368 },
2369 http::{DeriveHttpClient, client::DeriveCredentials},
2370 };
2371
2372 #[tokio::test]
2373 async fn command_during_sender_publication_reaches_initialized_handler() {
2374 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2375 let addr = listener.local_addr().unwrap();
2376 let (received_tx, mut received_rx) = tokio::sync::mpsc::unbounded_channel();
2377
2378 let server = tokio::spawn(async move {
2379 let (socket, _) = listener.accept().await.unwrap();
2380 let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap();
2381 while let Some(Ok(message)) = socket.next().await {
2382 if let tokio_tungstenite::tungstenite::Message::Text(text) = message {
2383 let payload: serde_json::Value = serde_json::from_str(text.as_str()).unwrap();
2384 let _ = received_tx.send(payload);
2385 }
2386 }
2387 });
2388
2389 let url = format!("ws://{addr}");
2390
2391 let mut ws_client = DeriveWebSocketClient::new(
2392 Some(url),
2393 DeriveEnvironment::Testnet,
2394 TransportBackend::default(),
2395 None,
2396 );
2397 let mode = Arc::clone(&ws_client.connection_mode);
2398 let (response_tx, _response_rx) = tokio::sync::oneshot::channel();
2399
2400 let command = HandlerCommand::Request {
2401 method: "public/subscribe",
2402 params: SecretString::from(
2403 serde_json::json!({
2404 "channels": [
2405 "trades.ETH-USDC"
2406 ]
2407 })
2408 .to_string(),
2409 ),
2410 connection_epoch: None,
2411 cancellation: None,
2412 response_tx,
2413 };
2414
2415 race_command_with_publication(
2416 Arc::clone(&ws_client.cmd_tx),
2417 ws_client.connect(),
2418 move || command,
2419 || ConnectionMode::from_atomic(&mode.load()).is_active(),
2420 )
2421 .await
2422 .expect("connect failed");
2423 let received = tokio::time::timeout(Duration::from_secs(2), received_rx.recv()).await;
2424 ws_client.disconnect().await.expect("disconnect failed");
2425 server.abort();
2426
2427 let payload = received
2428 .expect("startup command must reach server")
2429 .unwrap();
2430 assert_eq!(
2431 payload,
2432 serde_json::json!({
2433 "jsonrpc": "2.0",
2434 "id": 1,
2435 "method": "public/subscribe",
2436 "params": {
2437 "channels": [
2438 "trades.ETH-USDC"
2439 ]
2440 }
2441 })
2442 );
2443 }
2444
2445 #[rstest]
2446 #[case("wss://stream.example.com/feed", "wss://stream.example.com")]
2447 #[case(
2448 "wss://SYN-user:SYN-password@stream.example.com:9443/SYN-path?token=SYN-query#SYN-fragment",
2449 "wss://stream.example.com:9443"
2450 )]
2451 fn test_debug_preserves_url_origin(#[case] url: &str, #[case] origin: &str) {
2452 let client = DeriveWebSocketClient::new(
2453 Some(url.to_string()),
2454 DeriveEnvironment::Testnet,
2455 TransportBackend::default(),
2456 None,
2457 );
2458 let debug = format!("{client:?}");
2459 assert!(debug.contains(&format!("url: {origin:?}")));
2460 assert!(!debug.contains("SYN-"));
2461 assert_eq!(client.url.expose(), url);
2462 }
2463
2464 #[rstest]
2465 #[case::subscription_lock(false)]
2466 #[case::rate_limit(true)]
2467 #[tokio::test]
2468 async fn test_queued_book_subscribe_uses_reconnected_generation(#[case] paced: bool) {
2469 let tracker = BookSyncTracker::default();
2470 let channel = "orderbook.ETH-USDC.1.10";
2471 tracker.record_subscription(channel, InstrumentId::from("ETH-USDC.DERIVE"));
2472 let client = DeriveWebSocketClient::new(
2473 None,
2474 DeriveEnvironment::Testnet,
2475 TransportBackend::default(),
2476 None,
2477 )
2478 .with_book_sync(tracker.clone(), Duration::from_secs(10));
2479 let mut handle = client.subscription_handle();
2480
2481 handle.rate_limiter = Arc::new(RequestLimiter::new(
2482 RateLimits::new(None, None),
2483 MonotonicClock,
2484 ));
2485 handle.connection_epoch.load().store(1, Ordering::Release);
2486 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2487 *handle.cmd_tx.write().await = cmd_tx;
2488 if paced {
2489 while handle
2490 .rate_limiter
2491 .try_class_ready(RateClass::NonMatching, None)
2492 .is_ok()
2493 {}
2494 }
2495
2496 let lock = Arc::clone(&handle.subscription_lock);
2497
2498 let guard = if paced { None } else { Some(lock.lock().await) };
2499 let request = handle.subscribe_book_gated(channel.to_owned());
2500 tokio::pin!(request);
2501 assert!(futures_util::poll!(&mut request).is_pending());
2502
2503 tracker.reset_on_reconnect(Duration::from_secs(10));
2504 handle.connection_epoch.load().store(2, Ordering::Release);
2505 drop(guard);
2506 let ack: Value =
2507 serde_json::from_str(include_str!("../../test_data/spot/ws_subscribe_ack.json"))
2508 .unwrap();
2509 tokio::time::timeout(Duration::from_secs(8), async {
2510 tokio::join!(async { request.await.unwrap() }, async {
2511 let HandlerCommand::Request {
2512 method,
2513 params,
2514 connection_epoch,
2515 cancellation,
2516 response_tx,
2517 } = cmd_rx.recv().await.unwrap()
2518 else {
2519 panic!("expected book subscribe request");
2520 };
2521 let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
2522 response_tx
2523 .send(Ok(serde_json::value::to_raw_value(&ack["result"]).unwrap()))
2524 .unwrap();
2525
2526 assert_eq!(method, methods::PUBLIC_SUBSCRIBE);
2527 assert_eq!(params, serde_json::json!({"channels": [channel]}));
2528 assert_eq!(connection_epoch, Some(2));
2529 assert!(!cancellation.unwrap().is_cancelled());
2530 },);
2531 })
2532 .await
2533 .unwrap();
2534 }
2535
2536 #[rstest]
2537 #[tokio::test]
2538 async fn test_replay_ack_cannot_open_queued_book_replacement() {
2539 let tracker = BookSyncTracker::default();
2540 let channel = "orderbook.ETH-USDC.1.10";
2541 let now = Instant::now();
2542 tracker.record_subscription(channel, InstrumentId::from("ETH-USDC.DERIVE"));
2543 tracker.write(channel).unwrap().gate.open();
2544 assert!(tracker.record_snapshot(channel, 100, None, now));
2545 let recovery = tracker
2546 .recoveries(Duration::from_secs(1), now + Duration::from_secs(2))
2547 .pop()
2548 .unwrap()
2549 .2;
2550 assert!(recovery.begin_replacement());
2551 tracker.reset_on_reconnect(Duration::from_secs(10));
2552 let client = DeriveWebSocketClient::new(
2553 None,
2554 DeriveEnvironment::Testnet,
2555 TransportBackend::default(),
2556 None,
2557 )
2558 .with_book_sync(tracker.clone(), Duration::from_secs(10));
2559 let mut handle = client.subscription_handle();
2560
2561 handle.rate_limiter = Arc::new(RequestLimiter::new(
2562 RateLimits::new(None, None),
2563 MonotonicClock,
2564 ));
2565 handle.connection_epoch.load().store(2, Ordering::Release);
2566 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2567 *handle.cmd_tx.write().await = cmd_tx;
2568 let lock = Arc::clone(&handle.subscription_lock);
2569 let guard = lock.lock().await;
2570 let request = handle.replace_book_gated(
2571 channel.to_owned(),
2572 CancellationToken::new(),
2573 &recovery,
2574 recovery.gate.clone(),
2575 );
2576 tokio::pin!(request);
2577 assert!(futures_util::poll!(&mut request).is_pending());
2578 let replay = tracker.write(channel).unwrap();
2579 replay.gate.open();
2580 drop(guard);
2581 let ack: Value =
2582 serde_json::from_str(include_str!("../../test_data/spot/ws_subscribe_ack.json"))
2583 .unwrap();
2584 tokio::time::timeout(Duration::from_secs(1), async {
2585 tokio::join!(async { request.await.unwrap() }, async {
2586 for method in [methods::PUBLIC_UNSUBSCRIBE, methods::PUBLIC_SUBSCRIBE] {
2587 let HandlerCommand::Request {
2588 method: actual_method,
2589 params,
2590 connection_epoch,
2591 cancellation,
2592 response_tx,
2593 } = cmd_rx.recv().await.unwrap()
2594 else {
2595 panic!("expected book replacement request");
2596 };
2597 let current = tracker.write(channel).unwrap();
2598 assert!(current.gate.lock().is_closed());
2599 assert!(!tracker.record_snapshot(channel, 101, Some(¤t.generation), now));
2600 assert!(replay.generation.is_cancelled());
2601 assert_eq!(actual_method, method);
2602 assert_eq!(
2603 serde_json::from_str::<Value>(params.expose_secret()).unwrap(),
2604 serde_json::json!({"channels": [channel]}),
2605 );
2606 assert_eq!(connection_epoch, Some(2));
2607 assert!(!cancellation.unwrap().is_cancelled());
2608 let result = if method == methods::PUBLIC_SUBSCRIBE {
2609 current.gate.open();
2610 ack["result"].clone()
2611 } else {
2612 serde_json::json!({"status": {channel: "ok"}, "remaining_subscriptions": []})
2613 };
2614 response_tx
2615 .send(Ok(serde_json::value::to_raw_value(&result).unwrap()))
2616 .unwrap();
2617 }
2618 });
2619 })
2620 .await
2621 .unwrap();
2622
2623 assert!(tracker.record_snapshot(channel, 101, None, now));
2624 assert!(recovery.is_accepted());
2625 }
2626
2627 #[tokio::test]
2628 async fn test_expired_matching_reservation_has_local_failure_evidence() {
2629 let client = DeriveWebSocketClient::new(
2630 None,
2631 DeriveEnvironment::Testnet,
2632 TransportBackend::default(),
2633 None,
2634 );
2635 let handle = client.execution_handle();
2636 let quota = handle
2637 .rate_limiter
2638 .reserve_matching_ready(&Ustr::from("ETH-PERP"))
2639 .await;
2640 time::sleep(Duration::from_secs(DERIVE_RATE_WINDOW_SECS)).await;
2641 for _ in 0..DERIVE_RATE_WINDOW_SECS {
2642 handle
2643 .rate_limiter
2644 .await_class_ready(RateClass::Matching, None)
2645 .await;
2646 }
2647
2648 let reservation = MatchingRateLimitReservation {
2649 method: methods::PRIVATE_ORDER,
2650 quota,
2651 };
2652
2653 let error = handle.commit_matching_reservation(reservation).unwrap_err();
2654
2655 assert!(matches!(error, DeriveWsError::RateLimited { .. }));
2656 assert!(should_retry_ws_error(&error));
2657 assert!(!is_fatal_ws_error(&error));
2658 assert!(!is_write_outcome_ambiguous_ws(&error));
2659 }
2660
2661 #[rstest]
2662 fn test_replace_response_preserves_order_with_undecodable_trade() {
2663 let order: Value = serde_json::from_str(include_str!(
2664 "../../test_data/perps/http_order_eth_partially_filled.json"
2665 ))
2666 .unwrap();
2667 let trade: Value = serde_json::from_str(include_str!(
2668 "../../test_data/perps/http_private_trade_eth.json"
2669 ))
2670 .unwrap();
2671 let response = DeriveReplaceResponse::from_raw(
2672 &serde_json::value::to_raw_value(&serde_json::json!({
2673 "order": order,
2674 "trades": [{"trade_id": "bad-row"}, trade],
2675 }))
2676 .unwrap(),
2677 );
2678 let outcome = response
2679 .result
2680 .unwrap()
2681 .into_outcome("old-order", "alpha-strategy", 42, "ETH-PERP")
2682 .unwrap();
2683
2684 let DeriveReplaceOutcome::Replaced(order) = outcome else {
2685 panic!("expected replaced outcome");
2686 };
2687
2688 let trades = response.trades;
2689 assert_eq!(order.order_id, "abc-123");
2690 assert_eq!(order.label, "alpha-strategy");
2691 assert_eq!(trades.len(), 1);
2692 assert_eq!(trades[0].trade_id, "trade-xyz");
2693 assert_eq!(trades[0].order_id, "order-abc");
2694 }
2695
2696 const SESSION_KEY_HEX: &str =
2697 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd";
2698
2699 #[rstest]
2700 #[case(Value::Null)]
2701 #[case(serde_json::json!("bad-nonce"))]
2702 #[case(serde_json::json!("18446744073709551616"))]
2703 fn test_replace_response_preserves_financial_rows_with_invalid_order_nonce(
2704 #[case] nonce: Value,
2705 ) {
2706 let mut order: Value = serde_json::from_str(include_str!(
2707 "../../test_data/perps/http_order_eth_partially_filled.json"
2708 ))
2709 .unwrap();
2710 let trade: DeriveTrade = serde_json::from_str(include_str!(
2711 "../../test_data/perps/http_private_trade_eth.json"
2712 ))
2713 .unwrap();
2714 order["nonce"] = nonce;
2715 let response = DeriveReplaceResponse::from_raw(
2716 &serde_json::value::to_raw_value(
2717 &serde_json::json!({"order": order, "trades": [trade]}),
2718 )
2719 .unwrap(),
2720 );
2721 assert!(response.result.is_err());
2722 assert_eq!(
2723 serde_json::to_value(response.trades).unwrap(),
2724 serde_json::to_value(vec![trade]).unwrap()
2725 );
2726 }
2727
2728 #[tokio::test]
2729 async fn test_http_and_websocket_share_matching_consumption() {
2730 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2731 let address = listener.local_addr().unwrap();
2732
2733 let server = tokio::spawn(async move {
2734 axum::serve(
2735 listener,
2736 Router::new().route(
2737 "/private/cancel",
2738 post(|| async { Json(serde_json::json!({"result": true})) }),
2739 ),
2740 )
2741 .await
2742 .unwrap();
2743 });
2744
2745 let wallet = "0x000000000000000000000000000000000000ab01";
2746 let http = DeriveHttpClient::with_credentials(
2747 format!("http://{address}"),
2748 DeriveCredentials::new(wallet, SESSION_KEY_HEX).unwrap(),
2749 None,
2750 None,
2751 None,
2752 )
2753 .unwrap();
2754 let ws = DeriveWebSocketClient::with_credentials(
2755 Some(format!("ws://{address}/ws")),
2756 DeriveEnvironment::Testnet,
2757 TransportBackend::default(),
2758 None,
2759 DeriveWsCredentials::new(wallet.to_ascii_uppercase(), SESSION_KEY_HEX).unwrap(),
2760 None,
2761 None,
2762 );
2763
2764 for _ in 0..5 {
2765 let result: bool = http
2766 .send_private_once("private/cancel", &serde_json::json!({}))
2767 .await
2768 .unwrap();
2769 assert!(result);
2770 }
2771
2772 server.abort();
2773 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2774 let params = serde_json::json!({});
2775 let request = send_raw(
2776 &ws.rate_limiter,
2777 &cmd_tx,
2778 methods::PRIVATE_CANCEL,
2779 ¶ms,
2780 Duration::from_secs(1),
2781 );
2782 tokio::pin!(request);
2783 assert!(futures_util::poll!(&mut request).is_pending());
2784 assert!(matches!(
2785 cmd_rx.try_recv(),
2786 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
2787 ));
2788 }
2789
2790 #[tokio::test]
2791 async fn test_login_builds_auth_timestamp_after_public_quota_wait() {
2792 let rate_limiter = RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
2793 for _ in 0..25 {
2794 rate_limiter.await_public_ready().await;
2795 }
2796
2797 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2798
2799 let captured = tokio::spawn(async move {
2800 let Some(HandlerCommand::Request {
2801 method,
2802 params,
2803 response_tx,
2804 ..
2805 }) = cmd_rx.recv().await
2806 else {
2807 panic!("login request must enqueue");
2808 };
2809
2810 assert_eq!(method, methods::PUBLIC_LOGIN);
2811 let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
2812 let timestamp: u64 = params["timestamp"].as_str().unwrap().parse().unwrap();
2813 let now = SystemTime::now()
2814 .duration_since(SystemTime::UNIX_EPOCH)
2815 .unwrap()
2816 .as_millis();
2817 response_tx
2818 .send(Ok(serde_json::value::to_raw_value(&serde_json::json!([
2819 77
2820 ]))
2821 .unwrap()))
2822 .unwrap();
2823 now.saturating_sub(u128::from(timestamp))
2824 });
2825
2826 let started = tokio::time::Instant::now();
2827 send_login_request(
2828 &rate_limiter,
2829 &cmd_tx,
2830 &DeriveWsCredentials::new(
2831 "0x000000000000000000000000000000000000ab02",
2832 SESSION_KEY_HEX,
2833 )
2834 .unwrap(),
2835 Duration::from_secs(1),
2836 0,
2837 )
2838 .await
2839 .unwrap();
2840 assert!(started.elapsed() >= Duration::from_secs(4));
2841 assert!(
2842 captured.await.unwrap() < 1_000,
2843 "login timestamp must remain fresh after the pacing wait"
2844 );
2845 }
2846
2847 #[tokio::test]
2848 #[ignore = "requires Derive public testnet network access"]
2849 async fn test_live_testnet_unsubscribe_acknowledgement() {
2850 let mut ws = DeriveWebSocketClient::new(
2851 None,
2852 DeriveEnvironment::Testnet,
2853 TransportBackend::default(),
2854 None,
2855 );
2856 ws.connect().await.unwrap();
2857 let cmd_tx = ws.cmd_tx.read().await.clone();
2858 let params = serde_json::json!({"channels": ["ticker_slim.ETH-PERP.1000"]});
2859 let subscribe = send_raw(
2860 &ws.rate_limiter,
2861 &cmd_tx,
2862 methods::PUBLIC_SUBSCRIBE,
2863 ¶ms,
2864 Duration::from_secs(15),
2865 )
2866 .await;
2867 let unsubscribe = send_raw(
2868 &ws.rate_limiter,
2869 &cmd_tx,
2870 methods::PUBLIC_UNSUBSCRIBE,
2871 ¶ms,
2872 Duration::from_secs(15),
2873 )
2874 .await;
2875 ws.disconnect().await.unwrap();
2876 let subscribe: Value = serde_json::from_str(subscribe.unwrap().get()).unwrap();
2877 let unsubscribe: Value = serde_json::from_str(unsubscribe.unwrap().get()).unwrap();
2878 println!("Public testnet subscribe acknowledgement: {subscribe}");
2879 println!("Public testnet unsubscribe acknowledgement: {unsubscribe}");
2880 assert_eq!(subscribe["status"]["ticker_slim.ETH-PERP.1000"], "ok");
2881 assert_eq!(unsubscribe["status"]["ticker_slim.ETH-PERP.1000"], "ok");
2882 assert_eq!(
2883 unsubscribe["remaining_subscriptions"],
2884 serde_json::json!([])
2885 );
2886 }
2887
2888 #[tokio::test]
2889 #[ignore = "requires Derive testnet credentials and network access"]
2890 async fn test_live_testnet_request_budget_scopes() {
2891 let wallet = env::var("DERIVE_TESTNET_WALLET_ADDRESS").expect("testnet wallet configured");
2892 let key = env::var("DERIVE_TESTNET_SESSION_PRIVATE_KEY").expect("testnet key configured");
2893 let subaccount: u64 = env::var("DERIVE_TESTNET_SUBACCOUNT_ID")
2894 .expect("testnet subaccount configured")
2895 .parse()
2896 .unwrap();
2897 let http = DeriveHttpClient::with_credentials(
2898 REST_URL_TESTNET,
2899 DeriveCredentials::new(&wallet, &key).unwrap(),
2900 Some(15),
2901 None,
2902 None,
2903 )
2904 .unwrap();
2905 let mut ws = DeriveWebSocketClient::with_credentials(
2906 None,
2907 DeriveEnvironment::Testnet,
2908 TransportBackend::default(),
2909 None,
2910 DeriveWsCredentials::new(&wallet, &key).unwrap(),
2911 None,
2912 None,
2913 );
2914 ws.connect()
2915 .await
2916 .expect("testnet WebSocket login succeeds");
2917 let cmd_tx = ws.cmd_tx.read().await.clone();
2918 let before: Value = http
2919 .send_private("public/getRateLimits", &serde_json::json!({}))
2920 .await
2921 .expect("authenticated HTTP budget query succeeds");
2922 let ws_limits = send_raw(
2923 &ws.rate_limiter,
2924 &cmd_tx,
2925 "public/getRateLimits",
2926 &serde_json::json!({}),
2927 Duration::from_secs(15),
2928 )
2929 .await
2930 .expect("authenticated WebSocket budget query succeeds");
2931 println!("Testnet authenticated HTTP budgets: {before}");
2932 println!(
2933 "Testnet authenticated WebSocket budgets: {}",
2934 ws_limits.get()
2935 );
2936 let label = format!("nautilus-pacing-{}", UUID4::new());
2937
2938 for scoped in [true, false] {
2939 let mut params = serde_json::json!({"subaccount_id": subaccount, "label": label});
2940 if scoped {
2941 params["instrument_name"] = serde_json::json!("ETH-PERP");
2942 }
2943
2944 let result: Value = http
2945 .send_private_once("private/cancel_by_label", ¶ms)
2946 .await
2947 .expect("testnet cancellation of unique empty label succeeds");
2948 let budgets: Value = http
2949 .send_private("public/getRateLimits", &serde_json::json!({}))
2950 .await
2951 .unwrap();
2952 println!("Testnet cancel-by-label scoped={scoped}: {result}; budgets={budgets}");
2953 let ws_budgets = send_raw(
2954 &ws.rate_limiter,
2955 &cmd_tx,
2956 "public/getRateLimits",
2957 &serde_json::json!({}),
2958 Duration::from_secs(15),
2959 )
2960 .await
2961 .unwrap();
2962 println!(
2963 "Testnet wallet budgets after HTTP scoped={scoped}: {}",
2964 ws_budgets.get()
2965 );
2966 }
2967
2968 let started = tokio::time::Instant::now();
2969 let name = Ustr::from("ETH-PERP");
2970 let params = serde_json::json!({"subaccount_id": subaccount, "label": label, "instrument_name": name});
2971
2972 for _ in 0..6 {
2973 let result = send_raw_for_instrument(
2974 &ws.rate_limiter,
2975 &cmd_tx,
2976 methods::PRIVATE_CANCEL_BY_LABEL,
2977 ¶ms,
2978 Duration::from_secs(15),
2979 name,
2980 )
2981 .await
2982 .expect("paced testnet WebSocket scoped cancellation succeeds");
2983 let result: Value = serde_json::from_str(result.get()).unwrap();
2984 assert_eq!(result["cancelled_orders"], 0);
2985 }
2986
2987 let elapsed = started.elapsed();
2988 assert!(
2989 elapsed >= Duration::from_secs(1),
2990 "six same-instrument cancels must exceed the local five-token burst"
2991 );
2992 println!(
2993 "Testnet six scoped WebSocket cancellations pass with local pacing in {elapsed:?}"
2994 );
2995 let after = send_raw(
2996 &ws.rate_limiter,
2997 &cmd_tx,
2998 "public/getRateLimits",
2999 &serde_json::json!({}),
3000 Duration::from_secs(15),
3001 )
3002 .await
3003 .unwrap();
3004 println!(
3005 "Testnet wallet budgets after paced WebSocket requests: {}",
3006 after.get()
3007 );
3008 ws.disconnect().await.unwrap();
3009 ws.connect()
3010 .await
3011 .expect("testnet reconnect authenticates with preserved pacing state");
3012 ws.disconnect().await.unwrap();
3013 }
3014
3015 #[rstest]
3016 fn test_public_client_defaults_to_environment_url() {
3017 let client = DeriveWebSocketClient::new(
3018 None,
3019 DeriveEnvironment::Mainnet,
3020 TransportBackend::default(),
3021 None,
3022 );
3023 assert_eq!(client.url(), "wss://api.derive.xyz/v3/ws");
3024 assert!(!client.is_authenticated());
3025 assert!(!client.is_active());
3026 assert_eq!(client.subscription_count(), 0);
3027 }
3028
3029 #[tokio::test]
3030 async fn test_execution_auth_barrier_waits_for_authentication() {
3031 let client = DeriveWebSocketClient::with_credentials(
3032 None,
3033 DeriveEnvironment::Mainnet,
3034 TransportBackend::default(),
3035 None,
3036 DeriveWsCredentials::new(
3037 "0x000000000000000000000000000000000000aaaa",
3038 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3039 )
3040 .unwrap(),
3041 None,
3042 None,
3043 );
3044 let execution = client.execution_handle();
3045 let auth_tracker = execution.auth_tracker.clone();
3046 let _receiver = auth_tracker.begin();
3047 let tracker_for_task = auth_tracker.clone();
3048
3049 get_runtime().spawn(async move {
3050 tokio::time::sleep(Duration::from_millis(10)).await;
3051 tracker_for_task.succeed();
3052 });
3053
3054 execution
3055 .require_authenticated(methods::PRIVATE_ORDER)
3056 .await
3057 .expect("barrier should wait for successful authentication");
3058 }
3059
3060 #[tokio::test]
3061 async fn test_execution_auth_barrier_fails_on_terminal_auth_failure() {
3062 let client = DeriveWebSocketClient::with_credentials(
3063 None,
3064 DeriveEnvironment::Mainnet,
3065 TransportBackend::default(),
3066 None,
3067 DeriveWsCredentials::new(
3068 "0x000000000000000000000000000000000000aaaa",
3069 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3070 )
3071 .unwrap(),
3072 None,
3073 None,
3074 );
3075 let execution = client.execution_handle();
3076 let _receiver = execution.auth_tracker.begin();
3077 execution.auth_tracker.fail("bad signature");
3078
3079 let error = execution
3080 .require_authenticated(methods::PRIVATE_ORDER)
3081 .await
3082 .expect_err("terminal auth failure must reject private operations");
3083
3084 assert!(matches!(error, DeriveWsError::Authentication { .. }));
3085 }
3086
3087 #[tokio::test]
3088 async fn test_session_recovery_waits_for_reconnecting_transport() {
3089 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect as u8));
3090 let connection_epoch = Arc::new(AtomicU64::new(1));
3091 let mode_for_task = Arc::clone(&connection_mode);
3092 let epoch_for_task = Arc::clone(&connection_epoch);
3093
3094 get_runtime().spawn(async move {
3095 tokio::time::sleep(Duration::from_millis(10)).await;
3096 epoch_for_task.store(2, Ordering::Release);
3097 mode_for_task.store(ConnectionMode::Active as u8, Ordering::Release);
3098 });
3099
3100 let epoch = tokio::time::timeout(
3101 Duration::from_secs(1),
3102 wait_for_session_connection(&connection_mode, &connection_epoch),
3103 )
3104 .await
3105 .expect("session recovery should resume after reconnect")
3106 .expect("active replacement connection should be accepted");
3107
3108 assert_eq!(epoch, 2);
3109 }
3110
3111 #[rstest]
3112 fn test_testnet_client_routes_to_v3_url() {
3113 let client = DeriveWebSocketClient::new(
3114 None,
3115 DeriveEnvironment::Testnet,
3116 TransportBackend::default(),
3117 None,
3118 );
3119 assert_eq!(client.url(), "wss://testnet.api.derive.xyz/v3/ws");
3120 }
3121
3122 #[rstest]
3123 fn test_credentials_constructor_parses_session_key() {
3124 let creds = DeriveWsCredentials::new(
3125 "0x000000000000000000000000000000000000aaaa",
3126 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3127 )
3128 .unwrap();
3129 assert!(creds.wallet_address.starts_with("0x"));
3130 let client = DeriveWebSocketClient::with_credentials(
3131 None,
3132 DeriveEnvironment::Testnet,
3133 TransportBackend::default(),
3134 None,
3135 creds,
3136 None,
3137 None,
3138 );
3139 assert_eq!(client.url(), "wss://testnet.api.derive.xyz/v3/ws");
3140 assert!(!client.is_authenticated());
3141 }
3142
3143 #[rstest]
3144 fn test_credentials_debug_redacts_signer() {
3145 let creds = DeriveWsCredentials::new(
3146 "0xWALLET",
3147 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3148 )
3149 .unwrap();
3150 let debug = format!("{creds:?}");
3151 assert!(debug.contains("redacted"));
3152 assert!(debug.contains("0xWALLET"));
3153 assert!(!debug.contains("2ae8be44"));
3154 }
3155
3156 #[rstest]
3157 fn test_credentials_constructor_rejects_invalid_session_key() {
3158 let err = DeriveWsCredentials::new("0xWALLET", "not-a-hex-key").unwrap_err();
3159 assert!(err.to_string().contains("invalid session key"));
3160 }
3161
3162 #[rstest]
3163 #[tokio::test]
3164 async fn test_send_raw_closed_command_queue_is_definitive() {
3165 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3166 drop(cmd_rx);
3167 let rate_limiter: WsRateLimiter =
3168 RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3169 let error = send_raw(
3170 &rate_limiter,
3171 &cmd_tx,
3172 methods::PRIVATE_REPLACE,
3173 &serde_json::json!({}),
3174 Duration::from_millis(50),
3175 )
3176 .await
3177 .unwrap_err();
3178 assert!(matches!(error, DeriveWsError::NotConnected));
3179 assert!(!crate::common::retry::is_write_outcome_ambiguous_ws(&error));
3180 }
3181
3182 #[rstest]
3183 #[tokio::test]
3184 async fn test_send_raw_times_out_when_no_response_arrives() {
3185 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3188 let rate_limiter: WsRateLimiter =
3189 RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3190 let err = send_raw(
3191 &rate_limiter,
3192 &cmd_tx,
3193 methods::PRIVATE_ORDER,
3194 &serde_json::json!({}),
3195 Duration::from_millis(50),
3196 )
3197 .await
3198 .expect_err("must time out");
3199
3200 match err {
3201 DeriveWsError::Timeout { method } => assert_eq!(method, methods::PRIVATE_ORDER),
3202 other => panic!("expected Timeout, was {other:?}"),
3203 }
3204 }
3205
3206 #[rstest]
3207 #[tokio::test]
3208 async fn test_send_request_typed_rejects_null_result() {
3209 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3213 tokio::spawn(async move {
3214 if let Some(HandlerCommand::Request { response_tx, .. }) = cmd_rx.recv().await {
3215 let _ = response_tx.send(Ok(RawValue::NULL.to_owned()));
3216 }
3217 });
3218
3219 let rate_limiter: WsRateLimiter =
3220 RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3221 let result: Result<DeriveOrderResult> = send_request_typed(
3222 &rate_limiter,
3223 &cmd_tx,
3224 methods::PRIVATE_ORDER,
3225 &serde_json::json!({}),
3226 Duration::from_secs(1),
3227 )
3228 .await;
3229 assert!(matches!(result, Err(DeriveWsError::Serde(_))));
3230 }
3231
3232 #[rstest]
3233 #[case::pending(false)]
3234 #[case::fired_during_pacing(true)]
3235 #[tokio::test]
3236 async fn test_trigger_cancel_selects_endpoint_after_pacing(#[case] fires: bool) {
3237 let client = DeriveWebSocketClient::new(
3238 None,
3239 DeriveEnvironment::Testnet,
3240 TransportBackend::default(),
3241 None,
3242 );
3243 let mut handle = client.execution_handle();
3244 handle.auth_tracker.succeed();
3245
3246 handle.rate_limiter = Arc::new(RequestLimiter::new(
3247 RateLimits::new(None, None),
3248 MonotonicClock,
3249 ));
3250 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
3251 *handle.cmd_tx.write().await = cmd_tx;
3252 for _ in 0..crate::common::rate_limit::DERIVE_WEBSOCKET_NON_MATCHING_TPS
3253 * crate::common::rate_limit::DERIVE_RATE_BURST_MULTIPLIER
3254 {
3255 handle
3256 .rate_limiter
3257 .try_class_ready(RateClass::NonMatching, None)
3258 .unwrap();
3259 }
3260
3261 assert!(
3262 handle
3263 .rate_limiter
3264 .try_class_ready(RateClass::NonMatching, None)
3265 .is_err()
3266 );
3267 let active = AtomicBool::new(false);
3268 let trigger = DeriveCancelTriggerOrderParams::new(42, "paced-trigger");
3269 let regular = DeriveCancelParams::new(42, "ETH-PERP", "paced-trigger");
3270 let cancel = handle
3271 .cancel_trigger_order_current(&trigger, ®ular, || active.load(Ordering::Acquire));
3272 tokio::pin!(cancel);
3273 assert!(futures_util::poll!(cancel.as_mut()).is_pending());
3274 assert!(cmd_rx.try_recv().is_err());
3275 active.store(fires, Ordering::Release);
3276 let mut canceled: DeriveOrder = serde_json::from_str(include_str!(
3277 "../../test_data/perps/http_order_eth_partially_filled.json"
3278 ))
3279 .unwrap();
3280 canceled.order_id = "paced-trigger".into();
3281 canceled.order_status = crate::common::enums::DeriveOrderStatus::Cancelled;
3282 canceled.filled_amount = rust_decimal::Decimal::ZERO;
3283 canceled.average_price = rust_decimal::Decimal::ZERO;
3284 let expected = (!fires).then_some(canceled);
3285
3286 let response = async {
3287 let HandlerCommand::Request {
3288 method,
3289 params,
3290 response_tx,
3291 ..
3292 } = cmd_rx.recv().await.unwrap()
3293 else {
3294 panic!("expected one cancel request")
3295 };
3296
3297 let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
3298
3299 let expected_method = if fires {
3300 methods::PRIVATE_CANCEL
3301 } else {
3302 methods::PRIVATE_CANCEL_TRIGGER_ORDER
3303 };
3304
3305 assert_eq!(method, expected_method);
3306 assert_eq!(params["subaccount_id"], serde_json::json!(42));
3307 assert_eq!(params["order_id"], serde_json::json!("paced-trigger"));
3308 assert_eq!(
3309 params.get("instrument_name"),
3310 fires
3311 .then_some(®ular.instrument_name)
3312 .map(|name| serde_json::json!(name))
3313 .as_ref()
3314 );
3315 response_tx
3316 .send(Ok(serde_json::value::to_raw_value(&expected).unwrap()))
3317 .unwrap();
3318 };
3319
3320 let (result, ()) = tokio::join!(cancel, response);
3321 assert_eq!(
3322 serde_json::to_value(result.unwrap()).unwrap(),
3323 serde_json::to_value(expected).unwrap()
3324 );
3325 assert!(cmd_rx.try_recv().is_err());
3326 }
3327
3328 #[rstest]
3329 #[tokio::test]
3330 async fn test_reserved_send_does_not_wait_for_or_consume_second_quota_cell() {
3331 let rate_limiter: WsRateLimiter =
3332 RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3333
3334 for _ in 0..5 {
3335 rate_limiter
3336 .try_class_ready(RateClass::Matching, None)
3337 .expect("reservation consumes the matching window");
3338 }
3339
3340 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3341 tokio::spawn(async move {
3342 if let Some(HandlerCommand::Request { response_tx, .. }) = cmd_rx.recv().await {
3343 let _ = response_tx.send(Ok(serde_json::value::to_raw_value(
3344 &serde_json::json!({"accepted": true}),
3345 )
3346 .unwrap()));
3347 }
3348 });
3349
3350 let response = tokio::time::timeout(
3351 Duration::from_millis(100),
3352 send_raw_after_rate_limit(
3353 &rate_limiter,
3354 &cmd_tx,
3355 methods::PRIVATE_ORDER,
3356 &serde_json::json!({}),
3357 Duration::from_secs(1),
3358 ),
3359 )
3360 .await
3361 .expect("reserved send must not wait for quota")
3362 .expect("reserved send succeeds");
3363
3364 assert_eq!(
3365 serde_json::from_str::<Value>(response.get()).unwrap(),
3366 serde_json::json!({"accepted": true}),
3367 );
3368 assert!(
3369 rate_limiter
3370 .try_class_ready(RateClass::Matching, None)
3371 .is_err(),
3372 "reserved send must not consume a second cell",
3373 );
3374 }
3375}