1use std::{
25 fmt::Debug,
26 sync::{
27 Arc,
28 atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
29 },
30 time::Duration,
31};
32
33use alloy::signers::local::PrivateKeySigner;
34use arc_swap::ArcSwap;
35use nautilus_common::live::dst::time;
36#[cfg(test)]
37use nautilus_common::live::get_runtime;
38use nautilus_core::{
39 UUID4,
40 string::secret::{REDACTED, SecretString},
41};
42use nautilus_live::{
43 SocketControl,
44 book::snapshot::SnapshotGate,
45 task::{SharedTaskSlot, TaskJoinOutcome, TaskSlot, finish_task},
46};
47use nautilus_network::{
48 ConnectionUrl,
49 http::create_standard_nautilus_headers,
50 mode::ConnectionMode,
51 ratelimiter::clock::MonotonicClock,
52 websocket::{
53 AuthTracker, SubscriptionState, TransportBackend, WebSocketClient, WebSocketConfig,
54 channel_message_handler,
55 },
56};
57use serde::{Deserialize, Serialize, de::DeserializeOwned};
58use serde_json::value::RawValue;
59use tokio_util::sync::CancellationToken;
60use ustr::Ustr;
61use zeroize::Zeroizing;
62
63use super::{
64 error::{DeriveWsError, Result},
65 handler::{
66 DeriveWsMessage, FeedHandler, HandlerCommand, orderbook_subscribe_params,
67 ticker_subscribe_params, trades_subscribe_params,
68 },
69 messages::{
70 DeriveWsChannel, WsLoginParams, WsLoginResult, WsSetCancelOnDisconnectParams,
71 WsSetCancelOnDisconnectResult, WsSubscribeParams, WsSubscribeResult, WsUnsubscribeParams,
72 WsUnsubscribeResult, methods, orderbook_channel, ticker_channel, trades_channel,
73 },
74};
75use crate::{
76 book::{recovery::BookRecovery, sync::BookSyncTracker},
77 common::{
78 consts::{
79 RECONNECT_BACKOFF_FACTOR, RECONNECT_BASE_BACKOFF, RECONNECT_JITTER_MS,
80 RECONNECT_MAX_BACKOFF, RECONNECT_TIMEOUT, WS_HEARTBEAT_SECS, WS_HEARTBEAT_TIMEOUT,
81 WS_REQUEST_TIMEOUT,
82 },
83 enums::DeriveEnvironment,
84 parse::salvage_rows,
85 rate_limit::{DeriveRateLimiter, RateClass, RateReservation, rate_class_for_method},
86 retry::should_retry_ws_error,
87 urls,
88 },
89 http::{
90 models::{
91 DeriveCancelByInstrumentResult, DeriveCancelByLabelResult, DeriveEmptyResult,
92 DeriveOpenOrdersResult, DeriveOrder, DeriveOrderResult, DeriveReplaceOutcome,
93 DeriveReplaceResult, DeriveTrade,
94 },
95 query::{
96 DeriveCancelAllParams, DeriveCancelByInstrumentParams, DeriveCancelByLabelParams,
97 DeriveCancelParams, DeriveCancelTriggerOrderParams, DeriveGetTriggerOrdersParams,
98 DeriveOrderParams, DeriveReplaceParams, DeriveTriggerOrderParams,
99 },
100 },
101 signing::auth::build_ws_login,
102};
103
104#[derive(Clone)]
107pub struct DeriveWsCredentials {
108 pub wallet_address: String,
110 pub signer: PrivateKeySigner,
112}
113
114impl DeriveWsCredentials {
115 pub fn new(wallet_address: impl Into<String>, session_key_hex: &str) -> Result<Self> {
121 let signer: PrivateKeySigner = session_key_hex
122 .parse()
123 .map_err(|e| DeriveWsError::transport(format!("invalid session key: {e}")))?;
124 Ok(Self {
125 wallet_address: wallet_address.into(),
126 signer,
127 })
128 }
129}
130
131impl Debug for DeriveWsCredentials {
132 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
133 f.debug_struct(stringify!(DeriveWsCredentials))
134 .field("wallet_address", &self.wallet_address)
135 .field("signer", &REDACTED)
136 .finish()
137 }
138}
139
140type WsRateLimiter = DeriveRateLimiter;
143
144const MAX_SESSION_RECOVERY_ATTEMPTS: u32 = 3;
145const SUBSCRIPTION_ACCEPTED_STATUSES: &[&str] = &["ok"];
146const SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES: &[&str] = &["ok", "already subscribed"];
147pub(super) const UNAUTHENTICATED_CONNECTION_EPOCH: u64 = u64::MAX;
148
149#[derive(Debug)]
155pub struct DeriveWebSocketClient {
156 url: ConnectionUrl,
157 transport_backend: TransportBackend,
158 proxy_url: Option<SecretString>,
159 connection_mode: Arc<ArcSwap<AtomicU8>>,
160 connection_epoch: Arc<ArcSwap<AtomicU64>>,
161 signal: Arc<AtomicBool>,
162 auth_tracker: AuthTracker,
163 authenticated_epoch: Arc<AtomicU64>,
164 credentials: Option<DeriveWsCredentials>,
165 next_id: Arc<AtomicU64>,
166 cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
167 out_rx: Option<tokio::sync::mpsc::UnboundedReceiver<DeriveWsMessage>>,
168 subscriptions: Arc<SubscriptionState>,
169 subscription_lock: Arc<tokio::sync::Mutex<()>>,
170 book_sync: Option<(BookSyncTracker, Duration)>,
171 task_handle: TaskSlot<()>,
172 send_task: Arc<SharedTaskSlot<()>>,
173 recovery_task: Arc<SharedTaskSlot<()>>,
174 shutdown_errors: Vec<String>,
175 request_timeout: Duration,
176 conn_id: Arc<ArcSwap<String>>,
177 rate_limiter: Arc<WsRateLimiter>,
178 socket_control: Option<SocketControl>,
179}
180
181#[derive(Debug, Clone)]
183pub struct DeriveWebSocketSubscriptionHandle {
184 cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
185 subscriptions: Arc<SubscriptionState>,
186 subscription_lock: Arc<tokio::sync::Mutex<()>>,
187 book_sync: Option<(BookSyncTracker, Duration)>,
188 connection_epoch: Arc<ArcSwap<AtomicU64>>,
189 request_timeout: Duration,
190 rate_limiter: Arc<WsRateLimiter>,
191}
192
193#[derive(Debug, Clone)]
202pub struct DeriveWsExecutionHandle {
203 cmd_tx: Arc<tokio::sync::RwLock<tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>,
204 auth_tracker: AuthTracker,
205 request_timeout: Duration,
206 conn_id: Arc<ArcSwap<String>>,
207 rate_limiter: Arc<WsRateLimiter>,
208}
209
210#[derive(Debug)]
211pub(crate) struct MatchingRateLimitReservation {
212 method: &'static str,
213 quota: RateReservation<MonotonicClock>,
214}
215
216impl DeriveWebSocketClient {
217 #[must_use]
220 pub fn new(
221 url: Option<String>,
222 environment: DeriveEnvironment,
223 transport_backend: TransportBackend,
224 proxy_url: Option<String>,
225 ) -> Self {
226 let url = url.unwrap_or_else(|| urls::ws_url(environment).to_string());
227 Self::build(url, transport_backend, proxy_url, None, None, None)
228 }
229
230 #[must_use]
241 pub fn with_credentials(
242 url: Option<String>,
243 environment: DeriveEnvironment,
244 transport_backend: TransportBackend,
245 proxy_url: Option<String>,
246 credentials: DeriveWsCredentials,
247 max_matching_requests_per_second: Option<u32>,
248 max_per_instrument_matching_requests_per_second: Option<u32>,
249 ) -> Self {
250 let url = url.unwrap_or_else(|| urls::ws_url(environment).to_string());
251 Self::build(
252 url,
253 transport_backend,
254 proxy_url,
255 Some(credentials),
256 max_matching_requests_per_second,
257 max_per_instrument_matching_requests_per_second,
258 )
259 }
260
261 fn build(
262 url: String,
263 transport_backend: TransportBackend,
264 proxy_url: Option<String>,
265 credentials: Option<DeriveWsCredentials>,
266 matching_tps: Option<u32>,
267 instrument_tps: Option<u32>,
268 ) -> Self {
269 let connection_mode = Arc::new(ArcSwap::new(Arc::new(AtomicU8::new(
270 ConnectionMode::Closed as u8,
271 ))));
272 let connection_epoch = Arc::new(ArcSwap::new(Arc::new(AtomicU64::new(0))));
273
274 let (placeholder_tx, _) = tokio::sync::mpsc::unbounded_channel();
276
277 let rate_limiter = Arc::new(DeriveRateLimiter::shared(
278 &url,
279 credentials
280 .as_ref()
281 .map(|credentials| credentials.wallet_address.as_str()),
282 matching_tps,
283 instrument_tps,
284 ));
285
286 Self {
287 url: url.into(),
288 transport_backend,
289 proxy_url: proxy_url.map(SecretString::from),
290 connection_mode,
291 connection_epoch,
292 signal: Arc::new(AtomicBool::new(false)),
293 auth_tracker: AuthTracker::new(),
294 authenticated_epoch: Arc::new(AtomicU64::new(UNAUTHENTICATED_CONNECTION_EPOCH)),
295 credentials,
296 next_id: Arc::new(AtomicU64::new(1)),
297 cmd_tx: Arc::new(tokio::sync::RwLock::new(placeholder_tx)),
298 out_rx: None,
299 subscriptions: Arc::new(SubscriptionState::new('.')),
300 subscription_lock: Arc::new(tokio::sync::Mutex::new(())),
301 book_sync: None,
302 task_handle: TaskSlot::new(),
303 send_task: Arc::new(SharedTaskSlot::new()),
304 recovery_task: Arc::new(SharedTaskSlot::new()),
305 shutdown_errors: Vec::new(),
306 request_timeout: WS_REQUEST_TIMEOUT,
307 conn_id: Arc::new(ArcSwap::from_pointee(UUID4::new().to_string())),
308 rate_limiter,
309 socket_control: None,
310 }
311 }
312
313 #[must_use]
314 pub(crate) fn with_book_sync(mut self, tracker: BookSyncTracker, timeout: Duration) -> Self {
315 self.book_sync = Some((tracker, timeout));
316 self
317 }
318
319 #[must_use]
321 pub fn with_socket_control(mut self, control: SocketControl) -> Self {
322 self.socket_control = Some(control);
323 self
324 }
325
326 #[must_use]
328 pub fn url(&self) -> &str {
329 self.url.expose()
330 }
331
332 pub fn set_request_timeout(&mut self, timeout: Duration) {
336 self.request_timeout = timeout;
337 }
338
339 #[must_use]
342 pub fn is_authenticated(&self) -> bool {
343 self.auth_tracker.is_authenticated()
344 && self.is_active()
345 && self.authenticated_epoch.load(Ordering::Acquire)
346 == self.connection_epoch.load().load(Ordering::Acquire)
347 }
348
349 #[must_use]
351 pub fn is_active(&self) -> bool {
352 self.connection_mode.load().load(Ordering::Relaxed) == ConnectionMode::Active as u8
353 }
354
355 pub async fn connect(&mut self) -> Result<()> {
364 let auth_ok = self.credentials.is_none() || self.is_authenticated();
367 if self.is_active() && auth_ok && self.task_handle.is_some() {
368 log::warn!("Derive WebSocket already connected");
369 return Ok(());
370 }
371
372 if self.task_handle.is_some()
374 || !self.send_task.is_empty()
375 || !self.recovery_task.is_empty()
376 {
377 log::debug!("Tearing down stale Derive WebSocket state before connect");
378 self.teardown().await?;
379 }
380
381 self.signal.store(false, Ordering::Release);
382 let setup_guard = DeriveWebSocketSetupGuard::new(self.shutdown_handle());
383
384 self.authenticated_epoch
385 .store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
386
387 let (message_handler, raw_rx) = channel_message_handler();
388 let headers = create_standard_nautilus_headers();
389
390 let cfg = WebSocketConfig {
391 url: self.url.expose().to_owned(),
392 headers,
393 heartbeat_interval_secs: Some(WS_HEARTBEAT_SECS),
394 heartbeat_payload: None,
395 connect_timeout_ms: Some(RECONNECT_TIMEOUT.as_millis() as u64),
396 reconnect_delay_initial_ms: Some(RECONNECT_BASE_BACKOFF.as_millis() as u64),
397 reconnect_delay_max_ms: Some(RECONNECT_MAX_BACKOFF.as_millis() as u64),
398 reconnect_backoff_factor: Some(RECONNECT_BACKOFF_FACTOR),
399 reconnect_jitter_ms: Some(RECONNECT_JITTER_MS),
400 reconnect_max_attempts: None,
401 heartbeat_timeout_secs: Some(WS_HEARTBEAT_TIMEOUT.as_secs()),
402 idle_timeout_ms: None,
403 writer_capacity: None,
404 backend: self.transport_backend,
405 proxy_url: self
406 .proxy_url
407 .as_ref()
408 .map(|value| value.expose_secret().to_owned()),
409 max_message_size_bytes: None,
410 max_frame_size_bytes: None,
411 };
412
413 let client = WebSocketClient::builder()
417 .config(cfg)
418 .message_handler(message_handler)
419 .maybe_state_sink(self.socket_control.as_ref().map(SocketControl::sink))
420 .connect()
421 .await
422 .map_err(|e| DeriveWsError::transport(e.to_string()))?;
423
424 client.set_auth_tracker(self.auth_tracker.clone(), false);
428
429 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
430 let (out_tx, out_rx) = tokio::sync::mpsc::unbounded_channel::<DeriveWsMessage>();
431
432 self.out_rx = Some(out_rx);
433 self.conn_id.store(Arc::new(UUID4::new().to_string()));
434
435 let connection_mode = client.connection_mode_atomic();
436 let connection_epoch = client.connection_epoch_atomic();
437 let reconnect_handle = client.reconnect_handle();
438 if let Err(e) = cmd_tx.send(HandlerCommand::SetClient(client)) {
439 return Err(DeriveWsError::transport(format!(
440 "failed to send SetClient command: {e}",
441 )));
442 }
443
444 {
445 let mut command_sender = self.cmd_tx.write().await;
446 *command_sender = cmd_tx.clone();
447 self.connection_epoch.store(Arc::clone(&connection_epoch));
448 self.connection_mode.store(Arc::clone(&connection_mode));
449 }
450
451 log::debug!("Derive WebSocket connected: {}", self.url.origin());
452
453 let recovery = SessionRecovery {
454 subscription: self.subscription_handle(),
455 cmd_tx: cmd_tx.clone(),
456 auth_tracker: self.auth_tracker.clone(),
457 authenticated_epoch: Arc::clone(&self.authenticated_epoch),
458 connection_mode: Arc::clone(&connection_mode),
459 connection_epoch: Arc::clone(&connection_epoch),
460 credentials: self.credentials.clone(),
461 signal: Arc::clone(&self.signal),
462 };
463
464 let handler = FeedHandler::new_with_send_task(
465 Arc::clone(&self.signal),
466 cmd_rx,
467 raw_rx,
468 Arc::clone(&self.next_id),
469 self.auth_tracker.clone(),
470 Arc::clone(&self.authenticated_epoch),
471 Arc::clone(&self.send_task),
472 )
473 .with_book_sync(self.book_sync.clone());
474
475 if let Err(e) = self.task_handle.spawn(run_session_handler(
476 handler,
477 recovery,
478 out_tx,
479 Arc::clone(&self.recovery_task),
480 Arc::clone(&self.conn_id),
481 )) {
482 let shutdown_result = self.teardown().await;
483 return Err(DeriveWsError::transport(match shutdown_result {
484 Ok(()) => format!("failed to start WebSocket handler task: {e}"),
485 Err(shutdown_error) => format!(
486 "failed to start WebSocket handler task: {e}; startup rollback failed: \
487 {shutdown_error}"
488 ),
489 }));
490 }
491
492 if let Some(control) = &self.socket_control {
493 control.register(move || reconnect_handle.request_reconnect());
494 }
495
496 if let Some(creds) = self.credentials.clone()
497 && let Err(e) = login_via_handler(
498 &self.rate_limiter,
499 &cmd_tx,
500 &self.auth_tracker,
501 &self.authenticated_epoch,
502 &connection_mode,
503 &connection_epoch,
504 &creds,
505 self.request_timeout,
506 )
507 .await
508 {
509 log::warn!(
512 "Derive WebSocket login failed; tearing down transport: {}",
513 e.redacted_message()
514 );
515 self.teardown().await?;
516 return Err(e);
517 }
518
519 setup_guard.disarm();
520 Ok(())
521 }
522
523 pub(crate) fn begin_shutdown(&self) {
524 self.shutdown_handle().begin_shutdown();
525 }
526
527 pub(crate) fn shutdown_handle(&self) -> DeriveWebSocketShutdownHandle {
528 DeriveWebSocketShutdownHandle {
529 signal: Arc::clone(&self.signal),
530 recovery_task: Arc::clone(&self.recovery_task),
531 }
532 }
533
534 async fn teardown(&mut self) -> Result<()> {
538 self.begin_shutdown();
539
540 if let Err(e) = self.cmd_tx.read().await.send(HandlerCommand::Disconnect) {
541 log::debug!(
542 "Failed to enqueue Disconnect command (handler may already be shut down): {e}",
543 );
544 }
545
546 match finish_task(
547 &mut self.task_handle,
548 Duration::from_secs(2),
549 Duration::from_secs(2),
550 )
551 .await
552 {
553 None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => {}
554 Some(TaskJoinOutcome::Failed(error)) => self
555 .shutdown_errors
556 .push(format!("WebSocket handler task failed: {error}")),
557 Some(TaskJoinOutcome::Incomplete) => self
558 .shutdown_errors
559 .push("WebSocket handler task did not stop after abort".to_string()),
560 }
561
562 if self.task_handle.is_some() {
563 if let Some(control) = &self.socket_control {
564 control.deregister();
565 }
566
567 return self.take_shutdown_result();
568 }
569
570 for (name, slot) in [
571 ("recovery", Arc::clone(&self.recovery_task)),
572 ("send", Arc::clone(&self.send_task)),
573 ] {
574 match slot
575 .finish(Duration::from_secs(1), Duration::from_secs(2))
576 .await
577 {
578 None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => {}
579 Some(TaskJoinOutcome::Failed(error)) => self
580 .shutdown_errors
581 .push(format!("WebSocket {name} worker failed: {error}")),
582 Some(TaskJoinOutcome::Incomplete) => self
583 .shutdown_errors
584 .push(format!("WebSocket {name} worker did not stop after abort")),
585 }
586
587 if !slot.is_empty() {
588 if let Some(control) = &self.socket_control {
589 control.deregister();
590 }
591
592 return self.take_shutdown_result();
593 }
594 }
595
596 let (placeholder_tx, _) = tokio::sync::mpsc::unbounded_channel();
599 *self.cmd_tx.write().await = placeholder_tx;
600 self.out_rx = None;
601 self.connection_mode
602 .store(Arc::new(AtomicU8::new(ConnectionMode::Closed as u8)));
603 self.connection_epoch.store(Arc::new(AtomicU64::new(0)));
604 self.auth_tracker.invalidate();
605 self.authenticated_epoch
606 .store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
607 self.subscriptions.clear();
608
609 if let Some((tracker, _)) = &self.book_sync {
610 tracker.clear();
611 }
612
613 self.signal.store(false, Ordering::Relaxed);
614
615 if let Some(control) = &self.socket_control {
616 control.deregister();
617 }
618
619 self.take_shutdown_result()
620 }
621
622 fn take_shutdown_result(&mut self) -> Result<()> {
623 if self.shutdown_errors.is_empty() {
624 Ok(())
625 } else {
626 Err(DeriveWsError::transport(
627 std::mem::take(&mut self.shutdown_errors).join("; "),
628 ))
629 }
630 }
631
632 pub async fn disconnect(&mut self) -> Result<()> {
639 log::debug!("Disconnecting Derive WebSocket");
640 self.begin_shutdown();
641 self.teardown().await
642 }
643
644 pub async fn subscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
651 self.subscription_handle()
652 .subscribe_ticker(instrument_name, interval)
653 .await
654 }
655
656 pub async fn unsubscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
662 self.subscription_handle()
663 .unsubscribe_ticker(instrument_name, interval)
664 .await
665 }
666
667 pub async fn subscribe_orderbook(
673 &self,
674 instrument_name: &str,
675 group: &str,
676 depth: &str,
677 ) -> Result<()> {
678 self.subscription_handle()
679 .subscribe_orderbook(instrument_name, group, depth)
680 .await
681 }
682
683 pub async fn unsubscribe_orderbook(
689 &self,
690 instrument_name: &str,
691 group: &str,
692 depth: &str,
693 ) -> Result<()> {
694 self.subscription_handle()
695 .unsubscribe_orderbook(instrument_name, group, depth)
696 .await
697 }
698
699 pub async fn subscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
705 self.subscription_handle()
706 .subscribe_trades(instrument_type, currency)
707 .await
708 }
709
710 pub async fn unsubscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
716 self.subscription_handle()
717 .unsubscribe_trades(instrument_type, currency)
718 .await
719 }
720
721 pub async fn subscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
731 where
732 C: Into<DeriveWsChannel>,
733 {
734 self.subscription_handle()
735 .subscribe_channels(channels)
736 .await
737 }
738
739 pub async fn unsubscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
746 where
747 C: Into<DeriveWsChannel>,
748 {
749 self.subscription_handle()
750 .unsubscribe_channels(channels)
751 .await
752 }
753
754 pub async fn next_event(&mut self) -> Option<DeriveWsMessage> {
756 if let Some(rx) = self.out_rx.as_mut() {
757 rx.recv().await
758 } else {
759 None
760 }
761 }
762
763 #[must_use]
766 pub fn subscription_count(&self) -> usize {
767 self.subscriptions.len()
768 }
769
770 #[must_use]
772 pub fn subscription_handle(&self) -> DeriveWebSocketSubscriptionHandle {
773 DeriveWebSocketSubscriptionHandle {
774 cmd_tx: Arc::clone(&self.cmd_tx),
775 subscriptions: Arc::clone(&self.subscriptions),
776 subscription_lock: Arc::clone(&self.subscription_lock),
777 book_sync: self.book_sync.clone(),
778 connection_epoch: Arc::clone(&self.connection_epoch),
779 request_timeout: self.request_timeout,
780 rate_limiter: Arc::clone(&self.rate_limiter),
781 }
782 }
783
784 pub async fn set_cancel_on_disconnect(&self, enabled: bool) -> Result<()> {
806 let creds = self
807 .credentials
808 .as_ref()
809 .ok_or_else(|| DeriveWsError::MissingCredentials {
810 operation: methods::PRIVATE_SET_CANCEL_ON_DISCONNECT.to_string(),
811 })?;
812
813 self.execution_handle()
814 .require_authenticated(methods::PRIVATE_SET_CANCEL_ON_DISCONNECT)
815 .await?;
816
817 let params = WsSetCancelOnDisconnectParams {
818 wallet: creds.wallet_address.clone(),
819 enabled,
820 };
821
822 let cmd_tx = self.cmd_tx.read().await.clone();
823 let _: WsSetCancelOnDisconnectResult = send_request_typed(
824 &self.rate_limiter,
825 &cmd_tx,
826 methods::PRIVATE_SET_CANCEL_ON_DISCONNECT,
827 ¶ms,
828 self.request_timeout,
829 )
830 .await?;
831 Ok(())
832 }
833
834 #[must_use]
841 pub fn execution_handle(&self) -> DeriveWsExecutionHandle {
842 DeriveWsExecutionHandle {
843 cmd_tx: Arc::clone(&self.cmd_tx),
844 auth_tracker: self.auth_tracker.clone(),
845 request_timeout: self.request_timeout,
846 conn_id: Arc::clone(&self.conn_id),
847 rate_limiter: Arc::clone(&self.rate_limiter),
848 }
849 }
850
851 pub fn take_event_receiver(
856 &mut self,
857 ) -> Option<tokio::sync::mpsc::UnboundedReceiver<DeriveWsMessage>> {
858 self.out_rx.take()
859 }
860}
861
862impl Drop for DeriveWebSocketClient {
863 fn drop(&mut self) {
864 self.begin_shutdown();
865
866 if let Some(handle) = self.task_handle.as_ref() {
867 handle.abort();
868 }
869
870 if let Some(control) = &self.socket_control {
871 control.deregister();
872 }
873 }
874}
875
876struct SessionRecovery {
877 subscription: DeriveWebSocketSubscriptionHandle,
878 cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
879 auth_tracker: AuthTracker,
880 authenticated_epoch: Arc<AtomicU64>,
881 connection_mode: Arc<AtomicU8>,
882 connection_epoch: Arc<AtomicU64>,
883 credentials: Option<DeriveWsCredentials>,
884 signal: Arc<AtomicBool>,
885}
886
887async fn run_session_handler(
888 mut handler: FeedHandler,
889 recovery: SessionRecovery,
890 out_tx: tokio::sync::mpsc::UnboundedSender<DeriveWsMessage>,
891 recovery_task: Arc<SharedTaskSlot<()>>,
892 conn_id: Arc<ArcSwap<String>>,
893) {
894 let (recovery_tx, recovery_rx) = tokio::sync::mpsc::unbounded_channel();
895 let cmd_tx = recovery.cmd_tx.clone();
896 let connection_epoch = Arc::clone(&recovery.connection_epoch);
897 if let Err(e) = recovery_task.spawn(recovery.run(recovery_rx, out_tx.clone())) {
898 let _ = out_tx.send(DeriveWsMessage::SessionRecoveryFailed(format!(
899 "failed to start recovery worker: {e}"
900 )));
901 let _ = cmd_tx.send(HandlerCommand::Disconnect);
902 return;
903 }
904
905 loop {
906 match handler.next().await {
907 Some(DeriveWsMessage::Reconnected) => {
908 log::info!("Derive WebSocket re-establishing session after reconnect");
909 conn_id.store(Arc::new(UUID4::new().to_string()));
910 let epoch = connection_epoch.load(Ordering::Acquire);
911 if recovery_tx.send(epoch).is_err() {
912 log::error!("Derive WebSocket recovery task stopped unexpectedly");
913 let _ = cmd_tx.send(HandlerCommand::Disconnect);
914 }
915 }
916 Some(msg) => {
917 if out_tx.send(msg).is_err() {
918 log::debug!("Derive outer receiver dropped, exiting stream loop");
919 break;
920 }
921 }
922 None => {
923 log::debug!("Derive handler task ended");
924 break;
925 }
926 }
927 }
928
929 recovery_task.abort();
930}
931
932impl SessionRecovery {
933 async fn run(
934 self,
935 mut rx: tokio::sync::mpsc::UnboundedReceiver<u64>,
936 out_tx: tokio::sync::mpsc::UnboundedSender<DeriveWsMessage>,
937 ) {
938 if self.signal.load(Ordering::Acquire) {
939 return;
940 }
941
942 while let Some(mut requested_epoch) = rx.recv().await {
943 take_latest_epoch(&mut rx, &mut requested_epoch);
944 match self.recover_until_current(&mut rx, requested_epoch).await {
945 Ok(()) => {
946 if out_tx.send(DeriveWsMessage::Reconnected).is_err() {
947 log::debug!("Derive outer receiver dropped during recovery");
948 }
949 }
950 Err(e) => {
951 log::error!(
952 "Derive WebSocket session recovery failed: {}",
953 e.redacted_message()
954 );
955 let _ = out_tx.send(DeriveWsMessage::SessionRecoveryFailed(e.to_string()));
956 let _ = self.cmd_tx.send(HandlerCommand::Disconnect);
957 return;
958 }
959 }
960 }
961 }
962
963 async fn recover_until_current(
964 &self,
965 rx: &mut tokio::sync::mpsc::UnboundedReceiver<u64>,
966 mut requested_epoch: u64,
967 ) -> Result<()> {
968 loop {
969 match recover_session(
970 &self.subscription.rate_limiter,
971 &self.cmd_tx,
972 &self.auth_tracker,
973 &self.authenticated_epoch,
974 &self.connection_mode,
975 &self.connection_epoch,
976 self.credentials.as_ref(),
977 &self.subscription.subscriptions,
978 &self.subscription.subscription_lock,
979 self.subscription.request_timeout,
980 )
981 .await
982 {
983 Ok(recovered_epoch) => {
984 take_latest_epoch(rx, &mut requested_epoch);
985 if requested_epoch > recovered_epoch
986 || !session_connection_is_active(
987 &self.connection_mode,
988 &self.connection_epoch,
989 recovered_epoch,
990 )
991 {
992 continue;
993 }
994
995 return Ok(());
996 }
997 Err(e) => {
998 if take_latest_epoch(rx, &mut requested_epoch) {
999 continue;
1000 }
1001
1002 return Err(e);
1003 }
1004 }
1005 }
1006 }
1007}
1008
1009fn take_latest_epoch(
1010 rx: &mut tokio::sync::mpsc::UnboundedReceiver<u64>,
1011 requested_epoch: &mut u64,
1012) -> bool {
1013 let mut received = false;
1014
1015 while let Ok(epoch) = rx.try_recv() {
1016 *requested_epoch = (*requested_epoch).max(epoch);
1017 received = true;
1018 }
1019
1020 received
1021}
1022
1023#[derive(Clone, Debug)]
1024pub(crate) struct DeriveWebSocketShutdownHandle {
1025 signal: Arc<AtomicBool>,
1026 recovery_task: Arc<SharedTaskSlot<()>>,
1027}
1028
1029impl DeriveWebSocketShutdownHandle {
1030 pub(crate) fn begin_shutdown(&self) {
1031 self.signal.store(true, Ordering::Release);
1032 self.recovery_task.abort();
1033 }
1034}
1035
1036struct DeriveWebSocketSetupGuard {
1037 shutdown: DeriveWebSocketShutdownHandle,
1038 armed: bool,
1039}
1040
1041impl DeriveWebSocketSetupGuard {
1042 fn new(shutdown: DeriveWebSocketShutdownHandle) -> Self {
1043 Self {
1044 shutdown,
1045 armed: true,
1046 }
1047 }
1048
1049 fn disarm(mut self) {
1050 self.armed = false;
1051 }
1052}
1053
1054impl Drop for DeriveWebSocketSetupGuard {
1055 fn drop(&mut self) {
1056 if self.armed {
1057 self.shutdown.begin_shutdown();
1058 }
1059 }
1060}
1061
1062impl DeriveWebSocketSubscriptionHandle {
1063 pub(crate) async fn subscribe_book_gated(&self, channel: String) -> Result<()> {
1064 let _guard = self.subscription_lock.lock().await;
1065 self.book_request_locked(methods::PUBLIC_SUBSCRIBE, channel, None, None)
1066 .await
1067 }
1068
1069 pub(crate) async fn replace_book_gated(
1070 &self,
1071 channel: String,
1072 cancel: CancellationToken,
1073 recovery: &Arc<BookRecovery>,
1074 gate: SnapshotGate,
1075 ) -> Result<()> {
1076 let _guard = tokio::select! {
1077 biased;
1078 () = cancel.cancelled() => return Err(DeriveWsError::NotConnected),
1079 guard = self.subscription_lock.lock() => guard,
1080 };
1081 self.book_sync
1082 .as_ref()
1083 .and_then(|(tracker, _)| tracker.begin_recovery_write(&channel, recovery, gate))
1084 .ok_or(DeriveWsError::NotConnected)?;
1085 let epoch = self.connection_epoch.load().load(Ordering::Acquire);
1086
1087 self.remember_subscription(&channel);
1089 self.book_request_locked(
1090 methods::PUBLIC_UNSUBSCRIBE,
1091 channel.clone(),
1092 Some(cancel.clone()),
1093 Some(epoch),
1094 )
1095 .await?;
1096 self.book_request_locked(
1097 methods::PUBLIC_SUBSCRIBE,
1098 channel,
1099 Some(cancel),
1100 Some(epoch),
1101 )
1102 .await
1103 }
1104
1105 async fn book_request_locked(
1106 &self,
1107 method: &'static str,
1108 channel: String,
1109 cancel: Option<CancellationToken>,
1110 epoch: Option<u64>,
1111 ) -> Result<()> {
1112 let params = WsSubscribeParams {
1113 channels: vec![DeriveWsChannel::from(channel.clone())],
1114 };
1115
1116 self.remember_subscription(&channel);
1117 self.rate_limiter
1118 .await_class_ready(rate_class_for_method(method), None)
1119 .await;
1120
1121 let write = self
1122 .book_sync
1123 .as_ref()
1124 .and_then(|(tracker, _)| tracker.write(&channel))
1125 .ok_or(DeriveWsError::NotConnected)?;
1126 let generation = write.generation.clone();
1127 let cancel = cancel.unwrap_or_else(|| generation.clone());
1128 let epoch = epoch.unwrap_or_else(|| self.connection_epoch.load().load(Ordering::Acquire));
1129
1130 if cancel.is_cancelled() || generation.is_cancelled() {
1131 return Err(DeriveWsError::NotConnected);
1132 }
1133
1134 let cmd_tx = self.cmd_tx.read().await.clone();
1135 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
1136 cmd_tx
1137 .send(HandlerCommand::Request {
1138 method,
1139 params: SecretString::from(serde_json::to_string(¶ms)?),
1140 connection_epoch: Some(epoch),
1141 cancellation: Some(cancel.clone()),
1142 response_tx,
1143 })
1144 .map_err(|_| DeriveWsError::NotConnected)?;
1145
1146 let result = tokio::select! {
1147 biased;
1148 () = cancel.cancelled() => return Err(DeriveWsError::NotConnected),
1149 () = generation.cancelled() => return Err(DeriveWsError::NotConnected),
1150 result = time::timeout(self.request_timeout, response_rx) => result,
1151 };
1152
1153 let value = result
1154 .map_err(|_| DeriveWsError::Timeout {
1155 method: method.to_owned(),
1156 })?
1157 .map_err(|_| DeriveWsError::RequestCancelled {
1158 method: method.to_owned(),
1159 })??;
1160
1161 if method == methods::PUBLIC_SUBSCRIBE {
1162 let result: WsSubscribeResult = decode_default_result(&value)?;
1163 subscription_outcome(
1164 ¶ms.channels,
1165 &result,
1166 SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES,
1167 &self.subscriptions,
1168 )
1169 } else {
1170 let result: WsUnsubscribeResult = decode_default_result(&value)?;
1171
1172 let topic = DeriveWsChannel::from(channel.clone());
1174
1175 if result
1176 .status
1177 .get(&topic)
1178 .is_some_and(|status| status.as_str() == "ok")
1179 && !result.remaining_subscriptions.contains(&topic)
1180 {
1181 Ok(())
1182 } else {
1183 Err(DeriveWsError::Subscription {
1184 details: format!("{channel}: unsubscribe not confirmed"),
1185 })
1186 }
1187 }
1188 }
1189
1190 pub(crate) fn has_subscription(&self, channel: &str) -> bool {
1191 let (name, symbol) = channel.split_once('.').unwrap_or((channel, ""));
1192 self.subscriptions
1193 .is_subscribed(&Ustr::from(name), &Ustr::from(symbol))
1194 }
1195
1196 pub(crate) fn forget_subscription(&self, channel: &str) {
1197 self.subscriptions.mark_unsubscribe(channel);
1198 self.subscriptions.confirm_unsubscribe(channel);
1199 }
1200
1201 pub(crate) fn remember_subscription(&self, channel: &str) {
1202 self.subscriptions.mark_subscribe(channel);
1203 }
1204
1205 pub async fn subscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
1211 let params = ticker_subscribe_params(instrument_name, interval);
1212 self.subscribe_channels(params.channels).await
1213 }
1214
1215 pub async fn unsubscribe_ticker(&self, instrument_name: &str, interval: &str) -> Result<()> {
1221 let channel = ticker_channel(instrument_name, interval);
1222 self.send_unsubscribe(channel).await
1223 }
1224
1225 pub async fn subscribe_orderbook(
1231 &self,
1232 instrument_name: &str,
1233 group: &str,
1234 depth: &str,
1235 ) -> Result<()> {
1236 let params = orderbook_subscribe_params(instrument_name, group, depth);
1237 self.subscribe_channels(params.channels).await
1238 }
1239
1240 pub async fn unsubscribe_orderbook(
1246 &self,
1247 instrument_name: &str,
1248 group: &str,
1249 depth: &str,
1250 ) -> Result<()> {
1251 let channel = orderbook_channel(instrument_name, group, depth);
1252 self.send_unsubscribe(channel).await
1253 }
1254
1255 pub async fn subscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
1261 let params = trades_subscribe_params(instrument_type, currency);
1262 self.subscribe_channels(params.channels).await
1263 }
1264
1265 pub async fn unsubscribe_trades(&self, instrument_type: &str, currency: &str) -> Result<()> {
1271 let channel = trades_channel(instrument_type, currency);
1272 self.send_unsubscribe(channel).await
1273 }
1274
1275 pub async fn subscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
1281 where
1282 C: Into<DeriveWsChannel>,
1283 {
1284 let channels = channels.into_iter().map(Into::into).collect::<Vec<_>>();
1285 if channels.is_empty() {
1286 return Ok(());
1287 }
1288
1289 let _guard = self.subscription_lock.lock().await;
1290 let params = WsSubscribeParams { channels };
1291 for topic in channel_topics(¶ms.channels) {
1292 self.subscriptions.mark_subscribe(&topic);
1293 }
1294
1295 let cmd_tx = self.cmd_tx.read().await.clone();
1296 let result: WsSubscribeResult = send_request(
1297 &self.rate_limiter,
1298 &cmd_tx,
1299 methods::PUBLIC_SUBSCRIBE,
1300 ¶ms,
1301 self.request_timeout,
1302 )
1303 .await?;
1304
1305 subscription_outcome(
1306 ¶ms.channels,
1307 &result,
1308 SUBSCRIPTION_ACCEPTED_STATUSES,
1309 &self.subscriptions,
1310 )
1311 }
1312
1313 pub async fn unsubscribe_channels<C>(&self, channels: Vec<C>) -> Result<()>
1320 where
1321 C: Into<DeriveWsChannel>,
1322 {
1323 let channels = channels.into_iter().map(Into::into).collect::<Vec<_>>();
1324 if channels.is_empty() {
1325 return Ok(());
1326 }
1327
1328 let _guard = self.subscription_lock.lock().await;
1329
1330 for topic in channel_topics(&channels) {
1331 self.subscriptions.mark_unsubscribe(&topic);
1332 }
1333
1334 let params = WsUnsubscribeParams { channels };
1335 let cmd_tx = self.cmd_tx.read().await.clone();
1336 let result: WsUnsubscribeResult = send_request(
1337 &self.rate_limiter,
1338 &cmd_tx,
1339 methods::PUBLIC_UNSUBSCRIBE,
1340 ¶ms,
1341 self.request_timeout,
1342 )
1343 .await?;
1344
1345 unsubscribe_outcome(¶ms.channels, &result, &self.subscriptions)
1346 }
1347
1348 async fn send_unsubscribe(&self, channel: String) -> Result<()> {
1349 self.unsubscribe_channels(vec![DeriveWsChannel::from(channel)])
1350 .await
1351 }
1352}
1353
1354impl DeriveWsExecutionHandle {
1355 #[must_use]
1357 pub fn conn_id(&self) -> String {
1358 self.conn_id.load_full().as_ref().clone()
1359 }
1360
1361 pub async fn submit_order(&self, params: &DeriveOrderParams) -> Result<DeriveOrder> {
1372 let reservation = self
1373 .reserve_matching_request(methods::PRIVATE_ORDER, ¶ms.instrument_name)
1374 .await?;
1375 self.submit_order_after_rate_limit(params, reservation)
1376 .await
1377 .map(|result| result.order)
1378 }
1379
1380 pub(crate) async fn submit_order_after_rate_limit(
1381 &self,
1382 params: &DeriveOrderParams,
1383 reservation: MatchingRateLimitReservation,
1384 ) -> Result<DeriveOrderResult> {
1385 self.ensure_authenticated(methods::PRIVATE_ORDER)?;
1386 debug_assert_eq!(reservation.method, methods::PRIVATE_ORDER);
1387 let cmd_tx = self.cmd_tx.read().await.clone();
1388 self.ensure_authenticated(reservation.method)?;
1389 self.commit_matching_reservation(reservation)?;
1390 let result: DeriveOrderResult = send_request_typed_after_rate_limit(
1391 &self.rate_limiter,
1392 &cmd_tx,
1393 methods::PRIVATE_ORDER,
1394 params,
1395 self.request_timeout,
1396 )
1397 .await?;
1398 Ok(result)
1399 }
1400
1401 pub async fn submit_trigger_order(
1408 &self,
1409 params: &DeriveTriggerOrderParams,
1410 ) -> Result<DeriveOrder> {
1411 let reservation = self
1412 .reserve_matching_request(
1413 methods::PRIVATE_TRIGGER_ORDER,
1414 ¶ms.order.instrument_name,
1415 )
1416 .await?;
1417 self.submit_trigger_order_after_rate_limit(params, reservation)
1418 .await
1419 .map(|result| result.order)
1420 }
1421
1422 pub(crate) async fn submit_trigger_order_after_rate_limit(
1423 &self,
1424 params: &DeriveTriggerOrderParams,
1425 reservation: MatchingRateLimitReservation,
1426 ) -> Result<DeriveOrderResult> {
1427 self.submit_order_after_rate_limit(¶ms.order, reservation)
1428 .await
1429 }
1430
1431 pub async fn modify_order(&self, params: &DeriveReplaceParams) -> Result<DeriveReplaceOutcome> {
1439 let reservation = self
1440 .reserve_matching_request(methods::PRIVATE_REPLACE, ¶ms.order.instrument_name)
1441 .await?;
1442 self.modify_order_after_rate_limit(params, reservation)
1443 .await
1444 .and_then(|response| {
1445 response
1446 .result
1447 .and_then(|result| {
1448 result.into_outcome(
1449 ¶ms.order_id_to_cancel,
1450 ¶ms.order.label,
1451 params.order.envelope.subaccount_id,
1452 params.order.instrument_name.as_str(),
1453 )
1454 })
1455 .map_err(|message| {
1456 DeriveWsError::Serde(<serde_json::Error as serde::de::Error>::custom(
1457 message,
1458 ))
1459 })
1460 })
1461 }
1462
1463 pub(crate) async fn modify_order_after_rate_limit(
1464 &self,
1465 params: &DeriveReplaceParams,
1466 reservation: MatchingRateLimitReservation,
1467 ) -> Result<DeriveReplaceResponse> {
1468 self.ensure_authenticated(methods::PRIVATE_REPLACE)?;
1469 debug_assert_eq!(reservation.method, methods::PRIVATE_REPLACE);
1470 let cmd_tx = self.cmd_tx.read().await.clone();
1471 self.ensure_authenticated(reservation.method)?;
1472 self.commit_matching_reservation(reservation)?;
1473 let response = send_raw_after_rate_limit(
1474 &self.rate_limiter,
1475 &cmd_tx,
1476 methods::PRIVATE_REPLACE,
1477 params,
1478 self.request_timeout,
1479 )
1480 .await?;
1481 Ok(DeriveReplaceResponse::from_raw(&response))
1482 }
1483
1484 pub async fn cancel_order(&self, params: &DeriveCancelParams) -> Result<()> {
1490 self.require_authenticated(methods::PRIVATE_CANCEL).await?;
1491 let cmd_tx = self.cmd_tx.read().await.clone();
1492 let _: DeriveEmptyResult = send_request_for_instrument(
1493 &self.rate_limiter,
1494 &cmd_tx,
1495 methods::PRIVATE_CANCEL,
1496 params,
1497 self.request_timeout,
1498 params.instrument_name,
1499 )
1500 .await?;
1501 Ok(())
1502 }
1503
1504 pub async fn cancel_by_instrument(
1510 &self,
1511 params: &DeriveCancelByInstrumentParams,
1512 ) -> Result<DeriveCancelByInstrumentResult> {
1513 self.require_authenticated(methods::PRIVATE_CANCEL_BY_INSTRUMENT)
1514 .await?;
1515 let cmd_tx = self.cmd_tx.read().await.clone();
1516 send_request_typed_for_instrument(
1517 &self.rate_limiter,
1518 &cmd_tx,
1519 methods::PRIVATE_CANCEL_BY_INSTRUMENT,
1520 params,
1521 self.request_timeout,
1522 params.instrument_name,
1523 )
1524 .await
1525 }
1526
1527 pub async fn cancel_trigger_order(
1533 &self,
1534 params: &DeriveCancelTriggerOrderParams,
1535 ) -> Result<DeriveOrder> {
1536 self.require_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)
1537 .await?;
1538 let cmd_tx = self.cmd_tx.read().await.clone();
1539 send_request_typed(
1540 &self.rate_limiter,
1541 &cmd_tx,
1542 methods::PRIVATE_CANCEL_TRIGGER_ORDER,
1543 params,
1544 self.request_timeout,
1545 )
1546 .await
1547 }
1548
1549 pub(crate) async fn cancel_trigger_order_current(
1550 &self,
1551 trigger: &DeriveCancelTriggerOrderParams,
1552 regular: &DeriveCancelParams,
1553 is_active: impl Fn() -> bool,
1554 ) -> Result<Option<DeriveOrder>> {
1555 self.require_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)
1556 .await?;
1557 self.rate_limiter
1558 .await_class_ready(RateClass::NonMatching, None)
1559 .await;
1560 let cmd_tx = self.cmd_tx.read().await.clone();
1561
1562 if is_active() {
1563 return self.cancel_order(regular).await.map(|()| None);
1564 }
1565
1566 self.ensure_authenticated(methods::PRIVATE_CANCEL_TRIGGER_ORDER)?;
1567 send_request_typed_after_rate_limit(
1568 &self.rate_limiter,
1569 &cmd_tx,
1570 methods::PRIVATE_CANCEL_TRIGGER_ORDER,
1571 trigger,
1572 self.request_timeout,
1573 )
1574 .await
1575 .map(Some)
1576 }
1577
1578 pub async fn cancel_by_label(
1585 &self,
1586 params: &DeriveCancelByLabelParams,
1587 ) -> Result<DeriveCancelByLabelResult> {
1588 self.require_authenticated(methods::PRIVATE_CANCEL_BY_LABEL)
1589 .await?;
1590 let cmd_tx = self.cmd_tx.read().await.clone();
1591 send_request_typed(
1592 &self.rate_limiter,
1593 &cmd_tx,
1594 methods::PRIVATE_CANCEL_BY_LABEL,
1595 params,
1596 self.request_timeout,
1597 )
1598 .await
1599 }
1600
1601 pub async fn get_trigger_orders(
1608 &self,
1609 params: &DeriveGetTriggerOrdersParams,
1610 ) -> Result<DeriveOpenOrdersResult> {
1611 self.require_authenticated(methods::PRIVATE_GET_TRIGGER_ORDERS)
1612 .await?;
1613 let cmd_tx = self.cmd_tx.read().await.clone();
1614 send_request_typed(
1615 &self.rate_limiter,
1616 &cmd_tx,
1617 methods::PRIVATE_GET_TRIGGER_ORDERS,
1618 params,
1619 self.request_timeout,
1620 )
1621 .await
1622 }
1623
1624 pub async fn cancel_all_orders(&self, params: &DeriveCancelAllParams) -> Result<()> {
1631 self.require_authenticated(methods::PRIVATE_CANCEL_ALL)
1632 .await?;
1633 let cmd_tx = self.cmd_tx.read().await.clone();
1634 let _: DeriveEmptyResult = send_request(
1635 &self.rate_limiter,
1636 &cmd_tx,
1637 methods::PRIVATE_CANCEL_ALL,
1638 params,
1639 self.request_timeout,
1640 )
1641 .await?;
1642 Ok(())
1643 }
1644
1645 pub(crate) async fn reserve_matching_request(
1646 &self,
1647 operation: &'static str,
1648 instrument_name: &Ustr,
1649 ) -> Result<MatchingRateLimitReservation> {
1650 self.require_authenticated(operation).await?;
1651 debug_assert_eq!(rate_class_for_method(operation), RateClass::Matching);
1652 let quota = self
1653 .rate_limiter
1654 .reserve_matching_ready(instrument_name)
1655 .await;
1656 self.ensure_authenticated(operation)?;
1657 Ok(MatchingRateLimitReservation {
1658 method: operation,
1659 quota,
1660 })
1661 }
1662
1663 fn commit_matching_reservation(&self, reservation: MatchingRateLimitReservation) -> Result<()> {
1664 reservation
1665 .quota
1666 .commit()
1667 .map_err(|retry_after| DeriveWsError::RateLimited { retry_after })
1668 }
1669
1670 fn ensure_authenticated(&self, operation: &'static str) -> Result<()> {
1671 if self.auth_tracker.is_authenticated() {
1672 return Ok(());
1673 }
1674
1675 Err(DeriveWsError::Authentication {
1676 operation: operation.to_string(),
1677 reason: "WebSocket session is not authenticated".to_string(),
1678 })
1679 }
1680
1681 async fn require_authenticated(&self, operation: &'static str) -> Result<()> {
1682 if self
1683 .auth_tracker
1684 .wait_for_authenticated(self.request_timeout)
1685 .await
1686 {
1687 return Ok(());
1688 }
1689
1690 Err(DeriveWsError::Authentication {
1691 operation: operation.to_string(),
1692 reason: "WebSocket session is not authenticated".to_string(),
1693 })
1694 }
1695}
1696
1697#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1701enum RequestRateLimit {
1702 Await(Option<Ustr>),
1705 Reserved,
1708}
1709
1710async fn send_raw<P>(
1711 rate_limiter: &WsRateLimiter,
1712 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1713 method: &'static str,
1714 params: &P,
1715 timeout: Duration,
1716) -> Result<Box<RawValue>>
1717where
1718 P: Serialize + ?Sized,
1719{
1720 send_raw_with_rate_limit(
1721 rate_limiter,
1722 cmd_tx,
1723 method,
1724 params,
1725 timeout,
1726 RequestRateLimit::Await(None),
1727 None,
1728 )
1729 .await
1730}
1731
1732async fn send_raw_for_instrument<P>(
1735 rate_limiter: &WsRateLimiter,
1736 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1737 method: &'static str,
1738 params: &P,
1739 timeout: Duration,
1740 instrument_name: Ustr,
1741) -> Result<Box<RawValue>>
1742where
1743 P: Serialize + ?Sized,
1744{
1745 send_raw_with_rate_limit(
1746 rate_limiter,
1747 cmd_tx,
1748 method,
1749 params,
1750 timeout,
1751 RequestRateLimit::Await(Some(instrument_name)),
1752 None,
1753 )
1754 .await
1755}
1756
1757async fn send_raw_after_rate_limit<P>(
1758 rate_limiter: &WsRateLimiter,
1759 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1760 method: &'static str,
1761 params: &P,
1762 timeout: Duration,
1763) -> Result<Box<RawValue>>
1764where
1765 P: Serialize + ?Sized,
1766{
1767 send_raw_with_rate_limit(
1768 rate_limiter,
1769 cmd_tx,
1770 method,
1771 params,
1772 timeout,
1773 RequestRateLimit::Reserved,
1774 None,
1775 )
1776 .await
1777}
1778
1779async fn send_raw_with_rate_limit<P>(
1780 rate_limiter: &WsRateLimiter,
1781 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1782 method: &'static str,
1783 params: &P,
1784 timeout: Duration,
1785 rate_limit: RequestRateLimit,
1786 connection_epoch: Option<u64>,
1787) -> Result<Box<RawValue>>
1788where
1789 P: Serialize + ?Sized,
1790{
1791 let params = SecretString::from(serde_json::to_string(params)?);
1792
1793 if let RequestRateLimit::Await(instrument_name) = rate_limit {
1794 rate_limiter
1795 .await_class_ready(rate_class_for_method(method), instrument_name.as_ref())
1796 .await;
1797 }
1798
1799 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
1800 cmd_tx
1801 .send(HandlerCommand::Request {
1802 method,
1803 params,
1804 connection_epoch,
1805 cancellation: None,
1806 response_tx,
1807 })
1808 .map_err(|_| DeriveWsError::NotConnected)?;
1809
1810 match time::timeout(timeout, response_rx).await {
1814 Ok(Ok(outcome)) => outcome,
1815 Ok(Err(_)) => Err(DeriveWsError::RequestCancelled {
1816 method: method.to_owned(),
1817 }),
1818 Err(_) => Err(DeriveWsError::Timeout {
1819 method: method.to_owned(),
1820 }),
1821 }
1822}
1823
1824async fn send_request<P, R>(
1827 rate_limiter: &WsRateLimiter,
1828 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1829 method: &'static str,
1830 params: &P,
1831 timeout: Duration,
1832) -> Result<R>
1833where
1834 P: Serialize + ?Sized,
1835 R: Default + DeserializeOwned,
1836{
1837 let value = send_raw(rate_limiter, cmd_tx, method, params, timeout).await?;
1838 decode_default_result(&value)
1839}
1840
1841async fn send_request_for_instrument<P, R>(
1844 rate_limiter: &WsRateLimiter,
1845 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1846 method: &'static str,
1847 params: &P,
1848 timeout: Duration,
1849 instrument_name: Ustr,
1850) -> Result<R>
1851where
1852 P: Serialize + ?Sized,
1853 R: Default + DeserializeOwned,
1854{
1855 let value = send_raw_for_instrument(
1856 rate_limiter,
1857 cmd_tx,
1858 method,
1859 params,
1860 timeout,
1861 instrument_name,
1862 )
1863 .await?;
1864 decode_default_result(&value)
1865}
1866
1867async fn send_request_typed_for_instrument<P, R>(
1869 rate_limiter: &WsRateLimiter,
1870 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1871 method: &'static str,
1872 params: &P,
1873 timeout: Duration,
1874 instrument_name: Ustr,
1875) -> Result<R>
1876where
1877 P: Serialize + ?Sized,
1878 R: DeserializeOwned,
1879{
1880 let value = send_raw_for_instrument(
1881 rate_limiter,
1882 cmd_tx,
1883 method,
1884 params,
1885 timeout,
1886 instrument_name,
1887 )
1888 .await?;
1889 Ok(serde_json::from_str(value.get())?)
1890}
1891
1892fn decode_default_result<R>(value: &RawValue) -> Result<R>
1893where
1894 R: Default + DeserializeOwned,
1895{
1896 if value.get() == "null" {
1897 Ok(R::default())
1898 } else {
1899 Ok(serde_json::from_str(value.get())?)
1900 }
1901}
1902
1903async fn send_request_on_connection<P, R>(
1904 rate_limiter: &WsRateLimiter,
1905 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1906 method: &'static str,
1907 params: &P,
1908 timeout: Duration,
1909 connection_epoch: u64,
1910) -> Result<R>
1911where
1912 P: Serialize + ?Sized,
1913 R: Default + DeserializeOwned,
1914{
1915 let value = send_raw_with_rate_limit(
1916 rate_limiter,
1917 cmd_tx,
1918 method,
1919 params,
1920 timeout,
1921 RequestRateLimit::Await(None),
1922 Some(connection_epoch),
1923 )
1924 .await?;
1925
1926 decode_default_result(&value)
1927}
1928
1929async fn send_request_typed<P, R>(
1932 rate_limiter: &WsRateLimiter,
1933 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1934 method: &'static str,
1935 params: &P,
1936 timeout: Duration,
1937) -> Result<R>
1938where
1939 P: Serialize + ?Sized,
1940 R: DeserializeOwned,
1941{
1942 let value = send_raw(rate_limiter, cmd_tx, method, params, timeout).await?;
1943 Ok(serde_json::from_str(value.get())?)
1944}
1945
1946async fn send_request_typed_after_rate_limit<P, R>(
1947 rate_limiter: &WsRateLimiter,
1948 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1949 method: &'static str,
1950 params: &P,
1951 timeout: Duration,
1952) -> Result<R>
1953where
1954 P: Serialize + ?Sized,
1955 R: DeserializeOwned,
1956{
1957 let value = send_raw_after_rate_limit(rate_limiter, cmd_tx, method, params, timeout).await?;
1958 Ok(serde_json::from_str(value.get())?)
1959}
1960
1961fn channel_topics(channels: &[DeriveWsChannel]) -> Vec<String> {
1962 channels.iter().map(ToString::to_string).collect()
1963}
1964
1965#[expect(
1966 clippy::too_many_arguments,
1967 reason = "authentication state is passed explicitly for epoch fencing"
1968)]
1969async fn login_via_handler(
1970 rate_limiter: &WsRateLimiter,
1971 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1972 auth_tracker: &AuthTracker,
1973 authenticated_epoch: &AtomicU64,
1974 connection_mode: &AtomicU8,
1975 connection_epoch: &AtomicU64,
1976 creds: &DeriveWsCredentials,
1977 timeout: Duration,
1978) -> Result<()> {
1979 let _receiver = auth_tracker.begin();
1980 let expected_epoch = connection_epoch.load(Ordering::Acquire);
1981
1982 match send_login_request(rate_limiter, cmd_tx, creds, timeout, expected_epoch).await {
1983 Ok(())
1984 if complete_session_authentication(
1985 auth_tracker,
1986 authenticated_epoch,
1987 connection_mode,
1988 connection_epoch,
1989 expected_epoch,
1990 ) =>
1991 {
1992 log::debug!("Derive WebSocket authenticated");
1993
1994 Ok(())
1995 }
1996 Ok(()) => {
1997 let e = DeriveWsError::transport(
1998 "connection changed while completing WebSocket authentication",
1999 );
2000 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2001 auth_tracker.fail(e.to_string());
2002 Err(e)
2003 }
2004 Err(e) => {
2005 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2006 auth_tracker.fail(e.to_string());
2007 Err(e)
2008 }
2009 }
2010}
2011
2012async fn send_login_request(
2013 rate_limiter: &WsRateLimiter,
2014 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2015 creds: &DeriveWsCredentials,
2016 timeout: Duration,
2017 connection_epoch: u64,
2018) -> Result<()> {
2019 rate_limiter.await_public_ready().await;
2020 let login = build_ws_login(&creds.wallet_address, &creds.signer)?;
2021
2022 let params = Zeroizing::new(WsLoginParams {
2023 wallet: login.wallet,
2024 timestamp: login.timestamp,
2025 signature: login.signature,
2026 });
2027
2028 let value = send_raw_with_rate_limit(
2029 rate_limiter,
2030 cmd_tx,
2031 methods::PUBLIC_LOGIN,
2032 &*params,
2033 timeout,
2034 RequestRateLimit::Reserved,
2035 Some(connection_epoch),
2036 )
2037 .await?;
2038
2039 let _: WsLoginResult =
2040 serde_json::from_str(value.get()).map_err(|e| DeriveWsError::Authentication {
2041 operation: methods::PUBLIC_LOGIN.to_string(),
2042 reason: format!("invalid login result: {e}"),
2043 })?;
2044
2045 Ok(())
2046}
2047
2048#[expect(
2049 clippy::too_many_arguments,
2050 reason = "recovery state is passed explicitly for epoch fencing"
2051)]
2052async fn recover_session(
2053 rate_limiter: &WsRateLimiter,
2054 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2055 auth_tracker: &AuthTracker,
2056 authenticated_epoch: &AtomicU64,
2057 connection_mode: &AtomicU8,
2058 connection_epoch: &AtomicU64,
2059 creds: Option<&DeriveWsCredentials>,
2060 subscriptions: &SubscriptionState,
2061 subscription_lock: &tokio::sync::Mutex<()>,
2062 timeout: Duration,
2063) -> Result<u64> {
2064 let _guard = subscription_lock.lock().await;
2065 let mut channels = subscriptions.reset_after_reconnect();
2066 channels.sort();
2067 let _receiver = creds.map(|_| auth_tracker.begin());
2068
2069 for attempt in 1..=MAX_SESSION_RECOVERY_ATTEMPTS {
2070 let expected_epoch = wait_for_session_connection(connection_mode, connection_epoch).await?;
2071
2072 let result = recover_session_attempt(
2073 rate_limiter,
2074 cmd_tx,
2075 connection_mode,
2076 connection_epoch,
2077 creds,
2078 subscriptions,
2079 channels.clone(),
2080 timeout,
2081 expected_epoch,
2082 )
2083 .await;
2084
2085 match result {
2086 Ok(()) => {
2087 if creds.is_some()
2088 && !complete_session_authentication(
2089 auth_tracker,
2090 authenticated_epoch,
2091 connection_mode,
2092 connection_epoch,
2093 expected_epoch,
2094 )
2095 {
2096 continue;
2097 }
2098
2099 if creds.is_some() {
2100 log::info!("Derive WebSocket session re-authenticated");
2101 }
2102
2103 return Ok(expected_epoch);
2104 }
2105 Err(e) if attempt < MAX_SESSION_RECOVERY_ATTEMPTS && should_retry_ws_error(&e) => {
2106 let multiplier = 1_u32 << (attempt - 1);
2107 let delay = RECONNECT_BASE_BACKOFF
2108 .saturating_mul(multiplier)
2109 .min(RECONNECT_MAX_BACKOFF);
2110 log::warn!(
2111 "Derive WebSocket session recovery attempt {attempt}/{MAX_SESSION_RECOVERY_ATTEMPTS} failed: {}; retrying in {delay:?}",
2112 e.redacted_message(),
2113 );
2114 time::sleep(delay).await;
2115 }
2116 Err(e) => {
2117 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2118
2119 if creds.is_some() {
2120 auth_tracker.fail(e.to_string());
2121 }
2122
2123 return Err(e);
2124 }
2125 }
2126 }
2127
2128 let e = DeriveWsError::transport("WebSocket session changed while recovery completed");
2129 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2130
2131 if creds.is_some() {
2132 auth_tracker.fail(e.to_string());
2133 }
2134
2135 Err(e)
2136}
2137
2138#[expect(
2139 clippy::too_many_arguments,
2140 reason = "a session attempt retains its fixed connection epoch and native replay context"
2141)]
2142async fn recover_session_attempt(
2143 rate_limiter: &WsRateLimiter,
2144 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2145 connection_mode: &AtomicU8,
2146 connection_epoch: &AtomicU64,
2147 creds: Option<&DeriveWsCredentials>,
2148 subscriptions: &SubscriptionState,
2149 channels: Vec<String>,
2150 timeout: Duration,
2151 expected_epoch: u64,
2152) -> Result<()> {
2153 if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2154 return Err(DeriveWsError::transport(
2155 "connection changed before WebSocket session recovery",
2156 ));
2157 }
2158
2159 if let Some(creds) = creds {
2160 send_login_request(rate_limiter, cmd_tx, creds, timeout, expected_epoch).await?;
2161 }
2162
2163 subscribe_via_handler(
2164 rate_limiter,
2165 cmd_tx,
2166 channels,
2167 timeout,
2168 expected_epoch,
2169 subscriptions,
2170 )
2171 .await?;
2172
2173 if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2174 return Err(DeriveWsError::transport(
2175 "connection changed during WebSocket session recovery",
2176 ));
2177 }
2178
2179 Ok(())
2180}
2181
2182async fn wait_for_session_connection(
2183 connection_mode: &AtomicU8,
2184 connection_epoch: &AtomicU64,
2185) -> Result<u64> {
2186 loop {
2187 match ConnectionMode::from_atomic(connection_mode) {
2188 ConnectionMode::Active => return Ok(connection_epoch.load(Ordering::Acquire)),
2189 ConnectionMode::Reconnect => time::sleep(RECONNECT_BASE_BACKOFF).await,
2190 ConnectionMode::Disconnect | ConnectionMode::Closed => {
2191 return Err(DeriveWsError::transport(
2192 "WebSocket closed during session recovery",
2193 ));
2194 }
2195 }
2196 }
2197}
2198
2199fn complete_session_authentication(
2200 auth_tracker: &AuthTracker,
2201 authenticated_epoch: &AtomicU64,
2202 connection_mode: &AtomicU8,
2203 connection_epoch: &AtomicU64,
2204 expected_epoch: u64,
2205) -> bool {
2206 if !session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2207 return false;
2208 }
2209
2210 authenticated_epoch.store(expected_epoch, Ordering::Release);
2211 auth_tracker.succeed();
2212
2213 if session_connection_is_active(connection_mode, connection_epoch, expected_epoch) {
2214 true
2215 } else {
2216 authenticated_epoch.store(UNAUTHENTICATED_CONNECTION_EPOCH, Ordering::Release);
2217 auth_tracker.invalidate();
2218 false
2219 }
2220}
2221
2222fn session_connection_is_active(
2223 connection_mode: &AtomicU8,
2224 connection_epoch: &AtomicU64,
2225 expected_epoch: u64,
2226) -> bool {
2227 ConnectionMode::from_atomic(connection_mode).is_active()
2228 && connection_epoch.load(Ordering::Acquire) == expected_epoch
2229}
2230
2231async fn subscribe_via_handler(
2232 rate_limiter: &WsRateLimiter,
2233 cmd_tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
2234 channels: Vec<String>,
2235 timeout: Duration,
2236 connection_epoch: u64,
2237 subscriptions: &SubscriptionState,
2238) -> Result<()> {
2239 if channels.is_empty() {
2240 return Ok(());
2241 }
2242
2243 let params = WsSubscribeParams {
2244 channels: channels.into_iter().map(DeriveWsChannel::from).collect(),
2245 };
2246
2247 let result: WsSubscribeResult = send_request_on_connection(
2248 rate_limiter,
2249 cmd_tx,
2250 methods::PUBLIC_SUBSCRIBE,
2251 ¶ms,
2252 timeout,
2253 connection_epoch,
2254 )
2255 .await?;
2256
2257 subscription_outcome(
2258 ¶ms.channels,
2259 &result,
2260 SUBSCRIPTION_REPLAY_ACCEPTED_STATUSES,
2261 subscriptions,
2262 )
2263}
2264
2265fn subscription_outcome(
2266 requested: &[DeriveWsChannel],
2267 result: &WsSubscribeResult,
2268 accepted_statuses: &[&str],
2269 subscriptions: &SubscriptionState,
2270) -> Result<()> {
2271 let mut failures = Vec::new();
2272
2273 for channel in requested {
2274 let topic = channel.to_string();
2275 match result.status.get(channel) {
2276 Some(status)
2277 if accepted_statuses.contains(&status.as_str())
2278 && result.channels.contains(channel) =>
2279 {
2280 subscriptions.confirm_subscribe(&topic);
2281 }
2282 Some(status) if accepted_statuses.contains(&status.as_str()) => {
2283 subscriptions.mark_failure(&topic);
2284 failures.push(format!("{topic}: not subscribed"));
2285 }
2286 other => {
2287 subscriptions.mark_failure(&topic);
2288 let reason = other.map_or("missing channel status", |status| status.as_str());
2289 failures.push(format!("{topic}: {reason}"));
2290 }
2291 }
2292 }
2293
2294 if failures.is_empty() {
2295 Ok(())
2296 } else {
2297 Err(DeriveWsError::Subscription {
2298 details: failures.join(", "),
2299 })
2300 }
2301}
2302
2303fn unsubscribe_outcome(
2304 requested: &[DeriveWsChannel],
2305 result: &WsUnsubscribeResult,
2306 subscriptions: &SubscriptionState,
2307) -> Result<()> {
2308 let mut failures = Vec::new();
2309
2310 for channel in requested {
2311 let topic = channel.to_string();
2312 match result.status.get(channel) {
2313 Some(status) if status == "ok" && !result.remaining_subscriptions.contains(channel) => {
2314 subscriptions.confirm_unsubscribe(&topic);
2315 }
2316 Some(status) if status == "ok" => failures.push(format!("{topic}: still subscribed")),
2317 Some(status) => failures.push(format!("{topic}: {status}")),
2318 None => failures.push(format!("{topic}: missing channel status")),
2319 }
2320 }
2321
2322 if failures.is_empty() {
2323 Ok(())
2324 } else {
2325 Err(DeriveWsError::Subscription {
2326 details: failures.join(", "),
2327 })
2328 }
2329}
2330
2331pub(crate) struct DeriveReplaceResponse {
2332 pub result: std::result::Result<DeriveReplaceResult, String>,
2333 pub trades: Vec<DeriveTrade>,
2334}
2335
2336impl DeriveReplaceResponse {
2337 fn from_raw(response: &RawValue) -> Self {
2338 #[derive(Deserialize)]
2339 struct Trades {
2340 #[serde(default)]
2341 trades: Vec<Box<RawValue>>,
2342 }
2343 let trades = serde_json::from_str::<Trades>(response.get())
2344 .map(|response| salvage_rows(response.trades))
2345 .unwrap_or_default();
2346 let result = serde_json::from_str(response.get()).map_err(|e| e.to_string());
2347 Self { result, trades }
2348 }
2349}
2350
2351#[cfg(test)]
2352mod tests {
2353 use std::{env, time::SystemTime};
2354
2355 use axum::{Json, Router, routing::post};
2356 use futures_util::StreamExt;
2357 use nautilus_common::live::dst::time::Instant;
2358 use nautilus_model::identifiers::InstrumentId;
2359 use nautilus_testkit::connection::race_command_with_publication;
2360 use rstest::rstest;
2361 use serde_json::Value;
2362
2363 use super::*;
2364 use crate::{
2365 common::{
2366 consts::REST_URL_TESTNET,
2367 rate_limit::{DERIVE_RATE_WINDOW_SECS, RateLimits, RequestLimiter},
2368 retry::{is_fatal_ws_error, is_write_outcome_ambiguous_ws},
2369 },
2370 http::{DeriveHttpClient, client::DeriveCredentials},
2371 };
2372
2373 #[tokio::test]
2374 async fn command_during_sender_publication_reaches_initialized_handler() {
2375 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2376 let addr = listener.local_addr().unwrap();
2377 let (received_tx, mut received_rx) = tokio::sync::mpsc::unbounded_channel();
2378
2379 let server = tokio::spawn(async move {
2380 let (socket, _) = listener.accept().await.unwrap();
2381 let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap();
2382 while let Some(Ok(message)) = socket.next().await {
2383 if let tokio_tungstenite::tungstenite::Message::Text(text) = message {
2384 let payload: serde_json::Value = serde_json::from_str(text.as_str()).unwrap();
2385 let _ = received_tx.send(payload);
2386 }
2387 }
2388 });
2389
2390 let url = format!("ws://{addr}");
2391
2392 let mut ws_client = DeriveWebSocketClient::new(
2393 Some(url),
2394 DeriveEnvironment::Testnet,
2395 TransportBackend::default(),
2396 None,
2397 );
2398 let mode = Arc::clone(&ws_client.connection_mode);
2399 let (response_tx, _response_rx) = tokio::sync::oneshot::channel();
2400
2401 let command = HandlerCommand::Request {
2402 method: "public/subscribe",
2403 params: SecretString::from(
2404 serde_json::json!({
2405 "channels": [
2406 "trades.ETH-USDC"
2407 ]
2408 })
2409 .to_string(),
2410 ),
2411 connection_epoch: None,
2412 cancellation: None,
2413 response_tx,
2414 };
2415
2416 race_command_with_publication(
2417 Arc::clone(&ws_client.cmd_tx),
2418 ws_client.connect(),
2419 move || command,
2420 || ConnectionMode::from_atomic(&mode.load()).is_active(),
2421 )
2422 .await
2423 .expect("connect failed");
2424 let received = tokio::time::timeout(Duration::from_secs(2), received_rx.recv()).await;
2425 ws_client.disconnect().await.expect("disconnect failed");
2426 server.abort();
2427
2428 let payload = received
2429 .expect("startup command must reach server")
2430 .unwrap();
2431 assert_eq!(
2432 payload,
2433 serde_json::json!({
2434 "jsonrpc": "2.0",
2435 "id": 1,
2436 "method": "public/subscribe",
2437 "params": {
2438 "channels": [
2439 "trades.ETH-USDC"
2440 ]
2441 }
2442 })
2443 );
2444 }
2445
2446 #[rstest]
2447 #[case("wss://stream.example.com/feed", "wss://stream.example.com")]
2448 #[case(
2449 "wss://SYN-user:SYN-password@stream.example.com:9443/SYN-path?token=SYN-query#SYN-fragment",
2450 "wss://stream.example.com:9443"
2451 )]
2452 fn test_debug_preserves_url_origin(#[case] url: &str, #[case] origin: &str) {
2453 let client = DeriveWebSocketClient::new(
2454 Some(url.to_string()),
2455 DeriveEnvironment::Testnet,
2456 TransportBackend::default(),
2457 None,
2458 );
2459 let debug = format!("{client:?}");
2460 assert!(debug.contains(&format!("url: {origin:?}")));
2461 assert!(!debug.contains("SYN-"));
2462 assert_eq!(client.url.expose(), url);
2463 }
2464
2465 #[rstest]
2466 #[case::subscription_lock(false)]
2467 #[case::rate_limit(true)]
2468 #[tokio::test]
2469 async fn test_queued_book_subscribe_uses_reconnected_generation(#[case] paced: bool) {
2470 let tracker = BookSyncTracker::default();
2471 let channel = "orderbook.ETH-USDC.1.10";
2472 tracker.record_subscription(channel, InstrumentId::from("ETH-USDC.DERIVE"));
2473 let client = DeriveWebSocketClient::new(
2474 None,
2475 DeriveEnvironment::Testnet,
2476 TransportBackend::default(),
2477 None,
2478 )
2479 .with_book_sync(tracker.clone(), Duration::from_secs(10));
2480 let mut handle = client.subscription_handle();
2481
2482 handle.rate_limiter = Arc::new(RequestLimiter::new(
2483 RateLimits::new(None, None),
2484 MonotonicClock,
2485 ));
2486 handle.connection_epoch.load().store(1, Ordering::Release);
2487 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2488 *handle.cmd_tx.write().await = cmd_tx;
2489 if paced {
2490 while handle
2491 .rate_limiter
2492 .try_class_ready(RateClass::NonMatching, None)
2493 .is_ok()
2494 {}
2495 }
2496
2497 let lock = Arc::clone(&handle.subscription_lock);
2498
2499 let guard = if paced { None } else { Some(lock.lock().await) };
2500 let request = handle.subscribe_book_gated(channel.to_owned());
2501 tokio::pin!(request);
2502 assert!(futures_util::poll!(&mut request).is_pending());
2503
2504 tracker.reset_on_reconnect(Duration::from_secs(10));
2505 handle.connection_epoch.load().store(2, Ordering::Release);
2506 drop(guard);
2507 let ack: Value =
2508 serde_json::from_str(include_str!("../../test_data/spot/ws_subscribe_ack.json"))
2509 .unwrap();
2510 tokio::time::timeout(Duration::from_secs(8), async {
2511 tokio::join!(async { request.await.unwrap() }, async {
2512 let HandlerCommand::Request {
2513 method,
2514 params,
2515 connection_epoch,
2516 cancellation,
2517 response_tx,
2518 } = cmd_rx.recv().await.unwrap()
2519 else {
2520 panic!("expected book subscribe request");
2521 };
2522 let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
2523 response_tx
2524 .send(Ok(serde_json::value::to_raw_value(&ack["result"]).unwrap()))
2525 .unwrap();
2526
2527 assert_eq!(method, methods::PUBLIC_SUBSCRIBE);
2528 assert_eq!(params, serde_json::json!({"channels": [channel]}));
2529 assert_eq!(connection_epoch, Some(2));
2530 assert!(!cancellation.unwrap().is_cancelled());
2531 },);
2532 })
2533 .await
2534 .unwrap();
2535 }
2536
2537 #[rstest]
2538 #[tokio::test]
2539 async fn test_replay_ack_cannot_open_queued_book_replacement() {
2540 let tracker = BookSyncTracker::default();
2541 let channel = "orderbook.ETH-USDC.1.10";
2542 let now = Instant::now();
2543 tracker.record_subscription(channel, InstrumentId::from("ETH-USDC.DERIVE"));
2544 tracker.write(channel).unwrap().gate.open();
2545 assert!(tracker.record_snapshot(channel, 100, None, now));
2546 let recovery = tracker
2547 .recoveries(Duration::from_secs(1), now + Duration::from_secs(2))
2548 .pop()
2549 .unwrap()
2550 .2;
2551 assert!(recovery.begin_replacement());
2552 tracker.reset_on_reconnect(Duration::from_secs(10));
2553 let client = DeriveWebSocketClient::new(
2554 None,
2555 DeriveEnvironment::Testnet,
2556 TransportBackend::default(),
2557 None,
2558 )
2559 .with_book_sync(tracker.clone(), Duration::from_secs(10));
2560 let mut handle = client.subscription_handle();
2561
2562 handle.rate_limiter = Arc::new(RequestLimiter::new(
2563 RateLimits::new(None, None),
2564 MonotonicClock,
2565 ));
2566 handle.connection_epoch.load().store(2, Ordering::Release);
2567 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2568 *handle.cmd_tx.write().await = cmd_tx;
2569 let lock = Arc::clone(&handle.subscription_lock);
2570 let guard = lock.lock().await;
2571 let request = handle.replace_book_gated(
2572 channel.to_owned(),
2573 CancellationToken::new(),
2574 &recovery,
2575 recovery.gate.clone(),
2576 );
2577 tokio::pin!(request);
2578 assert!(futures_util::poll!(&mut request).is_pending());
2579 let replay = tracker.write(channel).unwrap();
2580 replay.gate.open();
2581 drop(guard);
2582 let ack: Value =
2583 serde_json::from_str(include_str!("../../test_data/spot/ws_subscribe_ack.json"))
2584 .unwrap();
2585 tokio::time::timeout(Duration::from_secs(1), async {
2586 tokio::join!(async { request.await.unwrap() }, async {
2587 for method in [methods::PUBLIC_UNSUBSCRIBE, methods::PUBLIC_SUBSCRIBE] {
2588 let HandlerCommand::Request {
2589 method: actual_method,
2590 params,
2591 connection_epoch,
2592 cancellation,
2593 response_tx,
2594 } = cmd_rx.recv().await.unwrap()
2595 else {
2596 panic!("expected book replacement request");
2597 };
2598 let current = tracker.write(channel).unwrap();
2599 assert!(current.gate.lock().is_closed());
2600 assert!(!tracker.record_snapshot(channel, 101, Some(¤t.generation), now));
2601 assert!(replay.generation.is_cancelled());
2602 assert_eq!(actual_method, method);
2603 assert_eq!(
2604 serde_json::from_str::<Value>(params.expose_secret()).unwrap(),
2605 serde_json::json!({"channels": [channel]}),
2606 );
2607 assert_eq!(connection_epoch, Some(2));
2608 assert!(!cancellation.unwrap().is_cancelled());
2609 let result = if method == methods::PUBLIC_SUBSCRIBE {
2610 current.gate.open();
2611 ack["result"].clone()
2612 } else {
2613 serde_json::json!({"status": {channel: "ok"}, "remaining_subscriptions": []})
2614 };
2615 response_tx
2616 .send(Ok(serde_json::value::to_raw_value(&result).unwrap()))
2617 .unwrap();
2618 }
2619 });
2620 })
2621 .await
2622 .unwrap();
2623
2624 assert!(tracker.record_snapshot(channel, 101, None, now));
2625 assert!(recovery.is_accepted());
2626 }
2627
2628 #[tokio::test]
2629 async fn test_expired_matching_reservation_has_local_failure_evidence() {
2630 let client = DeriveWebSocketClient::new(
2631 None,
2632 DeriveEnvironment::Testnet,
2633 TransportBackend::default(),
2634 None,
2635 );
2636 let handle = client.execution_handle();
2637 let quota = handle
2638 .rate_limiter
2639 .reserve_matching_ready(&Ustr::from("ETH-PERP"))
2640 .await;
2641 time::sleep(Duration::from_secs(DERIVE_RATE_WINDOW_SECS)).await;
2642 for _ in 0..DERIVE_RATE_WINDOW_SECS {
2643 handle
2644 .rate_limiter
2645 .await_class_ready(RateClass::Matching, None)
2646 .await;
2647 }
2648
2649 let reservation = MatchingRateLimitReservation {
2650 method: methods::PRIVATE_ORDER,
2651 quota,
2652 };
2653
2654 let error = handle.commit_matching_reservation(reservation).unwrap_err();
2655
2656 assert!(matches!(error, DeriveWsError::RateLimited { .. }));
2657 assert!(should_retry_ws_error(&error));
2658 assert!(!is_fatal_ws_error(&error));
2659 assert!(!is_write_outcome_ambiguous_ws(&error));
2660 }
2661
2662 #[rstest]
2663 fn test_replace_response_preserves_order_with_undecodable_trade() {
2664 let order: Value = serde_json::from_str(include_str!(
2665 "../../test_data/perps/http_order_eth_partially_filled.json"
2666 ))
2667 .unwrap();
2668 let trade: Value = serde_json::from_str(include_str!(
2669 "../../test_data/perps/http_private_trade_eth.json"
2670 ))
2671 .unwrap();
2672 let response = DeriveReplaceResponse::from_raw(
2673 &serde_json::value::to_raw_value(&serde_json::json!({
2674 "order": order,
2675 "trades": [{"trade_id": "bad-row"}, trade],
2676 }))
2677 .unwrap(),
2678 );
2679 let outcome = response
2680 .result
2681 .unwrap()
2682 .into_outcome("old-order", "alpha-strategy", 42, "ETH-PERP")
2683 .unwrap();
2684
2685 let DeriveReplaceOutcome::Replaced(order) = outcome else {
2686 panic!("expected replaced outcome");
2687 };
2688
2689 let trades = response.trades;
2690 assert_eq!(order.order_id, "abc-123");
2691 assert_eq!(order.label, "alpha-strategy");
2692 assert_eq!(trades.len(), 1);
2693 assert_eq!(trades[0].trade_id, "trade-xyz");
2694 assert_eq!(trades[0].order_id, "order-abc");
2695 }
2696
2697 const SESSION_KEY_HEX: &str =
2698 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd";
2699
2700 #[rstest]
2701 #[case(Value::Null)]
2702 #[case(serde_json::json!("bad-nonce"))]
2703 #[case(serde_json::json!("18446744073709551616"))]
2704 fn test_replace_response_preserves_financial_rows_with_invalid_order_nonce(
2705 #[case] nonce: Value,
2706 ) {
2707 let mut order: Value = serde_json::from_str(include_str!(
2708 "../../test_data/perps/http_order_eth_partially_filled.json"
2709 ))
2710 .unwrap();
2711 let trade: DeriveTrade = serde_json::from_str(include_str!(
2712 "../../test_data/perps/http_private_trade_eth.json"
2713 ))
2714 .unwrap();
2715 order["nonce"] = nonce;
2716 let response = DeriveReplaceResponse::from_raw(
2717 &serde_json::value::to_raw_value(
2718 &serde_json::json!({"order": order, "trades": [trade]}),
2719 )
2720 .unwrap(),
2721 );
2722 assert!(response.result.is_err());
2723 assert_eq!(
2724 serde_json::to_value(response.trades).unwrap(),
2725 serde_json::to_value(vec![trade]).unwrap()
2726 );
2727 }
2728
2729 #[tokio::test]
2730 async fn test_http_and_websocket_share_matching_consumption() {
2731 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2732 let address = listener.local_addr().unwrap();
2733
2734 let server = tokio::spawn(async move {
2735 axum::serve(
2736 listener,
2737 Router::new().route(
2738 "/private/cancel",
2739 post(|| async { Json(serde_json::json!({"result": true})) }),
2740 ),
2741 )
2742 .await
2743 .unwrap();
2744 });
2745
2746 let wallet = "0x000000000000000000000000000000000000ab01";
2747 let http = DeriveHttpClient::with_credentials(
2748 format!("http://{address}"),
2749 DeriveCredentials::new(wallet, SESSION_KEY_HEX).unwrap(),
2750 None,
2751 None,
2752 None,
2753 )
2754 .unwrap();
2755 let ws = DeriveWebSocketClient::with_credentials(
2756 Some(format!("ws://{address}/ws")),
2757 DeriveEnvironment::Testnet,
2758 TransportBackend::default(),
2759 None,
2760 DeriveWsCredentials::new(wallet.to_ascii_uppercase(), SESSION_KEY_HEX).unwrap(),
2761 None,
2762 None,
2763 );
2764
2765 for _ in 0..5 {
2766 let result: bool = http
2767 .send_private_once("private/cancel", &serde_json::json!({}))
2768 .await
2769 .unwrap();
2770 assert!(result);
2771 }
2772
2773 server.abort();
2774 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2775 let params = serde_json::json!({});
2776 let request = send_raw(
2777 &ws.rate_limiter,
2778 &cmd_tx,
2779 methods::PRIVATE_CANCEL,
2780 ¶ms,
2781 Duration::from_secs(1),
2782 );
2783 tokio::pin!(request);
2784 assert!(futures_util::poll!(&mut request).is_pending());
2785 assert!(matches!(
2786 cmd_rx.try_recv(),
2787 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
2788 ));
2789 }
2790
2791 #[tokio::test]
2792 async fn test_login_builds_auth_timestamp_after_public_quota_wait() {
2793 let rate_limiter = RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
2794 for _ in 0..25 {
2795 rate_limiter.await_public_ready().await;
2796 }
2797
2798 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
2799
2800 let captured = tokio::spawn(async move {
2801 let Some(HandlerCommand::Request {
2802 method,
2803 params,
2804 response_tx,
2805 ..
2806 }) = cmd_rx.recv().await
2807 else {
2808 panic!("login request must enqueue");
2809 };
2810
2811 assert_eq!(method, methods::PUBLIC_LOGIN);
2812 let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
2813 let timestamp: u64 = params["timestamp"].as_str().unwrap().parse().unwrap();
2814 let now = SystemTime::now()
2815 .duration_since(SystemTime::UNIX_EPOCH)
2816 .unwrap()
2817 .as_millis();
2818 response_tx
2819 .send(Ok(serde_json::value::to_raw_value(&serde_json::json!([
2820 77
2821 ]))
2822 .unwrap()))
2823 .unwrap();
2824 now.saturating_sub(u128::from(timestamp))
2825 });
2826
2827 let started = tokio::time::Instant::now();
2828 send_login_request(
2829 &rate_limiter,
2830 &cmd_tx,
2831 &DeriveWsCredentials::new(
2832 "0x000000000000000000000000000000000000ab02",
2833 SESSION_KEY_HEX,
2834 )
2835 .unwrap(),
2836 Duration::from_secs(1),
2837 0,
2838 )
2839 .await
2840 .unwrap();
2841 assert!(started.elapsed() >= Duration::from_secs(4));
2842 assert!(
2843 captured.await.unwrap() < 1_000,
2844 "login timestamp must remain fresh after the pacing wait"
2845 );
2846 }
2847
2848 #[tokio::test]
2849 #[ignore = "requires Derive public testnet network access"]
2850 async fn test_live_testnet_unsubscribe_acknowledgement() {
2851 let mut ws = DeriveWebSocketClient::new(
2852 None,
2853 DeriveEnvironment::Testnet,
2854 TransportBackend::default(),
2855 None,
2856 );
2857 ws.connect().await.unwrap();
2858 let cmd_tx = ws.cmd_tx.read().await.clone();
2859 let params = serde_json::json!({"channels": ["ticker_slim.ETH-PERP.1000"]});
2860 let subscribe = send_raw(
2861 &ws.rate_limiter,
2862 &cmd_tx,
2863 methods::PUBLIC_SUBSCRIBE,
2864 ¶ms,
2865 Duration::from_secs(15),
2866 )
2867 .await;
2868 let unsubscribe = send_raw(
2869 &ws.rate_limiter,
2870 &cmd_tx,
2871 methods::PUBLIC_UNSUBSCRIBE,
2872 ¶ms,
2873 Duration::from_secs(15),
2874 )
2875 .await;
2876 ws.disconnect().await.unwrap();
2877 let subscribe: Value = serde_json::from_str(subscribe.unwrap().get()).unwrap();
2878 let unsubscribe: Value = serde_json::from_str(unsubscribe.unwrap().get()).unwrap();
2879 println!("Public testnet subscribe acknowledgement: {subscribe}");
2880 println!("Public testnet unsubscribe acknowledgement: {unsubscribe}");
2881 assert_eq!(subscribe["status"]["ticker_slim.ETH-PERP.1000"], "ok");
2882 assert_eq!(unsubscribe["status"]["ticker_slim.ETH-PERP.1000"], "ok");
2883 assert_eq!(
2884 unsubscribe["remaining_subscriptions"],
2885 serde_json::json!([])
2886 );
2887 }
2888
2889 #[tokio::test]
2890 #[ignore = "requires Derive testnet credentials and network access"]
2891 async fn test_live_testnet_request_budget_scopes() {
2892 let wallet = env::var("DERIVE_TESTNET_WALLET_ADDRESS").expect("testnet wallet configured");
2893 let key = env::var("DERIVE_TESTNET_SESSION_PRIVATE_KEY").expect("testnet key configured");
2894 let subaccount: u64 = env::var("DERIVE_TESTNET_SUBACCOUNT_ID")
2895 .expect("testnet subaccount configured")
2896 .parse()
2897 .unwrap();
2898 let http = DeriveHttpClient::with_credentials(
2899 REST_URL_TESTNET,
2900 DeriveCredentials::new(&wallet, &key).unwrap(),
2901 Some(15),
2902 None,
2903 None,
2904 )
2905 .unwrap();
2906 let mut ws = DeriveWebSocketClient::with_credentials(
2907 None,
2908 DeriveEnvironment::Testnet,
2909 TransportBackend::default(),
2910 None,
2911 DeriveWsCredentials::new(&wallet, &key).unwrap(),
2912 None,
2913 None,
2914 );
2915 ws.connect()
2916 .await
2917 .expect("testnet WebSocket login succeeds");
2918 let cmd_tx = ws.cmd_tx.read().await.clone();
2919 let before: Value = http
2920 .send_private("public/getRateLimits", &serde_json::json!({}))
2921 .await
2922 .expect("authenticated HTTP budget query succeeds");
2923 let ws_limits = send_raw(
2924 &ws.rate_limiter,
2925 &cmd_tx,
2926 "public/getRateLimits",
2927 &serde_json::json!({}),
2928 Duration::from_secs(15),
2929 )
2930 .await
2931 .expect("authenticated WebSocket budget query succeeds");
2932 println!("Testnet authenticated HTTP budgets: {before}");
2933 println!(
2934 "Testnet authenticated WebSocket budgets: {}",
2935 ws_limits.get()
2936 );
2937 let label = format!("nautilus-pacing-{}", UUID4::new());
2938
2939 for scoped in [true, false] {
2940 let mut params = serde_json::json!({"subaccount_id": subaccount, "label": label});
2941 if scoped {
2942 params["instrument_name"] = serde_json::json!("ETH-PERP");
2943 }
2944
2945 let result: Value = http
2946 .send_private_once("private/cancel_by_label", ¶ms)
2947 .await
2948 .expect("testnet cancellation of unique empty label succeeds");
2949 let budgets: Value = http
2950 .send_private("public/getRateLimits", &serde_json::json!({}))
2951 .await
2952 .unwrap();
2953 println!("Testnet cancel-by-label scoped={scoped}: {result}; budgets={budgets}");
2954 let ws_budgets = send_raw(
2955 &ws.rate_limiter,
2956 &cmd_tx,
2957 "public/getRateLimits",
2958 &serde_json::json!({}),
2959 Duration::from_secs(15),
2960 )
2961 .await
2962 .unwrap();
2963 println!(
2964 "Testnet wallet budgets after HTTP scoped={scoped}: {}",
2965 ws_budgets.get()
2966 );
2967 }
2968
2969 let started = tokio::time::Instant::now();
2970 let name = Ustr::from("ETH-PERP");
2971 let params = serde_json::json!({"subaccount_id": subaccount, "label": label, "instrument_name": name});
2972
2973 for _ in 0..6 {
2974 let result = send_raw_for_instrument(
2975 &ws.rate_limiter,
2976 &cmd_tx,
2977 methods::PRIVATE_CANCEL_BY_LABEL,
2978 ¶ms,
2979 Duration::from_secs(15),
2980 name,
2981 )
2982 .await
2983 .expect("paced testnet WebSocket scoped cancellation succeeds");
2984 let result: Value = serde_json::from_str(result.get()).unwrap();
2985 assert_eq!(result["cancelled_orders"], 0);
2986 }
2987
2988 let elapsed = started.elapsed();
2989 assert!(
2990 elapsed >= Duration::from_secs(1),
2991 "six same-instrument cancels must exceed the local five-token burst"
2992 );
2993 println!(
2994 "Testnet six scoped WebSocket cancellations pass with local pacing in {elapsed:?}"
2995 );
2996 let after = send_raw(
2997 &ws.rate_limiter,
2998 &cmd_tx,
2999 "public/getRateLimits",
3000 &serde_json::json!({}),
3001 Duration::from_secs(15),
3002 )
3003 .await
3004 .unwrap();
3005 println!(
3006 "Testnet wallet budgets after paced WebSocket requests: {}",
3007 after.get()
3008 );
3009 ws.disconnect().await.unwrap();
3010 ws.connect()
3011 .await
3012 .expect("testnet reconnect authenticates with preserved pacing state");
3013 ws.disconnect().await.unwrap();
3014 }
3015
3016 #[rstest]
3017 fn test_public_client_defaults_to_environment_url() {
3018 let client = DeriveWebSocketClient::new(
3019 None,
3020 DeriveEnvironment::Mainnet,
3021 TransportBackend::default(),
3022 None,
3023 );
3024 assert_eq!(client.url(), "wss://api.derive.xyz/v3/ws");
3025 assert!(!client.is_authenticated());
3026 assert!(!client.is_active());
3027 assert_eq!(client.subscription_count(), 0);
3028 }
3029
3030 #[tokio::test]
3031 async fn test_execution_auth_barrier_waits_for_authentication() {
3032 let client = DeriveWebSocketClient::with_credentials(
3033 None,
3034 DeriveEnvironment::Mainnet,
3035 TransportBackend::default(),
3036 None,
3037 DeriveWsCredentials::new(
3038 "0x000000000000000000000000000000000000aaaa",
3039 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3040 )
3041 .unwrap(),
3042 None,
3043 None,
3044 );
3045 let execution = client.execution_handle();
3046 let auth_tracker = execution.auth_tracker.clone();
3047 let _receiver = auth_tracker.begin();
3048 let tracker_for_task = auth_tracker.clone();
3049
3050 get_runtime().spawn(async move {
3051 tokio::time::sleep(Duration::from_millis(10)).await;
3052 tracker_for_task.succeed();
3053 });
3054
3055 execution
3056 .require_authenticated(methods::PRIVATE_ORDER)
3057 .await
3058 .expect("barrier should wait for successful authentication");
3059 }
3060
3061 #[tokio::test]
3062 async fn test_execution_auth_barrier_fails_on_terminal_auth_failure() {
3063 let client = DeriveWebSocketClient::with_credentials(
3064 None,
3065 DeriveEnvironment::Mainnet,
3066 TransportBackend::default(),
3067 None,
3068 DeriveWsCredentials::new(
3069 "0x000000000000000000000000000000000000aaaa",
3070 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3071 )
3072 .unwrap(),
3073 None,
3074 None,
3075 );
3076 let execution = client.execution_handle();
3077 let _receiver = execution.auth_tracker.begin();
3078 execution.auth_tracker.fail("bad signature");
3079
3080 let error = execution
3081 .require_authenticated(methods::PRIVATE_ORDER)
3082 .await
3083 .expect_err("terminal auth failure must reject private operations");
3084
3085 assert!(matches!(error, DeriveWsError::Authentication { .. }));
3086 }
3087
3088 #[tokio::test]
3089 async fn test_session_recovery_waits_for_reconnecting_transport() {
3090 let connection_mode = Arc::new(AtomicU8::new(ConnectionMode::Reconnect as u8));
3091 let connection_epoch = Arc::new(AtomicU64::new(1));
3092 let mode_for_task = Arc::clone(&connection_mode);
3093 let epoch_for_task = Arc::clone(&connection_epoch);
3094
3095 get_runtime().spawn(async move {
3096 tokio::time::sleep(Duration::from_millis(10)).await;
3097 epoch_for_task.store(2, Ordering::Release);
3098 mode_for_task.store(ConnectionMode::Active as u8, Ordering::Release);
3099 });
3100
3101 let epoch = tokio::time::timeout(
3102 Duration::from_secs(1),
3103 wait_for_session_connection(&connection_mode, &connection_epoch),
3104 )
3105 .await
3106 .expect("session recovery should resume after reconnect")
3107 .expect("active replacement connection should be accepted");
3108
3109 assert_eq!(epoch, 2);
3110 }
3111
3112 #[rstest]
3113 fn test_testnet_client_routes_to_v3_url() {
3114 let client = DeriveWebSocketClient::new(
3115 None,
3116 DeriveEnvironment::Testnet,
3117 TransportBackend::default(),
3118 None,
3119 );
3120 assert_eq!(client.url(), "wss://testnet.api.derive.xyz/v3/ws");
3121 }
3122
3123 #[rstest]
3124 fn test_credentials_constructor_parses_session_key() {
3125 let creds = DeriveWsCredentials::new(
3126 "0x000000000000000000000000000000000000aaaa",
3127 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3128 )
3129 .unwrap();
3130 assert!(creds.wallet_address.starts_with("0x"));
3131 let client = DeriveWebSocketClient::with_credentials(
3132 None,
3133 DeriveEnvironment::Testnet,
3134 TransportBackend::default(),
3135 None,
3136 creds,
3137 None,
3138 None,
3139 );
3140 assert_eq!(client.url(), "wss://testnet.api.derive.xyz/v3/ws");
3141 assert!(!client.is_authenticated());
3142 }
3143
3144 #[rstest]
3145 fn test_credentials_debug_redacts_signer() {
3146 let creds = DeriveWsCredentials::new(
3147 "0xWALLET",
3148 "0x2ae8be44db8a590d20bffbe3b6872df9b569147d3bf6801a35a28281a4816bbd",
3149 )
3150 .unwrap();
3151 let debug = format!("{creds:?}");
3152 assert!(debug.contains("redacted"));
3153 assert!(debug.contains("0xWALLET"));
3154 assert!(!debug.contains("2ae8be44"));
3155 }
3156
3157 #[rstest]
3158 fn test_credentials_constructor_rejects_invalid_session_key() {
3159 let err = DeriveWsCredentials::new("0xWALLET", "not-a-hex-key").unwrap_err();
3160 assert!(err.to_string().contains("invalid session key"));
3161 }
3162
3163 #[rstest]
3164 #[tokio::test]
3165 async fn test_send_raw_closed_command_queue_is_definitive() {
3166 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3167 drop(cmd_rx);
3168 let rate_limiter: WsRateLimiter =
3169 RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3170 let error = send_raw(
3171 &rate_limiter,
3172 &cmd_tx,
3173 methods::PRIVATE_REPLACE,
3174 &serde_json::json!({}),
3175 Duration::from_millis(50),
3176 )
3177 .await
3178 .unwrap_err();
3179 assert!(matches!(error, DeriveWsError::NotConnected));
3180 assert!(!crate::common::retry::is_write_outcome_ambiguous_ws(&error));
3181 }
3182
3183 #[rstest]
3184 #[tokio::test]
3185 async fn test_send_raw_times_out_when_no_response_arrives() {
3186 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3189 let rate_limiter: WsRateLimiter =
3190 RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3191 let err = send_raw(
3192 &rate_limiter,
3193 &cmd_tx,
3194 methods::PRIVATE_ORDER,
3195 &serde_json::json!({}),
3196 Duration::from_millis(50),
3197 )
3198 .await
3199 .expect_err("must time out");
3200
3201 match err {
3202 DeriveWsError::Timeout { method } => assert_eq!(method, methods::PRIVATE_ORDER),
3203 other => panic!("expected Timeout, was {other:?}"),
3204 }
3205 }
3206
3207 #[rstest]
3208 #[tokio::test]
3209 async fn test_send_request_typed_rejects_null_result() {
3210 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3214 tokio::spawn(async move {
3215 if let Some(HandlerCommand::Request { response_tx, .. }) = cmd_rx.recv().await {
3216 let _ = response_tx.send(Ok(RawValue::NULL.to_owned()));
3217 }
3218 });
3219
3220 let rate_limiter: WsRateLimiter =
3221 RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3222 let result: Result<DeriveOrderResult> = send_request_typed(
3223 &rate_limiter,
3224 &cmd_tx,
3225 methods::PRIVATE_ORDER,
3226 &serde_json::json!({}),
3227 Duration::from_secs(1),
3228 )
3229 .await;
3230 assert!(matches!(result, Err(DeriveWsError::Serde(_))));
3231 }
3232
3233 #[rstest]
3234 #[case::pending(false)]
3235 #[case::fired_during_pacing(true)]
3236 #[tokio::test]
3237 async fn test_trigger_cancel_selects_endpoint_after_pacing(#[case] fires: bool) {
3238 let client = DeriveWebSocketClient::new(
3239 None,
3240 DeriveEnvironment::Testnet,
3241 TransportBackend::default(),
3242 None,
3243 );
3244 let mut handle = client.execution_handle();
3245 handle.auth_tracker.succeed();
3246
3247 handle.rate_limiter = Arc::new(RequestLimiter::new(
3248 RateLimits::new(None, None),
3249 MonotonicClock,
3250 ));
3251 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel();
3252 *handle.cmd_tx.write().await = cmd_tx;
3253 for _ in 0..crate::common::rate_limit::DERIVE_WEBSOCKET_NON_MATCHING_TPS
3254 * crate::common::rate_limit::DERIVE_RATE_BURST_MULTIPLIER
3255 {
3256 handle
3257 .rate_limiter
3258 .try_class_ready(RateClass::NonMatching, None)
3259 .unwrap();
3260 }
3261
3262 assert!(
3263 handle
3264 .rate_limiter
3265 .try_class_ready(RateClass::NonMatching, None)
3266 .is_err()
3267 );
3268 let active = AtomicBool::new(false);
3269 let trigger = DeriveCancelTriggerOrderParams::new(42, "paced-trigger");
3270 let regular = DeriveCancelParams::new(42, "ETH-PERP", "paced-trigger");
3271 let cancel = handle
3272 .cancel_trigger_order_current(&trigger, ®ular, || active.load(Ordering::Acquire));
3273 tokio::pin!(cancel);
3274 assert!(futures_util::poll!(cancel.as_mut()).is_pending());
3275 assert!(cmd_rx.try_recv().is_err());
3276 active.store(fires, Ordering::Release);
3277 let mut canceled: DeriveOrder = serde_json::from_str(include_str!(
3278 "../../test_data/perps/http_order_eth_partially_filled.json"
3279 ))
3280 .unwrap();
3281 canceled.order_id = "paced-trigger".into();
3282 canceled.order_status = crate::common::enums::DeriveOrderStatus::Cancelled;
3283 canceled.filled_amount = rust_decimal::Decimal::ZERO;
3284 canceled.average_price = rust_decimal::Decimal::ZERO;
3285 let expected = (!fires).then_some(canceled);
3286
3287 let response = async {
3288 let HandlerCommand::Request {
3289 method,
3290 params,
3291 response_tx,
3292 ..
3293 } = cmd_rx.recv().await.unwrap()
3294 else {
3295 panic!("expected one cancel request")
3296 };
3297
3298 let params: Value = serde_json::from_str(params.expose_secret()).unwrap();
3299
3300 let expected_method = if fires {
3301 methods::PRIVATE_CANCEL
3302 } else {
3303 methods::PRIVATE_CANCEL_TRIGGER_ORDER
3304 };
3305
3306 assert_eq!(method, expected_method);
3307 assert_eq!(params["subaccount_id"], serde_json::json!(42));
3308 assert_eq!(params["order_id"], serde_json::json!("paced-trigger"));
3309 assert_eq!(
3310 params.get("instrument_name"),
3311 fires
3312 .then_some(®ular.instrument_name)
3313 .map(|name| serde_json::json!(name))
3314 .as_ref()
3315 );
3316 response_tx
3317 .send(Ok(serde_json::value::to_raw_value(&expected).unwrap()))
3318 .unwrap();
3319 };
3320
3321 let (result, ()) = tokio::join!(cancel, response);
3322 assert_eq!(
3323 serde_json::to_value(result.unwrap()).unwrap(),
3324 serde_json::to_value(expected).unwrap()
3325 );
3326 assert!(cmd_rx.try_recv().is_err());
3327 }
3328
3329 #[rstest]
3330 #[tokio::test]
3331 async fn test_reserved_send_does_not_wait_for_or_consume_second_quota_cell() {
3332 let rate_limiter: WsRateLimiter =
3333 RequestLimiter::new(RateLimits::new(None, None), MonotonicClock);
3334
3335 for _ in 0..5 {
3336 rate_limiter
3337 .try_class_ready(RateClass::Matching, None)
3338 .expect("reservation consumes the matching window");
3339 }
3340
3341 let (cmd_tx, mut cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
3342 tokio::spawn(async move {
3343 if let Some(HandlerCommand::Request { response_tx, .. }) = cmd_rx.recv().await {
3344 let _ = response_tx.send(Ok(serde_json::value::to_raw_value(
3345 &serde_json::json!({"accepted": true}),
3346 )
3347 .unwrap()));
3348 }
3349 });
3350
3351 let response = tokio::time::timeout(
3352 Duration::from_millis(100),
3353 send_raw_after_rate_limit(
3354 &rate_limiter,
3355 &cmd_tx,
3356 methods::PRIVATE_ORDER,
3357 &serde_json::json!({}),
3358 Duration::from_secs(1),
3359 ),
3360 )
3361 .await
3362 .expect("reserved send must not wait for quota")
3363 .expect("reserved send succeeds");
3364
3365 assert_eq!(
3366 serde_json::from_str::<Value>(response.get()).unwrap(),
3367 serde_json::json!({"accepted": true}),
3368 );
3369 assert!(
3370 rate_limiter
3371 .try_class_ready(RateClass::Matching, None)
3372 .is_err(),
3373 "reserved send must not consume a second cell",
3374 );
3375 }
3376}