1use std::{
19 borrow::Cow,
20 num::NonZeroUsize,
21 str::FromStr,
22 sync::{
23 Arc, Weak,
24 atomic::{AtomicBool, Ordering},
25 },
26 time::Duration,
27};
28
29use ahash::{AHashMap, AHashSet};
30use anyhow::Context;
31use async_trait::async_trait;
32use dashmap::DashMap;
33use nautilus_common::{
34 cache::{InstrumentLookupError, fifo::FifoCache, quote::QuoteCache},
35 clients::{DataClient, execute_instrument_subscriptions},
36 live::{
37 dst::time::{self, Instant},
38 runner::get_data_event_sender,
39 sender::EventSender,
40 },
41 messages::{
42 DataEvent,
43 data::{
44 BarsResponse, DataResponse, FundingRatesResponse, InstrumentResponse,
45 InstrumentsResponse, OptionChainReferencePriceResponse, QuotesResponse, RequestBars,
46 RequestFundingRates, RequestInstrument, RequestInstruments,
47 RequestOptionChainReferencePrice, RequestQuotes, RequestTrades, SubscribeBookDeltas,
48 SubscribeBookDepth, SubscribeFundingRates, SubscribeIndexPrices, SubscribeMarkPrices,
49 SubscribeOptionGreeks, SubscribeQuotes, SubscribeTrades, TradesResponse,
50 UnsubscribeBookDeltas, UnsubscribeBookDepth, UnsubscribeFundingRates,
51 UnsubscribeIndexPrices, UnsubscribeMarkPrices, UnsubscribeOptionGreeks,
52 UnsubscribeQuotes, UnsubscribeTrades,
53 },
54 },
55 providers::InstrumentProvider,
56};
57use nautilus_core::{
58 AtomicMap, AtomicSet, Params, UnixNanos,
59 datetime::{NANOSECONDS_IN_SECOND, datetime_to_unix_nanos},
60 time::{AtomicTime, get_atomic_clock_realtime},
61};
62use nautilus_live::{
63 SocketControl,
64 task::{TaskGroup, TaskGroupGuard},
65};
66use nautilus_model::{
67 data::{Bar, Data, QuoteTick},
68 enums::{AggregationSource, BookType, PriceType},
69 identifiers::{ClientId, InstrumentId, TradeId, Venue},
70 instruments::{Instrument, InstrumentAny},
71 types::Price,
72};
73use parking_lot::Mutex;
74use rust_decimal::Decimal;
75use tokio_util::sync::CancellationToken;
76
77use crate::{
78 book::{recovery::spawn_recovery, sync::BookSyncTracker},
79 common::{
80 consts::{
81 DERIVE_CANDLES_DEFAULT_LIMIT, DERIVE_CANDLES_MAX_PAGES, DERIVE_TRADES_PAGE_SIZE,
82 DERIVE_VENUE,
83 },
84 enums::{
85 DeriveInstrumentType, DeriveOrderbookDepth, DeriveOrderbookGroup, DeriveTickerInterval,
86 },
87 parse::{format_instrument_id, format_venue_symbol, parse_derive_instrument_any},
88 },
89 config::DeriveDataClientConfig,
90 http::{DeriveHttpClient, query::PaginationCursor},
91 providers::{
92 DeriveInstrumentProvider, fetch_instrument_definitions, normalize_currency,
93 parse_instrument_definitions,
94 },
95 websocket::{
96 DEFAULT_ORDERBOOK_DEPTH, DEFAULT_ORDERBOOK_GROUP, DEFAULT_TICKER_INTERVAL,
97 DeriveOrderbookMsg, DerivePublicWsData, DeriveTickerMsg, DeriveTradesMsg,
98 DeriveWebSocketClient, DeriveWebSocketSubscriptionHandle, DeriveWsError, DeriveWsMessage,
99 WsMessageContext, bar_spec_to_derive_period, dispatch::TRADE_DEDUP_CAPACITY,
100 orderbook_channel, parse_candle_record, parse_funding_rate,
101 parse_funding_rate_history_record, parse_index_price, parse_mark_price,
102 parse_option_greeks, parse_orderbook_deltas, parse_orderbook_depth, parse_public_ws_data,
103 parse_ticker_quote, parse_ticker_quote_from_rest, parse_trade_tick,
104 parse_trade_tick_from_rest, ticker_channel, trades_channel,
105 },
106};
107
108#[derive(Debug)]
110pub struct DeriveDataClient {
111 client_id: ClientId,
112 config: DeriveDataClientConfig,
113 http_client: DeriveHttpClient,
114 provider: DeriveInstrumentProvider,
115 ws_client: DeriveWebSocketClient,
116 is_connected: Arc<AtomicBool>,
117 cancellation_token: CancellationToken,
118 session_tasks: TaskGroup,
119 pending_tasks: TaskGroup,
120 shutdown_errors: Vec<String>,
121 data_sender: EventSender<DataEvent>,
122 instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
123 active_book_delta_channels: Arc<AtomicMap<InstrumentId, String>>,
124 active_book_depth_channels: Arc<AtomicMap<InstrumentId, String>>,
125 active_ticker_channels: Arc<AtomicMap<InstrumentId, String>>,
126 active_quote_subs: Arc<AtomicSet<InstrumentId>>,
127 active_trade_subs: Arc<AtomicSet<InstrumentId>>,
128 active_mark_subs: Arc<AtomicSet<InstrumentId>>,
129 active_index_subs: Arc<AtomicSet<InstrumentId>>,
130 active_funding_subs: Arc<AtomicSet<InstrumentId>>,
131 active_greeks_subs: Arc<AtomicSet<InstrumentId>>,
132 channel_subscriptions: Arc<ChannelSubscriptionRegistry>,
133 subscription_lock: Arc<Mutex<()>>,
134 book_sync: BookSyncTracker,
135 quote_cache: Arc<Mutex<QuoteCache>>,
136 trades_emitted: Arc<Mutex<FifoCache<TradeId, TRADE_DEDUP_CAPACITY>>>,
137 clock: &'static AtomicTime,
138}
139
140impl DeriveDataClient {
141 pub fn new(client_id: ClientId, config: DeriveDataClientConfig) -> anyhow::Result<Self> {
147 config.validate()?;
148
149 let clock = get_atomic_clock_realtime();
150 let data_sender = get_data_event_sender();
151 let proxy_url = config
152 .proxy_url
153 .as_ref()
154 .map(|value| value.expose_secret().to_owned());
155
156 let http_client = DeriveHttpClient::new(
157 config.rest_url(),
158 Some(config.http_timeout_secs),
159 proxy_url.clone(),
160 None,
161 )?;
162 let provider = DeriveInstrumentProvider::with_expired(
163 http_client.clone(),
164 config.currencies.clone(),
165 config.include_expired,
166 );
167 let book_sync = BookSyncTracker::default();
168 let mut ws_client = DeriveWebSocketClient::new(
169 Some(config.ws_url()),
170 config.environment,
171 config.transport_backend,
172 proxy_url,
173 )
174 .with_book_sync(
175 book_sync.clone(),
176 Duration::from_secs(config.book_snapshot_timeout_secs),
177 )
178 .with_socket_control(SocketControl::new(
179 client_id,
180 Some(*DERIVE_VENUE),
181 "derive-data-streams",
182 ));
183
184 if let Some(secs) = config.ws_timeout_secs {
185 ws_client.set_request_timeout(Duration::from_secs(secs));
186 }
187
188 let session_tasks = TaskGroup::new();
189 let pending_tasks = TaskGroup::new();
190
191 Ok(Self {
192 client_id,
193 config,
194 http_client,
195 provider,
196 ws_client,
197 is_connected: Arc::new(AtomicBool::new(false)),
198 cancellation_token: CancellationToken::new(),
199 session_tasks,
200 pending_tasks,
201 shutdown_errors: Vec::new(),
202 data_sender,
203 instruments: Arc::new(AtomicMap::new()),
204 active_book_delta_channels: Arc::new(AtomicMap::new()),
205 active_book_depth_channels: Arc::new(AtomicMap::new()),
206 active_ticker_channels: Arc::new(AtomicMap::new()),
207 active_quote_subs: Arc::new(AtomicSet::new()),
208 active_trade_subs: Arc::new(AtomicSet::new()),
209 active_mark_subs: Arc::new(AtomicSet::new()),
210 active_index_subs: Arc::new(AtomicSet::new()),
211 active_funding_subs: Arc::new(AtomicSet::new()),
212 active_greeks_subs: Arc::new(AtomicSet::new()),
213 channel_subscriptions: Arc::new(ChannelSubscriptionRegistry::default()),
214 subscription_lock: Arc::new(Mutex::new(())),
215 book_sync,
216 quote_cache: Arc::new(Mutex::new(QuoteCache::new())),
217 trades_emitted: Arc::new(Mutex::new(FifoCache::new())),
218 clock,
219 })
220 }
221
222 fn spawn_task<F>(&self, description: impl Into<Cow<'static, str>>, fut: F)
223 where
224 F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
225 {
226 let description = description.into();
227 let task_description = description.clone();
228
229 let future = async move {
230 if fut.await.is_err() {
231 log::warn!("{task_description} failed");
232 }
233 };
234
235 if let Err(e) = self.pending_tasks.spawn(future) {
236 log::warn!("Skipping Derive {description} after shutdown began: {e}");
237 }
238 }
239
240 fn abort_pending_tasks(&self) {
241 self.pending_tasks.begin_shutdown();
242 }
243
244 fn abort_session_tasks(&self) {
245 self.session_tasks.begin_shutdown();
246 self.ws_client.begin_shutdown();
247 }
248
249 fn clear_subscription_state(&self) {
252 let _guard = self.subscription_lock.lock();
253 self.channel_subscriptions.clear();
254 self.book_sync.clear();
255 self.active_book_delta_channels.store(AHashMap::new());
256 self.active_book_depth_channels.store(AHashMap::new());
257 self.active_ticker_channels.store(AHashMap::new());
258 self.active_quote_subs.store(AHashSet::new());
259 self.active_trade_subs.store(AHashSet::new());
260 self.active_mark_subs.store(AHashSet::new());
261 self.active_index_subs.store(AHashSet::new());
262 self.active_funding_subs.store(AHashSet::new());
263 self.active_greeks_subs.store(AHashSet::new());
264 self.quote_cache.lock().clear();
265 self.trades_emitted.lock().clear();
266 }
267
268 async fn teardown_partial_connect(&mut self) -> anyhow::Result<()> {
269 self.cancellation_token.cancel();
270 self.abort_session_tasks();
271 self.abort_pending_tasks();
272
273 if let Err(e) = self.ws_client.disconnect().await {
274 self.shutdown_errors
275 .push(format!("Derive WebSocket shutdown failed: {e}"));
276 }
277
278 let (session_result, pending_result) =
279 tokio::join!(self.join_session_tasks(), self.join_pending_tasks());
280 self.clear_subscription_state();
281 self.channel_subscriptions.clear_transitions();
282 self.is_connected.store(false, Ordering::Release);
283
284 if let Err(e) = session_result {
285 self.shutdown_errors.push(e.to_string());
286 }
287
288 if let Err(e) = pending_result {
289 self.shutdown_errors.push(e.to_string());
290 }
291
292 if !self.shutdown_errors.is_empty() {
293 anyhow::bail!(std::mem::take(&mut self.shutdown_errors).join("; "));
294 }
295
296 Ok(())
297 }
298
299 fn spawn_stream_task(
300 &self,
301 mut rx: tokio::sync::mpsc::UnboundedReceiver<DeriveWsMessage>,
302 ) -> anyhow::Result<()> {
303 let ctx = WsMessageContext {
304 clock: self.clock,
305 data_sender: self.data_sender.clone(),
306 instruments: Arc::clone(&self.instruments),
307 active_book_delta_channels: Arc::clone(&self.active_book_delta_channels),
308 active_book_depth_channels: Arc::clone(&self.active_book_depth_channels),
309 active_ticker_channels: Arc::clone(&self.active_ticker_channels),
310 active_quote_subs: Arc::clone(&self.active_quote_subs),
311 active_trade_subs: Arc::clone(&self.active_trade_subs),
312 active_mark_subs: Arc::clone(&self.active_mark_subs),
313 active_index_subs: Arc::clone(&self.active_index_subs),
314 active_funding_subs: Arc::clone(&self.active_funding_subs),
315 active_greeks_subs: Arc::clone(&self.active_greeks_subs),
316 subscription_lock: Arc::clone(&self.subscription_lock),
317 book_sync: self.book_sync.clone(),
318 quote_cache: Arc::clone(&self.quote_cache),
319 trades_emitted: Arc::clone(&self.trades_emitted),
320 };
321
322 let cancellation = self.cancellation_token.clone();
323 let is_connected = Arc::clone(&self.is_connected);
324
325 self.session_tasks.spawn(async move {
326 loop {
327 tokio::select! {
328 biased;
329 () = cancellation.cancelled() => {
330 log::debug!("Derive WebSocket data stream task cancelled");
331 break;
332 }
333 maybe_msg = rx.recv() => {
334 match maybe_msg {
335 Some(msg) => {
336 if matches!(&msg, DeriveWsMessage::SessionRecoveryFailed(_)) {
337 is_connected.store(false, Ordering::Release);
338 }
339 Self::handle_ws_message(msg, &ctx);
340 }
341 None => {
342 log::debug!("Derive WebSocket data stream ended");
343 break;
344 }
345 }
346 }
347 }
348 }
349
350 is_connected.store(false, Ordering::Release);
351
352 if !cancellation.is_cancelled() {
353 log::warn!("Derive data WebSocket stream ended unexpectedly");
354 }
355 })?;
356
357 Ok(())
358 }
359
360 fn spawn_book_monitor(&self) -> anyhow::Result<()> {
361 let tracker = self.book_sync.clone();
362 let ws = self.ws_handle();
363 let registry = Arc::clone(&self.channel_subscriptions);
364 let stale_timeout = Duration::from_secs(self.config.book_stale_threshold_secs);
365 let snapshot_timeout = Duration::from_secs(self.config.book_snapshot_timeout_secs);
366 let tasks = self.pending_tasks.spawner()?;
367 let cancellation = self.cancellation_token.clone();
368
369 self.session_tasks.spawn(async move {
370 let mut interval = time::interval(Duration::from_secs(1));
371
372 loop {
373 tokio::select! {
374 biased;
375 () = cancellation.cancelled() => break,
376 _ = interval.tick() => {
377 for (channel, signal, recovery) in tracker.recoveries(stale_timeout, Instant::now()) {
378 signal.log();
379 let transition = registry.transition(&channel);
380 spawn_recovery(
381 channel,
382 signal.instrument_id,
383 recovery,
384 ws.clone(),
385 transition,
386 snapshot_timeout,
387 &tasks,
388 );
389 }
390 }
391 }
392 }
393 })?;
394
395 Ok(())
396 }
397
398 fn spawn_instrument_refresh(&self) -> anyhow::Result<()> {
399 if let Some(future) = self.instrument_refresh() {
400 self.session_tasks.spawn(future)?;
401 }
402
403 Ok(())
404 }
405
406 fn instrument_refresh(&self) -> Option<impl Future<Output = ()> + Send + 'static> {
407 let minutes = self.config.update_instruments_interval_mins;
408 let currencies: Vec<_> = self
409 .config
410 .currencies
411 .iter()
412 .filter_map(|currency| normalize_currency(currency).map(ToOwned::to_owned))
413 .collect();
414
415 if minutes == 0 || currencies.is_empty() {
416 return None;
417 }
418
419 let interval = Duration::from_secs(minutes.saturating_mul(60));
420 let cancellation = self.cancellation_token.clone();
421 let http_client = self.http_client.clone();
422 let include_expired = self.config.include_expired;
423 let instruments = Arc::clone(&self.instruments);
424 let sender = self.data_sender.clone();
425 let subscription_lock = Arc::clone(&self.subscription_lock);
426
427 Some(async move {
428 loop {
429 tokio::select! {
430 biased;
431 () = cancellation.cancelled() => break,
432 () = time::sleep(interval) => {}
433 }
434
435 for currency in ¤cies {
436 let result = tokio::select! {
437 biased;
438 () = cancellation.cancelled() => return,
439 result = fetch_instrument_definitions(&http_client, currency, include_expired) => result,
440 };
441
442 match result {
443 Ok(definitions) => {
444 let loaded = parse_instrument_definitions(definitions);
445 cache_instruments(
446 loaded,
447 &instruments,
448 &sender,
449 &cancellation,
450 &subscription_lock,
451 );
452 }
453 Err(e) => {
454 log::warn!("Failed to refresh Derive instruments for {currency}: {e}");
455 }
456 }
457 }
458 }
459 })
460 }
461
462 fn handle_ws_message(message: DeriveWsMessage, ctx: &WsMessageContext) {
463 match message {
464 DeriveWsMessage::Subscription(payload) => match parse_public_ws_data(&payload) {
465 Ok(data) => Self::handle_public_ws_data(data, ctx),
466 Err(e) => {
467 let snippet = truncated_payload_snippet(payload.data.get());
471 log::warn!(
472 "Failed to parse Derive public WS data on channel `{}`: {e}; payload: {snippet}",
473 payload.channel,
474 );
475 }
476 },
477 DeriveWsMessage::BookSubscription {
478 payload,
479 generation,
480 } => {
481 let _guard = ctx.subscription_lock.lock();
482
483 if generation.is_cancelled() {
484 return;
485 }
486
487 match parse_public_ws_data(&payload) {
488 Ok(DerivePublicWsData::Orderbook(msg)) => {
489 Self::handle_public_book(&msg, ctx, Some(&generation));
490 }
491 Err(e) => {
492 ctx.book_sync.reject_snapshot(
493 payload.channel.as_str(),
494 Arc::new(DeriveWsError::Transport(e.to_string())),
495 Some(&generation),
496 );
497 log::warn!("Failed to parse Derive book snapshot: {e}");
498 }
499 Ok(_) => {}
500 }
501 }
502 DeriveWsMessage::Reconnected => {
503 let _guard = ctx.subscription_lock.lock();
504 ctx.quote_cache.lock().clear();
505 log::info!("Derive WebSocket reconnected");
506 }
507 DeriveWsMessage::SessionRecoveryFailed(_) => {
508 log::error!("Derive WebSocket session recovery failed");
509 }
510 DeriveWsMessage::Authenticated => log::debug!("Derive WebSocket authenticated"),
511 }
512 }
513
514 fn handle_public_ws_data(data: DerivePublicWsData, ctx: &WsMessageContext) {
515 let _guard = ctx.subscription_lock.lock();
518 match data {
519 DerivePublicWsData::Orderbook(msg) => Self::handle_public_book(&msg, ctx, None),
520 DerivePublicWsData::Trades(msg) => Self::handle_public_trades(&msg, ctx),
521 DerivePublicWsData::Ticker(msg) => Self::handle_public_ticker(&msg, ctx),
522 }
523 }
524
525 fn handle_public_book(
526 msg: &DeriveOrderbookMsg,
527 ctx: &WsMessageContext,
528 generation: Option<&CancellationToken>,
529 ) {
530 let instrument_id = match msg.data.instrument_id() {
531 Ok(instrument_id) => instrument_id,
532 Err(e) => {
533 log::warn!("Invalid Derive feed instrument: {e:#}");
534 return;
535 }
536 };
537
538 let channel = msg.channel.as_str();
539 let deltas_active =
540 channel_is_active(&ctx.active_book_delta_channels, instrument_id, channel);
541 let depth_active =
542 channel_is_active(&ctx.active_book_depth_channels, instrument_id, channel);
543
544 if !deltas_active && !depth_active {
545 return;
546 }
547
548 let Some(instrument) = ctx.instruments.get_cloned(&instrument_id) else {
549 log::warn!("Orderbook message received for unknown instrument {instrument_id}");
550 return;
551 };
552
553 let ts_init = ctx.clock.get_time_ns();
554
555 let deltas = if deltas_active {
556 match parse_orderbook_deltas(
557 msg,
558 instrument.price_precision(),
559 instrument.size_precision(),
560 ts_init,
561 ) {
562 Ok(deltas) => Some(deltas),
563 Err(e) => {
564 ctx.book_sync.reject_snapshot(
565 channel,
566 Arc::new(DeriveWsError::Transport(e.to_string())),
567 generation,
568 );
569
570 log::warn!("Failed to parse Derive book deltas: {e}");
571 return;
572 }
573 }
574 } else {
575 None
576 };
577
578 let depth = if depth_active {
579 match parse_orderbook_depth(
580 msg,
581 instrument.price_precision(),
582 instrument.size_precision(),
583 ts_init,
584 ) {
585 Ok(depth) => Some(depth),
586 Err(e) => {
587 ctx.book_sync.reject_snapshot(
588 channel,
589 Arc::new(DeriveWsError::Transport(e.to_string())),
590 generation,
591 );
592
593 log::warn!("Failed to parse Derive book depth: {e}");
594 return;
595 }
596 }
597 } else {
598 None
599 };
600
601 if !ctx
602 .book_sync
603 .record_snapshot(channel, msg.data.timestamp, generation, Instant::now())
604 {
605 return;
606 }
607
608 if let Some(deltas) = deltas {
609 Self::send_data(ctx, Data::BookDeltas(Box::new(deltas)));
610 }
611
612 if let Some(depth) = depth {
613 Self::send_data(ctx, Data::BookDepth(Box::new(depth)));
614 }
615 }
616
617 fn handle_public_ticker(msg: &DeriveTickerMsg, ctx: &WsMessageContext) {
618 let instrument_id = match msg.data.instrument_id() {
619 Ok(instrument_id) => instrument_id,
620 Err(e) => {
621 log::warn!("Invalid Derive feed instrument: {e:#}");
622 return;
623 }
624 };
625
626 if !channel_is_active(
627 &ctx.active_ticker_channels,
628 instrument_id,
629 msg.channel.as_str(),
630 ) {
631 return;
632 }
633
634 let Some(instrument) = ctx.instruments.get_cloned(&instrument_id) else {
635 log::warn!("Ticker message received for unknown instrument {instrument_id}");
636 return;
637 };
638
639 let ts_init = ctx.clock.get_time_ns();
640 let price_precision = instrument.price_precision();
641
642 if ctx.active_quote_subs.contains(&instrument_id) {
643 let mut quote_cache = ctx.quote_cache.lock();
644
645 match process_ticker_quote(
646 msg,
647 price_precision,
648 instrument.size_precision(),
649 ts_init,
650 &mut quote_cache,
651 ) {
652 Ok(Some(quote)) => {
653 Self::send_data(ctx, Data::Quote(quote));
654 }
655 Ok(None) => {}
656 Err(e) => log::warn!("Failed to parse Derive ticker quote: {e}"),
657 }
658 }
659
660 Self::handle_ticker_prices(msg, ctx, instrument_id, price_precision, ts_init);
661 Self::handle_ticker_derivatives(msg, ctx, instrument_id, ts_init);
662 }
663
664 fn handle_ticker_prices(
665 msg: &DeriveTickerMsg,
666 ctx: &WsMessageContext,
667 instrument_id: InstrumentId,
668 price_precision: u8,
669 ts_init: UnixNanos,
670 ) {
671 if ctx.active_mark_subs.contains(&instrument_id) {
672 match parse_mark_price(msg, price_precision, ts_init) {
673 Ok(Some(update)) => {
674 Self::send_data(ctx, Data::MarkPrice(update));
675 }
676 Ok(None) => {}
677 Err(e) => log::warn!("Failed to parse Derive mark price: {e}"),
678 }
679 }
680
681 if ctx.active_index_subs.contains(&instrument_id) {
682 match parse_index_price(msg, price_precision, ts_init) {
683 Ok(Some(update)) => {
684 Self::send_data(ctx, Data::IndexPrice(update));
685 }
686 Ok(None) => {}
687 Err(e) => log::warn!("Failed to parse Derive index price: {e}"),
688 }
689 }
690 }
691
692 fn handle_ticker_derivatives(
693 msg: &DeriveTickerMsg,
694 ctx: &WsMessageContext,
695 instrument_id: InstrumentId,
696 ts_init: UnixNanos,
697 ) {
698 if ctx.active_funding_subs.contains(&instrument_id) {
699 match parse_funding_rate(msg, ts_init) {
700 Ok(Some(update)) => {
701 if let Err(e) = ctx
702 .data_sender
703 .send(DataEvent::Data(Data::FundingRate(update)))
704 {
705 log::error!("Failed to send Derive funding rate: {e}");
706 }
707 }
708 Ok(None) => {}
709 Err(e) => log::warn!("Failed to parse Derive funding rate: {e}"),
710 }
711 }
712
713 if ctx.active_greeks_subs.contains(&instrument_id) {
714 match parse_option_greeks(msg, ts_init) {
715 Ok(Some(greeks)) => {
716 if let Err(e) = ctx
717 .data_sender
718 .send(DataEvent::Data(Data::OptionGreeks(greeks)))
719 {
720 log::error!("Failed to send Derive option greeks: {e}");
721 }
722 }
723 Ok(None) => {}
724 Err(e) => log::warn!("Failed to parse Derive option greeks: {e}"),
725 }
726 }
727 }
728
729 fn handle_public_trades(msg: &DeriveTradesMsg, ctx: &WsMessageContext) {
730 let ts_init = ctx.clock.get_time_ns();
731
732 for trade in &msg.trades {
733 let instrument_id = match format_instrument_id(trade.instrument_name) {
734 Ok(instrument_id) => instrument_id,
735 Err(e) => {
736 log::warn!("Invalid Derive trade instrument: {e:#}");
737 continue;
738 }
739 };
740
741 if !ctx.active_trade_subs.contains(&instrument_id) {
742 continue;
743 }
744
745 let Some(instrument) = ctx.instruments.get_cloned(&instrument_id) else {
746 log::warn!("Trade message received for unknown instrument {instrument_id}");
747 continue;
748 };
749
750 match parse_trade_tick(
751 trade,
752 instrument.price_precision(),
753 instrument.size_precision(),
754 ts_init,
755 ) {
756 Ok(tick) => {
757 let mut emitted = ctx.trades_emitted.lock();
758 if emitted.contains(&tick.trade_id) {
759 continue;
760 }
761
762 if Self::send_data(ctx, Data::Trade(tick)) {
763 emitted.add(tick.trade_id);
764 }
765 }
766 Err(e) => log::warn!("Failed to parse Derive trade tick: {e}"),
767 }
768 }
769 }
770
771 fn send_data(ctx: &WsMessageContext, data: Data) -> bool {
772 match ctx.data_sender.send(DataEvent::Data(data)) {
773 Ok(()) => true,
774 Err(e) => {
775 log::error!("Failed to send Derive data event: {e}");
776 false
777 }
778 }
779 }
780
781 fn cache_provider_instruments(&self) {
782 let instruments = self
783 .provider
784 .store()
785 .get_all()
786 .values()
787 .cloned()
788 .collect::<Vec<_>>();
789
790 cache_instruments(
791 instruments,
792 &self.instruments,
793 &self.data_sender,
794 &self.cancellation_token,
795 &self.subscription_lock,
796 );
797 }
798
799 fn prepare_subscribe(&self, instrument_id: InstrumentId) -> anyhow::Result<bool> {
800 if self.instruments.contains_key(&instrument_id) {
801 return Ok(false);
802 }
803
804 if !self.config.auto_load_missing_instruments {
805 anyhow::bail!(
806 "Instrument {instrument_id} not found and `auto_load_missing_instruments` is disabled"
807 );
808 }
809
810 Ok(true)
811 }
812
813 async fn lazy_load_instrument(
814 http_client: DeriveHttpClient,
815 instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
816 instrument_id: InstrumentId,
817 include_expired: bool,
818 ) -> anyhow::Result<()> {
819 let currency = currency_from_instrument_id(&instrument_id)?;
820 let definitions = fetch_instrument_definitions(&http_client, currency, include_expired)
821 .await
822 .with_context(|| format!("failed to lazy-load Derive instruments for {currency}"))?;
823 let loaded = parse_instrument_definitions(definitions);
824 instruments.rcu(|cache| {
825 for instrument in &loaded {
826 cache.insert(instrument.id(), instrument.clone());
827 }
828 });
829
830 let found = loaded
831 .iter()
832 .any(|instrument| instrument.id() == instrument_id);
833
834 if !found {
835 anyhow::bail!(InstrumentLookupError::not_found(instrument_id));
836 }
837
838 Ok(())
839 }
840
841 fn ws_handle(&self) -> DeriveWebSocketSubscriptionHandle {
842 self.ws_client.subscription_handle()
843 }
844
845 fn subscription_lifecycle(&self) -> SubscriptionLifecycle {
846 SubscriptionLifecycle {
847 registry: Arc::clone(&self.channel_subscriptions),
848 lock: Arc::clone(&self.subscription_lock),
849 snapshot_timeout: Duration::from_secs(self.config.book_snapshot_timeout_secs),
850 dispatch: SubscriptionDispatchState {
851 book_sync: self.book_sync.clone(),
852 active_book_delta_channels: Arc::clone(&self.active_book_delta_channels),
853 active_book_depth_channels: Arc::clone(&self.active_book_depth_channels),
854 active_ticker_channels: Arc::clone(&self.active_ticker_channels),
855 active_quote_subs: Arc::clone(&self.active_quote_subs),
856 active_trade_subs: Arc::clone(&self.active_trade_subs),
857 active_mark_subs: Arc::clone(&self.active_mark_subs),
858 active_index_subs: Arc::clone(&self.active_index_subs),
859 active_funding_subs: Arc::clone(&self.active_funding_subs),
860 active_greeks_subs: Arc::clone(&self.active_greeks_subs),
861 quote_cache: Arc::clone(&self.quote_cache),
862 },
863 }
864 }
865
866 fn subscribe_ticker_feed(
867 &self,
868 instrument_id: InstrumentId,
869 params: &Option<Params>,
870 feed: TickerFeed,
871 label: &'static str,
872 ) -> anyhow::Result<()> {
873 let owner = ChannelOwner::Ticker {
874 instrument_id,
875 feed,
876 };
877
878 let lifecycle = self.subscription_lifecycle();
879 if lifecycle.is_active(owner) {
880 return Ok(());
881 }
882
883 let channel = match self.active_ticker_channels.get_cloned(&instrument_id) {
884 Some(channel) => channel,
885 None => {
886 let instrument_name = format_venue_symbol(&instrument_id)?.to_string();
887 let interval = ticker_interval(params)?;
888 ticker_channel(&instrument_name, &interval)
889 }
890 };
891
892 let request = ChannelRequest::from_channel(&channel)?;
893 let needs_load = self.prepare_subscribe(instrument_id)?;
894
895 let Some(generation) = lifecycle.activate(owner, Some(&channel)) else {
896 return Ok(());
897 };
898
899 let ws = self.ws_handle();
900 let http_client = self.http_client.clone();
901 let include_expired = self.config.include_expired;
902 let instruments = Arc::clone(&self.instruments);
903
904 self.spawn_task("subscribe_ticker_feed", async move {
905 if needs_load
906 && let Err(e) = Self::lazy_load_instrument(
907 http_client,
908 instruments,
909 instrument_id,
910 include_expired,
911 )
912 .await
913 {
914 lifecycle.rollback(owner, generation);
915 log::error!("Lazy-load failed for {instrument_id} ({label}): {e}");
916 return Ok(());
917 }
918
919 run_channel_subscribe(lifecycle, owner, generation, request, ws).await
920 });
921
922 Ok(())
923 }
924
925 fn unsubscribe_channel_owner(&self, owner: ChannelOwner) -> anyhow::Result<()> {
926 let lifecycle = self.subscription_lifecycle();
927
928 let Some(removed) = lifecycle.remove(owner) else {
929 return Ok(());
930 };
931
932 if !removed.channel_empty {
933 return Ok(());
934 }
935
936 let Some(channel) = removed.channel else {
937 return Ok(());
938 };
939
940 let request = ChannelRequest::from_channel(&channel)?;
941 let ws = self.ws_handle();
942 self.spawn_task("unsubscribe_channel", async move {
943 run_channel_unsubscribe(lifecycle, request, ws).await
944 });
945
946 Ok(())
947 }
948}
949
950#[async_trait(?Send)]
951impl DataClient for DeriveDataClient {
952 fn client_id(&self) -> ClientId {
953 self.client_id
954 }
955
956 fn venue(&self) -> Option<Venue> {
957 Some(*DERIVE_VENUE)
958 }
959
960 fn start(&mut self) -> anyhow::Result<()> {
961 log::info!("Starting Derive data client: {}", self.client_id);
962 Ok(())
963 }
964
965 fn stop(&mut self) -> anyhow::Result<()> {
966 log::info!("Stopping Derive data client: {}", self.client_id);
967 self.cancellation_token.cancel();
968 let _guard = self.subscription_lock.lock();
969 self.abort_session_tasks();
970 self.abort_pending_tasks();
971 self.book_sync.clear();
972 self.is_connected.store(false, Ordering::Relaxed);
973 Ok(())
974 }
975
976 fn reset(&mut self) -> anyhow::Result<()> {
977 log::info!("Resetting Derive data client: {}", self.client_id);
978 self.cancellation_token.cancel();
979
980 self.abort_session_tasks();
981 self.abort_pending_tasks();
982 self.is_connected.store(false, Ordering::Relaxed);
983
984 {
987 let _guard = self.subscription_lock.lock();
988 self.instruments.store(AHashMap::new());
989 }
990
991 self.clear_subscription_state();
992 self.provider.store_mut().clear();
993 Ok(())
994 }
995
996 fn dispose(&mut self) -> anyhow::Result<()> {
997 log::debug!("Disposing Derive data client: {}", self.client_id);
998 self.stop()
999 }
1000
1001 fn is_connected(&self) -> bool {
1002 self.is_connected.load(Ordering::SeqCst)
1003 }
1004
1005 fn is_disconnected(&self) -> bool {
1006 !self.is_connected()
1007 }
1008
1009 async fn connect(&mut self) -> anyhow::Result<()> {
1010 if self.is_connected()
1011 && !self.cancellation_token.is_cancelled()
1012 && self.session_tasks.is_open()
1013 && self.pending_tasks.is_open()
1014 {
1015 return Ok(());
1016 }
1017
1018 if self.cancellation_token.is_cancelled()
1020 || !self.session_tasks.is_open()
1021 || !self.pending_tasks.is_open()
1022 || !self.session_tasks.is_empty()
1023 {
1024 self.teardown_partial_connect().await?;
1025 self.cancellation_token = CancellationToken::new();
1026 self.session_tasks.start_generation().map_err(|e| {
1027 anyhow::anyhow!("Failed to start Derive data session generation: {e}")
1028 })?;
1029
1030 self.pending_tasks
1031 .start_generation()
1032 .map_err(|e| anyhow::anyhow!("Failed to start Derive data task generation: {e}"))?;
1033 }
1034
1035 let cancellation_token = self.cancellation_token.clone();
1036 let ws_shutdown = self.ws_client.shutdown_handle();
1037
1038 let setup_guard =
1039 TaskGroupGuard::new(&[&self.session_tasks, &self.pending_tasks], move || {
1040 cancellation_token.cancel();
1041 ws_shutdown.begin_shutdown();
1042 });
1043
1044 if !self.config.currencies.is_empty() {
1045 self.provider
1046 .load_all(None)
1047 .await
1048 .context("failed to load Derive instruments")?;
1049 self.cache_provider_instruments();
1050 }
1051
1052 self.ws_client
1053 .connect()
1054 .await
1055 .context("failed to connect Derive WebSocket")?;
1056 let session_result = self
1057 .ws_client
1058 .take_event_receiver()
1059 .ok_or_else(|| anyhow::anyhow!("Derive WebSocket event receiver not initialized"))
1060 .and_then(|rx| self.spawn_stream_task(rx))
1061 .and_then(|()| self.spawn_book_monitor())
1062 .and_then(|()| self.spawn_instrument_refresh());
1063
1064 if let Err(e) = session_result {
1065 if let Err(teardown_error) = self.teardown_partial_connect().await {
1066 return Err(e.context(format!(
1067 "Derive data startup teardown failed: {teardown_error}"
1068 )));
1069 }
1070
1071 return Err(e);
1072 }
1073
1074 self.is_connected.store(true, Ordering::Release);
1075 setup_guard.disarm();
1076 log::info!(
1077 "Connected Derive data client ({:?})",
1078 self.config.environment
1079 );
1080 Ok(())
1081 }
1082
1083 async fn disconnect(&mut self) -> anyhow::Result<()> {
1084 self.teardown_partial_connect().await?;
1085 log::info!("Disconnected Derive data client");
1086 Ok(())
1087 }
1088
1089 fn subscribe_book_deltas(&mut self, cmd: SubscribeBookDeltas) -> anyhow::Result<()> {
1090 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1091 let cmd = cmd.with_instrument_ids(instrument_id);
1092 if cmd.book_type != BookType::L2_MBP {
1093 anyhow::bail!("Derive only supports L2_MBP order book deltas");
1094 }
1095
1096 let owner = ChannelOwner::BookDeltas(instrument_id);
1097 let lifecycle = self.subscription_lifecycle();
1098 if lifecycle.is_active(owner) {
1099 return Ok(());
1100 }
1101
1102 let instrument_name = format_venue_symbol(&instrument_id)?.to_string();
1103 let group = orderbook_group(&cmd.params)?;
1104 let depth = orderbook_depth(cmd.depth.map(|d| d.get()), &cmd.params)?;
1105 let channel = orderbook_channel(&instrument_name, &group, &depth);
1106 let request = ChannelRequest::from_channel(&channel)?;
1107 let needs_load = self.prepare_subscribe(instrument_id)?;
1108
1109 let Some(generation) = lifecycle.activate(owner, Some(&channel)) else {
1110 return Ok(());
1111 };
1112
1113 let ws = self.ws_handle();
1114 let http_client = self.http_client.clone();
1115 let include_expired = self.config.include_expired;
1116 let instruments = Arc::clone(&self.instruments);
1117
1118 self.spawn_task("subscribe_book_deltas", async move {
1119 if needs_load
1120 && let Err(e) = Self::lazy_load_instrument(
1121 http_client,
1122 instruments,
1123 instrument_id,
1124 include_expired,
1125 )
1126 .await
1127 {
1128 lifecycle.rollback(owner, generation);
1129 log::error!("Lazy-load failed for {instrument_id} (book deltas): {e}");
1130 return Ok(());
1131 }
1132
1133 run_channel_subscribe(lifecycle, owner, generation, request, ws).await
1134 });
1135
1136 Ok(())
1137 })
1138 }
1139
1140 fn subscribe_book_depth(&mut self, cmd: SubscribeBookDepth) -> anyhow::Result<()> {
1141 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1142 let cmd = cmd.with_instrument_ids(instrument_id);
1143 if cmd.book_type != BookType::L2_MBP {
1144 anyhow::bail!("Derive only supports L2_MBP order book depth");
1145 }
1146
1147 let owner = ChannelOwner::BookDepth(instrument_id);
1148 let lifecycle = self.subscription_lifecycle();
1149 if lifecycle.is_active(owner) {
1150 return Ok(());
1151 }
1152
1153 let instrument_name = format_venue_symbol(&instrument_id)?.to_string();
1154 let group = orderbook_group(&cmd.params)?;
1155 let depth = DeriveOrderbookDepth::D10.to_string();
1156 let channel = orderbook_channel(&instrument_name, &group, &depth);
1157 let request = ChannelRequest::from_channel(&channel)?;
1158 let needs_load = self.prepare_subscribe(instrument_id)?;
1159
1160 let Some(generation) = lifecycle.activate(owner, Some(&channel)) else {
1161 return Ok(());
1162 };
1163
1164 let ws = self.ws_handle();
1165 let http_client = self.http_client.clone();
1166 let include_expired = self.config.include_expired;
1167 let instruments = Arc::clone(&self.instruments);
1168
1169 self.spawn_task("subscribe_book_depth", async move {
1170 if needs_load
1171 && let Err(e) = Self::lazy_load_instrument(
1172 http_client,
1173 instruments,
1174 instrument_id,
1175 include_expired,
1176 )
1177 .await
1178 {
1179 lifecycle.rollback(owner, generation);
1180 log::error!("Lazy-load failed for {instrument_id} (book depth): {e}");
1181 return Ok(());
1182 }
1183
1184 run_channel_subscribe(lifecycle, owner, generation, request, ws).await
1185 });
1186
1187 Ok(())
1188 })
1189 }
1190
1191 fn subscribe_quotes(&mut self, cmd: SubscribeQuotes) -> anyhow::Result<()> {
1192 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1193 let cmd = cmd.with_instrument_ids(instrument_id);
1194 self.subscribe_ticker_feed(instrument_id, &cmd.params, TickerFeed::Quote, "quotes")
1195 })
1196 }
1197
1198 fn subscribe_trades(&mut self, cmd: SubscribeTrades) -> anyhow::Result<()> {
1199 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1200 let owner = ChannelOwner::Trades(instrument_id);
1201 let lifecycle = self.subscription_lifecycle();
1202 if lifecycle.is_active(owner) {
1203 return Ok(());
1204 }
1205
1206 let needs_load = self.prepare_subscribe(instrument_id)?;
1207
1208 let Some(generation) = lifecycle.activate(owner, None) else {
1209 return Ok(());
1210 };
1211
1212 let ws = self.ws_handle();
1213 let http_client = self.http_client.clone();
1214 let include_expired = self.config.include_expired;
1215 let instruments = Arc::clone(&self.instruments);
1216
1217 self.spawn_task("subscribe_trades", async move {
1218 if needs_load
1219 && let Err(e) = Self::lazy_load_instrument(
1220 http_client,
1221 Arc::clone(&instruments),
1222 instrument_id,
1223 include_expired,
1224 )
1225 .await
1226 {
1227 lifecycle.rollback(owner, generation);
1228 log::error!("Lazy-load failed for {instrument_id} (trades): {e}");
1229 return Ok(());
1230 }
1231
1232 if !lifecycle.is_current(owner, generation) {
1233 return Ok(());
1234 }
1235
1236 let Some(instrument) = instruments.get_cloned(&instrument_id) else {
1237 lifecycle.rollback(owner, generation);
1238 log::error!("Instrument {instrument_id} not found for Derive trades");
1239 return Ok(());
1240 };
1241
1242 let channel = match trade_channel(&instrument) {
1243 Ok(channel) => channel,
1244 Err(e) => {
1245 lifecycle.rollback(owner, generation);
1246 log::error!("Failed to resolve Derive trades channel: {e}");
1247 return Ok(());
1248 }
1249 };
1250
1251 let request = match ChannelRequest::from_channel(&channel) {
1252 Ok(request) => request,
1253 Err(e) => {
1254 lifecycle.rollback(owner, generation);
1255 log::error!("Invalid Derive trades channel `{channel}`: {e}");
1256 return Ok(());
1257 }
1258 };
1259
1260 if !lifecycle.attach_channel(owner, generation, channel) {
1261 return Ok(());
1262 }
1263
1264 run_channel_subscribe(lifecycle, owner, generation, request, ws).await
1265 });
1266
1267 Ok(())
1268 })
1269 }
1270
1271 fn subscribe_mark_prices(&mut self, cmd: SubscribeMarkPrices) -> anyhow::Result<()> {
1272 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1273 let cmd = cmd.with_instrument_ids(instrument_id);
1274 self.subscribe_ticker_feed(instrument_id, &cmd.params, TickerFeed::Mark, "mark prices")
1275 })
1276 }
1277
1278 fn subscribe_index_prices(&mut self, cmd: SubscribeIndexPrices) -> anyhow::Result<()> {
1279 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1280 let cmd = cmd.with_instrument_ids(instrument_id);
1281 self.subscribe_ticker_feed(
1282 instrument_id,
1283 &cmd.params,
1284 TickerFeed::Index,
1285 "index prices",
1286 )
1287 })
1288 }
1289
1290 fn subscribe_funding_rates(&mut self, cmd: SubscribeFundingRates) -> anyhow::Result<()> {
1291 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1292 let cmd = cmd.with_instrument_ids(instrument_id);
1293 self.subscribe_ticker_feed(
1294 instrument_id,
1295 &cmd.params,
1296 TickerFeed::Funding,
1297 "funding rates",
1298 )
1299 })
1300 }
1301
1302 fn subscribe_option_greeks(&mut self, cmd: SubscribeOptionGreeks) -> anyhow::Result<()> {
1303 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1304 let cmd = cmd.with_instrument_ids(instrument_id);
1305 self.subscribe_ticker_feed(
1306 instrument_id,
1307 &cmd.params,
1308 TickerFeed::Greeks,
1309 "option greeks",
1310 )
1311 })
1312 }
1313
1314 fn unsubscribe_book_deltas(&mut self, cmd: &UnsubscribeBookDeltas) -> anyhow::Result<()> {
1315 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1316 self.unsubscribe_channel_owner(ChannelOwner::BookDeltas(instrument_id))
1317 })
1318 }
1319
1320 fn unsubscribe_book_depth(&mut self, cmd: &UnsubscribeBookDepth) -> anyhow::Result<()> {
1321 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1322 self.unsubscribe_channel_owner(ChannelOwner::BookDepth(instrument_id))
1323 })
1324 }
1325
1326 fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<()> {
1327 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1328 self.unsubscribe_channel_owner(ChannelOwner::Ticker {
1329 instrument_id,
1330 feed: TickerFeed::Quote,
1331 })
1332 })
1333 }
1334
1335 fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> anyhow::Result<()> {
1336 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1337 self.unsubscribe_channel_owner(ChannelOwner::Trades(instrument_id))
1338 })
1339 }
1340
1341 fn unsubscribe_mark_prices(&mut self, cmd: &UnsubscribeMarkPrices) -> anyhow::Result<()> {
1342 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1343 self.unsubscribe_channel_owner(ChannelOwner::Ticker {
1344 instrument_id,
1345 feed: TickerFeed::Mark,
1346 })
1347 })
1348 }
1349
1350 fn unsubscribe_index_prices(&mut self, cmd: &UnsubscribeIndexPrices) -> anyhow::Result<()> {
1351 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1352 self.unsubscribe_channel_owner(ChannelOwner::Ticker {
1353 instrument_id,
1354 feed: TickerFeed::Index,
1355 })
1356 })
1357 }
1358
1359 fn unsubscribe_funding_rates(&mut self, cmd: &UnsubscribeFundingRates) -> anyhow::Result<()> {
1360 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1361 self.unsubscribe_channel_owner(ChannelOwner::Ticker {
1362 instrument_id,
1363 feed: TickerFeed::Funding,
1364 })
1365 })
1366 }
1367
1368 fn unsubscribe_option_greeks(&mut self, cmd: &UnsubscribeOptionGreeks) -> anyhow::Result<()> {
1369 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1370 self.unsubscribe_channel_owner(ChannelOwner::Ticker {
1371 instrument_id,
1372 feed: TickerFeed::Greeks,
1373 })
1374 })
1375 }
1376
1377 fn request_quotes(&self, request: RequestQuotes) -> anyhow::Result<()> {
1378 let instrument_id = request.single_instrument_id()?;
1379
1380 let instrument = self
1382 .instruments
1383 .get_cloned(&instrument_id)
1384 .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1385 let venue_symbol = format_venue_symbol(&instrument_id)?.to_string();
1386 let price_precision = instrument.price_precision();
1387 let size_precision = instrument.size_precision();
1388
1389 let http_client = self.http_client.clone();
1390 let sender = self.data_sender.clone();
1391 let clock = self.clock;
1392 let client_id = request.client_id.unwrap_or(self.client_id);
1393 let request_id = request.request_id;
1394 let params = request.params;
1395 let start_nanos = datetime_to_unix_nanos(request.start);
1396 let end_nanos = datetime_to_unix_nanos(request.end);
1397
1398 self.spawn_task(format!("request_quotes {instrument_id}"), async move {
1399 let ticker = match http_client.get_ticker(&venue_symbol).await {
1400 Ok(ticker) => ticker,
1401 Err(e) => {
1402 log::error!("Failed to fetch Derive ticker for {instrument_id}: {e:?}");
1403 return Ok(());
1404 }
1405 };
1406
1407 let ts_init = clock.get_time_ns();
1408
1409 let quotes = match parse_ticker_quote_from_rest(
1410 &ticker,
1411 price_precision,
1412 size_precision,
1413 ts_init,
1414 ) {
1415 Ok(quote) => {
1416 let within_start = start_nanos.is_none_or(|nanos| quote.ts_event >= nanos);
1418 let within_end = end_nanos.is_none_or(|nanos| quote.ts_event <= nanos);
1419 if within_start && within_end {
1420 vec![quote]
1421 } else {
1422 Vec::new()
1423 }
1424 }
1425 Err(e) => {
1426 log::warn!("Failed to parse Derive ticker for {instrument_id}: {e}");
1427 Vec::new()
1428 }
1429 };
1430
1431 let response = DataResponse::Quotes(QuotesResponse::new(
1432 request_id,
1433 client_id,
1434 instrument_id,
1435 quotes,
1436 start_nanos,
1437 end_nanos,
1438 clock.get_time_ns(),
1439 params,
1440 ));
1441
1442 if let Err(e) = sender.send(DataEvent::Response(response)) {
1443 log::error!("Failed to send Derive quotes response: {e}");
1444 }
1445
1446 Ok(())
1447 });
1448
1449 Ok(())
1450 }
1451
1452 fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
1453 let instrument_id = request.single_instrument_id()?;
1454 let instrument = self
1455 .instruments
1456 .get_cloned(&instrument_id)
1457 .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1458 let venue_symbol = format_venue_symbol(&instrument_id)?.to_string();
1459 let price_precision = instrument.price_precision();
1460 let size_precision = instrument.size_precision();
1461
1462 let http_client = self.http_client.clone();
1463 let sender = self.data_sender.clone();
1464 let clock = self.clock;
1465 let client_id = request.client_id.unwrap_or(self.client_id);
1466 let request_id = request.request_id;
1467 let params = request.params;
1468 let start = request.start;
1469 let end = request.end;
1470 let limit = request.limit.map(NonZeroUsize::get);
1471 let start_nanos = datetime_to_unix_nanos(start);
1472 let end_nanos = datetime_to_unix_nanos(end);
1473 let from_timestamp = start.map(|dt| dt.as_millisecond());
1474
1475 let to_timestamp = Some(match end {
1476 Some(dt) => dt.as_millisecond(),
1477 None => i64::try_from(clock.get_time_ms())
1478 .context("Derive current time exceeds i64 milliseconds")?,
1479 });
1480
1481 self.spawn_task("request_trades", async move {
1482 let page_size = limit.map_or(DERIVE_TRADES_PAGE_SIZE, |cap| {
1486 cap.min(DERIVE_TRADES_PAGE_SIZE as usize) as u32
1487 });
1488
1489 let mut trades = Vec::new();
1490 let mut seen_trade_ids = AHashSet::new();
1491 let mut pages = PaginationCursor::new_public();
1492
1493 loop {
1494 let result = match http_client
1495 .get_trade_history(
1496 &venue_symbol,
1497 from_timestamp,
1498 to_timestamp,
1499 pages.page(),
1500 page_size,
1501 )
1502 .await
1503 {
1504 Ok(result) => result,
1505 Err(e) => {
1506 log::error!("Failed to fetch Derive trades for {instrument_id}: {e:?}");
1507 return Ok(());
1508 }
1509 };
1510
1511 let row_ids: Vec<String> = result
1512 .trades
1513 .iter()
1514 .map(|trade| {
1515 format!(
1516 "{}:{:?}:{:?}",
1517 trade.trade_id, trade.liquidity_role, trade.subaccount_id
1518 )
1519 })
1520 .collect();
1521
1522 let more =
1523 match pages.advance(&result.pagination, row_ids.iter().map(String::as_str)) {
1524 Ok(more) => more,
1525 Err(e) => {
1526 log::error!("Failed to collect Derive trades for {instrument_id}: {e}");
1527 return Ok(());
1528 }
1529 };
1530
1531 let ts_init = clock.get_time_ns();
1532
1533 for trade in &result.trades {
1534 match parse_trade_tick_from_rest(
1535 trade,
1536 price_precision,
1537 size_precision,
1538 ts_init,
1539 ) {
1540 Ok(tick) => {
1541 let within_start =
1542 start_nanos.is_none_or(|nanos| tick.ts_event >= nanos);
1543 let within_end = end_nanos.is_none_or(|nanos| tick.ts_event <= nanos);
1544 if within_start && within_end && seen_trade_ids.insert(tick.trade_id) {
1545 trades.push(tick);
1546 }
1547 }
1548 Err(e) => log::warn!(
1549 "Failed to parse Derive trade {} for {instrument_id}: {e}",
1550 trade.trade_id,
1551 ),
1552 }
1553 }
1554
1555 if let Some(cap) = limit
1556 && trades.len() >= cap
1557 {
1558 break;
1559 }
1560
1561 if !more {
1562 break;
1563 }
1564 }
1565
1566 trades.sort_by_key(|trade| trade.ts_event);
1567 if let Some(cap) = limit
1568 && trades.len() > cap
1569 {
1570 trades.drain(..trades.len() - cap);
1571 }
1572
1573 let response = DataResponse::Trades(TradesResponse::new(
1574 request_id,
1575 client_id,
1576 instrument_id,
1577 trades,
1578 start_nanos,
1579 end_nanos,
1580 clock.get_time_ns(),
1581 params,
1582 ));
1583
1584 if let Err(e) = sender.send(DataEvent::Response(response)) {
1585 log::error!("Failed to send Derive trades response: {e}");
1586 }
1587
1588 Ok(())
1589 });
1590
1591 Ok(())
1592 }
1593
1594 fn request_funding_rates(&self, request: RequestFundingRates) -> anyhow::Result<()> {
1595 let instrument_id = request.single_instrument_id()?;
1596 let instrument = self
1597 .instruments
1598 .get_cloned(&instrument_id)
1599 .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1600 anyhow::ensure!(
1601 matches!(instrument, InstrumentAny::CryptoPerpetual(_)),
1602 "Funding rates are only available for Derive perpetual instruments (got {instrument_id})",
1603 );
1604 let venue_symbol = format_venue_symbol(&instrument_id)?.to_string();
1605
1606 let http_client = self.http_client.clone();
1607 let sender = self.data_sender.clone();
1608 let clock = self.clock;
1609 let client_id = request.client_id.unwrap_or(self.client_id);
1610 let request_id = request.request_id;
1611 let params = request.params;
1612 let start = request.start;
1613 let end = request.end;
1614 let limit = request.limit.map(NonZeroUsize::get);
1615 let start_nanos = datetime_to_unix_nanos(start);
1616 let end_nanos = datetime_to_unix_nanos(end);
1617 let start_ms = start.map(|dt| dt.as_millisecond());
1618 let end_ms = end.map(|dt| dt.as_millisecond());
1619
1620 self.spawn_task("request_funding_rates", async move {
1621 let result = match http_client
1622 .get_funding_rate_history(&venue_symbol, start_ms, end_ms, None)
1623 .await
1624 {
1625 Ok(result) => result,
1626 Err(e) => {
1627 log::error!(
1628 "Failed to fetch Derive funding rate history for {instrument_id}: {e:?}",
1629 );
1630 return Ok(());
1631 }
1632 };
1633
1634 let ts_init = clock.get_time_ns();
1635 let mut updates = Vec::with_capacity(result.funding_rate_history.len());
1636
1637 for record in &result.funding_rate_history {
1638 match parse_funding_rate_history_record(record, instrument_id, None, ts_init) {
1639 Ok(update) => {
1640 let within_start = start_nanos.is_none_or(|nanos| update.ts_event >= nanos);
1641 let within_end = end_nanos.is_none_or(|nanos| update.ts_event <= nanos);
1642 if within_start && within_end {
1643 updates.push(update);
1644 }
1645 }
1646 Err(e) => log::warn!(
1647 "Failed to parse Derive funding rate record for {instrument_id} at {}: {e}",
1648 record.timestamp,
1649 ),
1650 }
1651 }
1652
1653 updates.sort_by_key(|update| update.ts_event);
1654 if let Some(cap) = limit
1655 && updates.len() > cap
1656 {
1657 updates.drain(..updates.len() - cap);
1658 }
1659
1660 let response = DataResponse::FundingRates(FundingRatesResponse::new(
1661 request_id,
1662 client_id,
1663 instrument_id,
1664 updates,
1665 start_nanos,
1666 end_nanos,
1667 clock.get_time_ns(),
1668 params,
1669 ));
1670
1671 if let Err(e) = sender.send(DataEvent::Response(response)) {
1672 log::error!("Failed to send Derive funding rates response: {e}");
1673 }
1674
1675 Ok(())
1676 });
1677
1678 Ok(())
1679 }
1680
1681 fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
1682 let bar_type = request.single_bar_type()?;
1683
1684 anyhow::ensure!(
1685 bar_type.aggregation_source() == AggregationSource::External,
1686 "Derive only supports EXTERNAL aggregation source (got {bar_type})",
1687 );
1688 let spec = bar_type.spec();
1689 anyhow::ensure!(
1690 spec.price_type == PriceType::Last,
1691 "Derive candles are trade-based; only PriceType::Last is supported (got {bar_type})",
1692 );
1693
1694 let instrument_id = bar_type.instrument_id();
1695 let instrument = self
1696 .instruments
1697 .get_cloned(&instrument_id)
1698 .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1699 let venue_symbol = format_venue_symbol(&instrument_id)?.to_string();
1700 let price_precision = instrument.price_precision();
1701 let size_precision = instrument.size_precision();
1702
1703 let period = bar_spec_to_derive_period(spec.aggregation, spec.step.get() as u64)
1704 .with_context(|| format!("unsupported Derive bar spec for {bar_type}"))?;
1705
1706 let http_client = self.http_client.clone();
1707 let sender = self.data_sender.clone();
1708 let clock = self.clock;
1709 let client_id = request.client_id.unwrap_or(self.client_id);
1710 let request_id = request.request_id;
1711 let params = request.params;
1712 let start = request.start;
1713 let end = request.end;
1714 let limit = request.limit.map(NonZeroUsize::get);
1715 let start_nanos = datetime_to_unix_nanos(start);
1716 let end_nanos = datetime_to_unix_nanos(end);
1717
1718 let request_time = clock.get_time_ns();
1721 let now_secs = (request_time.as_u64() / NANOSECONDS_IN_SECOND) as i64;
1722 let end_ts = end.map_or(now_secs, |dt| dt.as_second());
1723 let default_span = i64::from(period) * limit.unwrap_or(DERIVE_CANDLES_DEFAULT_LIMIT) as i64;
1724 let start_ts = start.map_or(end_ts - default_span, |dt| dt.as_second());
1725
1726 self.spawn_task("request_bars", async move {
1727 let mut seen_timestamps: AHashSet<i64> = AHashSet::new();
1730 let mut pages: Vec<Vec<Bar>> = Vec::new();
1731 let mut total_bars = 0usize;
1732 let mut current_end = end_ts;
1733 let mut page_count = 0;
1734 loop {
1735 page_count += 1;
1736
1737 let mut records = match http_client
1738 .get_candles(&venue_symbol, start_ts, current_end, period)
1739 .await
1740 {
1741 Ok(records) => records,
1742 Err(e) => {
1743 log::error!("Failed to fetch Derive candles for {bar_type}: {e:?}");
1744 return Ok(());
1745 }
1746 };
1747
1748 if records.is_empty() {
1749 break;
1750 }
1751
1752 records.sort_by_key(|r| r.timestamp_bucket);
1753
1754 let has_new = records
1755 .iter()
1756 .any(|r| !seen_timestamps.contains(&r.timestamp_bucket));
1757
1758 if !has_new {
1759 break;
1760 }
1761
1762 let ts_init = clock.get_time_ns();
1763 let mut page_bars = Vec::with_capacity(records.len());
1764 let mut earliest_ts: Option<i64> = None;
1765
1766 for record in &records {
1767 let bucket = record.timestamp_bucket;
1768 if earliest_ts.is_none_or(|ts| bucket < ts) {
1769 earliest_ts = Some(bucket);
1770 }
1771
1772 if seen_timestamps.contains(&bucket) {
1773 continue;
1774 }
1775
1776 match parse_candle_record(
1777 record,
1778 bar_type,
1779 price_precision,
1780 size_precision,
1781 ts_init,
1782 ) {
1783 Ok(bar) => {
1784 seen_timestamps.insert(bucket);
1785
1786 if bar.ts_event <= request_time {
1787 page_bars.push(bar);
1788 }
1789 }
1790 Err(e) => log::warn!(
1791 "Failed to parse Derive candle for {bar_type} at {bucket}: {e}",
1792 ),
1793 }
1794 }
1795
1796 total_bars += page_bars.len();
1797 pages.push(page_bars);
1798
1799 if let Some(cap) = limit
1800 && total_bars >= cap
1801 {
1802 break;
1803 }
1804
1805 let Some(earliest) = earliest_ts else {
1806 break;
1807 };
1808
1809 if earliest <= start_ts {
1810 break;
1811 }
1812
1813 current_end = earliest - 1;
1814
1815 if page_count >= DERIVE_CANDLES_MAX_PAGES {
1816 log::warn!(
1817 "Derive bars pagination hit safety cap of {DERIVE_CANDLES_MAX_PAGES} pages for {bar_type}",
1818 );
1819 break;
1820 }
1821 }
1822
1823 let mut bars: Vec<Bar> = Vec::with_capacity(total_bars);
1824 for page in pages.into_iter().rev() {
1825 bars.extend(page);
1826 }
1827
1828 if let Some(cap) = limit
1829 && bars.len() > cap
1830 {
1831 let drop_count = bars.len() - cap;
1832 bars.drain(..drop_count);
1833 }
1834
1835 let response = DataResponse::Bars(BarsResponse::new(
1836 request_id,
1837 client_id,
1838 bar_type,
1839 bars,
1840 start_nanos,
1841 end_nanos,
1842 clock.get_time_ns(),
1843 params,
1844 ));
1845
1846 if let Err(e) = sender.send(DataEvent::Response(response)) {
1847 log::error!("Failed to send Derive bars response: {e}");
1848 }
1849
1850 Ok(())
1851 });
1852
1853 Ok(())
1854 }
1855
1856 fn request_option_chain_reference_price(
1857 &self,
1858 request: RequestOptionChainReferencePrice,
1859 ) -> anyhow::Result<()> {
1860 let series_id = request.series_id;
1861 let instrument_id = request.instrument_id;
1862 let instrument = self
1863 .instruments
1864 .get_cloned(&instrument_id)
1865 .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1866 anyhow::ensure!(
1867 matches!(instrument, InstrumentAny::CryptoOption(_)),
1868 "Derive option-chain reference prices require an option instrument (got {instrument_id})",
1869 );
1870 let venue_symbol = format_venue_symbol(&instrument_id)?.to_string();
1871
1872 let http_client = self.http_client.clone();
1873 let sender = self.data_sender.clone();
1874 let clock = self.clock;
1875 let client_id = request.client_id.unwrap_or(self.client_id);
1876 let request_id = request.request_id;
1877 let params = request.params;
1878
1879 self.spawn_task("request_option_chain_reference_price", async move {
1880 let price = match http_client.get_ticker(&venue_symbol).await {
1881 Ok(ticker) => match ticker.option_pricing.as_ref() {
1882 Some(pricing) if pricing.forward_price > Decimal::ZERO => {
1883 match Price::from_decimal(pricing.forward_price) {
1884 Ok(price) => Some(price),
1885 Err(e) => {
1886 log::warn!(
1887 "Invalid Derive option-chain reference price for {instrument_id}: {e}"
1888 );
1889 None
1890 }
1891 }
1892 }
1893 None => {
1894 log::warn!(
1895 "Derive ticker for {instrument_id} has no option pricing reference"
1896 );
1897 None
1898 }
1899 Some(_) => None,
1900 },
1901 Err(e) => {
1902 log::error!(
1903 "Option-chain reference price request failed for {series_id}: {e:?}"
1904 );
1905 None
1906 }
1907 };
1908
1909 let response = DataResponse::OptionChainReferencePrice(
1910 OptionChainReferencePriceResponse::new(
1911 request_id,
1912 client_id,
1913 series_id,
1914 price,
1915 clock.get_time_ns(),
1916 params,
1917 ),
1918 );
1919
1920 if let Err(e) = sender.send(DataEvent::Response(response)) {
1921 log::error!("Failed to send option-chain reference price response: {e}");
1922 }
1923
1924 Ok(())
1925 });
1926
1927 Ok(())
1928 }
1929
1930 fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
1931 let currencies = self.config.currencies.clone();
1932 if currencies.is_empty() {
1933 anyhow::bail!(
1934 "Derive request_instruments requires at least one configured currency \
1935 (DeriveDataClientConfig::currencies)"
1936 );
1937 }
1938
1939 let http_client = self.http_client.clone();
1940 let include_expired = self.config.include_expired;
1941 let instruments_cache = Arc::clone(&self.instruments);
1942 let sender = self.data_sender.clone();
1943 let clock = self.clock;
1944 let venue = self.venue().unwrap_or(*DERIVE_VENUE);
1945 let client_id = request.client_id.unwrap_or(self.client_id);
1946 let request_id = request.request_id;
1947 let start_nanos = datetime_to_unix_nanos(request.start);
1948 let end_nanos = datetime_to_unix_nanos(request.end);
1949 let params = request.params;
1950
1951 self.spawn_task("request_instruments", async move {
1952 let mut all_instruments = Vec::new();
1953
1954 for currency in currencies {
1955 match fetch_instrument_definitions(&http_client, ¤cy, include_expired).await {
1956 Ok(definitions) => {
1957 let instruments = parse_instrument_definitions(definitions);
1958 instruments_cache.rcu(|cache| {
1959 for instrument in &instruments {
1960 cache.insert(instrument.id(), instrument.clone());
1961 }
1962 });
1963
1964 all_instruments.extend(instruments);
1965 }
1966 Err(e) => {
1967 log::error!("Failed to fetch Derive instruments for {currency}: {e:?}");
1968 }
1969 }
1970 }
1971
1972 let response = DataResponse::Instruments(InstrumentsResponse::new(
1973 request_id,
1974 client_id,
1975 venue,
1976 all_instruments,
1977 start_nanos,
1978 end_nanos,
1979 clock.get_time_ns(),
1980 params,
1981 ));
1982
1983 if let Err(e) = sender.send(DataEvent::Response(response)) {
1984 log::error!("Failed to send Derive instruments response: {e}");
1985 }
1986
1987 Ok(())
1988 });
1989
1990 Ok(())
1991 }
1992
1993 fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
1994 let instrument_id = request.single_instrument_id()?;
1995
1996 let venue_symbol = format_venue_symbol(&instrument_id)?.to_string();
1997
1998 let http_client = self.http_client.clone();
1999 let instruments_cache = Arc::clone(&self.instruments);
2000 let sender = self.data_sender.clone();
2001 let clock = self.clock;
2002 let client_id = request.client_id.unwrap_or(self.client_id);
2003 let request_id = request.request_id;
2004 let start_nanos = datetime_to_unix_nanos(request.start);
2005 let end_nanos = datetime_to_unix_nanos(request.end);
2006 let params = request.params;
2007
2008 self.spawn_task("request_instrument", async move {
2009 let definition = match http_client.get_instrument(&venue_symbol).await {
2010 Ok(definition) => definition,
2011 Err(e) => {
2012 log::error!("Failed to fetch Derive instrument {instrument_id}: {e:?}");
2013 return Ok(());
2014 }
2015 };
2016
2017 let ts_init = clock.get_time_ns();
2018
2019 let instrument = match parse_derive_instrument_any(&definition, ts_init) {
2020 Ok(Some(instrument)) => instrument,
2021 Ok(None) => {
2022 log::warn!(
2023 "Derive instrument {instrument_id} resolved to an unsupported type ({:?})",
2024 definition.instrument_type,
2025 );
2026 return Ok(());
2027 }
2028 Err(e) => {
2029 log::error!("Failed to parse Derive instrument {instrument_id}: {e}");
2030 return Ok(());
2031 }
2032 };
2033
2034 cache_instrument(&instruments_cache, &instrument);
2035
2036 let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
2037 request_id,
2038 client_id,
2039 instrument.id(),
2040 instrument,
2041 start_nanos,
2042 end_nanos,
2043 clock.get_time_ns(),
2044 params,
2045 )));
2046
2047 if let Err(e) = sender.send(DataEvent::Response(response)) {
2048 log::error!("Failed to send Derive instrument response: {e}");
2049 }
2050
2051 Ok(())
2052 });
2053
2054 Ok(())
2055 }
2056}
2057
2058#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
2059enum TickerFeed {
2060 Quote,
2061 Mark,
2062 Index,
2063 Funding,
2064 Greeks,
2065}
2066
2067#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
2068enum ChannelOwner {
2069 BookDeltas(InstrumentId),
2070 BookDepth(InstrumentId),
2071 Ticker {
2072 instrument_id: InstrumentId,
2073 feed: TickerFeed,
2074 },
2075 Trades(InstrumentId),
2076}
2077
2078#[derive(Debug)]
2079struct OwnedChannel {
2080 generation: u64,
2081 channel: Option<String>,
2082}
2083
2084#[derive(Debug, Default)]
2085struct ChannelSubscriptionState {
2086 next_generation: u64,
2087 owners: AHashMap<ChannelOwner, OwnedChannel>,
2088 channels: AHashMap<String, AHashSet<ChannelOwner>>,
2089}
2090
2091#[derive(Debug, Default)]
2092struct ChannelSubscriptionRegistry {
2093 state: Mutex<ChannelSubscriptionState>,
2094 transitions: DashMap<String, Weak<tokio::sync::Mutex<()>>>,
2095}
2096
2097const TRANSITION_GC_THRESHOLD: usize = 256;
2098
2099#[derive(Debug)]
2100struct RemovedSubscription {
2101 channel: Option<String>,
2102 channel_empty: bool,
2103}
2104
2105#[derive(Debug, Clone)]
2106struct SubscriptionDispatchState {
2107 book_sync: BookSyncTracker,
2108 active_book_delta_channels: Arc<AtomicMap<InstrumentId, String>>,
2109 active_book_depth_channels: Arc<AtomicMap<InstrumentId, String>>,
2110 active_ticker_channels: Arc<AtomicMap<InstrumentId, String>>,
2111 active_quote_subs: Arc<AtomicSet<InstrumentId>>,
2112 active_trade_subs: Arc<AtomicSet<InstrumentId>>,
2113 active_mark_subs: Arc<AtomicSet<InstrumentId>>,
2114 active_index_subs: Arc<AtomicSet<InstrumentId>>,
2115 active_funding_subs: Arc<AtomicSet<InstrumentId>>,
2116 active_greeks_subs: Arc<AtomicSet<InstrumentId>>,
2117 quote_cache: Arc<Mutex<QuoteCache>>,
2118}
2119
2120#[derive(Debug, Clone)]
2121struct SubscriptionLifecycle {
2122 snapshot_timeout: Duration,
2123 registry: Arc<ChannelSubscriptionRegistry>,
2124 lock: Arc<Mutex<()>>,
2125 dispatch: SubscriptionDispatchState,
2126}
2127
2128#[derive(Debug)]
2129enum ChannelRequest {
2130 Orderbook {
2131 channel: String,
2132 instrument_name: String,
2133 group: String,
2134 depth: String,
2135 },
2136 Ticker {
2137 channel: String,
2138 instrument_name: String,
2139 interval: String,
2140 },
2141 Trades {
2142 channel: String,
2143 instrument_type: String,
2144 currency: String,
2145 },
2146}
2147
2148async fn run_channel_subscribe(
2149 lifecycle: SubscriptionLifecycle,
2150 owner: ChannelOwner,
2151 generation: u64,
2152 request: ChannelRequest,
2153 ws: DeriveWebSocketSubscriptionHandle,
2154) -> anyhow::Result<()> {
2155 let channel = request.channel().to_string();
2156 let transition = lifecycle.registry.transition(&channel);
2157 let _guard = transition.lock().await;
2158
2159 if !lifecycle.is_current_channel(owner, generation, &channel) {
2160 return Ok(());
2161 }
2162
2163 let book_pending = lifecycle
2164 .dispatch
2165 .book_sync
2166 .write(&channel)
2167 .is_some_and(|write| write.gate.lock().is_closed());
2168
2169 if ws.has_subscription(&channel) && !book_pending {
2170 return Ok(());
2171 }
2172
2173 let result = if matches!(&request, ChannelRequest::Orderbook { .. }) {
2174 ws.subscribe_book_gated(channel.clone()).await
2175 } else {
2176 request.subscribe(&ws).await
2177 };
2178
2179 if let Err(e) = result {
2180 lifecycle.rollback(owner, generation);
2181 log::error!(
2182 "Failed to subscribe to Derive channel `{channel}`: {}",
2183 e.redacted_message()
2184 );
2185
2186 let cleanup_error = match request.unsubscribe(&ws).await {
2187 Ok(()) => None,
2188 Err(cleanup_error) => {
2189 log::error!(
2190 "Failed to clean up uncertain Derive channel `{channel}`: {}",
2191 cleanup_error.redacted_message(),
2192 );
2193 Some(cleanup_error)
2194 }
2195 };
2196
2197 if retain_channel_for_reconnect(
2198 &ws,
2199 &channel,
2200 lifecycle.has_owners(&channel),
2201 cleanup_error.as_ref(),
2202 ) {
2203 log::error!(
2204 "Derive channel `{channel}` remains uncertain and will replay on reconnect",
2205 );
2206 }
2207
2208 return Ok(());
2209 }
2210
2211 lifecycle
2212 .dispatch
2213 .book_sync
2214 .subscription_written(&channel, lifecycle.snapshot_timeout);
2215 if !lifecycle.has_owners(&channel)
2216 && let Err(e) = request.unsubscribe(&ws).await
2217 {
2218 log::error!("Failed to clean up stale Derive channel `{channel}`: {e}");
2219 ws.forget_subscription(&channel);
2220 }
2221
2222 Ok(())
2223}
2224
2225async fn run_channel_unsubscribe(
2226 lifecycle: SubscriptionLifecycle,
2227 request: ChannelRequest,
2228 ws: DeriveWebSocketSubscriptionHandle,
2229) -> anyhow::Result<()> {
2230 let channel = request.channel().to_string();
2231 let transition = lifecycle.registry.transition(&channel);
2232 let _guard = transition.lock().await;
2233
2234 if lifecycle.has_owners(&channel) || !ws.has_subscription(&channel) {
2235 return Ok(());
2236 }
2237
2238 if let Err(e) = request.unsubscribe(&ws).await {
2239 log::error!("Failed to unsubscribe from Derive channel `{channel}`: {e}");
2240 ws.forget_subscription(&channel);
2241 }
2242
2243 Ok(())
2244}
2245
2246impl SubscriptionLifecycle {
2247 fn is_active(&self, owner: ChannelOwner) -> bool {
2248 let _guard = self.lock.lock();
2249 self.registry.state.lock().owners.contains_key(&owner)
2250 }
2251
2252 fn activate(&self, owner: ChannelOwner, channel: Option<&str>) -> Option<u64> {
2253 let _guard = self.lock.lock();
2254 let mut state = self.registry.state.lock();
2255 let generation = state.activate(owner, channel)?;
2256 self.dispatch.activate(owner, channel);
2257
2258 if let ChannelOwner::BookDeltas(instrument_id) | ChannelOwner::BookDepth(instrument_id) =
2259 owner
2260 {
2261 self.dispatch
2262 .book_sync
2263 .record_subscription(channel.expect("book channel present"), instrument_id);
2264 }
2265
2266 Some(generation)
2267 }
2268
2269 fn attach_channel(&self, owner: ChannelOwner, generation: u64, channel: String) -> bool {
2270 let _guard = self.lock.lock();
2271 self.registry
2272 .state
2273 .lock()
2274 .attach_channel(owner, generation, channel)
2275 }
2276
2277 fn is_current(&self, owner: ChannelOwner, generation: u64) -> bool {
2278 let _guard = self.lock.lock();
2279 self.registry.state.lock().is_current(owner, generation)
2280 }
2281
2282 fn is_current_channel(&self, owner: ChannelOwner, generation: u64, channel: &str) -> bool {
2283 let _guard = self.lock.lock();
2284 self.registry
2285 .state
2286 .lock()
2287 .is_current_channel(owner, generation, channel)
2288 }
2289
2290 fn rollback(&self, owner: ChannelOwner, generation: u64) -> bool {
2291 let _guard = self.lock.lock();
2292 let mut state = self.registry.state.lock();
2293
2294 let Some(removed) = state.remove_if_generation(owner, generation) else {
2295 return false;
2296 };
2297
2298 self.dispatch.deactivate(owner, removed.channel_empty);
2299 if removed.channel_empty
2300 && let Some(channel) = &removed.channel
2301 {
2302 self.dispatch.book_sync.remove(channel);
2303 }
2304
2305 true
2306 }
2307
2308 fn remove(&self, owner: ChannelOwner) -> Option<RemovedSubscription> {
2309 let _guard = self.lock.lock();
2310 let mut state = self.registry.state.lock();
2311 let removed = state.remove(owner)?;
2312 self.dispatch.deactivate(owner, removed.channel_empty);
2313 if removed.channel_empty
2314 && let Some(channel) = &removed.channel
2315 {
2316 self.dispatch.book_sync.remove(channel);
2317 }
2318
2319 Some(removed)
2320 }
2321
2322 fn has_owners(&self, channel: &str) -> bool {
2323 let _guard = self.lock.lock();
2324 self.registry.state.lock().has_owners(channel)
2325 }
2326}
2327
2328impl SubscriptionDispatchState {
2329 fn activate(&self, owner: ChannelOwner, channel: Option<&str>) {
2330 match owner {
2331 ChannelOwner::BookDeltas(instrument_id) => {
2332 self.active_book_delta_channels.insert(
2333 instrument_id,
2334 channel.expect("book channel present").to_string(),
2335 );
2336 }
2337 ChannelOwner::BookDepth(instrument_id) => {
2338 self.active_book_depth_channels.insert(
2339 instrument_id,
2340 channel.expect("book channel present").to_string(),
2341 );
2342 }
2343 ChannelOwner::Ticker {
2344 instrument_id,
2345 feed,
2346 } => {
2347 self.active_ticker_channels.insert(
2348 instrument_id,
2349 channel.expect("ticker channel present").to_string(),
2350 );
2351 self.ticker_subscriptions(feed).insert(instrument_id);
2352 }
2353 ChannelOwner::Trades(instrument_id) => {
2354 self.active_trade_subs.insert(instrument_id);
2355 }
2356 }
2357 }
2358
2359 fn deactivate(&self, owner: ChannelOwner, channel_empty: bool) {
2360 match owner {
2361 ChannelOwner::BookDeltas(instrument_id) => {
2362 self.active_book_delta_channels.remove(&instrument_id);
2363 }
2364 ChannelOwner::BookDepth(instrument_id) => {
2365 self.active_book_depth_channels.remove(&instrument_id);
2366 }
2367 ChannelOwner::Ticker {
2368 instrument_id,
2369 feed,
2370 } => {
2371 self.ticker_subscriptions(feed).remove(&instrument_id);
2372 if feed == TickerFeed::Quote {
2373 self.quote_cache.lock().remove(&instrument_id);
2374 }
2375
2376 if channel_empty {
2377 self.active_ticker_channels.remove(&instrument_id);
2378 }
2379 }
2380 ChannelOwner::Trades(instrument_id) => {
2381 self.active_trade_subs.remove(&instrument_id);
2382 }
2383 }
2384 }
2385
2386 fn ticker_subscriptions(&self, feed: TickerFeed) -> &AtomicSet<InstrumentId> {
2387 match feed {
2388 TickerFeed::Quote => &self.active_quote_subs,
2389 TickerFeed::Mark => &self.active_mark_subs,
2390 TickerFeed::Index => &self.active_index_subs,
2391 TickerFeed::Funding => &self.active_funding_subs,
2392 TickerFeed::Greeks => &self.active_greeks_subs,
2393 }
2394 }
2395}
2396
2397impl ChannelSubscriptionRegistry {
2398 fn transition(&self, channel: &str) -> Arc<tokio::sync::Mutex<()>> {
2399 if self.transitions.len() >= TRANSITION_GC_THRESHOLD {
2400 self.transitions
2401 .retain(|_, transition| transition.strong_count() > 0);
2402 }
2403
2404 let mut entry = self.transitions.entry(channel.to_string()).or_default();
2405 if let Some(transition) = entry.value().upgrade() {
2406 return transition;
2407 }
2408
2409 let transition = Arc::new(tokio::sync::Mutex::new(()));
2410 *entry.value_mut() = Arc::downgrade(&transition);
2411 transition
2412 }
2413
2414 fn clear(&self) {
2415 let mut state = self.state.lock();
2416 let next_generation = state.next_generation;
2417
2418 *state = ChannelSubscriptionState {
2419 next_generation,
2420 ..Default::default()
2421 };
2422 }
2423
2424 fn clear_transitions(&self) {
2425 self.transitions.clear();
2426 }
2427}
2428
2429impl ChannelSubscriptionState {
2430 fn activate(&mut self, owner: ChannelOwner, channel: Option<&str>) -> Option<u64> {
2431 if self.owners.contains_key(&owner) {
2432 return None;
2433 }
2434
2435 self.next_generation = self
2436 .next_generation
2437 .checked_add(1)
2438 .expect("subscription generation overflow");
2439 let generation = self.next_generation;
2440
2441 if let Some(channel) = channel {
2442 self.channels
2443 .entry(channel.to_string())
2444 .or_default()
2445 .insert(owner);
2446 }
2447
2448 self.owners.insert(
2449 owner,
2450 OwnedChannel {
2451 generation,
2452 channel: channel.map(ToOwned::to_owned),
2453 },
2454 );
2455
2456 Some(generation)
2457 }
2458
2459 fn attach_channel(&mut self, owner: ChannelOwner, generation: u64, channel: String) -> bool {
2460 let Some(owned) = self.owners.get(&owner) else {
2461 return false;
2462 };
2463
2464 if owned.generation != generation {
2465 return false;
2466 }
2467
2468 if let Some(active_channel) = &owned.channel {
2469 return active_channel == &channel;
2470 }
2471
2472 self.owners.get_mut(&owner).expect("owner present").channel = Some(channel.clone());
2473 self.channels.entry(channel).or_default().insert(owner);
2474 true
2475 }
2476
2477 fn is_current(&self, owner: ChannelOwner, generation: u64) -> bool {
2478 self.owners
2479 .get(&owner)
2480 .is_some_and(|owned| owned.generation == generation)
2481 }
2482
2483 fn is_current_channel(&self, owner: ChannelOwner, generation: u64, channel: &str) -> bool {
2484 self.owners.get(&owner).is_some_and(|owned| {
2485 owned.generation == generation && owned.channel.as_deref() == Some(channel)
2486 })
2487 }
2488
2489 fn remove_if_generation(
2490 &mut self,
2491 owner: ChannelOwner,
2492 generation: u64,
2493 ) -> Option<RemovedSubscription> {
2494 if !self.is_current(owner, generation) {
2495 return None;
2496 }
2497
2498 self.remove(owner)
2499 }
2500
2501 fn remove(&mut self, owner: ChannelOwner) -> Option<RemovedSubscription> {
2502 let owned = self.owners.remove(&owner)?;
2503
2504 let channel_empty = owned.channel.as_ref().is_some_and(|channel| {
2505 let Some(owners) = self.channels.get_mut(channel) else {
2506 return true;
2507 };
2508
2509 owners.remove(&owner);
2510 owners.is_empty()
2511 });
2512
2513 if channel_empty && let Some(channel) = &owned.channel {
2514 self.channels.remove(channel);
2515 }
2516
2517 Some(RemovedSubscription {
2518 channel: owned.channel,
2519 channel_empty,
2520 })
2521 }
2522
2523 fn has_owners(&self, channel: &str) -> bool {
2524 self.channels
2525 .get(channel)
2526 .is_some_and(|owners| !owners.is_empty())
2527 }
2528}
2529
2530impl ChannelRequest {
2531 fn from_channel(channel: &str) -> anyhow::Result<Self> {
2532 if channel.starts_with("orderbook.") {
2533 let (instrument_name, group, depth) = orderbook_channel_parts(channel)?;
2534 return Ok(Self::Orderbook {
2535 channel: channel.to_string(),
2536 instrument_name,
2537 group,
2538 depth,
2539 });
2540 }
2541
2542 if channel.starts_with("ticker_slim.") || channel.starts_with("ticker.") {
2543 let (instrument_name, interval) = ticker_channel_parts(channel)?;
2544 return Ok(Self::Ticker {
2545 channel: channel.to_string(),
2546 instrument_name,
2547 interval,
2548 });
2549 }
2550
2551 if let Some((instrument_type, currency)) = channel
2552 .strip_prefix("trades.")
2553 .and_then(|value| value.split_once('.'))
2554 {
2555 return Ok(Self::Trades {
2556 channel: channel.to_string(),
2557 instrument_type: instrument_type.to_string(),
2558 currency: currency.to_string(),
2559 });
2560 }
2561
2562 anyhow::bail!("invalid Derive subscription channel `{channel}`")
2563 }
2564
2565 fn channel(&self) -> &str {
2566 match self {
2567 Self::Orderbook { channel, .. }
2568 | Self::Ticker { channel, .. }
2569 | Self::Trades { channel, .. } => channel,
2570 }
2571 }
2572
2573 async fn subscribe(&self, ws: &DeriveWebSocketSubscriptionHandle) -> Result<(), DeriveWsError> {
2574 match self {
2575 Self::Orderbook {
2576 instrument_name,
2577 group,
2578 depth,
2579 ..
2580 } => {
2581 ws.subscribe_orderbook(instrument_name, group, depth)
2582 .await?;
2583 }
2584 Self::Ticker {
2585 instrument_name,
2586 interval,
2587 ..
2588 } => ws.subscribe_ticker(instrument_name, interval).await?,
2589 Self::Trades {
2590 instrument_type,
2591 currency,
2592 ..
2593 } => ws.subscribe_trades(instrument_type, currency).await?,
2594 }
2595
2596 Ok(())
2597 }
2598
2599 async fn unsubscribe(
2600 &self,
2601 ws: &DeriveWebSocketSubscriptionHandle,
2602 ) -> Result<(), DeriveWsError> {
2603 match self {
2604 Self::Orderbook {
2605 instrument_name,
2606 group,
2607 depth,
2608 ..
2609 } => {
2610 ws.unsubscribe_orderbook(instrument_name, group, depth)
2611 .await?;
2612 }
2613 Self::Ticker {
2614 instrument_name,
2615 interval,
2616 ..
2617 } => ws.unsubscribe_ticker(instrument_name, interval).await?,
2618 Self::Trades {
2619 instrument_type,
2620 currency,
2621 ..
2622 } => ws.unsubscribe_trades(instrument_type, currency).await?,
2623 }
2624
2625 Ok(())
2626 }
2627}
2628
2629fn retain_channel_for_reconnect(
2630 ws: &DeriveWebSocketSubscriptionHandle,
2631 channel: &str,
2632 has_surviving_owner: bool,
2633 cleanup_error: Option<&DeriveWsError>,
2634) -> bool {
2635 let replay = has_surviving_owner
2636 && cleanup_error.is_some_and(|e| {
2637 matches!(
2638 e,
2639 DeriveWsError::Transport(_)
2640 | DeriveWsError::RequestCancelled { .. }
2641 | DeriveWsError::Timeout { .. }
2642 | DeriveWsError::NotConnected
2643 )
2644 });
2645
2646 if replay {
2647 ws.remember_subscription(channel);
2648 } else {
2649 ws.forget_subscription(channel);
2650 }
2651
2652 replay
2653}
2654
2655impl DeriveDataClient {
2656 async fn join_session_tasks(&self) -> anyhow::Result<()> {
2657 self.session_tasks.begin_shutdown();
2658 self.session_tasks
2659 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
2660 .await
2661 .map_err(|e| anyhow::anyhow!("Failed to terminate Derive data session tasks: {e}"))?;
2662 Ok(())
2663 }
2664
2665 async fn join_pending_tasks(&self) -> anyhow::Result<()> {
2666 self.pending_tasks.begin_shutdown();
2667 self.pending_tasks
2668 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
2669 .await
2670 .map_err(|e| anyhow::anyhow!("Failed to terminate Derive data tasks: {e}"))?;
2671 Ok(())
2672 }
2673}
2674
2675fn cache_instruments(
2676 mut loaded: Vec<InstrumentAny>,
2677 instruments: &AtomicMap<InstrumentId, InstrumentAny>,
2678 sender: &EventSender<DataEvent>,
2679 cancellation: &CancellationToken,
2680 subscription_lock: &Mutex<()>,
2681) {
2682 loaded.sort_by_key(|instrument| instrument.id());
2683 let _guard = subscription_lock.lock();
2684
2685 if cancellation.is_cancelled() {
2686 return;
2687 }
2688
2689 instruments.rcu(|cache| {
2690 for instrument in &loaded {
2691 cache.insert(instrument.id(), instrument.clone());
2692 }
2693 });
2694
2695 for instrument in loaded {
2696 if let Err(e) = sender.send(DataEvent::Data(instrument.into())) {
2697 log::warn!("Failed to send Derive instrument: {e}");
2698 }
2699 }
2700}
2701
2702fn cache_instrument(
2703 instruments: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
2704 instrument: &InstrumentAny,
2705) {
2706 instruments.insert(instrument.id(), instrument.clone());
2707}
2708
2709fn process_ticker_quote(
2710 msg: &DeriveTickerMsg,
2711 price_precision: u8,
2712 size_precision: u8,
2713 ts_init: UnixNanos,
2714 quote_cache: &mut QuoteCache,
2715) -> anyhow::Result<Option<QuoteTick>> {
2716 let quote = parse_ticker_quote(msg, price_precision, size_precision, ts_init)?;
2717 if quote.bid_price.is_zero()
2718 || quote.ask_price.is_zero()
2719 || quote.bid_size.is_zero()
2720 || quote.ask_size.is_zero()
2721 {
2722 quote_cache.remove("e.instrument_id);
2724
2725 log::debug!(
2726 "Skipping Derive ticker quote with an empty side for {}",
2727 msg.data.instrument_name(),
2728 );
2729 return Ok(None);
2730 }
2731
2732 quote_cache.insert(quote.instrument_id, quote);
2733 Ok(Some(quote))
2734}
2735
2736fn channel_is_active(
2737 channels: &AtomicMap<InstrumentId, String>,
2738 instrument_id: InstrumentId,
2739 channel: &str,
2740) -> bool {
2741 channels
2742 .get_cloned(&instrument_id)
2743 .is_some_and(|active_channel| active_channel == channel)
2744}
2745
2746fn orderbook_channel_parts(channel: &str) -> anyhow::Result<(String, String, String)> {
2747 let rest = channel
2748 .strip_prefix("orderbook.")
2749 .ok_or_else(|| anyhow::anyhow!("invalid Derive orderbook channel `{channel}`"))?;
2750 let mut parts = rest.rsplitn(3, '.');
2751 let depth = parts
2752 .next()
2753 .filter(|value| !value.is_empty())
2754 .ok_or_else(|| anyhow::anyhow!("invalid Derive orderbook channel `{channel}`"))?;
2755 let group = parts
2756 .next()
2757 .filter(|value| !value.is_empty())
2758 .ok_or_else(|| anyhow::anyhow!("invalid Derive orderbook channel `{channel}`"))?;
2759 let instrument_name = parts
2760 .next()
2761 .filter(|value| !value.is_empty())
2762 .ok_or_else(|| anyhow::anyhow!("invalid Derive orderbook channel `{channel}`"))?;
2763
2764 Ok((
2765 instrument_name.to_string(),
2766 group.to_string(),
2767 depth.to_string(),
2768 ))
2769}
2770
2771fn ticker_channel_parts(channel: &str) -> anyhow::Result<(String, String)> {
2772 let rest = channel
2773 .strip_prefix("ticker_slim.")
2774 .or_else(|| channel.strip_prefix("ticker."))
2775 .ok_or_else(|| anyhow::anyhow!("invalid Derive ticker channel `{channel}`"))?;
2776 let (instrument_name, interval) = rest
2777 .rsplit_once('.')
2778 .ok_or_else(|| anyhow::anyhow!("invalid Derive ticker channel `{channel}`"))?;
2779 anyhow::ensure!(
2780 !instrument_name.is_empty() && !interval.is_empty(),
2781 "invalid Derive ticker channel `{channel}`"
2782 );
2783
2784 Ok((instrument_name.to_string(), interval.to_string()))
2785}
2786
2787fn orderbook_group(params: &Option<Params>) -> anyhow::Result<String> {
2788 let group = params
2789 .as_ref()
2790 .and_then(|p| {
2791 p.get_str("group")
2792 .map(ToOwned::to_owned)
2793 .or_else(|| p.get_u64("group").map(|value| value.to_string()))
2794 })
2795 .unwrap_or_else(|| DEFAULT_ORDERBOOK_GROUP.to_string());
2796
2797 DeriveOrderbookGroup::from_str(&group)
2798 .with_context(|| format!("invalid Derive orderbook group `{group}`"))?;
2799 Ok(group)
2800}
2801
2802fn orderbook_depth(depth: Option<usize>, params: &Option<Params>) -> anyhow::Result<String> {
2803 let depth = depth
2804 .map(|value| value.to_string())
2805 .or_else(|| {
2806 params.as_ref().and_then(|p| {
2807 p.get_str("depth")
2808 .map(ToOwned::to_owned)
2809 .or_else(|| p.get_u64("depth").map(|value| value.to_string()))
2810 })
2811 })
2812 .unwrap_or_else(|| DEFAULT_ORDERBOOK_DEPTH.to_string());
2813
2814 DeriveOrderbookDepth::from_str(&depth)
2815 .with_context(|| format!("invalid Derive orderbook depth `{depth}`"))?;
2816 Ok(depth)
2817}
2818
2819fn ticker_interval(params: &Option<Params>) -> anyhow::Result<String> {
2820 let interval = params
2821 .as_ref()
2822 .and_then(|p| {
2823 p.get_str("interval")
2824 .map(ToOwned::to_owned)
2825 .or_else(|| p.get_u64("interval").map(|value| value.to_string()))
2826 })
2827 .unwrap_or_else(|| DEFAULT_TICKER_INTERVAL.to_string());
2828
2829 DeriveTickerInterval::from_str(&interval)
2830 .with_context(|| format!("invalid Derive ticker interval `{interval}`"))?;
2831 Ok(interval)
2832}
2833
2834fn trade_channel(instrument: &InstrumentAny) -> anyhow::Result<String> {
2835 let instrument_type = derive_instrument_type(instrument)?.to_string();
2836 let instrument_id = instrument.id();
2837 let currency = currency_from_instrument_id(&instrument_id)?;
2838 Ok(trades_channel(&instrument_type, currency))
2839}
2840
2841fn derive_instrument_type(instrument: &InstrumentAny) -> anyhow::Result<DeriveInstrumentType> {
2842 match instrument {
2843 InstrumentAny::CryptoPerpetual(_) => Ok(DeriveInstrumentType::Perp),
2844 InstrumentAny::CryptoOption(_) => Ok(DeriveInstrumentType::Option),
2845 InstrumentAny::CurrencyPair(_) => Ok(DeriveInstrumentType::Erc20),
2846 other => anyhow::bail!("unsupported Derive instrument type for trades: {other:?}"),
2847 }
2848}
2849
2850fn currency_from_instrument_id(instrument_id: &InstrumentId) -> anyhow::Result<&str> {
2851 anyhow::ensure!(
2852 instrument_id.venue == *DERIVE_VENUE,
2853 "instrument ID `{instrument_id}` is not for venue {}",
2854 DERIVE_VENUE.as_str(),
2855 );
2856
2857 instrument_id
2858 .symbol
2859 .as_str()
2860 .split_once('-')
2861 .and_then(|(currency, _)| (!currency.is_empty()).then_some(currency))
2862 .ok_or_else(|| anyhow::anyhow!("cannot derive currency from {instrument_id}"))
2863}
2864
2865fn truncated_payload_snippet(raw: &str) -> String {
2869 const MAX_LEN: usize = 512;
2870
2871 if raw.len() <= MAX_LEN {
2872 return raw.to_string();
2873 }
2874
2875 let mut end = MAX_LEN;
2876 while end > 0 && !raw.is_char_boundary(end) {
2877 end -= 1;
2878 }
2879
2880 format!("{}...(truncated)", &raw[..end])
2881}
2882
2883#[cfg(test)]
2884#[path = "../tests/integration/data_client.rs"]
2885mod data_client_tests;
2886
2887#[cfg(test)]
2888mod tests {
2889 use std::{path::PathBuf, time::Duration};
2890
2891 use nautilus_common::{live::runner::replace_data_event_sender, testing::wait_until_async};
2892 use nautilus_core::{UUID4, UnixNanos};
2893 use nautilus_model::{
2894 enums::AggressorSide,
2895 identifiers::InstrumentId,
2896 types::{Price, Quantity},
2897 };
2898 use nautilus_network::websocket::TransportBackend;
2899 use rstest::rstest;
2900 use rust_decimal_macros::dec;
2901 use serde_json::{Value, json};
2902
2903 use super::*;
2904 use crate::{
2905 client_logging,
2906 common::{
2907 consts::DERIVE_CLIENT_ID, enums::DeriveEnvironment, parse::parse_derive_instrument_any,
2908 urls,
2909 },
2910 http::models::DeriveInstrument,
2911 websocket::{DeriveWsFrame, WsSubscriptionPayload, parse_orderbook_msg, parse_ticker_msg},
2912 };
2913
2914 fn data_path() -> PathBuf {
2915 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test_data")
2916 }
2917
2918 fn load_json(filename: &str) -> Value {
2919 let content = std::fs::read_to_string(data_path().join(filename))
2920 .unwrap_or_else(|_| panic!("failed to read {filename}"));
2921 serde_json::from_str(&content).expect("invalid json")
2922 }
2923
2924 #[rstest]
2925 #[case::stop("stop")]
2926 #[case::reset("reset")]
2927 #[case::dispose("dispose")]
2928 #[tokio::test]
2929 async fn test_cache_instruments_rejects_completion_after_shutdown(#[case] shutdown: &str) {
2930 let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
2931 replace_data_event_sender(sender);
2932 let mut client =
2933 DeriveDataClient::new(*DERIVE_CLIENT_ID, DeriveDataClientConfig::default()).unwrap();
2934 let original = perp_instrument();
2935 let instrument_id = original.id();
2936 client.instruments.insert(instrument_id, original.clone());
2937 let mut definition: DeriveInstrument =
2938 serde_json::from_value(load_json("perps/instrument_eth.json")).unwrap();
2939 definition.amount_step = dec!(0.007);
2940 let refreshed = parse_derive_instrument_any(&definition, UnixNanos::from(751))
2941 .unwrap()
2942 .unwrap();
2943 let cancellation = client.cancellation_token.clone();
2944 match shutdown {
2945 "stop" => client.stop().unwrap(),
2946 "reset" => client.reset().unwrap(),
2947 "dispose" => client.dispose().unwrap(),
2948 _ => unreachable!(),
2949 }
2950
2951 cache_instruments(
2952 vec![refreshed],
2953 &client.instruments,
2954 &client.data_sender,
2955 &cancellation,
2956 &client.subscription_lock,
2957 );
2958
2959 let expected = (shutdown != "reset").then_some(original);
2960 assert_eq!(client.instruments.get_cloned(&instrument_id), expected);
2961 assert!(receiver.try_recv().is_err());
2962 }
2963
2964 #[tokio::test]
2965 async fn test_data_task_log_omits_provider_error() {
2966 let _capture = client_logging::capture_logs().await;
2967 let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel();
2968 replace_data_event_sender(sender);
2969 let client =
2970 DeriveDataClient::new(*DERIVE_CLIENT_ID, DeriveDataClientConfig::default()).unwrap();
2971 client.spawn_task("Synthetic subscription", async {
2972 anyhow::bail!("SYN-secret-provider-error")
2973 });
2974
2975 wait_until_async(
2976 || async {
2977 client_logging::messages().iter().any(|(level, message)| {
2978 *level == log::Level::Warn && message == "Synthetic subscription failed"
2979 })
2980 },
2981 Duration::from_secs(2),
2982 )
2983 .await;
2984
2985 client.join_pending_tasks().await.unwrap();
2986 let messages = client_logging::messages();
2987 assert!(messages.contains(&(
2988 log::Level::Warn,
2989 "Synthetic subscription failed".to_string()
2990 )));
2991 assert!(
2992 !messages
2993 .iter()
2994 .any(|(_, message)| message.contains("SYN-secret"))
2995 );
2996 }
2997
2998 #[tokio::test]
2999 async fn test_session_failure_log_omits_provider_reason() {
3000 let _capture = client_logging::capture_logs().await;
3001 let (ctx, _receiver) = make_ctx(None);
3002 DeriveDataClient::handle_ws_message(
3003 DeriveWsMessage::SessionRecoveryFailed("SYN-secret-provider-reason".to_string()),
3004 &ctx,
3005 );
3006 assert_eq!(
3007 client_logging::messages()
3008 .into_iter()
3009 .filter(|(_, message)| {
3010 message.starts_with("Derive WebSocket session recovery failed")
3011 })
3012 .collect::<Vec<_>>(),
3013 vec![(
3014 log::Level::Error,
3015 "Derive WebSocket session recovery failed".to_string()
3016 )]
3017 );
3018 }
3019
3020 #[tokio::test]
3021 async fn test_public_stream_closure_marks_disconnected() {
3022 let (sender, mut events) = tokio::sync::mpsc::unbounded_channel();
3023 replace_data_event_sender(sender);
3024 let client =
3025 DeriveDataClient::new(*DERIVE_CLIENT_ID, DeriveDataClientConfig::default()).unwrap();
3026 let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
3027 client.is_connected.store(true, Ordering::Release);
3028 client.spawn_stream_task(rx).unwrap();
3029 assert!(client.is_connected());
3030 drop(tx);
3031 client.session_tasks.begin_shutdown();
3032 client
3033 .session_tasks
3034 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(1))
3035 .await
3036 .unwrap();
3037 assert!(!client.is_connected());
3038 assert!(events.try_recv().is_err());
3039 }
3040
3041 #[rstest]
3042 fn test_truncated_payload_snippet_short_payload_is_unchanged() {
3043 let short = json!({"ok": true}).to_string();
3044 assert_eq!(truncated_payload_snippet(&short), r#"{"ok":true}"#);
3045 }
3046
3047 #[rstest]
3048 fn test_truncated_payload_snippet_truncates_long_ascii_payload() {
3049 let big = json!({"msg": "x".repeat(1024)}).to_string();
3050 let snippet = truncated_payload_snippet(&big);
3051 assert!(snippet.ends_with("...(truncated)"));
3052 assert!(snippet.len() <= 512 + "...(truncated)".len());
3054 }
3055
3056 #[rstest]
3057 fn test_truncated_payload_snippet_handles_multibyte_at_boundary() {
3058 let value: String = format!("x{}", "\u{00E9}".repeat(1024));
3066 let big = json!({"a": value});
3067 let raw = big.to_string();
3068 assert!(
3069 !raw.is_char_boundary(512),
3070 "test premise: 512 must be mid-codepoint",
3071 );
3072
3073 let snippet = truncated_payload_snippet(&raw);
3074 assert!(snippet.ends_with("...(truncated)"));
3075 let body_len = snippet.len() - "...(truncated)".len();
3076 assert!(body_len <= 512);
3077 }
3078
3079 fn subscription_payload(channel: &str, data: &Value) -> WsSubscriptionPayload {
3080 let frame = json!({
3081 "jsonrpc": "2.0",
3082 "method": "subscription",
3083 "params": {
3084 "channel": channel,
3085 "data": data
3086 }
3087 });
3088
3089 match DeriveWsFrame::parse(&frame.to_string()).unwrap() {
3090 DeriveWsFrame::Subscription(payload) => payload,
3091 other => panic!("expected subscription frame, was {other:?}"),
3092 }
3093 }
3094
3095 fn make_ctx(
3096 instrument: Option<InstrumentAny>,
3097 ) -> (
3098 WsMessageContext,
3099 tokio::sync::mpsc::UnboundedReceiver<DataEvent>,
3100 ) {
3101 let (data_sender, data_rx) = tokio::sync::mpsc::unbounded_channel();
3102 let instruments = Arc::new(AtomicMap::new());
3103 if let Some(instrument) = instrument {
3104 cache_instrument(&instruments, &instrument);
3105 }
3106
3107 (
3108 WsMessageContext {
3109 clock: get_atomic_clock_realtime(),
3110 data_sender: data_sender.into(),
3111 instruments,
3112 active_book_delta_channels: Arc::new(AtomicMap::new()),
3113 active_book_depth_channels: Arc::new(AtomicMap::new()),
3114 active_ticker_channels: Arc::new(AtomicMap::new()),
3115 active_quote_subs: Arc::new(AtomicSet::new()),
3116 active_trade_subs: Arc::new(AtomicSet::new()),
3117 active_mark_subs: Arc::new(AtomicSet::new()),
3118 active_index_subs: Arc::new(AtomicSet::new()),
3119 active_funding_subs: Arc::new(AtomicSet::new()),
3120 active_greeks_subs: Arc::new(AtomicSet::new()),
3121 subscription_lock: Arc::new(Mutex::new(())),
3122 book_sync: {
3123 let tracker = BookSyncTracker::default();
3124 let id = InstrumentId::from("ETH-PERP.DERIVE");
3125 tracker.record_subscription("orderbook.ETH-PERP.1.10", id);
3126 tracker
3127 .write("orderbook.ETH-PERP.1.10")
3128 .unwrap()
3129 .gate
3130 .open();
3131 tracker
3132 },
3133 quote_cache: Arc::new(Mutex::new(QuoteCache::new())),
3134 trades_emitted: Arc::new(Mutex::new(FifoCache::new())),
3135 },
3136 data_rx,
3137 )
3138 }
3139
3140 fn perp_instrument() -> InstrumentAny {
3141 parse_derive_instrument_any(&perp_definition("ETH-PERP", "ETH"), UnixNanos::from(1))
3142 .unwrap()
3143 .unwrap()
3144 }
3145
3146 fn btc_perp_instrument() -> InstrumentAny {
3147 parse_derive_instrument_any(&perp_definition("BTC-PERP", "BTC"), UnixNanos::from(1))
3148 .unwrap()
3149 .unwrap()
3150 }
3151
3152 fn perp_definition(name: &str, currency: &str) -> DeriveInstrument {
3153 let mut value = load_json("perps/instrument_eth.json");
3154 value["base_currency"] = json!(currency);
3155 value["instrument_name"] = json!(name);
3156 value["perp_details"]["index"] = json!(format!("{currency}-USD"));
3157
3158 serde_json::from_value(value).unwrap()
3159 }
3160
3161 fn option_instrument() -> InstrumentAny {
3162 let definition: DeriveInstrument =
3163 serde_json::from_value(load_json("options/instrument_eth.json")).unwrap();
3164 parse_derive_instrument_any(&definition, UnixNanos::from(1))
3165 .unwrap()
3166 .unwrap()
3167 }
3168
3169 fn spot_instrument() -> InstrumentAny {
3170 let definition: DeriveInstrument =
3171 serde_json::from_value(load_json("spot/instrument_eth.json")).unwrap();
3172 parse_derive_instrument_any(&definition, UnixNanos::from(1))
3173 .unwrap()
3174 .unwrap()
3175 }
3176
3177 fn ticker_json(timestamp: i64) -> Value {
3178 let mut value = load_json("perps/ws_ticker_eth.json");
3179 value["timestamp"] = json!(timestamp);
3180 value
3181 }
3182
3183 fn option_ticker_json(timestamp: i64) -> Value {
3184 let mut value = load_json("options/http_ticker_eth_snapshot.json");
3185 value["timestamp"] = json!(timestamp);
3186 value
3187 }
3188
3189 fn spot_ticker_slim_json() -> Value {
3190 load_json("spot/ws_ticker_slim_eth.json")
3191 }
3192
3193 fn orderbook_json() -> Value {
3194 load_json("perps/ws_orderbook_eth.json")
3195 }
3196
3197 fn trade_json(instrument_name: &str, trade_id: &str) -> Value {
3198 let mut value = load_json("perps/ws_trade_eth.json");
3199 value["instrument_name"] = json!(instrument_name);
3200 value["trade_id"] = json!(trade_id);
3201 value
3202 }
3203
3204 #[rstest]
3205 fn test_handle_ticker_subscription_emits_quote_with_instrument_precision() {
3206 let instrument = perp_instrument();
3207 let instrument_id = instrument.id();
3208 let (ctx, mut rx) = make_ctx(Some(instrument));
3209 ctx.active_ticker_channels
3210 .insert(instrument_id, "ticker_slim.ETH-PERP.1000".to_string());
3211 ctx.active_quote_subs.insert(instrument_id);
3212 let payload = subscription_payload(
3213 "ticker_slim.ETH-PERP.1000",
3214 &json!({
3215 "timestamp": 1_700_000_000_010_i64,
3216 "instrument_ticker": ticker_json(1_700_000_000_000)
3217 }),
3218 );
3219
3220 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3221
3222 match rx.try_recv().unwrap() {
3223 DataEvent::Data(Data::Quote(quote)) => {
3224 assert_eq!(quote.instrument_id, instrument_id);
3225 assert_eq!(quote.bid_price, Price::from("3500.00"));
3226 assert_eq!(quote.ask_price, Price::from("3501.00"));
3227 assert_eq!(quote.bid_size, Quantity::from("1.000"));
3228 assert_eq!(quote.ask_size, Quantity::from("2.000"));
3229 assert_eq!(quote.bid_price.precision, 2);
3230 assert_eq!(quote.bid_size.precision, 3);
3231 }
3232 other => panic!("expected quote data event, was {other:?}"),
3233 }
3234 }
3235
3236 #[rstest]
3237 fn test_handle_ticker_partial_quote_without_cache_emits_no_quote() {
3238 let instrument = spot_instrument();
3239 let instrument_id = instrument.id();
3240 let channel = "ticker_slim.ETH-USDC.1000";
3241 let (ctx, mut rx) = make_ctx(Some(instrument));
3242 install_ticker(&ctx, instrument_id, channel);
3243 ctx.active_quote_subs.insert(instrument_id);
3244 let payload = subscription_payload(channel, &spot_ticker_slim_json());
3245
3246 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3247
3248 assert!(rx.try_recv().is_err());
3249 }
3250
3251 #[rstest]
3252 fn test_handle_ticker_empty_side_clears_cached_quote() {
3253 let instrument = spot_instrument();
3254 let instrument_id = instrument.id();
3255 let channel = "ticker_slim.ETH-USDC.1000";
3256 let (ctx, mut rx) = make_ctx(Some(instrument));
3257
3258 let cached_quote = QuoteTick::new(
3259 instrument_id,
3260 Price::from("0.1"),
3261 Price::from("0.3"),
3262 Quantity::from("10.00"),
3263 Quantity::from("20.00"),
3264 UnixNanos::from(1),
3265 UnixNanos::from(1),
3266 );
3267 ctx.quote_cache.lock().insert(instrument_id, cached_quote);
3268 install_ticker(&ctx, instrument_id, channel);
3269 ctx.active_quote_subs.insert(instrument_id);
3270 let payload = subscription_payload(channel, &spot_ticker_slim_json());
3271
3272 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3273
3274 assert!(rx.try_recv().is_err());
3275 assert_eq!(ctx.quote_cache.lock().get(&instrument_id), None);
3276 }
3277
3278 #[rstest]
3279 #[case::bid_price("0", "0.4", "45", "67")]
3280 #[case::bid_size("0.2", "0.4", "0", "67")]
3281 #[case::ask_price("0.2", "0", "45", "67")]
3282 #[case::ask_size("0.2", "0.4", "45", "0")]
3283 #[case::both_empty("0", "0", "0", "0")]
3284 fn test_process_ticker_empty_sides_and_recovery(
3285 #[case] bid_price: &str,
3286 #[case] ask_price: &str,
3287 #[case] bid_size: &str,
3288 #[case] ask_size: &str,
3289 #[values(false, true)] populated: bool,
3290 #[values(false, true)] slim: bool,
3291 ) {
3292 let (instrument, channel, mut data) = if slim {
3293 (
3294 spot_instrument(),
3295 "ticker_slim.ETH-USDC.1000",
3296 spot_ticker_slim_json(),
3297 )
3298 } else {
3299 (perp_instrument(), "ticker.ETH-PERP.1000", ticker_json(1234))
3300 };
3301
3302 let instrument_id = instrument.id();
3303 let mut quote_cache = QuoteCache::new();
3304 let ts_init = UnixNanos::from(9_876_543_210);
3305
3306 let expected = QuoteTick::new(
3307 instrument_id,
3308 Price::from_decimal_dp(dec!(0.2), instrument.price_precision()).unwrap(),
3309 Price::from_decimal_dp(dec!(0.4), instrument.price_precision()).unwrap(),
3310 Quantity::from_decimal_dp(dec!(45), instrument.size_precision()).unwrap(),
3311 Quantity::from_decimal_dp(dec!(67), instrument.size_precision()).unwrap(),
3312 UnixNanos::from(1_238_000_000),
3313 ts_init,
3314 );
3315
3316 let unrelated_quote = QuoteTick {
3317 instrument_id: InstrumentId::from("BTC-PERP.DERIVE"),
3318 ..expected
3319 };
3320
3321 quote_cache.insert(unrelated_quote.instrument_id, unrelated_quote);
3322 if populated {
3323 quote_cache.insert(instrument_id, expected);
3324 }
3325
3326 let keys = if slim {
3327 ["b", "a", "B", "A"]
3328 } else {
3329 [
3330 "best_bid_price",
3331 "best_ask_price",
3332 "best_bid_amount",
3333 "best_ask_amount",
3334 ]
3335 };
3336
3337 for (values, timestamp, expected_result) in [
3338 ([bid_price, ask_price, bid_size, ask_size], 1235, None),
3339 (["0.6", "0", "89", "0"], 1236, None),
3340 (["0", "0.8", "0", "91"], 1237, None),
3341 (["0.2", "0.4", "45", "67"], 1238, Some(expected)),
3342 ] {
3343 data["timestamp"] = json!(timestamp);
3344 let snapshot = if slim {
3345 &mut data["instrument_ticker"]
3346 } else {
3347 &mut data
3348 };
3349
3350 for (key, value) in keys.into_iter().zip(values) {
3351 snapshot[key] = json!(value);
3352 }
3353
3354 let payload = subscription_payload(channel, &data);
3355 let msg = parse_ticker_msg(&payload).unwrap();
3356 let result = process_ticker_quote(
3357 &msg,
3358 instrument.price_precision(),
3359 instrument.size_precision(),
3360 ts_init,
3361 &mut quote_cache,
3362 )
3363 .unwrap();
3364
3365 assert_eq!(result, expected_result);
3366 assert_eq!(quote_cache.get(&instrument_id), expected_result.as_ref());
3367 assert_eq!(
3368 quote_cache.get(&unrelated_quote.instrument_id),
3369 Some(&unrelated_quote)
3370 );
3371 }
3372 }
3373
3374 #[rstest]
3375 #[case::testnet_spot(DeriveEnvironment::Testnet, "ETH-USDC")]
3376 #[case::testnet_perp(DeriveEnvironment::Testnet, "ETH-PERP")]
3377 #[case::mainnet_spot(DeriveEnvironment::Mainnet, "ETH-USDC")]
3378 #[case::mainnet_perp(DeriveEnvironment::Mainnet, "ETH-PERP")]
3379 #[tokio::test]
3380 #[ignore = "requires Derive public network access; mainnet requires an isolated NZ route"]
3381 async fn test_live_ticker_complete_snapshots(
3382 #[case] environment: DeriveEnvironment,
3383 #[case] instrument_name: &str,
3384 ) {
3385 let http =
3386 DeriveHttpClient::new(urls::rest_url(environment), Some(15), None, None).unwrap();
3387 let definition = http.get_instrument(instrument_name).await.unwrap();
3388 let instrument = parse_derive_instrument_any(&definition, UnixNanos::from(1))
3389 .unwrap()
3390 .unwrap();
3391 let instrument_id = instrument.id();
3392 let mut ws_client =
3393 DeriveWebSocketClient::new(None, environment, TransportBackend::default(), None);
3394 ws_client.connect().await.unwrap();
3395 let mut events = ws_client.take_event_receiver().unwrap();
3396 let validation = async {
3397 ws_client
3398 .subscribe_ticker(instrument_name, "1000")
3399 .await
3400 .unwrap();
3401 let mut quote_cache = QuoteCache::new();
3402
3403 let seed = QuoteTick::new(
3404 instrument_id,
3405 Price::from("0.1"),
3406 Price::from("0.3"),
3407 Quantity::from("10.00"),
3408 Quantity::from("20.00"),
3409 UnixNanos::from(1),
3410 UnixNanos::from(1),
3411 );
3412 quote_cache.insert(instrument_id, seed);
3413 let mut published = 0;
3414 let mut suppressed = 0;
3415
3416 for _ in 0..3 {
3417 let payload = loop {
3418 if let DeriveWsMessage::Subscription(payload) = events.recv().await.unwrap() {
3419 break payload;
3420 }
3421 };
3422
3423 let msg = parse_ticker_msg(&payload).unwrap();
3424 let ts_init = UnixNanos::from(9_876_543_210);
3425 let expected = parse_ticker_quote(
3426 &msg,
3427 instrument.price_precision(),
3428 instrument.size_precision(),
3429 ts_init,
3430 )
3431 .unwrap();
3432 let result = process_ticker_quote(
3433 &msg,
3434 instrument.price_precision(),
3435 instrument.size_precision(),
3436 ts_init,
3437 &mut quote_cache,
3438 )
3439 .unwrap();
3440
3441 if expected.bid_price.is_zero()
3442 || expected.ask_price.is_zero()
3443 || expected.bid_size.is_zero()
3444 || expected.ask_size.is_zero()
3445 {
3446 assert_eq!(result, None);
3447 assert_eq!(quote_cache.get(&instrument_id), None);
3448 suppressed += 1;
3449 } else {
3450 assert_eq!(result, Some(expected));
3451 assert_eq!(quote_cache.get(&instrument_id), Some(&expected));
3452 assert_eq!(expected.instrument_id, instrument_id);
3453 assert_eq!(
3454 expected.ts_event,
3455 UnixNanos::from(msg.data.timestamp() as u64 * 1_000_000)
3456 );
3457 assert_eq!(expected.ts_init, ts_init);
3458 published += 1;
3459 }
3460 }
3461
3462 ws_client
3463 .unsubscribe_ticker(instrument_name, "1000")
3464 .await
3465 .unwrap();
3466 (published, suppressed)
3467 };
3468
3469 let result = tokio::time::timeout(Duration::from_secs(30), validation).await;
3470 tokio::time::timeout(Duration::from_secs(5), ws_client.disconnect())
3471 .await
3472 .expect("ticker validation shutdown exceeds five seconds")
3473 .unwrap();
3474 let (published, suppressed) = result.expect("ticker validation exceeds 30 seconds");
3475 println!(
3476 "Derive {environment:?} {instrument_name}: {published} complete quotes, {suppressed} empty snapshots suppressed; unsubscribed and disconnected"
3477 );
3478 }
3479
3480 #[rstest]
3481 fn test_handle_reconnected_clears_quote_cache() {
3482 let instrument = spot_instrument();
3483 let instrument_id = instrument.id();
3484 let channel = "ticker_slim.ETH-USDC.1000";
3485 let (ctx, mut rx) = make_ctx(Some(instrument));
3486
3487 let cached_quote = QuoteTick::new(
3488 instrument_id,
3489 Price::from("0.1"),
3490 Price::from("0.3"),
3491 Quantity::from("10.00"),
3492 Quantity::from("20.00"),
3493 UnixNanos::from(1),
3494 UnixNanos::from(1),
3495 );
3496 ctx.quote_cache.lock().insert(instrument_id, cached_quote);
3497 install_ticker(&ctx, instrument_id, channel);
3498 ctx.active_quote_subs.insert(instrument_id);
3499
3500 DeriveDataClient::handle_ws_message(DeriveWsMessage::Reconnected, &ctx);
3501
3502 assert_eq!(ctx.quote_cache.lock().get(&instrument_id), None);
3503
3504 let payload = subscription_payload(channel, &spot_ticker_slim_json());
3505 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3506
3507 assert!(rx.try_recv().is_err());
3508 }
3509
3510 #[rstest]
3511 #[tokio::test]
3512 async fn test_session_recovery_failure_marks_client_disconnected() {
3513 let _capture = client_logging::capture_logs().await;
3514 let (data_tx, _data_rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
3515 replace_data_event_sender(data_tx);
3516
3517 let config = DeriveDataClientConfig {
3518 environment: DeriveEnvironment::Mainnet,
3519 ..Default::default()
3520 };
3521
3522 let client = DeriveDataClient::new(*DERIVE_CLIENT_ID, config).unwrap();
3523 let (ws_tx, ws_rx) = tokio::sync::mpsc::unbounded_channel();
3524 client.is_connected.store(true, Ordering::Release);
3525 client.spawn_stream_task(ws_rx).unwrap();
3526
3527 ws_tx
3528 .send(DeriveWsMessage::SessionRecoveryFailed(
3529 "subscription replay failed".to_string(),
3530 ))
3531 .unwrap();
3532 wait_until_async(|| async { !client.is_connected() }, Duration::from_secs(2)).await;
3533
3534 assert!(!client.is_connected());
3535
3536 client.cancellation_token.cancel();
3537 client.join_session_tasks().await.unwrap();
3538 }
3539
3540 #[rstest]
3541 fn test_handle_orderbook_subscription_emits_snapshot_deltas() {
3542 let instrument = perp_instrument();
3543 let instrument_id = instrument.id();
3544 let (ctx, mut rx) = make_ctx(Some(instrument));
3545 ctx.active_book_delta_channels
3546 .insert(instrument_id, "orderbook.ETH-PERP.1.10".to_string());
3547 let payload = subscription_payload("orderbook.ETH-PERP.1.10", &orderbook_json());
3548
3549 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3550
3551 match rx.try_recv().unwrap() {
3552 DataEvent::Data(Data::BookDeltas(deltas)) => {
3553 assert_eq!(deltas.instrument_id, instrument_id);
3554 assert_eq!(deltas.deltas.len(), 3);
3555 assert_eq!(deltas.deltas[1].order.price, Price::from("3500.00"));
3556 assert_eq!(deltas.deltas[1].order.size, Quantity::from("1.000"));
3557 assert_eq!(deltas.deltas[2].order.price, Price::from("3501.00"));
3558 assert_eq!(deltas.deltas[2].order.size, Quantity::from("2.000"));
3559 }
3560 other => panic!("expected deltas data event, was {other:?}"),
3561 }
3562 }
3563
3564 #[rstest]
3565 fn test_handle_orderbook_subscription_emits_for_depth_subscription() {
3566 let instrument = perp_instrument();
3567 let instrument_id = instrument.id();
3568 let (ctx, mut rx) = make_ctx(Some(instrument));
3569 ctx.active_book_depth_channels
3570 .insert(instrument_id, "orderbook.ETH-PERP.1.10".to_string());
3571 let payload = subscription_payload("orderbook.ETH-PERP.1.10", &orderbook_json());
3572
3573 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3574
3575 match rx.try_recv().unwrap() {
3576 DataEvent::Data(Data::BookDepth(depth)) => {
3577 assert_eq!(depth.instrument_id, instrument_id);
3578 assert_eq!(depth.bids[0].price, Price::from("3500.00"));
3579 assert_eq!(depth.bids[0].size, Quantity::from("1.000"));
3580 assert_eq!(depth.asks[0].price, Price::from("3501.00"));
3581 assert_eq!(depth.asks[0].size, Quantity::from("2.000"));
3582 }
3583 other => panic!("expected depth data event, was {other:?}"),
3584 }
3585 }
3586
3587 #[rstest]
3588 #[case::null_price(0, Value::Null)]
3589 #[case::empty_price(0, json!(""))]
3590 #[case::null_amount(1, Value::Null)]
3591 #[case::empty_amount(1, json!(""))]
3592 fn test_orderbook_missing_numeric_values_preserve_recovery_and_watermark(
3593 #[case] field: usize,
3594 #[case] value: Value,
3595 #[values("bids", "asks")] side: &str,
3596 #[values(false, true)] depth: bool,
3597 ) {
3598 let instrument = perp_instrument();
3599 let instrument_id = instrument.id();
3600 let price_precision = instrument.price_precision();
3601 let size_precision = instrument.size_precision();
3602 let (ctx, mut rx) = make_ctx(Some(instrument));
3603 let channel = "orderbook.ETH-PERP.1.10";
3604
3605 let channels = if depth {
3606 &ctx.active_book_depth_channels
3607 } else {
3608 &ctx.active_book_delta_channels
3609 };
3610
3611 channels.insert(instrument_id, channel.to_string());
3612
3613 let assert_snapshot = |event, msg: &DeriveOrderbookMsg| match event {
3614 DataEvent::Data(Data::BookDeltas(deltas)) if !depth => {
3615 let expected =
3616 parse_orderbook_deltas(msg, price_precision, size_precision, deltas.ts_init)
3617 .unwrap();
3618 assert_eq!(*deltas, expected);
3619 }
3620 DataEvent::Data(Data::BookDepth(snapshot)) if depth => {
3621 let expected =
3622 parse_orderbook_depth(msg, price_precision, size_precision, snapshot.ts_init)
3623 .unwrap();
3624 assert_eq!(*snapshot, expected);
3625 }
3626 other => panic!("expected book data event, was {other:?}"),
3627 };
3628
3629 let initial = orderbook_json();
3630 let timestamp = initial["timestamp"].as_i64().unwrap();
3631 let write = ctx.book_sync.write(channel).unwrap();
3632 let payload = subscription_payload(channel, &initial);
3633 let msg = parse_orderbook_msg(&payload).unwrap();
3634 DeriveDataClient::handle_ws_message(
3635 DeriveWsMessage::BookSubscription {
3636 payload,
3637 generation: write.generation.clone(),
3638 },
3639 &ctx,
3640 );
3641
3642 assert_snapshot(rx.try_recv().unwrap(), &msg);
3643
3644 let mut malformed = initial.clone();
3645 malformed["timestamp"] = json!(timestamp + 2);
3646 malformed[side][0][field] = value;
3647 let payload = subscription_payload(channel, &malformed);
3648 DeriveDataClient::handle_ws_message(
3649 DeriveWsMessage::BookSubscription {
3650 payload,
3651 generation: write.generation,
3652 },
3653 &ctx,
3654 );
3655
3656 assert!(rx.try_recv().is_err());
3657 let recoveries = ctx.book_sync.recoveries(Duration::ZERO, Instant::now());
3658 assert_eq!(recoveries.len(), 1);
3659 let recovery = &recoveries[0].2;
3660 let write = ctx
3661 .book_sync
3662 .begin_recovery_write(channel, recovery, recovery.gate.clone())
3663 .unwrap();
3664 write.gate.open();
3665 let payload = subscription_payload(channel, &malformed);
3666 DeriveDataClient::handle_ws_message(
3667 DeriveWsMessage::BookSubscription {
3668 payload,
3669 generation: write.generation.clone(),
3670 },
3671 &ctx,
3672 );
3673
3674 assert!(rx.try_recv().is_err());
3675 assert!(recovery.is_running());
3676 assert!(!recovery.is_accepted());
3677
3678 let mut valid = initial;
3679 valid["timestamp"] = json!(timestamp + 1);
3680 let payload = subscription_payload(channel, &valid);
3681 let msg = parse_orderbook_msg(&payload).unwrap();
3682 DeriveDataClient::handle_ws_message(
3683 DeriveWsMessage::BookSubscription {
3684 payload,
3685 generation: write.generation,
3686 },
3687 &ctx,
3688 );
3689
3690 assert_snapshot(rx.try_recv().unwrap(), &msg);
3691 assert!(recovery.is_accepted());
3692 assert!(!recovery.is_running());
3693 assert!(rx.try_recv().is_err());
3694 assert!(
3695 ctx.book_sync
3696 .recoveries(Duration::ZERO, Instant::now())
3697 .is_empty()
3698 );
3699 }
3700
3701 #[rstest]
3702 fn test_orderbook_frame_ignored_for_inactive_channel() {
3703 let instrument = perp_instrument();
3704 let instrument_id = instrument.id();
3705 let (ctx, mut rx) = make_ctx(Some(instrument));
3706 ctx.active_book_delta_channels
3707 .insert(instrument_id, "orderbook.ETH-PERP.1.20".to_string());
3708 let payload = subscription_payload("orderbook.ETH-PERP.1.10", &orderbook_json());
3709
3710 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3711
3712 assert!(rx.try_recv().is_err());
3713 }
3714
3715 #[rstest]
3716 fn test_handle_trades_subscription_filters_and_emits_active_instrument() {
3717 let instrument = perp_instrument();
3718 let other = btc_perp_instrument();
3719 let instrument_id = instrument.id();
3720 let (ctx, mut rx) = make_ctx(Some(instrument));
3721 cache_instrument(&ctx.instruments, &other);
3722 ctx.active_trade_subs.insert(instrument_id);
3723 let payload = subscription_payload(
3724 "trades.perp.ETH",
3725 &json!([
3726 trade_json("ETH-PERP", "trade-1"),
3727 trade_json("BTC-PERP", "trade-2")
3728 ]),
3729 );
3730
3731 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3732
3733 match rx.try_recv().unwrap() {
3734 DataEvent::Data(Data::Trade(trade)) => {
3735 assert_eq!(trade.instrument_id, instrument_id);
3736 assert_eq!(trade.trade_id.to_string(), "trade-1");
3737 assert_eq!(trade.price, Price::from("3500.00"));
3738 assert_eq!(trade.size, Quantity::from("1.000"));
3739 }
3740 other => panic!("expected trade data event, was {other:?}"),
3741 }
3742
3743 assert!(rx.try_recv().is_err());
3744 }
3745
3746 #[rstest]
3747 fn test_public_trade_replay_after_failed_delivery() {
3748 let instrument = perp_instrument();
3749 let instrument_id = instrument.id();
3750 let (mut ctx, rx) = make_ctx(Some(instrument));
3751 ctx.clock = Box::leak(Box::new(AtomicTime::new(false, UnixNanos::from(42))));
3752 ctx.active_trade_subs.insert(instrument_id);
3753 drop(rx);
3754 let payload = subscription_payload(
3755 "trades.perp.ETH",
3756 &json!([trade_json("ETH-PERP", "delivery-replay")]),
3757 );
3758 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload.clone()), &ctx);
3759 let (sender, mut rx) = tokio::sync::mpsc::unbounded_channel();
3760 ctx.data_sender = sender.into();
3761
3762 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload.clone()), &ctx);
3763 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3764
3765 let DataEvent::Data(Data::Trade(trade)) =
3766 rx.try_recv().expect("undelivered trade must replay")
3767 else {
3768 panic!("expected trade data event");
3769 };
3770
3771 assert_eq!(trade.instrument_id, instrument_id);
3772 assert_eq!(trade.trade_id, TradeId::from("delivery-replay"));
3773 assert_eq!(trade.price, Price::from("3500.00"));
3774 assert_eq!(trade.size, Quantity::from("1.000"));
3775 assert_eq!(trade.aggressor_side, AggressorSide::Buy);
3776 assert_eq!(trade.ts_event, UnixNanos::from(1_700_000_000_001_000_000));
3777 assert_eq!(trade.ts_init, UnixNanos::from(42));
3778 assert!(rx.try_recv().is_err());
3779 }
3780
3781 #[rstest]
3782 fn test_public_trades_continue_after_invalid_id() {
3783 let instrument = perp_instrument();
3784 let instrument_id = instrument.id();
3785 let (ctx, mut rx) = make_ctx(Some(instrument));
3786 ctx.active_trade_subs.insert(instrument_id);
3787 let payload = subscription_payload(
3788 "trades.perp.ETH",
3789 &json!([
3790 trade_json("ETH-PERP", "1234567890123456789012345678901234567"),
3791 trade_json("ETH-PERP", "valid-first")
3792 ]),
3793 );
3794
3795 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3796 let first_event = rx
3797 .try_recv()
3798 .expect("valid row in the same frame must emit");
3799 assert!(rx.try_recv().is_err());
3800 let payload = subscription_payload(
3801 "trades.perp.ETH",
3802 &json!([
3803 trade_json("ETH-PERP", "valid-first"),
3804 trade_json("ETH-PERP", "valid-next")
3805 ]),
3806 );
3807 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3808
3809 for (event, expected_id) in [
3810 (first_event, "valid-first"),
3811 (rx.try_recv().unwrap(), "valid-next"),
3812 ] {
3813 let DataEvent::Data(Data::Trade(trade)) = event else {
3814 panic!("expected trade data event");
3815 };
3816
3817 assert_eq!(trade.instrument_id, instrument_id);
3818 assert_eq!(trade.trade_id.to_string(), expected_id);
3819 assert_eq!(trade.price, Price::from("3500.00"));
3820 assert_eq!(trade.size, Quantity::from("1.000"));
3821 assert_eq!(trade.aggressor_side, AggressorSide::Buy);
3822 assert_eq!(trade.ts_event, UnixNanos::from(1_700_000_000_001_000_000));
3823 }
3824
3825 assert!(rx.try_recv().is_err());
3826 }
3827
3828 #[rstest]
3829 #[case("perps/http_public_trades_result_eth_paired_maker_first.json")]
3830 #[case("perps/http_public_trades_result_eth_paired_taker_first.json")]
3831 fn test_v3_public_paired_rows_deduplicate_across_frames(#[case] filename: &str) {
3832 let instrument = perp_instrument();
3833 let instrument_id = instrument.id();
3834 let (ctx, mut rx) = make_ctx(Some(instrument));
3835 ctx.active_trade_subs.insert(instrument_id);
3836 let rows = load_json(filename)["trades"].as_array().unwrap().clone();
3837 for row in &rows {
3838 let payload = subscription_payload("trades.perp.ETH", &json!([row]));
3839 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3840 }
3841
3842 DeriveDataClient::handle_ws_message(DeriveWsMessage::Reconnected, &ctx);
3843 let payload = subscription_payload("trades.perp.ETH", &json!(rows));
3844 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3845
3846 match rx.try_recv().unwrap() {
3847 DataEvent::Data(Data::Trade(trade)) => {
3848 assert_eq!(trade.instrument_id, instrument_id);
3849 assert_eq!(trade.aggressor_side, AggressorSide::Buy);
3850 assert_eq!(trade.trade_id, TradeId::new("pub-pair-1"));
3851 assert_eq!(trade.price, Price::from("3500.00"));
3852 assert_eq!(trade.size, Quantity::from("0.250"));
3853 assert_eq!(trade.ts_event, UnixNanos::from(1_700_000_000_000_000_000));
3854 }
3855 other => panic!("expected trade data event, was {other:?}"),
3856 }
3857
3858 assert!(rx.try_recv().is_err());
3859 }
3860
3861 #[rstest]
3862 fn test_handle_subscription_without_cached_instrument_emits_no_event() {
3863 let (ctx, mut rx) = make_ctx(None);
3864 let instrument_id = InstrumentId::from("ETH-PERP.DERIVE");
3865 ctx.active_ticker_channels
3866 .insert(instrument_id, "ticker_slim.ETH-PERP.1000".to_string());
3867 ctx.active_quote_subs.insert(instrument_id);
3868 let payload =
3869 subscription_payload("ticker_slim.ETH-PERP.1000", &ticker_json(1_700_000_000_000));
3870
3871 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3872
3873 assert!(rx.try_recv().is_err());
3874 }
3875
3876 #[rstest]
3877 fn test_ticker_frame_ignored_without_quote_subscription() {
3878 let instrument = perp_instrument();
3879 let (ctx, mut rx) = make_ctx(Some(instrument));
3880 let payload =
3881 subscription_payload("ticker_slim.ETH-PERP.1000", &ticker_json(1_700_000_000_000));
3882
3883 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3884
3885 assert!(rx.try_recv().is_err());
3886 }
3887
3888 #[rstest]
3889 fn test_ticker_frame_ignored_for_inactive_channel() {
3890 let instrument = perp_instrument();
3891 let instrument_id = instrument.id();
3892 let (ctx, mut rx) = make_ctx(Some(instrument));
3893 ctx.active_ticker_channels
3894 .insert(instrument_id, "ticker_slim.ETH-PERP.100".to_string());
3895 ctx.active_quote_subs.insert(instrument_id);
3896 let payload =
3897 subscription_payload("ticker_slim.ETH-PERP.1000", &ticker_json(1_700_000_000_000));
3898
3899 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
3900
3901 assert!(rx.try_recv().is_err());
3902 }
3903
3904 #[rstest]
3905 fn test_trade_channel_uses_instrument_type_and_currency() {
3906 let instrument = perp_instrument();
3907
3908 assert_eq!(trade_channel(&instrument).unwrap(), "trades.perp.ETH");
3909 }
3910
3911 #[rstest]
3912 fn test_trade_channel_uses_erc20_for_spot() {
3913 let instrument = spot_instrument();
3914
3915 assert_eq!(
3916 derive_instrument_type(&instrument).unwrap(),
3917 DeriveInstrumentType::Erc20
3918 );
3919 assert_eq!(trade_channel(&instrument).unwrap(), "trades.erc20.ETH");
3920 }
3921
3922 #[rstest]
3923 fn test_param_defaults_match_derive_public_channels() {
3924 assert_eq!(orderbook_group(&None).unwrap(), DEFAULT_ORDERBOOK_GROUP);
3925 assert_eq!(
3926 orderbook_depth(None, &None).unwrap(),
3927 DEFAULT_ORDERBOOK_DEPTH
3928 );
3929 assert_eq!(ticker_interval(&None).unwrap(), DEFAULT_TICKER_INTERVAL);
3930 }
3931
3932 #[rstest]
3933 fn test_orderbook_channel_parts_splits_from_right() {
3934 assert_eq!(
3935 orderbook_channel_parts("orderbook.ETH.TEST-PERP.10.100").unwrap(),
3936 (
3937 "ETH.TEST-PERP".to_string(),
3938 "10".to_string(),
3939 "100".to_string()
3940 )
3941 );
3942 }
3943
3944 #[rstest]
3945 fn test_ticker_channel_parts_splits_from_right() {
3946 assert_eq!(
3947 ticker_channel_parts("ticker_slim.ETH.TEST-PERP.1000").unwrap(),
3948 ("ETH.TEST-PERP".to_string(), "1000".to_string())
3949 );
3950 }
3951
3952 #[rstest]
3953 fn test_ticker_channel_parts_accepts_legacy_ticker_channel() {
3954 assert_eq!(
3955 ticker_channel_parts("ticker.ETH.TEST-PERP.1000").unwrap(),
3956 ("ETH.TEST-PERP".to_string(), "1000".to_string())
3957 );
3958 }
3959
3960 #[rstest]
3961 fn test_stale_generation_rollback_preserves_resubscription() {
3962 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
3963 replace_data_event_sender(tx);
3964
3965 let client = DeriveDataClient::new(
3966 *DERIVE_CLIENT_ID,
3967 DeriveDataClientConfig {
3968 environment: DeriveEnvironment::Mainnet,
3969 ..Default::default()
3970 },
3971 )
3972 .unwrap();
3973
3974 let lifecycle = client.subscription_lifecycle();
3975 let instrument_id = InstrumentId::from("ETH-PERP.DERIVE");
3976
3977 let owner = ChannelOwner::Ticker {
3978 instrument_id,
3979 feed: TickerFeed::Quote,
3980 };
3981
3982 let channel = "ticker_slim.ETH-PERP.1000".to_string();
3983
3984 let first_generation = lifecycle.activate(owner, Some(&channel)).unwrap();
3985 lifecycle.remove(owner).unwrap();
3986 let second_generation = lifecycle.activate(owner, Some(&channel)).unwrap();
3987
3988 assert!(!lifecycle.rollback(owner, first_generation));
3989 assert!(lifecycle.is_current_channel(owner, second_generation, &channel));
3990 assert!(client.active_quote_subs.contains(&instrument_id));
3991 assert!(channel_is_active(
3992 &client.active_ticker_channels,
3993 instrument_id,
3994 &channel,
3995 ));
3996 }
3997
3998 #[rstest]
3999 fn test_subscription_registry_clear_does_not_reuse_generation() {
4000 let registry = ChannelSubscriptionRegistry::default();
4001 let owner = ChannelOwner::Trades(InstrumentId::from("ETH-20260627-3500-C.DERIVE"));
4002 let first_generation = registry.state.lock().activate(owner, None).unwrap();
4003
4004 registry.clear();
4005 let second_generation = registry.state.lock().activate(owner, None).unwrap();
4006
4007 assert!(second_generation > first_generation);
4008 }
4009
4010 #[rstest]
4011 fn test_subscription_registry_clear_preserves_inflight_transition() {
4012 let registry = ChannelSubscriptionRegistry::default();
4013 let first = registry.transition("ticker_slim.ETH-PERP.1000");
4014
4015 registry.clear();
4016 let second = registry.transition("ticker_slim.ETH-PERP.1000");
4017
4018 assert!(Arc::ptr_eq(&first, &second));
4019 }
4020
4021 #[rstest]
4022 fn test_subscription_transition_registry_collects_inactive_channels() {
4023 let registry = ChannelSubscriptionRegistry::default();
4024 let live = registry.transition("ticker_slim.ETH-PERP.1000");
4025 for index in 0..TRANSITION_GC_THRESHOLD + 16 {
4026 drop(registry.transition(&format!("ticker_slim.ETH-OPTION-{index}.1000")));
4027 }
4028
4029 let same_live = registry.transition("ticker_slim.ETH-PERP.1000");
4030
4031 assert!(Arc::ptr_eq(&live, &same_live));
4032 assert!(registry.transitions.len() < TRANSITION_GC_THRESHOLD);
4033 }
4034
4035 #[rstest]
4036 fn test_ambiguous_cleanup_retains_survivor_for_reconnect() {
4037 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
4038 replace_data_event_sender(tx);
4039
4040 let client = DeriveDataClient::new(
4041 *DERIVE_CLIENT_ID,
4042 DeriveDataClientConfig {
4043 environment: DeriveEnvironment::Mainnet,
4044 ..Default::default()
4045 },
4046 )
4047 .unwrap();
4048
4049 let lifecycle = client.subscription_lifecycle();
4050 let instrument_id = InstrumentId::from("ETH-20260627-3600-C.DERIVE");
4051 let owner = ChannelOwner::Trades(instrument_id);
4052 lifecycle.activate(owner, Some("trades.option.ETH"));
4053
4054 let error = DeriveWsError::Timeout {
4055 method: "unsubscribe".to_string(),
4056 };
4057
4058 let ws = client.ws_handle();
4059
4060 let retained = retain_channel_for_reconnect(&ws, "trades.option.ETH", true, Some(&error));
4061
4062 assert!(retained);
4063 assert!(lifecycle.is_active(owner));
4064 assert!(client.active_trade_subs.contains(&instrument_id));
4065 assert!(ws.has_subscription("trades.option.ETH"));
4066 }
4067
4068 #[rstest]
4069 fn test_explicit_cleanup_rejection_preserves_surviving_channel_owner() {
4070 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
4071 replace_data_event_sender(tx);
4072
4073 let client = DeriveDataClient::new(
4074 *DERIVE_CLIENT_ID,
4075 DeriveDataClientConfig {
4076 environment: DeriveEnvironment::Mainnet,
4077 ..Default::default()
4078 },
4079 )
4080 .unwrap();
4081
4082 let lifecycle = client.subscription_lifecycle();
4083 let instrument_id = InstrumentId::from("ETH-20260627-3600-C.DERIVE");
4084 let owner = ChannelOwner::Trades(instrument_id);
4085 lifecycle.activate(owner, Some("trades.option.ETH"));
4086
4087 let error = DeriveWsError::JsonRpc {
4088 code: -32603,
4089 message: "not subscribed".to_string(),
4090 data: None,
4091 };
4092
4093 let ws = client.ws_handle();
4094 ws.remember_subscription("trades.option.ETH");
4095
4096 let retained = retain_channel_for_reconnect(&ws, "trades.option.ETH", true, Some(&error));
4097
4098 assert!(!retained);
4099 assert!(lifecycle.is_active(owner));
4100 assert!(client.active_trade_subs.contains(&instrument_id));
4101 assert!(!ws.has_subscription("trades.option.ETH"));
4102 }
4103
4104 #[rstest]
4105 fn test_ambiguous_cleanup_without_survivor_is_not_replayed() {
4106 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
4107 replace_data_event_sender(tx);
4108
4109 let client = DeriveDataClient::new(
4110 *DERIVE_CLIENT_ID,
4111 DeriveDataClientConfig {
4112 environment: DeriveEnvironment::Mainnet,
4113 ..Default::default()
4114 },
4115 )
4116 .unwrap();
4117
4118 let ws = client.ws_handle();
4119 ws.remember_subscription("trades.option.ETH");
4120
4121 let error = DeriveWsError::Timeout {
4122 method: "unsubscribe".to_string(),
4123 };
4124
4125 let retained = retain_channel_for_reconnect(&ws, "trades.option.ETH", false, Some(&error));
4126
4127 assert!(!retained);
4128 assert!(!ws.has_subscription("trades.option.ETH"));
4129 }
4130
4131 #[rstest]
4132 fn test_unsubscribe_quotes_prunes_cached_quote() {
4133 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
4134 replace_data_event_sender(tx);
4135
4136 let mut client = DeriveDataClient::new(
4137 *DERIVE_CLIENT_ID,
4138 DeriveDataClientConfig {
4139 environment: DeriveEnvironment::Mainnet,
4140 ..Default::default()
4141 },
4142 )
4143 .unwrap();
4144
4145 let instrument_id = InstrumentId::from("ETH-PERP.DERIVE");
4146
4147 let owner = ChannelOwner::Ticker {
4148 instrument_id,
4149 feed: TickerFeed::Quote,
4150 };
4151
4152 client
4153 .subscription_lifecycle()
4154 .activate(owner, Some("ticker_slim.ETH-PERP.1000"));
4155 client.quote_cache.lock().insert(
4156 instrument_id,
4157 QuoteTick::new(
4158 instrument_id,
4159 Price::from("3500.00"),
4160 Price::from("3501.00"),
4161 Quantity::from("1.000"),
4162 Quantity::from("2.000"),
4163 UnixNanos::from(1),
4164 UnixNanos::from(1),
4165 ),
4166 );
4167
4168 let command = UnsubscribeQuotes::new(
4169 instrument_id,
4170 Some(*DERIVE_CLIENT_ID),
4171 None,
4172 UUID4::new(),
4173 UnixNanos::default(),
4174 None,
4175 None,
4176 );
4177
4178 client.unsubscribe_quotes(&command).unwrap();
4179
4180 assert!(!client.quote_cache.lock().contains(&instrument_id));
4181 }
4182
4183 #[rstest]
4184 #[tokio::test]
4185 async fn test_unsubscribe_trades_uses_recorded_channel_without_cached_instrument() {
4186 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
4187 replace_data_event_sender(tx);
4188
4189 let mut client = DeriveDataClient::new(
4190 *DERIVE_CLIENT_ID,
4191 DeriveDataClientConfig {
4192 environment: DeriveEnvironment::Mainnet,
4193 ..Default::default()
4194 },
4195 )
4196 .unwrap();
4197
4198 let instrument_id = InstrumentId::from("ETH-20260627-3500-C.DERIVE");
4199 let owner = ChannelOwner::Trades(instrument_id);
4200 let lifecycle = client.subscription_lifecycle();
4201 let generation = lifecycle.activate(owner, None).unwrap();
4202 assert!(lifecycle.attach_channel(owner, generation, "trades.option.ETH".to_string(),));
4203
4204 let command = UnsubscribeTrades::new(
4205 instrument_id,
4206 Some(*DERIVE_CLIENT_ID),
4207 None,
4208 UUID4::new(),
4209 UnixNanos::default(),
4210 None,
4211 None,
4212 );
4213
4214 client.unsubscribe_trades(&command).unwrap();
4215 wait_until_async(
4216 || {
4217 let registry = Arc::clone(&client.channel_subscriptions);
4218 async move { registry.transitions.contains_key("trades.option.ETH") }
4219 },
4220 Duration::from_secs(1),
4221 )
4222 .await;
4223
4224 assert!(!lifecycle.is_active(owner));
4225 assert!(!client.active_trade_subs.contains(&instrument_id));
4226 }
4227
4228 fn perp_ticker_payload(instrument_id: InstrumentId) -> WsSubscriptionPayload {
4229 let channel = "ticker_slim.ETH-PERP.1000";
4230 let payload = subscription_payload(
4231 channel,
4232 &json!({
4233 "timestamp": 1_700_000_000_010_i64,
4234 "instrument_ticker": ticker_json(1_700_000_000_000)
4235 }),
4236 );
4237 assert_eq!(payload.channel, channel);
4238 let _ = instrument_id;
4239 payload
4240 }
4241
4242 fn install_ticker(ctx: &WsMessageContext, instrument_id: InstrumentId, channel: &str) {
4243 ctx.active_ticker_channels
4244 .insert(instrument_id, channel.to_string());
4245 }
4246
4247 #[rstest]
4248 fn test_ticker_emits_mark_price_when_mark_subscribed() {
4249 let instrument = perp_instrument();
4250 let instrument_id = instrument.id();
4251 let (ctx, mut rx) = make_ctx(Some(instrument));
4252 install_ticker(&ctx, instrument_id, "ticker_slim.ETH-PERP.1000");
4253 ctx.active_mark_subs.insert(instrument_id);
4254
4255 DeriveDataClient::handle_ws_message(
4256 DeriveWsMessage::Subscription(perp_ticker_payload(instrument_id)),
4257 &ctx,
4258 );
4259
4260 match rx.try_recv().unwrap() {
4261 DataEvent::Data(Data::MarkPrice(mark)) => {
4262 assert_eq!(mark.instrument_id, instrument_id);
4263 assert_eq!(mark.value, Price::from("3500.50"));
4264 }
4265 other => panic!("expected MarkPriceUpdate, was {other:?}"),
4266 }
4267
4268 assert!(rx.try_recv().is_err());
4269 }
4270
4271 #[rstest]
4272 fn test_ticker_emits_index_price_when_index_subscribed() {
4273 let instrument = perp_instrument();
4274 let instrument_id = instrument.id();
4275 let (ctx, mut rx) = make_ctx(Some(instrument));
4276 install_ticker(&ctx, instrument_id, "ticker_slim.ETH-PERP.1000");
4277 ctx.active_index_subs.insert(instrument_id);
4278
4279 DeriveDataClient::handle_ws_message(
4280 DeriveWsMessage::Subscription(perp_ticker_payload(instrument_id)),
4281 &ctx,
4282 );
4283
4284 match rx.try_recv().unwrap() {
4285 DataEvent::Data(Data::IndexPrice(index)) => {
4286 assert_eq!(index.instrument_id, instrument_id);
4287 assert_eq!(index.value, Price::from("3500.00"));
4288 }
4289 other => panic!("expected IndexPriceUpdate, was {other:?}"),
4290 }
4291
4292 assert!(rx.try_recv().is_err());
4293 }
4294
4295 #[rstest]
4296 fn test_ticker_emits_funding_rate_for_perp_when_subscribed() {
4297 let instrument = perp_instrument();
4298 let instrument_id = instrument.id();
4299 let (ctx, mut rx) = make_ctx(Some(instrument));
4300 install_ticker(&ctx, instrument_id, "ticker_slim.ETH-PERP.1000");
4301 ctx.active_funding_subs.insert(instrument_id);
4302
4303 DeriveDataClient::handle_ws_message(
4304 DeriveWsMessage::Subscription(perp_ticker_payload(instrument_id)),
4305 &ctx,
4306 );
4307
4308 match rx.try_recv().unwrap() {
4309 DataEvent::Data(Data::FundingRate(update)) => {
4310 assert_eq!(update.instrument_id, instrument_id);
4311 assert_eq!(update.rate, "0.0002".parse().unwrap());
4312 }
4313 other => panic!("expected FundingRateUpdate, was {other:?}"),
4314 }
4315
4316 assert!(rx.try_recv().is_err());
4317 }
4318
4319 #[rstest]
4320 fn test_ticker_skips_funding_rate_when_not_perp() {
4321 let instrument = option_instrument();
4322 let instrument_id = instrument.id();
4323 let channel = format!("ticker_slim.{}.1000", instrument_id.symbol.as_str());
4324 let (ctx, mut rx) = make_ctx(Some(instrument));
4325 install_ticker(&ctx, instrument_id, &channel);
4326 ctx.active_funding_subs.insert(instrument_id);
4327
4328 let mut option_data = option_ticker_json(1_700_000_000_000);
4329 option_data["instrument_name"] = json!(instrument_id.symbol.as_str());
4330 let payload = subscription_payload(
4331 &channel,
4332 &json!({
4333 "timestamp": 1_700_000_000_010_i64,
4334 "instrument_ticker": option_data
4335 }),
4336 );
4337
4338 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
4339
4340 assert!(rx.try_recv().is_err());
4341 }
4342
4343 #[rstest]
4344 fn test_ticker_emits_option_greeks_when_subscribed() {
4345 let instrument = option_instrument();
4346 let instrument_id = instrument.id();
4347 let channel = format!("ticker_slim.{}.1000", instrument_id.symbol.as_str());
4348 let (ctx, mut rx) = make_ctx(Some(instrument));
4349 install_ticker(&ctx, instrument_id, &channel);
4350 ctx.active_greeks_subs.insert(instrument_id);
4351
4352 let mut option_data = option_ticker_json(1_700_000_000_000);
4353 option_data["instrument_name"] = json!(instrument_id.symbol.as_str());
4354 let payload = subscription_payload(
4355 &channel,
4356 &json!({
4357 "timestamp": 1_700_000_000_010_i64,
4358 "instrument_ticker": option_data
4359 }),
4360 );
4361
4362 DeriveDataClient::handle_ws_message(DeriveWsMessage::Subscription(payload), &ctx);
4363
4364 match rx.try_recv().unwrap() {
4365 DataEvent::Data(Data::OptionGreeks(greeks)) => {
4366 assert_eq!(greeks.instrument_id, instrument_id);
4367 assert!((greeks.greeks.delta - 0.55).abs() < 1e-9);
4368 assert!((greeks.greeks.gamma - 0.0008).abs() < 1e-9);
4369 assert!((greeks.greeks.vega - 4.5).abs() < 1e-9);
4370 assert!((greeks.greeks.theta + 2.1).abs() < 1e-9);
4371 assert!((greeks.greeks.rho - 1.2).abs() < 1e-9);
4372 assert_eq!(greeks.mark_iv, Some(0.60));
4373 assert_eq!(greeks.bid_iv, Some(0.58));
4374 assert_eq!(greeks.ask_iv, Some(0.62));
4375 assert_eq!(greeks.underlying_price, Some(3505.0));
4376 assert_eq!(greeks.open_interest, Some(1000.0));
4377 }
4378 other => panic!("expected OptionGreeks, was {other:?}"),
4379 }
4380
4381 assert!(rx.try_recv().is_err());
4382 }
4383
4384 #[rstest]
4385 fn test_ticker_emits_all_subscribed_feeds_in_one_frame() {
4386 let instrument = perp_instrument();
4387 let instrument_id = instrument.id();
4388 let (ctx, mut rx) = make_ctx(Some(instrument));
4389 install_ticker(&ctx, instrument_id, "ticker_slim.ETH-PERP.1000");
4390 ctx.active_quote_subs.insert(instrument_id);
4391 ctx.active_mark_subs.insert(instrument_id);
4392 ctx.active_index_subs.insert(instrument_id);
4393 ctx.active_funding_subs.insert(instrument_id);
4394
4395 DeriveDataClient::handle_ws_message(
4396 DeriveWsMessage::Subscription(perp_ticker_payload(instrument_id)),
4397 &ctx,
4398 );
4399
4400 let mut quote = None;
4401 let mut mark = None;
4402 let mut index = None;
4403 let mut funding = None;
4404
4405 while let Ok(event) = rx.try_recv() {
4406 match event {
4407 DataEvent::Data(Data::Quote(q)) => {
4408 assert!(quote.replace(q).is_none(), "duplicate Quote emission");
4409 }
4410 DataEvent::Data(Data::MarkPrice(m)) => {
4411 assert!(
4412 mark.replace(m).is_none(),
4413 "duplicate MarkPriceUpdate emission"
4414 );
4415 }
4416 DataEvent::Data(Data::IndexPrice(i)) => {
4417 assert!(
4418 index.replace(i).is_none(),
4419 "duplicate IndexPriceUpdate emission"
4420 );
4421 }
4422 DataEvent::Data(Data::FundingRate(f)) => {
4423 assert!(
4424 funding.replace(f).is_none(),
4425 "duplicate FundingRate emission"
4426 );
4427 }
4428 other => panic!("unexpected event: {other:?}"),
4429 }
4430 }
4431
4432 let quote = quote.expect("Quote event missing");
4433 let mark = mark.expect("MarkPriceUpdate missing");
4434 let index = index.expect("IndexPriceUpdate missing");
4435 let funding = funding.expect("FundingRateUpdate missing");
4436
4437 assert_eq!(quote.instrument_id, instrument_id);
4438 assert_eq!(quote.bid_price, Price::from("3500.00"));
4439 assert_eq!(quote.ask_price, Price::from("3501.00"));
4440 assert_eq!(mark.instrument_id, instrument_id);
4441 assert_eq!(mark.value, Price::from("3500.50"));
4442 assert_eq!(index.instrument_id, instrument_id);
4443 assert_eq!(index.value, Price::from("3500.00"));
4444 assert_eq!(funding.instrument_id, instrument_id);
4445 assert_eq!(funding.rate, "0.0002".parse().unwrap());
4446 }
4447
4448 #[rstest]
4449 fn test_reset_clears_all_subscription_state() {
4450 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
4451 replace_data_event_sender(tx);
4452
4453 let config = DeriveDataClientConfig {
4454 environment: DeriveEnvironment::Mainnet,
4455 ..Default::default()
4456 };
4457
4458 let mut client = DeriveDataClient::new(*DERIVE_CLIENT_ID, config).unwrap();
4459 let instrument = perp_instrument();
4460 let instrument_id = instrument.id();
4461 cache_instrument(&client.instruments, &instrument);
4462
4463 client
4464 .active_book_delta_channels
4465 .insert(instrument_id, "orderbook.ETH-PERP.1.10".to_string());
4466 client
4467 .active_book_depth_channels
4468 .insert(instrument_id, "orderbook.ETH-PERP.1.10".to_string());
4469 client
4470 .active_ticker_channels
4471 .insert(instrument_id, "ticker_slim.ETH-PERP.1000".to_string());
4472 client.active_quote_subs.insert(instrument_id);
4473 client.active_trade_subs.insert(instrument_id);
4474 client
4475 .channel_subscriptions
4476 .state
4477 .lock()
4478 .activate(ChannelOwner::Trades(instrument_id), Some("trades.perp.ETH"));
4479 client.channel_subscriptions.transition("trades.perp.ETH");
4480 client.active_mark_subs.insert(instrument_id);
4481 client.active_index_subs.insert(instrument_id);
4482 client.active_funding_subs.insert(instrument_id);
4483 client.active_greeks_subs.insert(instrument_id);
4484
4485 client.reset().unwrap();
4486
4487 assert!(!client.instruments.contains_key(&instrument_id));
4488 assert!(
4489 !client
4490 .active_book_delta_channels
4491 .contains_key(&instrument_id)
4492 );
4493 assert!(
4494 !client
4495 .active_book_depth_channels
4496 .contains_key(&instrument_id)
4497 );
4498 assert!(!client.active_ticker_channels.contains_key(&instrument_id));
4499 assert!(!client.active_quote_subs.contains(&instrument_id));
4500 assert!(!client.active_trade_subs.contains(&instrument_id));
4501 assert!(client.channel_subscriptions.state.lock().owners.is_empty());
4502
4503 assert_eq!(client.channel_subscriptions.transitions.len(), 1);
4506 assert!(!client.active_mark_subs.contains(&instrument_id));
4507 assert!(!client.active_index_subs.contains(&instrument_id));
4508 assert!(!client.active_funding_subs.contains(&instrument_id));
4509 assert!(!client.active_greeks_subs.contains(&instrument_id));
4510 assert!(!client.is_connected());
4511 }
4512
4513 #[tokio::test]
4514 async fn test_disconnect_clears_subscription_state() {
4515 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
4521 replace_data_event_sender(tx);
4522
4523 let config = DeriveDataClientConfig {
4524 environment: DeriveEnvironment::Mainnet,
4525 ..Default::default()
4526 };
4527
4528 let mut client = DeriveDataClient::new(*DERIVE_CLIENT_ID, config).unwrap();
4529 let instrument = perp_instrument();
4530 let instrument_id = instrument.id();
4531 cache_instrument(&client.instruments, &instrument);
4532
4533 client
4534 .active_book_delta_channels
4535 .insert(instrument_id, "orderbook.ETH-PERP.1.10".to_string());
4536 client
4537 .active_book_depth_channels
4538 .insert(instrument_id, "orderbook.ETH-PERP.1.10".to_string());
4539 client
4540 .active_ticker_channels
4541 .insert(instrument_id, "ticker_slim.ETH-PERP.1000".to_string());
4542 client.active_quote_subs.insert(instrument_id);
4543 client.active_trade_subs.insert(instrument_id);
4544 client
4545 .channel_subscriptions
4546 .state
4547 .lock()
4548 .activate(ChannelOwner::Trades(instrument_id), Some("trades.perp.ETH"));
4549 client.channel_subscriptions.transition("trades.perp.ETH");
4550 client.active_mark_subs.insert(instrument_id);
4551 client.active_index_subs.insert(instrument_id);
4552 client.active_funding_subs.insert(instrument_id);
4553 client.active_greeks_subs.insert(instrument_id);
4554 client.is_connected.store(true, Ordering::Relaxed);
4555 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
4556 let (drop_tx, mut drop_rx) = tokio::sync::oneshot::channel::<()>();
4557 client
4558 .pending_tasks
4559 .spawn(async move {
4560 let _drop_tx = drop_tx;
4561 let _ = started_tx.send(());
4562 std::future::pending::<()>().await;
4563 })
4564 .unwrap();
4565
4566 started_rx.await.unwrap();
4567
4568 client.disconnect().await.unwrap();
4569
4570 assert!(client.instruments.contains_key(&instrument_id));
4573 assert!(
4574 !client
4575 .active_book_delta_channels
4576 .contains_key(&instrument_id)
4577 );
4578 assert!(
4579 !client
4580 .active_book_depth_channels
4581 .contains_key(&instrument_id)
4582 );
4583 assert!(!client.active_ticker_channels.contains_key(&instrument_id));
4584 assert!(!client.active_quote_subs.contains(&instrument_id));
4585 assert!(!client.active_trade_subs.contains(&instrument_id));
4586 assert!(client.channel_subscriptions.state.lock().owners.is_empty());
4587 assert!(client.channel_subscriptions.transitions.is_empty());
4588 assert!(!client.active_mark_subs.contains(&instrument_id));
4589 assert!(!client.active_index_subs.contains(&instrument_id));
4590 assert!(!client.active_funding_subs.contains(&instrument_id));
4591 assert!(!client.active_greeks_subs.contains(&instrument_id));
4592 assert!(!client.is_connected());
4593 assert_eq!(
4594 drop_rx.try_recv(),
4595 Err(tokio::sync::oneshot::error::TryRecvError::Closed),
4596 );
4597 }
4598
4599 #[tokio::test]
4600 async fn test_task_group_unregisters_finished_tasks_before_drain() {
4601 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
4602 replace_data_event_sender(tx);
4603
4604 let config = DeriveDataClientConfig {
4605 environment: DeriveEnvironment::Mainnet,
4606 ..Default::default()
4607 };
4608
4609 let client = DeriveDataClient::new(*DERIVE_CLIENT_ID, config).unwrap();
4610 for _ in 0..100 {
4611 client.spawn_task("test_noop", async { Ok(()) });
4612 }
4613
4614 wait_until_async(
4615 || async { client.pending_tasks.all_finished() },
4616 Duration::from_secs(2),
4617 )
4618 .await;
4619
4620 assert!(client.pending_tasks.is_empty());
4621 client.pending_tasks.begin_shutdown();
4622 client
4623 .pending_tasks
4624 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
4625 .await
4626 .unwrap();
4627 assert!(client.pending_tasks.is_empty());
4628 }
4629}