1use std::{
17 str::FromStr,
18 sync::{
19 Arc,
20 atomic::{AtomicBool, Ordering},
21 },
22 time::Duration,
23};
24
25use ahash::{AHashMap, AHashSet};
26use anyhow::Context;
27use jiff::Timestamp;
28use nautilus_common::{
29 cache::InstrumentLookupError,
30 clients::{DataClient, execute_bar_subscriptions, execute_instrument_subscriptions},
31 live::{
32 dst::time::{Instant, timeout},
33 runner::get_data_event_sender,
34 sender::EventSender,
35 },
36 messages::{
37 DataEvent,
38 data::{
39 BarsResponse, BookResponse, CustomDataResponse, DataRequestFailed, DataResponse,
40 FundingRatesResponse, InstrumentResponse, InstrumentsResponse, RequestBars,
41 RequestBookSnapshot, RequestCustomData, RequestFundingRates, RequestInstrument,
42 RequestInstruments, RequestTrades, SubscribeBars, SubscribeBookDeltas,
43 SubscribeBookDepth, SubscribeCustomData, SubscribeFundingRates, SubscribeIndexPrices,
44 SubscribeInstrument, SubscribeInstrumentStatus, SubscribeMarkPrices, SubscribeQuotes,
45 SubscribeTrades, TradesResponse, UnsubscribeBars, UnsubscribeBookDeltas,
46 UnsubscribeBookDepth, UnsubscribeCustomData, UnsubscribeFundingRates,
47 UnsubscribeIndexPrices, UnsubscribeInstrument, UnsubscribeInstrumentStatus,
48 UnsubscribeInstruments, UnsubscribeMarkPrices, UnsubscribeQuotes, UnsubscribeTrades,
49 },
50 },
51};
52use nautilus_core::{
53 AtomicMap, Params, UUID4, UnixNanos,
54 datetime::{datetime_to_unix_nanos, unix_nanos_to_iso8601},
55 time::{AtomicTime, get_atomic_clock_realtime},
56};
57use nautilus_live::{
58 SocketControl,
59 task::{TaskGroup, TaskGroupGuard, TaskSpawner},
60};
61use nautilus_model::{
62 data::{Bar, BarType, BookOrder, CustomData, Data, DataType, FundingRateUpdate, TradeTick},
63 enums::{BarAggregation, BookType, OrderSide},
64 identifiers::{ClientId, InstrumentId, Venue},
65 instruments::{Instrument, InstrumentAny},
66 orderbook::OrderBook,
67 types::{Price, Quantity},
68};
69use parking_lot::Mutex;
70use rust_decimal::Decimal;
71use tokio_util::sync::CancellationToken;
72use ustr::Ustr;
73
74use crate::{
75 book::{self, sync::BookSyncTracker},
76 common::{
77 consts::HYPERLIQUID_VENUE,
78 credential::{Secrets, credential_env_vars},
79 parse::{bar_type_to_interval, millis_to_nanos},
80 status::InstrumentStatusState,
81 },
82 config::{HyperliquidBookFeed, HyperliquidDataClientConfig},
83 data_types::register_hyperliquid_custom_data,
84 http::{
85 client::HyperliquidHttpClient,
86 models::{HyperliquidCandle, HyperliquidFundingHistoryEntry, HyperliquidL2Book},
87 parse::parse_recent_trade,
88 },
89 websocket::{
90 DATA_STREAMS_ENDPOINT,
91 book::{BookStreamOptions, BookStreamUse},
92 client::HyperliquidWebSocketClient,
93 messages::NautilusWsMessage,
94 },
95};
96
97#[derive(Debug)]
98pub struct HyperliquidDataClient {
99 clock: &'static AtomicTime,
100 client_id: ClientId,
101 config: HyperliquidDataClientConfig,
102 http_client: HyperliquidHttpClient,
103 ws_client: HyperliquidWebSocketClient,
104 is_connected: AtomicBool,
105 cancellation_token: CancellationToken,
106 session_tasks: TaskGroup,
107 pending_tasks: TaskGroup,
108 shutdown_errors: Vec<String>,
109 data_sender: EventSender<DataEvent>,
110 instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
111 coin_to_instrument_id: Arc<AtomicMap<Ustr, InstrumentId>>,
112 instrument_update_lock: Arc<tokio::sync::Mutex<()>>,
114 instrument_status: Arc<Mutex<InstrumentStatusState>>,
115 stream_health: Arc<Mutex<MarketDataStreamHealthMonitor>>,
116 book_sync: BookSyncTracker,
117 compact_book_publications: AHashMap<(InstrumentId, BookStreamUse), CancellationToken>,
118 compact_book_command_tail: Option<tokio::sync::oneshot::Receiver<()>>,
119}
120
121impl HyperliquidDataClient {
122 pub fn new(client_id: ClientId, config: HyperliquidDataClientConfig) -> anyhow::Result<Self> {
128 let clock = get_atomic_clock_realtime();
129 let data_sender = get_data_event_sender();
130
131 let (pk_var, _) = credential_env_vars(config.environment);
134 let has_credentials = config.has_credentials() || std::env::var(pk_var).is_ok();
135 let proxy_url = config
136 .proxy_url
137 .as_ref()
138 .map(|value| value.expose_secret().to_owned());
139
140 let mut http_client = if has_credentials {
141 let secrets = Secrets::resolve(
142 config
143 .private_key
144 .as_ref()
145 .map(|value| value.expose_secret()),
146 None,
147 config.environment,
148 )?;
149 HyperliquidHttpClient::with_secrets(
150 &secrets,
151 config.http_timeout_secs,
152 proxy_url.clone(),
153 )?
154 } else {
155 HyperliquidHttpClient::new(
156 config.environment,
157 config.http_timeout_secs,
158 proxy_url.clone(),
159 )?
160 };
161
162 if let Some(url) = &config.base_url_http {
163 http_client.set_base_info_url(url.clone());
164 }
165
166 let ws_url = config.base_url_ws.clone();
167 let ws_client = HyperliquidWebSocketClient::new_with_book_feed(
168 ws_url,
169 config.environment,
170 None,
171 config.transport_backend,
172 proxy_url,
173 config.book_feed,
174 );
175 let ws_client = ws_client.with_socket_control(SocketControl::new(
176 client_id,
177 Some(*HYPERLIQUID_VENUE),
178 DATA_STREAMS_ENDPOINT,
179 ));
180 let mut stream_health_monitor = MarketDataStreamHealthMonitor::new(
181 Duration::from_secs(config.stale_stream_receive_timeout_secs),
182 Duration::from_secs(config.stale_stream_warning_cooldown_secs),
183 );
184
185 stream_health_monitor.compact_books = config.book_feed == HyperliquidBookFeed::L2;
186
187 if config.stale_stream_recovery_enabled {
188 if config.stale_stream_recovery_cooldown_secs > 0 {
189 stream_health_monitor = stream_health_monitor.with_recovery(
190 Duration::from_secs(config.stale_stream_recovery_cooldown_secs),
191 config.stale_stream_max_targeted_resubscribes,
192 );
193 } else {
194 log::warn!(
195 "Hyperliquid stale stream recovery disabled: \
196 stale_stream_recovery_cooldown_secs must be positive"
197 );
198 }
199 }
200
201 let stream_health = Arc::new(Mutex::new(stream_health_monitor));
202
203 let session_tasks = TaskGroup::new();
204 let pending_tasks = TaskGroup::new();
205
206 Ok(Self {
207 clock,
208 client_id,
209 config,
210 http_client,
211 ws_client,
212 is_connected: AtomicBool::new(false),
213 cancellation_token: CancellationToken::new(),
214 session_tasks,
215 pending_tasks,
216 shutdown_errors: Vec::new(),
217 data_sender,
218 instruments: Arc::new(AtomicMap::new()),
219 coin_to_instrument_id: Arc::new(AtomicMap::new()),
220 instrument_update_lock: Arc::new(tokio::sync::Mutex::new(())),
221 instrument_status: Arc::new(Mutex::new(InstrumentStatusState::default())),
222 stream_health,
223 book_sync: BookSyncTracker::default(),
224 compact_book_publications: AHashMap::new(),
225 compact_book_command_tail: None,
226 })
227 }
228
229 fn spawn_task<F>(&self, description: &'static str, fut: F)
230 where
231 F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
232 {
233 let future = async move {
234 if let Err(e) = fut.await {
235 log::warn!("{description} failed: {e:?}");
236 }
237 };
238
239 if let Err(e) = self.pending_tasks.spawn(future) {
240 log::warn!("Skipping Hyperliquid {description} after shutdown began: {e}");
241 }
242 }
243
244 fn spawn_instrument_request(
245 &self,
246 request_id: UUID4,
247 request: impl std::future::Future<Output = anyhow::Result<DataResponse>> + Send + 'static,
248 ) -> anyhow::Result<()> {
249 let tasks = self.pending_tasks.spawner()?;
250 let cancelled = tasks.cancellation_token();
251 let sender = self.data_sender.clone();
252
253 let deadline = Duration::from_secs(self.config.http_timeout_secs);
255
256 tasks.spawn(async move {
257 let result = tokio::select! {
258 biased;
259 () = cancelled.cancelled() => Err(anyhow::anyhow!("Hyperliquid instrument request cancelled")),
260 result = timeout(deadline, request) => match result {
261 Ok(result) => result,
262 Err(_) => Err(anyhow::anyhow!("Hyperliquid instrument request timed out")),
263 },
264 };
265
266 let event = match result {
267 Ok(response) => DataEvent::Response(response),
268 Err(e) => DataEvent::RequestFailed(DataRequestFailed {
269 request_id,
270 reason: format!("{e:#}"),
271 response: None,
272 }),
273 };
274
275 if let Err(e) = sender.send(event) {
276 log::error!("Failed to complete instrument request {request_id}: {e}");
277 }
278 })?;
279
280 Ok(())
281 }
282
283 fn chain_compact_book_command<F>(
285 &mut self,
286 command: F,
287 ) -> impl std::future::Future<Output = F::Output> + Send + 'static
288 where
289 F: std::future::Future + Send + 'static,
290 {
291 let (done, next) = tokio::sync::oneshot::channel();
292 let previous = self.compact_book_command_tail.replace(next);
293 async move {
294 if let Some(previous) = previous {
295 let _ = previous.await;
296 }
297
298 let result = command.await;
299 let _ = done.send(());
300 result
301 }
302 }
303
304 fn subscribe_compact_book(
305 &mut self,
306 instrument_id: InstrumentId,
307 stream_use: BookStreamUse,
308 options: BookStreamOptions,
309 ) {
310 let Ok(tasks) = self.pending_tasks.spawner() else {
311 return;
312 };
313 let publication = self
314 .compact_book_publications
315 .entry((instrument_id, stream_use))
316 .or_default()
317 .clone();
318 let ws = self.ws_client.clone();
319 let book_sync = self.book_sync.clone();
320 let snapshot_timeout = self.book_snapshot_timeout();
321 let command = self.chain_compact_book_command({
322 let ws = ws.clone();
323 let book_sync = book_sync.clone();
324 async move {
325 ws.subscribe_compact_book(
326 instrument_id,
327 stream_use,
328 options,
329 publication,
330 &book_sync,
331 )
332 .await
333 }
334 });
335 self.spawn_task("subscribe_compact_book", async move {
336 if let Some(write) = command.await? {
337 book::recovery::watch_compact_subscription(
338 instrument_id,
339 write,
340 &book_sync,
341 ws,
342 snapshot_timeout,
343 &tasks,
344 )
345 .await;
346 }
347
348 Ok(())
349 });
350 }
351
352 fn unsubscribe_compact_book(&mut self, instrument_id: InstrumentId, stream_use: BookStreamUse) {
353 if let Some(publication) = self
354 .compact_book_publications
355 .remove(&(instrument_id, stream_use))
356 {
357 self.book_sync.retire_compact_publication(&publication);
358 }
359
360 let ws = self.ws_client.clone();
361 let book_sync = self.book_sync.clone();
362 let command = self.chain_compact_book_command(async move {
363 ws.unsubscribe_compact_book(instrument_id, stream_use, &book_sync)
364 .await
365 });
366 self.spawn_task("unsubscribe_compact_book", command);
367 }
368
369 fn clear_book_sync(&mut self) {
370 for (_, publication) in self.compact_book_publications.drain() {
371 publication.cancel();
372 }
373
374 self.book_sync.clear();
375 }
376
377 fn abort_pending_tasks(&self) {
378 self.pending_tasks.begin_shutdown();
379 }
380
381 fn abort_session_tasks(&self) {
382 self.session_tasks.begin_shutdown();
383 self.ws_client.begin_shutdown();
384 }
385
386 async fn teardown_partial_connect(&mut self) -> anyhow::Result<()> {
387 self.cancellation_token.cancel();
388 self.abort_session_tasks();
389 self.abort_pending_tasks();
390
391 if let Err(e) = self.ws_client.disconnect().await {
392 self.shutdown_errors
393 .push(format!("Hyperliquid WebSocket shutdown failed: {e}"));
394 }
395
396 if let Err(e) = self.await_session_tasks().await {
397 self.shutdown_errors.push(e.to_string());
398 }
399
400 if let Err(e) = self.await_pending_tasks().await {
401 self.shutdown_errors.push(e.to_string());
402 }
403 self.clear_stream_health();
404 self.clear_book_sync();
405 self.is_connected.store(false, Ordering::Release);
406
407 if !self.shutdown_errors.is_empty() {
408 anyhow::bail!(std::mem::take(&mut self.shutdown_errors).join("; "));
409 }
410 Ok(())
411 }
412
413 async fn await_pending_tasks(&self) -> anyhow::Result<()> {
414 self.pending_tasks.begin_shutdown();
415 self.pending_tasks
416 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
417 .await
418 .map_err(|e| anyhow::anyhow!("Failed to terminate Hyperliquid data tasks: {e}"))?;
419 Ok(())
420 }
421
422 async fn await_session_tasks(&self) -> anyhow::Result<()> {
423 self.session_tasks.begin_shutdown();
424 self.session_tasks
425 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
426 .await
427 .map_err(|e| {
428 anyhow::anyhow!("Failed to terminate Hyperliquid data session tasks: {e}")
429 })?;
430 Ok(())
431 }
432
433 fn clear_stream_health(&self) {
434 self.stream_health.lock().clear();
435 }
436
437 fn register_stream_health(&self, channel: MarketDataChannel, instrument_id: InstrumentId) {
438 if !self.stream_health_monitor_enabled() {
439 return;
440 }
441
442 self.stream_health
443 .lock()
444 .subscribe(channel, instrument_id, Instant::now());
445 }
446
447 fn remove_stream_health(&self, channel: MarketDataChannel, instrument_id: InstrumentId) {
448 self.stream_health
449 .lock()
450 .unsubscribe(channel, instrument_id);
451 }
452
453 fn stream_health_monitor_enabled(&self) -> bool {
454 self.config.stale_stream_receive_timeout_secs > 0
455 && self.config.stream_health_check_interval_secs > 0
456 }
457
458 fn spawn_stream_health_monitor(&self) -> anyhow::Result<()> {
459 if !self.stream_health_monitor_enabled() {
460 return Ok(());
461 }
462
463 let stream_health = Arc::clone(&self.stream_health);
464 let cancellation_token = self.cancellation_token.clone();
465 let interval = Duration::from_secs(self.config.stream_health_check_interval_secs);
466 let clock = self.clock;
467 let ws_client = self.ws_client.clone();
468 let book_sync = self.book_sync.clone();
469 let snapshot_timeout = self.book_snapshot_timeout();
470 let book_tasks = self.pending_tasks.spawner()?;
471
472 self.session_tasks.spawn(async move {
473 log::debug!("Hyperliquid stream health monitor started");
474
475 loop {
476 tokio::select! {
477 () = cancellation_token.cancelled() => {
478 log::debug!("Hyperliquid stream health monitor cancelled");
479 break;
480 }
481 () = tokio::time::sleep(interval) => {
482 let events = stream_health
483 .lock()
484 .check_stale(Instant::now(), clock.get_time_ns());
485
486 handle_stream_health_events(
487 &ws_client,
488 &events,
489 &book_sync,
490 snapshot_timeout,
491 &book_tasks,
492 )
493 .await;
494 }
495 }
496 }
497
498 log::debug!("Hyperliquid stream health monitor stopped");
499 })?;
500
501 Ok(())
502 }
503
504 fn book_snapshot_timeout(&self) -> Duration {
505 Duration::from_secs(self.config.book_snapshot_timeout_secs)
506 }
507
508 fn venue(&self) -> Venue {
509 *HYPERLIQUID_VENUE
510 }
511
512 fn custom_instrument_id(data_type: &DataType) -> anyhow::Result<Option<InstrumentId>> {
513 let Some(raw_instrument_id) = data_type
514 .metadata()
515 .and_then(|m| m.get("instrument_id"))
516 .and_then(|v| v.as_str())
517 .map(str::trim)
518 .filter(|value| !value.is_empty())
519 else {
520 return Ok(None);
521 };
522
523 let instrument_id = InstrumentId::from_str(raw_instrument_id)
524 .with_context(|| format!("invalid instrument_id metadata `{raw_instrument_id}`"))?;
525
526 Ok(Some(instrument_id))
527 }
528
529 fn custom_user(data_type: &DataType) -> anyhow::Result<Option<String>> {
530 let Some(user) = data_type
531 .metadata()
532 .and_then(|m| m.get("user"))
533 .and_then(|v| v.as_str())
534 .filter(|value| !value.is_empty())
535 else {
536 return Ok(None);
537 };
538
539 anyhow::ensure!(
540 user == user.trim(),
541 "metadata['user'] must not contain surrounding whitespace",
542 );
543
544 Ok(Some(user.to_string()))
545 }
546
547 async fn bootstrap_instruments(&self) -> anyhow::Result<Vec<InstrumentAny>> {
548 let _update_guard = self.instrument_update_lock.lock().await;
550
551 let snapshot = self
552 .http_client
553 .request_instruments_with_statuses()
554 .await
555 .context("failed to fetch instruments during bootstrap")?;
556 let instruments = snapshot.instruments;
557
558 cache_instruments(
559 &instruments,
560 &self.instruments,
561 &self.coin_to_instrument_id,
562 &self.http_client,
563 &self.ws_client,
564 );
565 self.instrument_status.lock().store(snapshot.statuses);
566 rebuild_all_dex_asset_ctxs_mapping(&self.http_client, &self.ws_client).await;
567
568 log::debug!(
569 "Bootstrapped {} instruments with {} coin mappings",
570 self.instruments.len(),
571 self.coin_to_instrument_id.len()
572 );
573 Ok(instruments)
574 }
575
576 fn spawn_instrument_refresh(&self) -> anyhow::Result<()> {
580 if let Some(future) = self.instrument_refresh() {
581 self.session_tasks.spawn(future)?;
582 }
583
584 Ok(())
585 }
586
587 fn instrument_refresh(&self) -> Option<impl Future<Output = ()> + Send + 'static> {
588 let minutes = self.config.update_instruments_interval_mins;
589
590 if minutes == 0 {
591 log::debug!(
592 "Hyperliquid instrument refresh disabled (update_instruments_interval_mins=0)"
593 );
594 return None;
595 }
596
597 let interval = Duration::from_secs(minutes.saturating_mul(60));
598 let cancellation_token = self.cancellation_token.clone();
599 let http_client = self.http_client.clone();
600 let ws_client = self.ws_client.clone();
601 let instruments = Arc::clone(&self.instruments);
602 let coin_to_instrument_id = Arc::clone(&self.coin_to_instrument_id);
603 let instrument_update_lock = Arc::clone(&self.instrument_update_lock);
604 let instrument_status = Arc::clone(&self.instrument_status);
605 let data_sender = self.data_sender.clone();
606 let client_id = self.client_id;
607
608 Some(async move {
609 log::info!("Hyperliquid instrument refresh started, interval={interval:?}");
610
611 loop {
612 tokio::select! {
613 () = cancellation_token.cancelled() => {
614 log::debug!("Hyperliquid instrument refresh cancelled");
615 break;
616 }
617 () = tokio::time::sleep(interval) => {}
618 }
619
620 let result = tokio::select! {
623 () = cancellation_token.cancelled() => {
624 log::debug!("Hyperliquid instrument refresh cancelled");
625 break;
626 }
627 result = refresh_instruments(
628 &instrument_update_lock,
629 &http_client,
630 &ws_client,
631 &instruments,
632 &coin_to_instrument_id,
633 &instrument_status,
634 &data_sender,
635 ) => result,
636 };
637
638 match result {
639 Ok(summary) => summary.log(client_id),
640 Err(e) => log::warn!(
641 "Failed to refresh Hyperliquid instruments: client_id={client_id}, error={e:?}"
642 ),
643 }
644 }
645
646 log::debug!("Hyperliquid instrument refresh stopped");
647 })
648 }
649
650 async fn spawn_ws(&self) -> anyhow::Result<()> {
651 let mut ws_client = self.ws_client.clone();
653
654 ws_client
655 .connect()
656 .await
657 .context("failed to connect to Hyperliquid WebSocket")?;
658
659 let data_sender = self.data_sender.clone();
660 let cancellation_token = self.cancellation_token.clone();
661 let stream_health = Arc::clone(&self.stream_health);
662 let book_sync = self.book_sync.clone();
663 let snapshot_timeout = self.book_snapshot_timeout();
664 let book_tasks = self.pending_tasks.spawner()?;
665
666 self.session_tasks.spawn(async move {
667 log::debug!("Hyperliquid WebSocket consumption loop started");
668
669 loop {
670 tokio::select! {
671 () = cancellation_token.cancelled() => {
672 log::debug!("WebSocket consumption loop cancelled");
673 break;
674 }
675 msg_opt = ws_client.next_event() => {
676 if let Some(msg) = msg_opt {
677 if let Some((channel, instrument_id, ts_event)) =
678 stream_health_update(&msg)
679 {
680 record_stream_receive(
681 &stream_health,
682 channel,
683 instrument_id,
684 ts_event,
685 );
686 }
687
688 match msg {
689 NautilusWsMessage::Trades(trades) => {
690 for trade in trades {
691 if let Err(e) = data_sender
692 .send(DataEvent::Data(Data::Trade(trade)))
693 {
694 log::error!("Failed to send trade tick: {e}");
695 }
696 }
697 }
698 NautilusWsMessage::Quote(quote) => {
699 if let Err(e) = data_sender
700 .send(DataEvent::Data(Data::Quote(quote)))
701 {
702 log::error!("Failed to send quote tick: {e}");
703 }
704 }
705 NautilusWsMessage::Deltas(deltas) => {
706 if book_sync
707 .record_snapshot(deltas.instrument_id, Instant::now())
708 && let Err(e) = data_sender.send(DataEvent::Data(
709 Data::BookDeltas(Box::new(deltas)),
710 ))
711 {
712 log::error!("Failed to send order book deltas: {e}");
713 }
714 }
715 NautilusWsMessage::CompactBook {
716 deltas, depth, is_snapshot, output_gate,
717 } => {
718 let output = output_gate.lock();
720
721 if output.is_closed() {
722 continue;
723 }
724
725 record_stream_receive(
726 &stream_health,
727 MarketDataChannel::Deltas,
728 deltas.instrument_id,
729 deltas.ts_event,
730 );
731 book_sync.record_compact(
732 deltas.instrument_id,
733 is_snapshot,
734 Instant::now(),
735 |publish_deltas, publish_depth| {
736 if publish_deltas
737 && let Err(e) = data_sender.send(DataEvent::Data(
738 Data::BookDeltas(Box::new(deltas)),
739 ))
740 {
741 log::error!("Failed to send compact book snapshot: {e}");
742 }
743
744 if publish_depth && let Some(depth) = depth {
745 record_stream_receive(
746 &stream_health,
747 MarketDataChannel::Depth,
748 depth.instrument_id,
749 depth.ts_event,
750 );
751
752 if let Err(e) = data_sender.send(
753 DataEvent::Data(Data::BookDepth(depth)),
754 ) {
755 log::error!("Failed to send compact book depth: {e}");
756 }
757 }
758 },
759 );
760 }
761 NautilusWsMessage::Depth(depth) => {
762 if let Err(e) =
763 data_sender.send(DataEvent::Data(Data::BookDepth(depth)))
764 {
765 log::error!("Failed to send order book depth: {e}");
766 }
767 }
768 NautilusWsMessage::BookInvalid { instrument_id, output_gate } => {
769 let output = output_gate.as_ref().map(|gate| gate.lock());
770
771 if output.as_ref().is_some_and(|gate| gate.is_closed()) {
772 continue;
773 }
774
775 book::recovery::reject_snapshot(
776 instrument_id,
777 &book_sync,
778 &ws_client,
779 snapshot_timeout,
780 &book_tasks,
781 );
782 }
783 NautilusWsMessage::Candle(bar) => {
784 if let Err(e) = data_sender
785 .send(DataEvent::Data(Data::Bar(bar)))
786 {
787 log::error!("Failed to send bar: {e}");
788 }
789 }
790 NautilusWsMessage::MarkPrice(update) => {
791 if let Err(e) = data_sender
792 .send(DataEvent::Data(Data::MarkPrice(update)))
793 {
794 log::error!("Failed to send mark price update: {e}");
795 }
796 }
797 NautilusWsMessage::IndexPrice(update) => {
798 if let Err(e) = data_sender
799 .send(DataEvent::Data(Data::IndexPrice(update)))
800 {
801 log::error!("Failed to send index price update: {e}");
802 }
803 }
804 NautilusWsMessage::FundingRate(update) => {
805 if let Err(e) = data_sender
806 .send(DataEvent::Data(Data::FundingRate(update)))
807 {
808 log::error!("Failed to send funding rate update: {e}");
809 }
810 }
811 NautilusWsMessage::CustomData(data) => {
812 if let Err(e) = data_sender.send(DataEvent::Data(data)) {
813 log::error!("Failed to send custom data: {e}");
814 }
815 }
816 NautilusWsMessage::Reconnected => {
817 log::info!("WebSocket reconnected");
818 book::recovery::reset_on_reconnect(
819 &book_sync,
820 &ws_client,
821 snapshot_timeout,
822 &book_tasks,
823 );
824 }
825 NautilusWsMessage::Error(e) => {
826 log::warn!("WebSocket error: {e}");
827 }
828 NautilusWsMessage::ExecutionReports(_)
829 | NautilusWsMessage::ClearinghouseState(_)
830 | NautilusWsMessage::SpotState(_) => {
831 }
833 }
834 } else {
835 log::debug!("WebSocket next_event returned None, stream closed");
836 break;
837 }
838 }
839 }
840 }
841
842 log::debug!("Hyperliquid WebSocket consumption loop finished");
843 })?;
844
845 log::debug!("WebSocket consumption task spawned");
846
847 Ok(())
848 }
849}
850
851#[async_trait::async_trait(?Send)]
852impl DataClient for HyperliquidDataClient {
853 fn client_id(&self) -> ClientId {
854 self.client_id
855 }
856
857 fn venue(&self) -> Option<Venue> {
858 Some(self.venue())
859 }
860
861 fn start(&mut self) -> anyhow::Result<()> {
862 log::info!(
863 "Starting Hyperliquid data client: client_id={}, environment={:?}, proxy_url={:?}",
864 self.client_id,
865 self.config.environment,
866 self.config.proxy_url,
867 );
868 Ok(())
869 }
870
871 fn stop(&mut self) -> anyhow::Result<()> {
872 log::info!("Stopping Hyperliquid data client {}", self.client_id);
873 self.cancellation_token.cancel();
874 self.abort_session_tasks();
875 self.abort_pending_tasks();
876 self.is_connected.store(false, Ordering::Relaxed);
877 Ok(())
878 }
879
880 fn reset(&mut self) -> anyhow::Result<()> {
881 log::debug!("Resetting Hyperliquid data client {}", self.client_id);
882 self.cancellation_token.cancel();
886 self.abort_session_tasks();
887 self.abort_pending_tasks();
888 self.is_connected.store(false, Ordering::Relaxed);
889 self.instruments.store(AHashMap::new());
890 self.coin_to_instrument_id.store(AHashMap::new());
891 self.instrument_status.lock().clear();
892 Ok(())
893 }
894
895 fn dispose(&mut self) -> anyhow::Result<()> {
896 log::debug!("Disposing Hyperliquid data client {}", self.client_id);
897 self.stop()
898 }
899
900 fn is_connected(&self) -> bool {
901 self.is_connected.load(Ordering::Acquire)
902 }
903
904 fn is_disconnected(&self) -> bool {
905 !self.is_connected()
906 }
907
908 async fn connect(&mut self) -> anyhow::Result<()> {
909 if self.is_connected()
910 && !self.cancellation_token.is_cancelled()
911 && self.session_tasks.is_open()
912 && self.pending_tasks.is_open()
913 {
914 return Ok(());
915 }
916
917 if self.cancellation_token.is_cancelled()
918 || !self.session_tasks.is_open()
919 || !self.pending_tasks.is_open()
920 {
921 self.ws_client.begin_shutdown();
926 self.ws_client
927 .disconnect()
928 .await
929 .context("failed to tear down Hyperliquid WebSocket before reconnect")?;
930 self.ws_client.reset_runtime_state();
931 self.clear_book_sync();
932 self.abort_session_tasks();
933 self.abort_pending_tasks();
934 let (session_result, pending_result) =
935 tokio::join!(self.await_session_tasks(), self.await_pending_tasks());
936 session_result?;
937 pending_result?;
938 self.session_tasks.start_generation().map_err(|e| {
939 anyhow::anyhow!("Failed to start Hyperliquid data session generation: {e}")
940 })?;
941 self.pending_tasks.start_generation().map_err(|e| {
942 anyhow::anyhow!("Failed to start Hyperliquid data task generation: {e}")
943 })?;
944 self.cancellation_token = CancellationToken::new();
945 }
946 let cancellation_token = self.cancellation_token.clone();
947 let ws_client = self.ws_client.clone();
948 let setup_guard =
949 TaskGroupGuard::new(&[&self.session_tasks, &self.pending_tasks], move || {
950 cancellation_token.cancel();
951 ws_client.begin_shutdown();
952 });
953
954 register_hyperliquid_custom_data();
955
956 let instruments = self
957 .bootstrap_instruments()
958 .await
959 .context("failed to bootstrap instruments")?;
960
961 for instrument in instruments {
962 if let Err(e) = self.data_sender.send(DataEvent::Data(instrument.into())) {
963 log::warn!("Failed to send instrument: {e}");
964 }
965 }
966
967 let session_result = async {
968 self.spawn_ws()
969 .await
970 .context("failed to spawn WebSocket client")?;
971 self.spawn_stream_health_monitor()?;
972 self.spawn_instrument_refresh()?;
973 Ok::<(), anyhow::Error>(())
974 }
975 .await;
976
977 if let Err(e) = session_result {
978 if let Err(teardown_error) = self.teardown_partial_connect().await {
979 return Err(e.context(format!(
980 "Hyperliquid data startup teardown failed: {teardown_error}"
981 )));
982 }
983 return Err(e);
984 }
985
986 self.is_connected.store(true, Ordering::Relaxed);
987 setup_guard.disarm();
988 log::info!("Connected: client_id={}", self.client_id);
989
990 Ok(())
991 }
992
993 async fn disconnect(&mut self) -> anyhow::Result<()> {
994 self.teardown_partial_connect().await?;
995 self.instruments.store(AHashMap::new());
996 self.instrument_status.lock().clear();
997 log::info!("Disconnected: client_id={}", self.client_id);
998
999 Ok(())
1000 }
1001
1002 fn subscribe(&mut self, cmd: SubscribeCustomData) -> anyhow::Result<()> {
1003 let data_type = cmd.data_type.type_name();
1004
1005 if data_type == "HyperliquidAllMids" {
1006 let ws = self.ws_client.clone();
1007 let dex = cmd
1008 .data_type
1009 .metadata()
1010 .as_ref()
1011 .and_then(|m| m.get("dex"))
1012 .and_then(|v| v.as_str())
1013 .map(str::trim)
1014 .filter(|value| !value.is_empty())
1015 .map(ToString::to_string);
1016
1017 log::debug!("Subscribing to all mids (dex: {:?})", dex.as_deref());
1018
1019 self.spawn_task("subscribe_all_mids", async move {
1020 ws.subscribe_all_mids_with_dex(dex.as_deref()).await
1021 });
1022
1023 return Ok(());
1024 }
1025
1026 if data_type == "HyperliquidAllDexsAssetCtxs" {
1027 let ws = self.ws_client.clone();
1028
1029 self.spawn_task("subscribe_all_dexs_asset_ctxs", async move {
1030 ws.subscribe_all_dexs_asset_ctxs().await
1031 });
1032
1033 return Ok(());
1034 }
1035
1036 if data_type == "HyperliquidOpenInterest" {
1037 let ws = self.ws_client.clone();
1038 let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
1039 "HyperliquidOpenInterest subscriptions require metadata['instrument_id']",
1040 )?;
1041
1042 self.spawn_task("subscribe_open_interest", async move {
1043 ws.subscribe_open_interest(instrument_id).await
1044 });
1045
1046 return Ok(());
1047 }
1048
1049 if data_type == "HyperliquidPublicTrade" {
1050 let ws = self.ws_client.clone();
1051 let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
1052 "HyperliquidPublicTrade subscriptions require metadata['instrument_id']",
1053 )?;
1054
1055 self.spawn_task("subscribe_public_trades", async move {
1056 ws.subscribe_public_trades(instrument_id).await
1057 });
1058
1059 return Ok(());
1060 }
1061
1062 if data_type == "HyperliquidTwapHistory" {
1063 let ws = self.ws_client.clone();
1064 let user = Self::custom_user(&cmd.data_type)?
1065 .context("HyperliquidTwapHistory subscriptions require metadata['user']")?;
1066
1067 self.spawn_task("subscribe_user_twap_history", async move {
1068 ws.subscribe_user_twap_history(&user).await
1069 });
1070
1071 return Ok(());
1072 }
1073
1074 if data_type == "HyperliquidTwapSliceFill" {
1075 let ws = self.ws_client.clone();
1076 let user = Self::custom_user(&cmd.data_type)?
1077 .context("HyperliquidTwapSliceFill subscriptions require metadata['user']")?;
1078
1079 self.spawn_task("subscribe_user_twap_slice_fills", async move {
1080 ws.subscribe_user_twap_slice_fills(&user).await
1081 });
1082
1083 return Ok(());
1084 }
1085
1086 log::warn!("Unsupported custom data subscription: {data_type}");
1087 Ok(())
1088 }
1089
1090 fn unsubscribe(&mut self, cmd: &UnsubscribeCustomData) -> anyhow::Result<()> {
1091 let data_type = cmd.data_type.type_name();
1092
1093 if data_type == "HyperliquidAllMids" {
1094 let ws = self.ws_client.clone();
1095 let dex = cmd
1096 .data_type
1097 .metadata()
1098 .as_ref()
1099 .and_then(|m| m.get("dex"))
1100 .and_then(|v| v.as_str())
1101 .map(str::trim)
1102 .filter(|value| !value.is_empty())
1103 .map(ToString::to_string);
1104
1105 log::debug!("Unsubscribing from all mids (dex: {:?})", dex.as_deref());
1106
1107 self.spawn_task("unsubscribe_all_mids", async move {
1108 ws.unsubscribe_all_mids_with_dex(dex.as_deref()).await
1109 });
1110
1111 return Ok(());
1112 }
1113
1114 if data_type == "HyperliquidAllDexsAssetCtxs" {
1115 let ws = self.ws_client.clone();
1116
1117 self.spawn_task("unsubscribe_all_dexs_asset_ctxs", async move {
1118 ws.unsubscribe_all_dexs_asset_ctxs().await
1119 });
1120
1121 return Ok(());
1122 }
1123
1124 if data_type == "HyperliquidOpenInterest" {
1125 let ws = self.ws_client.clone();
1126 let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
1127 "HyperliquidOpenInterest unsubscriptions require metadata['instrument_id']",
1128 )?;
1129
1130 self.spawn_task("unsubscribe_open_interest", async move {
1131 ws.unsubscribe_open_interest(instrument_id).await
1132 });
1133
1134 return Ok(());
1135 }
1136
1137 if data_type == "HyperliquidPublicTrade" {
1138 let ws = self.ws_client.clone();
1139 let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
1140 "HyperliquidPublicTrade unsubscriptions require metadata['instrument_id']",
1141 )?;
1142
1143 self.spawn_task("unsubscribe_public_trades", async move {
1144 ws.unsubscribe_public_trades(instrument_id).await
1145 });
1146
1147 return Ok(());
1148 }
1149
1150 if data_type == "HyperliquidTwapHistory" {
1151 let ws = self.ws_client.clone();
1152 let user = Self::custom_user(&cmd.data_type)?
1153 .context("HyperliquidTwapHistory unsubscriptions require metadata['user']")?;
1154
1155 self.spawn_task("unsubscribe_user_twap_history", async move {
1156 ws.unsubscribe_user_twap_history(&user).await
1157 });
1158
1159 return Ok(());
1160 }
1161
1162 if data_type == "HyperliquidTwapSliceFill" {
1163 let ws = self.ws_client.clone();
1164 let user = Self::custom_user(&cmd.data_type)?
1165 .context("HyperliquidTwapSliceFill unsubscriptions require metadata['user']")?;
1166
1167 self.spawn_task("unsubscribe_user_twap_slice_fills", async move {
1168 ws.unsubscribe_user_twap_slice_fills(&user).await
1169 });
1170
1171 return Ok(());
1172 }
1173
1174 log::warn!("Unsupported custom data unsubscription: {data_type}");
1175 Ok(())
1176 }
1177
1178 fn subscribe_instrument(&mut self, cmd: SubscribeInstrument) -> anyhow::Result<()> {
1179 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1180 let instruments = self.instruments.load();
1181 if let Some(instrument) = instruments.get(&instrument_id) {
1182 if let Err(e) = self
1183 .data_sender
1184 .send(DataEvent::Data(instrument.clone().into()))
1185 {
1186 log::error!("Failed to send instrument {instrument_id}: {e}");
1187 }
1188 } else {
1189 log::warn!("Instrument {instrument_id} not found in cache");
1190 }
1191
1192 Ok(())
1193 })
1194 }
1195
1196 fn subscribe_book_deltas(&mut self, subscription: SubscribeBookDeltas) -> anyhow::Result<()> {
1197 execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1198 let subscription = subscription.with_instrument_ids(instrument_id);
1199 if subscription.book_type != BookType::L2_MBP {
1200 anyhow::bail!("Hyperliquid only supports L2_MBP order book deltas");
1201 }
1202
1203 let (n_sig_figs, mantissa) = parse_book_precision_params(subscription.params.as_ref())?;
1204 self.register_stream_health(MarketDataChannel::Deltas, instrument_id);
1205
1206 if self.config.book_feed == HyperliquidBookFeed::L2 {
1207 self.subscribe_compact_book(
1208 instrument_id,
1209 BookStreamUse::Deltas,
1210 BookStreamOptions {
1211 n_sig_figs,
1212 mantissa,
1213 },
1214 );
1215 return Ok(());
1216 }
1217
1218 match self.pending_tasks.spawner() {
1219 Ok(tasks) => book::recovery::spawn_subscription_task(
1220 instrument_id,
1221 n_sig_figs,
1222 mantissa,
1223 self.book_sync.clone(),
1224 self.ws_client.clone(),
1225 self.book_snapshot_timeout(),
1226 &tasks,
1227 ),
1228 Err(e) => {
1229 log::warn!(
1230 "Skipping Hyperliquid subscribe_book_deltas after shutdown began: {e}"
1231 );
1232 }
1233 }
1234
1235 Ok(())
1236 })
1237 }
1238
1239 fn subscribe_book_depth(&mut self, subscription: SubscribeBookDepth) -> anyhow::Result<()> {
1240 execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1241 let subscription = subscription.with_instrument_ids(instrument_id);
1242 log::debug!("Subscribing to book depth: {instrument_id}");
1243
1244 if subscription.book_type != BookType::L2_MBP {
1245 anyhow::bail!("Hyperliquid only supports L2_MBP order book depth");
1246 }
1247
1248 let ws = self.ws_client.clone();
1249 let (n_sig_figs, mantissa) = parse_book_precision_params(subscription.params.as_ref())?;
1250 self.register_stream_health(MarketDataChannel::Depth, instrument_id);
1251
1252 if self.config.book_feed == HyperliquidBookFeed::L2 {
1253 self.subscribe_compact_book(
1254 instrument_id,
1255 BookStreamUse::Depth,
1256 BookStreamOptions {
1257 n_sig_figs,
1258 mantissa,
1259 },
1260 );
1261 return Ok(());
1262 }
1263
1264 self.spawn_task("subscribe_book_depth", async move {
1265 ws.subscribe_book_depth_with_options(instrument_id, n_sig_figs, mantissa)
1266 .await
1267 });
1268
1269 Ok(())
1270 })
1271 }
1272
1273 fn subscribe_quotes(&mut self, subscription: SubscribeQuotes) -> anyhow::Result<()> {
1274 execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1275 let ws = self.ws_client.clone();
1276 self.register_stream_health(MarketDataChannel::Quote, instrument_id);
1277 self.spawn_task("subscribe_quotes", async move {
1278 ws.subscribe_quotes(instrument_id).await
1279 });
1280
1281 Ok(())
1282 })
1283 }
1284
1285 fn subscribe_trades(&mut self, subscription: SubscribeTrades) -> anyhow::Result<()> {
1286 execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1287 let ws = self.ws_client.clone();
1288 self.spawn_task("subscribe_trades", async move {
1289 ws.subscribe_trades(instrument_id).await
1290 });
1291
1292 Ok(())
1293 })
1294 }
1295
1296 fn subscribe_mark_prices(&mut self, cmd: SubscribeMarkPrices) -> anyhow::Result<()> {
1297 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1298 let ws = self.ws_client.clone();
1299 self.spawn_task("subscribe_mark_prices", async move {
1300 ws.subscribe_mark_prices(instrument_id).await
1301 });
1302
1303 Ok(())
1304 })
1305 }
1306
1307 fn subscribe_index_prices(&mut self, cmd: SubscribeIndexPrices) -> anyhow::Result<()> {
1308 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1309 let ws = self.ws_client.clone();
1310 self.spawn_task("subscribe_index_prices", async move {
1311 ws.subscribe_index_prices(instrument_id).await
1312 });
1313
1314 Ok(())
1315 })
1316 }
1317
1318 fn subscribe_funding_rates(&mut self, cmd: SubscribeFundingRates) -> anyhow::Result<()> {
1319 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1320 let ws = self.ws_client.clone();
1321 self.spawn_task("subscribe_funding_rates", async move {
1322 ws.subscribe_funding_rates(instrument_id).await
1323 });
1324
1325 Ok(())
1326 })
1327 }
1328
1329 fn subscribe_bars(&mut self, subscription: SubscribeBars) -> anyhow::Result<()> {
1330 execute_bar_subscriptions(&subscription.bar_types, |bar_type| {
1331 let instrument_id = bar_type.instrument_id();
1332 if !self.instruments.contains_key(&instrument_id) {
1333 anyhow::bail!(InstrumentLookupError::not_found(instrument_id));
1334 }
1335
1336 let ws = self.ws_client.clone();
1337 self.spawn_task("subscribe_bars", async move {
1338 ws.subscribe_bars(bar_type).await
1339 });
1340
1341 Ok(())
1342 })
1343 }
1344
1345 fn subscribe_instrument_status(
1346 &mut self,
1347 cmd: SubscribeInstrumentStatus,
1348 ) -> anyhow::Result<()> {
1349 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1350 log::debug!(
1351 "Subscribing to instrument status: {instrument_id} (detected by the instrument refresh)"
1352 );
1353 let ts = self.clock.get_time_ns();
1354 self.instrument_status
1355 .lock()
1356 .subscribe(instrument_id, &self.data_sender, ts);
1357
1358 Ok(())
1359 })
1360 }
1361
1362 fn unsubscribe_instrument_status(
1363 &mut self,
1364 cmd: &UnsubscribeInstrumentStatus,
1365 ) -> anyhow::Result<()> {
1366 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1367 log::debug!("Unsubscribing from instrument status: {instrument_id}");
1368 self.instrument_status.lock().unsubscribe(&instrument_id);
1369 Ok(())
1370 })
1371 }
1372
1373 fn unsubscribe_instrument(&mut self, _cmd: &UnsubscribeInstrument) -> anyhow::Result<()> {
1374 Ok(())
1377 }
1378
1379 fn unsubscribe_instruments(&mut self, _cmd: &UnsubscribeInstruments) -> anyhow::Result<()> {
1380 Ok(())
1383 }
1384
1385 fn unsubscribe_book_deltas(
1386 &mut self,
1387 unsubscription: &UnsubscribeBookDeltas,
1388 ) -> anyhow::Result<()> {
1389 execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1390 log::debug!("Unsubscribing from book deltas: {instrument_id}");
1391
1392 let ws = self.ws_client.clone();
1393 self.remove_stream_health(MarketDataChannel::Deltas, instrument_id);
1394
1395 if self.config.book_feed == HyperliquidBookFeed::L2 {
1396 self.unsubscribe_compact_book(instrument_id, BookStreamUse::Deltas);
1397 return Ok(());
1398 }
1399
1400 self.book_sync.remove(instrument_id);
1401 self.spawn_task("unsubscribe_book_deltas", async move {
1402 ws.unsubscribe_book(instrument_id).await
1403 });
1404
1405 Ok(())
1406 })
1407 }
1408
1409 fn unsubscribe_book_depth(
1410 &mut self,
1411 unsubscription: &UnsubscribeBookDepth,
1412 ) -> anyhow::Result<()> {
1413 execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1414 log::debug!("Unsubscribing from book depth: {instrument_id}");
1415
1416 let ws = self.ws_client.clone();
1417 self.remove_stream_health(MarketDataChannel::Depth, instrument_id);
1418
1419 if self.config.book_feed == HyperliquidBookFeed::L2 {
1420 self.unsubscribe_compact_book(instrument_id, BookStreamUse::Depth);
1421 return Ok(());
1422 }
1423
1424 self.spawn_task("unsubscribe_book_depth", async move {
1425 ws.unsubscribe_book_depth(instrument_id).await
1426 });
1427
1428 Ok(())
1429 })
1430 }
1431
1432 fn unsubscribe_quotes(&mut self, unsubscription: &UnsubscribeQuotes) -> anyhow::Result<()> {
1433 execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1434 log::debug!("Unsubscribing from quotes: {instrument_id}");
1435
1436 let ws = self.ws_client.clone();
1437 self.remove_stream_health(MarketDataChannel::Quote, instrument_id);
1438 self.spawn_task("unsubscribe_quotes", async move {
1439 ws.unsubscribe_quotes(instrument_id).await
1440 });
1441
1442 Ok(())
1443 })
1444 }
1445
1446 fn unsubscribe_trades(&mut self, unsubscription: &UnsubscribeTrades) -> anyhow::Result<()> {
1447 execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1448 log::debug!("Unsubscribing from trades: {instrument_id}");
1449
1450 let ws = self.ws_client.clone();
1451 self.spawn_task("unsubscribe_trades", async move {
1452 ws.unsubscribe_trades(instrument_id).await
1453 });
1454
1455 Ok(())
1456 })
1457 }
1458
1459 fn unsubscribe_mark_prices(&mut self, cmd: &UnsubscribeMarkPrices) -> anyhow::Result<()> {
1460 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1461 let ws = self.ws_client.clone();
1462 self.spawn_task("unsubscribe_mark_prices", async move {
1463 ws.unsubscribe_mark_prices(instrument_id).await
1464 });
1465
1466 Ok(())
1467 })
1468 }
1469
1470 fn unsubscribe_index_prices(&mut self, cmd: &UnsubscribeIndexPrices) -> anyhow::Result<()> {
1471 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1472 let ws = self.ws_client.clone();
1473 self.spawn_task("unsubscribe_index_prices", async move {
1474 ws.unsubscribe_index_prices(instrument_id).await
1475 });
1476
1477 Ok(())
1478 })
1479 }
1480
1481 fn unsubscribe_funding_rates(&mut self, cmd: &UnsubscribeFundingRates) -> anyhow::Result<()> {
1482 execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1483 let ws = self.ws_client.clone();
1484 self.spawn_task("unsubscribe_funding_rates", async move {
1485 ws.unsubscribe_funding_rates(instrument_id).await
1486 });
1487
1488 Ok(())
1489 })
1490 }
1491
1492 fn unsubscribe_bars(&mut self, unsubscription: &UnsubscribeBars) -> anyhow::Result<()> {
1493 execute_bar_subscriptions(&unsubscription.bar_types, |bar_type| {
1494 let ws = self.ws_client.clone();
1495 self.spawn_task("unsubscribe_bars", async move {
1496 ws.unsubscribe_bars(bar_type).await
1497 });
1498
1499 Ok(())
1500 })
1501 }
1502
1503 fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
1504 log::debug!("Requesting all instruments");
1505
1506 let http = self.http_client.clone();
1507 let ws = self.ws_client.clone();
1508 let sender = self.data_sender.clone();
1509 let instruments_cache = self.instruments.clone();
1510 let coin_map = self.coin_to_instrument_id.clone();
1511 let update_lock = Arc::clone(&self.instrument_update_lock);
1512 let instrument_status = Arc::clone(&self.instrument_status);
1513 let request_id = request.request_id;
1514 let client_id = request.client_id.unwrap_or(self.client_id);
1515 let venue = self.venue();
1516 let start_nanos = datetime_to_unix_nanos(request.start);
1517 let end_nanos = datetime_to_unix_nanos(request.end);
1518 let params = request.params;
1519 let clock = self.clock;
1520
1521 self.spawn_instrument_request(request_id, async move {
1522 let refresh = refresh_instruments(
1526 &update_lock,
1527 &http,
1528 &ws,
1529 &instruments_cache,
1530 &coin_map,
1531 &instrument_status,
1532 &sender,
1533 )
1534 .await?;
1535 refresh.log(client_id);
1536 let instruments = refresh.fetched;
1537
1538 Ok(DataResponse::Instruments(InstrumentsResponse::new(
1539 request_id,
1540 client_id,
1541 venue,
1542 instruments,
1543 start_nanos,
1544 end_nanos,
1545 clock.get_time_ns(),
1546 params,
1547 )))
1548 })
1549 }
1550
1551 fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
1552 let instrument_id = request.single_instrument_id()?;
1553
1554 log::debug!("Requesting instrument: {instrument_id}");
1555
1556 let http = self.http_client.clone();
1557 let ws = self.ws_client.clone();
1558 let sender = self.data_sender.clone();
1559 let instruments_cache = self.instruments.clone();
1560 let coin_map = self.coin_to_instrument_id.clone();
1561 let update_lock = Arc::clone(&self.instrument_update_lock);
1562 let instrument_status = Arc::clone(&self.instrument_status);
1563 let request_id = request.request_id;
1564 let client_id = request.client_id.unwrap_or(self.client_id);
1565 let start_nanos = datetime_to_unix_nanos(request.start);
1566 let end_nanos = datetime_to_unix_nanos(request.end);
1567 let params = request.params;
1568 let clock = self.clock;
1569
1570 self.spawn_instrument_request(request_id, async move {
1571 let refresh = refresh_instruments(
1574 &update_lock,
1575 &http,
1576 &ws,
1577 &instruments_cache,
1578 &coin_map,
1579 &instrument_status,
1580 &sender,
1581 )
1582 .await?;
1583 refresh.log(client_id);
1584 let instrument = refresh
1585 .fetched
1586 .into_iter()
1587 .find(|i| i.id() == instrument_id)
1588 .with_context(|| format!("Hyperliquid instrument {instrument_id} not found"))?;
1589 Ok(DataResponse::Instrument(Box::new(InstrumentResponse::new(
1590 request_id,
1591 client_id,
1592 instrument_id,
1593 instrument,
1594 start_nanos,
1595 end_nanos,
1596 clock.get_time_ns(),
1597 params,
1598 ))))
1599 })
1600 }
1601
1602 fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
1603 let bar_type = request.single_bar_type()?;
1604
1605 log::debug!("Requesting bars for {bar_type}");
1606
1607 let http = self.http_client.clone();
1608 let sender = self.data_sender.clone();
1609 let start = request.start;
1610 let end = request.end;
1611 let limit = request.limit.map(|n| n.get() as u32);
1612 let request_id = request.request_id;
1613 let client_id = request.client_id.unwrap_or(self.client_id);
1614 let params = request.params;
1615 let clock = self.clock;
1616 let start_nanos = datetime_to_unix_nanos(start);
1617 let end_nanos = datetime_to_unix_nanos(end);
1618 let instruments = Arc::clone(&self.instruments);
1619
1620 self.spawn_task("request_bars", async move {
1621 let bars = request_bars_from_http(http, bar_type, start, end, limit, instruments)
1622 .await
1623 .context("bar request failed")?;
1624
1625 let response = DataResponse::Bars(BarsResponse::new(
1626 request_id,
1627 client_id,
1628 bar_type,
1629 bars,
1630 start_nanos,
1631 end_nanos,
1632 clock.get_time_ns(),
1633 params,
1634 ));
1635
1636 if let Err(e) = sender.send(DataEvent::Response(response)) {
1637 log::error!("Failed to send bars response: {e}");
1638 }
1639 Ok(())
1640 });
1641
1642 Ok(())
1643 }
1644
1645 fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
1646 let instrument_id = request.single_instrument_id()?;
1647
1648 log::debug!("Requesting trades for {instrument_id}");
1649
1650 let instruments = self.instruments.load();
1651 let instrument = instruments
1652 .get(&instrument_id)
1653 .cloned()
1654 .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1655
1656 let coin = instrument.raw_symbol().to_string();
1657 let http = self.http_client.clone();
1658 let sender = self.data_sender.clone();
1659 let client_id = request.client_id.unwrap_or(self.client_id);
1660 let request_id = request.request_id;
1661 let params = request.params;
1662 let clock = self.clock;
1663 let limit = request.limit.map(|n| n.get());
1664 let start_nanos = datetime_to_unix_nanos(request.start);
1665 let end_nanos = datetime_to_unix_nanos(request.end);
1666
1667 self.spawn_task("request_trades", async move {
1668 let raw_trades = match http.info_recent_trades(&coin).await {
1672 Ok(trades) => trades,
1673 Err(e) if e.is_unprocessable_entity() => {
1674 log::warn!(
1675 "Recent trades endpoint unavailable for {instrument_id} \
1676 (requires the Hyperliquid indexer); sending empty response"
1677 );
1678 Vec::new()
1679 }
1680 Err(e) => {
1681 return Err(anyhow::Error::new(e))
1682 .with_context(|| format!("trades request failed for {instrument_id}"));
1683 }
1684 };
1685
1686 let mut trades: Vec<TradeTick> = Vec::with_capacity(raw_trades.len());
1687 for raw in &raw_trades {
1688 match parse_recent_trade(raw, &instrument) {
1689 Ok(trade) => trades.push(trade),
1690 Err(e) => log::warn!("Skipping recent trade for {instrument_id}: {e}"),
1691 }
1692 }
1693 trades.sort_by_key(|trade| trade.ts_event);
1694
1695 let trades = filter_recent_trades(trades, start_nanos, end_nanos, limit, instrument_id);
1696
1697 log::debug!("Fetched {} trades for {instrument_id}", trades.len());
1698
1699 let response = DataResponse::Trades(TradesResponse::new(
1700 request_id,
1701 client_id,
1702 instrument_id,
1703 trades,
1704 start_nanos,
1705 end_nanos,
1706 clock.get_time_ns(),
1707 params,
1708 ));
1709
1710 if let Err(e) = sender.send(DataEvent::Response(response)) {
1711 log::error!("Failed to send trades response: {e}");
1712 }
1713 Ok(())
1714 });
1715
1716 Ok(())
1717 }
1718
1719 fn request_data(&self, request: RequestCustomData) -> anyhow::Result<()> {
1720 if request.data_type.type_name() != "HyperliquidPublicTrade" {
1721 log::warn!(
1722 "Unsupported custom data request: {}",
1723 request.data_type.type_name()
1724 );
1725 return Ok(());
1726 }
1727
1728 let instrument_id = Self::custom_instrument_id(&request.data_type)?
1729 .context("HyperliquidPublicTrade requests require metadata['instrument_id']")?;
1730 let data_type = DataType::new(
1731 request.data_type.type_name(),
1732 request.data_type.metadata().cloned(),
1733 Some(instrument_id.to_string()),
1734 );
1735 let http = self.http_client.clone();
1736 let sender = self.data_sender.clone();
1737 let request_id = request.request_id;
1738 let client_id = request.client_id;
1739 let params = request.params;
1740 let clock = self.clock;
1741 let limit = request.limit.map(|limit| limit.get());
1742 let start = request.start;
1743 let end = request.end;
1744 let start_nanos = datetime_to_unix_nanos(start);
1745 let end_nanos = datetime_to_unix_nanos(end);
1746 let venue = self.venue();
1747
1748 self.spawn_task("request_public_trades", async move {
1749 let trades = http
1750 .request_public_trades(instrument_id, start, end, limit)
1751 .await
1752 .map_err(anyhow::Error::new)
1753 .with_context(|| format!("public trades request failed for {instrument_id}"))?;
1754 let data: Vec<CustomData> = trades
1755 .into_iter()
1756 .map(|trade| CustomData::new(Arc::new(trade), data_type.clone()))
1757 .collect();
1758
1759 let response = DataResponse::Data(CustomDataResponse::new(
1760 request_id,
1761 client_id,
1762 Some(venue),
1763 data_type,
1764 data,
1765 start_nanos,
1766 end_nanos,
1767 clock.get_time_ns(),
1768 params,
1769 ));
1770
1771 if let Err(e) = sender.send(DataEvent::Response(response)) {
1772 log::error!("Failed to send public trades response: {e}");
1773 }
1774 Ok(())
1775 });
1776
1777 Ok(())
1778 }
1779
1780 fn request_funding_rates(&self, request: RequestFundingRates) -> anyhow::Result<()> {
1781 let instrument_id = request.single_instrument_id()?;
1782
1783 log::debug!("Requesting funding rates for {instrument_id}");
1784
1785 let instruments = self.instruments.load();
1786 let instrument = instruments
1787 .get(&instrument_id)
1788 .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1789
1790 if !matches!(instrument, InstrumentAny::CryptoPerpetual(_)) {
1791 anyhow::bail!("Funding rates are only available for perpetual instruments");
1792 }
1793
1794 let instrument = instrument.clone();
1795 let http = self.http_client.clone();
1796 let sender = self.data_sender.clone();
1797 let client_id = request.client_id.unwrap_or(self.client_id);
1798 let request_id = request.request_id;
1799 let params = request.params;
1800 let clock = self.clock;
1801 let limit = request.limit.map(|n| n.get());
1802 let start_dt = request.start;
1803 let end_dt = request.end;
1804 let start_nanos = datetime_to_unix_nanos(start_dt);
1805 let end_nanos = datetime_to_unix_nanos(end_dt);
1806
1807 self.spawn_task("request_funding_rates", async move {
1808 let funding_rates = http
1809 .request_funding_rates(&instrument, start_dt, end_dt, limit)
1810 .await
1811 .with_context(|| format!("funding rates request failed for {instrument_id}"))?;
1812
1813 log::debug!(
1814 "Fetched {} funding rates for {instrument_id}",
1815 funding_rates.len(),
1816 );
1817
1818 let response = DataResponse::FundingRates(FundingRatesResponse::new(
1819 request_id,
1820 client_id,
1821 instrument_id,
1822 funding_rates,
1823 start_nanos,
1824 end_nanos,
1825 clock.get_time_ns(),
1826 params,
1827 ));
1828
1829 if let Err(e) = sender.send(DataEvent::Response(response)) {
1830 log::error!("Failed to send funding rates response: {e}");
1831 }
1832 Ok(())
1833 });
1834
1835 Ok(())
1836 }
1837
1838 fn request_book_snapshot(&self, request: RequestBookSnapshot) -> anyhow::Result<()> {
1839 let instrument_id = request.single_instrument_id()?;
1840
1841 let instruments = self.instruments.load();
1842 let instrument = instruments
1843 .get(&instrument_id)
1844 .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1845
1846 let raw_symbol = instrument.raw_symbol().to_string();
1847 let price_precision = instrument.price_precision();
1848 let size_precision = instrument.size_precision();
1849 let depth = request.depth.map(|d| d.get());
1850
1851 let http = self.http_client.clone();
1852 let sender = self.data_sender.clone();
1853 let client_id = request.client_id.unwrap_or(self.client_id);
1854 let request_id = request.request_id;
1855 let params = request.params;
1856 let clock = self.clock;
1857
1858 self.spawn_task("request_book_snapshot", async move {
1859 let l2_book = http
1860 .info_l2_book(&raw_symbol)
1861 .await
1862 .with_context(|| format!("book snapshot request failed for {instrument_id}"))?;
1863
1864 let book = parse_l2_book_snapshot(
1865 &l2_book,
1866 instrument_id,
1867 price_precision,
1868 size_precision,
1869 depth,
1870 );
1871
1872 let response = DataResponse::Book(BookResponse::new(
1873 request_id,
1874 client_id,
1875 instrument_id,
1876 book,
1877 None,
1878 None,
1879 clock.get_time_ns(),
1880 params,
1881 ));
1882
1883 if let Err(e) = sender.send(DataEvent::Response(response)) {
1884 log::error!("Failed to send book snapshot response: {e}");
1885 }
1886 Ok(())
1887 });
1888
1889 Ok(())
1890 }
1891}
1892
1893fn cache_instruments(
1895 instruments: &[InstrumentAny],
1896 instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1897 coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1898 http_client: &HyperliquidHttpClient,
1899 ws_client: &HyperliquidWebSocketClient,
1900) {
1901 instruments_by_id.rcu(|m| {
1902 for instrument in instruments {
1903 m.insert(instrument.id(), instrument.clone());
1904 }
1905 });
1906
1907 coin_to_instrument_id.rcu(|m| {
1908 for instrument in instruments {
1909 m.insert(instrument.raw_symbol().inner(), instrument.id());
1910 }
1911 });
1912
1913 http_client.merge_instruments(instruments);
1914 ws_client.merge_instruments(instruments);
1915}
1916
1917async fn rebuild_all_dex_asset_ctxs_mapping(
1925 http_client: &HyperliquidHttpClient,
1926 ws_client: &HyperliquidWebSocketClient,
1927) {
1928 match http_client.build_all_dex_asset_ctxs_instrument_ids().await {
1929 Ok(mapping) => {
1930 let mapping = mapping
1931 .into_iter()
1932 .map(|(dex, instrument_ids)| (Ustr::from(dex.as_str()), instrument_ids))
1933 .collect();
1934 ws_client.cache_all_dex_asset_ctxs_instrument_ids(mapping);
1935 }
1936 Err(e) => {
1937 log::warn!("Failed to build Hyperliquid allDexsAssetCtxs mapping: {e}");
1938 }
1939 }
1940}
1941
1942#[derive(Debug)]
1944struct InstrumentRefresh {
1945 fetched: Vec<InstrumentAny>,
1947 added: Vec<Ustr>,
1949 changed: usize,
1951}
1952
1953impl InstrumentRefresh {
1954 fn log(&self, client_id: ClientId) {
1955 if self.added.is_empty() {
1958 log::debug!(
1959 "Hyperliquid instruments refreshed: client_id={client_id}, fetched={}, changed={}",
1960 self.fetched.len(),
1961 self.changed,
1962 );
1963 } else {
1964 log::info!(
1965 "Hyperliquid instruments refreshed: client_id={client_id}, fetched={}, changed={}, added={:?}",
1966 self.fetched.len(),
1967 self.changed,
1968 self.added,
1969 );
1970 }
1971 }
1972}
1973
1974async fn refresh_instruments(
1983 update_lock: &tokio::sync::Mutex<()>,
1984 http_client: &HyperliquidHttpClient,
1985 ws_client: &HyperliquidWebSocketClient,
1986 instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1987 coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1988 instrument_status: &Mutex<InstrumentStatusState>,
1989 data_sender: &EventSender<DataEvent>,
1990) -> anyhow::Result<InstrumentRefresh> {
1991 let _update_guard = update_lock.lock().await;
1992
1993 let snapshot = http_client
1994 .request_instruments_with_statuses()
1995 .await
1996 .context("failed to fetch Hyperliquid instruments")?;
1997
1998 anyhow::ensure!(
2000 !snapshot.instruments.is_empty() || !snapshot.loaded.is_empty(),
2001 "failed to fetch Hyperliquid instruments: no usable metadata"
2002 );
2003
2004 let refresh = reconcile_instruments(
2007 snapshot.instruments,
2008 http_client,
2009 ws_client,
2010 instruments_by_id,
2011 coin_to_instrument_id,
2012 data_sender,
2013 )
2014 .await;
2015
2016 let ts = get_atomic_clock_realtime().get_time_ns();
2017 instrument_status
2018 .lock()
2019 .apply(&snapshot.statuses, &snapshot.loaded, data_sender, ts);
2020
2021 Ok(refresh)
2022}
2023
2024async fn reconcile_instruments(
2037 fetched: Vec<InstrumentAny>,
2038 http_client: &HyperliquidHttpClient,
2039 ws_client: &HyperliquidWebSocketClient,
2040 instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
2041 coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
2042 data_sender: &EventSender<DataEvent>,
2043) -> InstrumentRefresh {
2044 let changed = changed_definitions(&fetched, instruments_by_id);
2045 let added = added_symbols(&changed, instruments_by_id);
2046
2047 cache_instruments(
2048 &changed,
2049 instruments_by_id,
2050 coin_to_instrument_id,
2051 http_client,
2052 ws_client,
2053 );
2054
2055 for instrument in &changed {
2058 if let Err(e) = data_sender.send(DataEvent::Data(instrument.clone().into())) {
2059 log::warn!("Failed to send instrument: {e}");
2060 }
2061 }
2062
2063 rebuild_all_dex_asset_ctxs_mapping(http_client, ws_client).await;
2064
2065 InstrumentRefresh {
2066 added,
2067 changed: changed.len(),
2068 fetched,
2069 }
2070}
2071
2072fn changed_definitions(
2074 fetched: &[InstrumentAny],
2075 instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
2076) -> Vec<InstrumentAny> {
2077 fetched
2078 .iter()
2079 .filter(|instrument| {
2080 instruments_by_id
2081 .get_cloned(&instrument.id())
2082 .is_none_or(|cached| !instrument_definitions_match(&cached, instrument))
2083 })
2084 .cloned()
2085 .collect()
2086}
2087
2088fn added_symbols(
2095 changed: &[InstrumentAny],
2096 instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
2097) -> Vec<Ustr> {
2098 changed
2099 .iter()
2100 .filter(|instrument| instruments_by_id.get_cloned(&instrument.id()).is_none())
2101 .map(|instrument| instrument.symbol().inner())
2102 .collect()
2103}
2104
2105fn instrument_definitions_match(a: &InstrumentAny, b: &InstrumentAny) -> bool {
2111 fn normalized(instrument: &InstrumentAny) -> Option<serde_json::Value> {
2112 let mut value = serde_json::to_value(instrument).ok()?;
2113
2114 if let Some(definition) = value
2115 .as_object_mut()
2116 .and_then(|obj| obj.values_mut().next())
2117 .and_then(serde_json::Value::as_object_mut)
2118 {
2119 definition.remove("ts_event");
2120 definition.remove("ts_init");
2121 }
2122
2123 Some(value)
2124 }
2125
2126 match (normalized(a), normalized(b)) {
2128 (Some(a), Some(b)) => a == b,
2129 _ => false,
2130 }
2131}
2132
2133#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
2134enum MarketDataChannel {
2135 Deltas,
2136 Depth,
2137 Quote,
2138}
2139
2140impl MarketDataChannel {
2141 const fn as_str(self) -> &'static str {
2142 match self {
2143 Self::Deltas => "deltas",
2144 Self::Depth => "depth",
2145 Self::Quote => "quote",
2146 }
2147 }
2148}
2149
2150type MarketDataStreamKey = (MarketDataChannel, InstrumentId);
2151
2152#[derive(Debug, Clone)]
2153struct MarketDataStreamHealth {
2154 last_receive_at: Instant,
2155 last_venue_ts_event: Option<UnixNanos>,
2156 consecutive_stale_count: u32,
2157 last_warning_at: Option<Instant>,
2158 last_recovery_at: Option<Instant>,
2159 resubscribe_attempts: u32,
2160}
2161
2162impl MarketDataStreamHealth {
2163 fn new(receive_at: Instant) -> Self {
2164 Self {
2165 last_receive_at: receive_at,
2166 last_venue_ts_event: None,
2167 consecutive_stale_count: 0,
2168 last_warning_at: None,
2169 last_recovery_at: None,
2170 resubscribe_attempts: 0,
2171 }
2172 }
2173
2174 fn record_receive(&mut self, receive_at: Instant, venue_ts_event: UnixNanos) {
2175 self.last_receive_at = receive_at;
2176 self.last_venue_ts_event = Some(venue_ts_event);
2177 self.consecutive_stale_count = 0;
2178 self.last_warning_at = None;
2179 self.last_recovery_at = None;
2180 self.resubscribe_attempts = 0;
2181 }
2182
2183 fn record_recovery(
2185 &mut self,
2186 recovery_at: Instant,
2187 book_recovery: bool,
2188 max_targeted_resubscribes: u32,
2189 ) -> StaleStreamAction {
2190 self.last_recovery_at = Some(recovery_at);
2191 self.last_warning_at = Some(recovery_at);
2192
2193 if book_recovery {
2195 return StaleStreamAction::Recover;
2196 }
2197
2198 if self.resubscribe_attempts < max_targeted_resubscribes {
2199 self.resubscribe_attempts += 1;
2200 return StaleStreamAction::Resubscribe;
2201 }
2202
2203 self.resubscribe_attempts = 0;
2205 StaleStreamAction::Reconnect
2206 }
2207}
2208
2209#[derive(Debug, Clone, Copy)]
2210struct StreamRecoveryConfig {
2211 cooldown: Duration,
2212 max_targeted_resubscribes: u32,
2213}
2214
2215#[derive(Debug)]
2216struct MarketDataStreamHealthMonitor {
2217 stale_receive_threshold: Duration,
2218 compact_books: bool,
2219 warning_cooldown: Duration,
2220 recovery: Option<StreamRecoveryConfig>,
2221 streams: AHashMap<MarketDataStreamKey, MarketDataStreamHealth>,
2222}
2223
2224impl MarketDataStreamHealthMonitor {
2225 fn new(stale_receive_threshold: Duration, warning_cooldown: Duration) -> Self {
2226 Self {
2227 stale_receive_threshold,
2228 compact_books: false,
2229 warning_cooldown,
2230 recovery: None,
2231 streams: AHashMap::new(),
2232 }
2233 }
2234
2235 fn with_recovery(mut self, cooldown: Duration, max_targeted_resubscribes: u32) -> Self {
2236 self.recovery = Some(StreamRecoveryConfig {
2237 cooldown,
2238 max_targeted_resubscribes,
2239 });
2240 self
2241 }
2242
2243 fn subscribe(
2244 &mut self,
2245 channel: MarketDataChannel,
2246 instrument_id: InstrumentId,
2247 receive_at: Instant,
2248 ) {
2249 self.streams.insert(
2250 (channel, instrument_id),
2251 MarketDataStreamHealth::new(receive_at),
2252 );
2253 }
2254
2255 fn unsubscribe(&mut self, channel: MarketDataChannel, instrument_id: InstrumentId) {
2256 self.streams.remove(&(channel, instrument_id));
2257 }
2258
2259 fn clear(&mut self) {
2260 self.streams.clear();
2261 }
2262
2263 fn record_receive(
2264 &mut self,
2265 channel: MarketDataChannel,
2266 instrument_id: InstrumentId,
2267 receive_at: Instant,
2268 venue_ts_event: UnixNanos,
2269 ) {
2270 if let Some(stream) = self.streams.get_mut(&(channel, instrument_id)) {
2271 stream.record_receive(receive_at, venue_ts_event);
2272 }
2273 }
2274
2275 fn check_stale(
2276 &mut self,
2277 now: Instant,
2278 wall_clock_now: UnixNanos,
2279 ) -> Vec<MarketDataStaleEvent> {
2280 let fresh_quote_instruments: AHashSet<InstrumentId> = self
2282 .streams
2283 .iter()
2284 .filter(|((channel, _), stream)| {
2285 *channel == MarketDataChannel::Quote
2286 && now.saturating_duration_since(stream.last_receive_at)
2287 < self.stale_receive_threshold
2288 })
2289 .map(|((_, instrument_id), _)| *instrument_id)
2290 .collect();
2291
2292 let delta_book_instruments: AHashSet<InstrumentId> = self
2294 .streams
2295 .keys()
2296 .filter(|(channel, _)| *channel == MarketDataChannel::Deltas)
2297 .map(|(_, instrument_id)| *instrument_id)
2298 .collect();
2299
2300 let mut events = Vec::new();
2301
2302 for ((channel, instrument_id), stream) in &mut self.streams {
2303 let receive_age = now.saturating_duration_since(stream.last_receive_at);
2304 if receive_age < self.stale_receive_threshold {
2305 stream.consecutive_stale_count = 0;
2306 continue;
2307 }
2308
2309 stream.consecutive_stale_count = stream.consecutive_stale_count.saturating_add(1);
2310
2311 let quote_is_fresh = matches!(
2312 channel,
2313 MarketDataChannel::Deltas | MarketDataChannel::Depth
2314 ) && fresh_quote_instruments.contains(instrument_id);
2315
2316 let venue_age = stream.last_venue_ts_event.map(|ts_event| {
2317 Duration::from_nanos(wall_clock_now.as_u64().saturating_sub(ts_event.as_u64()))
2318 });
2319
2320 if let Some(recovery) = self.recovery {
2321 let stale_since = stream.last_receive_at + self.stale_receive_threshold;
2323 let anchor = stream.last_recovery_at.unwrap_or(stale_since);
2324
2325 if stream.last_warning_at.is_some()
2326 && now.saturating_duration_since(anchor) >= recovery.cooldown
2327 {
2328 let book_recovery = matches!(
2329 channel,
2330 MarketDataChannel::Deltas | MarketDataChannel::Depth
2331 ) && (self.compact_books
2332 || delta_book_instruments.contains(instrument_id));
2333 let action = stream.record_recovery(
2334 now,
2335 book_recovery,
2336 recovery.max_targeted_resubscribes,
2337 );
2338
2339 events.push(MarketDataStaleEvent {
2340 channel: *channel,
2341 instrument_id: *instrument_id,
2342 receive_age,
2343 venue_age,
2344 stale_count: stream.consecutive_stale_count,
2345 action,
2346 cooldown: recovery.cooldown,
2347 quote_is_fresh,
2348 });
2349 continue;
2350 }
2351 }
2352
2353 let should_warn = stream.last_warning_at.is_none_or(|last_warning_at| {
2354 now.saturating_duration_since(last_warning_at) >= self.warning_cooldown
2355 });
2356
2357 if !should_warn {
2358 continue;
2359 }
2360
2361 stream.last_warning_at = Some(now);
2362 events.push(MarketDataStaleEvent {
2363 channel: *channel,
2364 instrument_id: *instrument_id,
2365 receive_age,
2366 venue_age,
2367 stale_count: stream.consecutive_stale_count,
2368 action: StaleStreamAction::Warn,
2369 cooldown: self.warning_cooldown,
2370 quote_is_fresh,
2371 });
2372 }
2373
2374 events
2375 }
2376}
2377
2378#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2379enum StaleStreamAction {
2380 Warn,
2381 Recover,
2382 Resubscribe,
2383 Reconnect,
2384}
2385
2386impl StaleStreamAction {
2387 const fn as_str(self) -> &'static str {
2388 match self {
2389 Self::Warn => "warn",
2390 Self::Recover => "recover",
2391 Self::Resubscribe => "resubscribe",
2392 Self::Reconnect => "reconnect",
2393 }
2394 }
2395}
2396
2397#[derive(Debug, Clone, PartialEq, Eq)]
2398struct MarketDataStaleEvent {
2399 channel: MarketDataChannel,
2400 instrument_id: InstrumentId,
2401 receive_age: Duration,
2402 venue_age: Option<Duration>,
2403 stale_count: u32,
2404 action: StaleStreamAction,
2405 cooldown: Duration,
2406 quote_is_fresh: bool,
2407}
2408
2409fn stream_health_update(
2410 msg: &NautilusWsMessage,
2411) -> Option<(MarketDataChannel, InstrumentId, UnixNanos)> {
2412 match msg {
2413 NautilusWsMessage::Quote(quote) => Some((
2414 MarketDataChannel::Quote,
2415 quote.instrument_id,
2416 quote.ts_event,
2417 )),
2418 NautilusWsMessage::Deltas(deltas) => Some((
2419 MarketDataChannel::Deltas,
2420 deltas.instrument_id,
2421 deltas.ts_event,
2422 )),
2423 NautilusWsMessage::Depth(depth) => Some((
2424 MarketDataChannel::Depth,
2425 depth.instrument_id,
2426 depth.ts_event,
2427 )),
2428 _ => None,
2429 }
2430}
2431
2432fn record_stream_receive(
2433 stream_health: &Arc<Mutex<MarketDataStreamHealthMonitor>>,
2434 channel: MarketDataChannel,
2435 instrument_id: InstrumentId,
2436 venue_ts_event: UnixNanos,
2437) {
2438 stream_health
2439 .lock()
2440 .record_receive(channel, instrument_id, Instant::now(), venue_ts_event);
2441}
2442
2443fn log_stream_health_event(event: &MarketDataStaleEvent) {
2444 let venue_age_ms = event
2445 .venue_age
2446 .map_or_else(|| "n/a".to_string(), |age| age.as_millis().to_string());
2447 let prefix = if event.quote_is_fresh {
2448 "Hyperliquid book stream stale while bbo advances"
2449 } else {
2450 "Hyperliquid market data stream stale"
2451 };
2452
2453 log::warn!(
2454 "{prefix}: channel={}, instrument_id={}, receive_age_ms={}, venue_age_ms={}, \
2455 stale_count={}, action={}, cooldown_secs={}",
2456 event.channel.as_str(),
2457 event.instrument_id,
2458 event.receive_age.as_millis(),
2459 venue_age_ms,
2460 event.stale_count,
2461 event.action.as_str(),
2462 event.cooldown.as_secs(),
2463 );
2464}
2465
2466async fn handle_stream_health_events(
2467 ws_client: &HyperliquidWebSocketClient,
2468 events: &[MarketDataStaleEvent],
2469 book_sync: &BookSyncTracker,
2470 snapshot_timeout: Duration,
2471 book_tasks: &TaskSpawner,
2472) {
2473 let mut resubscribed_books: AHashSet<InstrumentId> = AHashSet::new();
2475 let mut reconnect_requested = false;
2476
2477 for event in events {
2478 log_stream_health_event(event);
2479
2480 match event.action {
2481 StaleStreamAction::Warn => {}
2482 StaleStreamAction::Recover => book::recovery::start_recovery(
2483 event.instrument_id,
2484 book_sync,
2485 ws_client,
2486 snapshot_timeout,
2487 book_tasks,
2488 ),
2489 StaleStreamAction::Resubscribe => match event.channel {
2490 MarketDataChannel::Deltas | MarketDataChannel::Depth => {
2491 if resubscribed_books.insert(event.instrument_id)
2492 && let Err(e) = ws_client.resubscribe_book(event.instrument_id).await
2493 {
2494 log::warn!(
2495 "Failed targeted l2Book resubscribe for {}: {e}",
2496 event.instrument_id,
2497 );
2498 }
2499 }
2500 MarketDataChannel::Quote => {
2501 if let Err(e) = ws_client.resubscribe_quotes(event.instrument_id).await {
2502 log::warn!(
2503 "Failed targeted bbo resubscribe for {}: {e}",
2504 event.instrument_id,
2505 );
2506 }
2507 }
2508 },
2509 StaleStreamAction::Reconnect => reconnect_requested = true,
2510 }
2511 }
2512
2513 if reconnect_requested {
2514 if ws_client.request_reconnect() {
2515 log::warn!("Requested full WebSocket reconnect after failed targeted stream recovery");
2516 } else {
2517 log::debug!("Skipping reconnect request: connection not active");
2518 }
2519 }
2520}
2521
2522fn filter_recent_trades(
2529 trades: Vec<TradeTick>,
2530 start: Option<UnixNanos>,
2531 end: Option<UnixNanos>,
2532 limit: Option<usize>,
2533 instrument_id: InstrumentId,
2534) -> Vec<TradeTick> {
2535 let Some(floor) = trades.first().map(|trade| trade.ts_event) else {
2536 return Vec::new();
2537 };
2538
2539 if let Some(end) = end
2540 && end < floor
2541 {
2542 log::warn!(
2543 "Recent trades for {instrument_id} are entirely older than the requested window; \
2544 snapshot only covers back to {}",
2545 unix_nanos_to_iso8601(floor),
2546 );
2547 return Vec::new();
2548 }
2549
2550 if let Some(start) = start
2551 && start < floor
2552 {
2553 log::warn!(
2554 "Recent trades for {instrument_id} only cover back to {}; \
2555 the requested start is earlier and cannot be served",
2556 unix_nanos_to_iso8601(floor),
2557 );
2558 }
2559
2560 let mut filtered: Vec<TradeTick> = trades
2561 .into_iter()
2562 .filter(|trade| start.is_none_or(|s| trade.ts_event >= s))
2563 .filter(|trade| end.is_none_or(|e| trade.ts_event <= e))
2564 .collect();
2565
2566 if let Some(limit) = limit
2567 && filtered.len() > limit
2568 {
2569 filtered.drain(0..filtered.len() - limit);
2571 }
2572
2573 filtered
2574}
2575
2576pub(crate) fn parse_l2_book_snapshot(
2580 l2_book: &HyperliquidL2Book,
2581 instrument_id: InstrumentId,
2582 price_precision: u8,
2583 size_precision: u8,
2584 depth: Option<usize>,
2585) -> OrderBook {
2586 let mut book = OrderBook::new(instrument_id, BookType::L2_MBP);
2587 let ts_event = UnixNanos::from(l2_book.time * 1_000_000);
2588
2589 let all_bids = l2_book
2590 .levels
2591 .first()
2592 .map_or([].as_slice(), |v| v.as_slice());
2593 let all_asks = l2_book
2594 .levels
2595 .get(1)
2596 .map_or([].as_slice(), |v| v.as_slice());
2597
2598 let bids = match depth {
2599 Some(d) if d < all_bids.len() => &all_bids[..d],
2600 _ => all_bids,
2601 };
2602 let asks = match depth {
2603 Some(d) if d < all_asks.len() => &all_asks[..d],
2604 _ => all_asks,
2605 };
2606
2607 for (i, level) in bids.iter().enumerate() {
2608 if level.sz <= Decimal::ZERO {
2609 continue;
2610 }
2611 let Ok(price) = Price::from_decimal_dp(level.px, price_precision) else {
2612 continue;
2613 };
2614 let Ok(size) = Quantity::from_decimal_dp(level.sz, size_precision) else {
2615 continue;
2616 };
2617
2618 let order = BookOrder::new(OrderSide::Buy, price, size, i as u64);
2619 book.add(order, 0, i as u64, ts_event);
2620 }
2621
2622 let bids_len = bids.len();
2623
2624 for (i, level) in asks.iter().enumerate() {
2625 if level.sz <= Decimal::ZERO {
2626 continue;
2627 }
2628 let Ok(price) = Price::from_decimal_dp(level.px, price_precision) else {
2629 continue;
2630 };
2631 let Ok(size) = Quantity::from_decimal_dp(level.sz, size_precision) else {
2632 continue;
2633 };
2634
2635 let order = BookOrder::new(OrderSide::Sell, price, size, (bids_len + i) as u64);
2636 book.add(order, 0, (bids_len + i) as u64, ts_event);
2637 }
2638
2639 log::debug!(
2640 "Built order book for {instrument_id} with {} bids and {} asks",
2641 bids.len(),
2642 asks.len(),
2643 );
2644
2645 book
2646}
2647
2648pub(crate) fn parse_book_precision_params(
2651 params: Option<&Params>,
2652) -> anyhow::Result<(Option<u32>, Option<u32>)> {
2653 let Some(params) = params else {
2654 return Ok((None, None));
2655 };
2656
2657 let read_u32 = |key: &str| -> anyhow::Result<Option<u32>> {
2658 match params.get(key) {
2659 None => Ok(None),
2660 Some(v) => v
2661 .as_u64()
2662 .and_then(|n| u32::try_from(n).ok())
2663 .ok_or_else(|| anyhow::anyhow!("`{key}` must be a positive u32"))
2664 .map(Some),
2665 }
2666 };
2667
2668 Ok((read_u32("n_sig_figs")?, read_u32("mantissa")?))
2669}
2670
2671pub(crate) fn funding_entry_to_update(
2674 entry: &HyperliquidFundingHistoryEntry,
2675 instrument_id: InstrumentId,
2676) -> FundingRateUpdate {
2677 let rate = entry.funding_rate;
2678 let ts = UnixNanos::from(entry.time * 1_000_000);
2679 FundingRateUpdate::new(instrument_id, rate, Some(60), None, ts, ts)
2680}
2681
2682pub(crate) fn candle_to_bar(
2683 candle: &HyperliquidCandle,
2684 bar_type: BarType,
2685 price_precision: u8,
2686 size_precision: u8,
2687) -> anyhow::Result<Bar> {
2688 let ts_event = millis_to_nanos(candle.timestamp)?;
2689 let close_boundary = candle
2690 .end_timestamp
2691 .checked_add(1)
2692 .context("candle close boundary overflow")?;
2693 let ts_init = millis_to_nanos(close_boundary)?;
2694
2695 let open = Price::from_decimal_dp(candle.open, price_precision)
2696 .map_err(|e| anyhow::anyhow!("invalid open price: {e}"))?;
2697 let high = Price::from_decimal_dp(candle.high, price_precision)
2698 .map_err(|e| anyhow::anyhow!("invalid high price: {e}"))?;
2699 let low = Price::from_decimal_dp(candle.low, price_precision)
2700 .map_err(|e| anyhow::anyhow!("invalid low price: {e}"))?;
2701 let close = Price::from_decimal_dp(candle.close, price_precision)
2702 .map_err(|e| anyhow::anyhow!("invalid close price: {e}"))?;
2703 let volume = Quantity::from_decimal_dp(candle.volume, size_precision)
2704 .map_err(|e| anyhow::anyhow!("invalid volume: {e}"))?;
2705
2706 Ok(Bar::new(
2707 bar_type, open, high, low, close, volume, ts_event, ts_init,
2708 ))
2709}
2710
2711async fn request_bars_from_http(
2713 http_client: HyperliquidHttpClient,
2714 bar_type: BarType,
2715 start: Option<Timestamp>,
2716 end: Option<Timestamp>,
2717 limit: Option<u32>,
2718 instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
2719) -> anyhow::Result<Vec<Bar>> {
2720 let instrument_id = bar_type.instrument_id();
2722 let instrument = instruments
2723 .load()
2724 .get(&instrument_id)
2725 .cloned()
2726 .context("instrument not found in cache")?;
2727
2728 let price_precision = instrument.price_precision();
2729 let size_precision = instrument.size_precision();
2730 let raw_symbol = instrument.raw_symbol();
2731 let coin = raw_symbol.as_str();
2732
2733 let interval = bar_type_to_interval(&bar_type)?;
2734
2735 let now = Timestamp::now();
2737 let end_time = end.unwrap_or(now).as_millisecond() as u64;
2738 let start_time = if let Some(start) = start {
2739 start.as_millisecond() as u64
2740 } else {
2741 let spec = bar_type.spec();
2743 let step_ms = match spec.aggregation {
2744 BarAggregation::Minute => spec.step.get() as u64 * 60_000,
2745 BarAggregation::Hour => spec.step.get() as u64 * 3_600_000,
2746 BarAggregation::Day => spec.step.get() as u64 * 86_400_000,
2747 _ => 60_000,
2748 };
2749 end_time.saturating_sub(1000 * step_ms)
2750 };
2751
2752 let candles = http_client
2753 .info_candle_snapshot(coin, interval, start_time, end_time)
2754 .await
2755 .context("failed to fetch candle snapshot from Hyperliquid")?;
2756
2757 let now_ms = now.as_millisecond() as u64;
2758 let mut bars: Vec<Bar> = candles
2759 .iter()
2760 .filter(|candle| candle.end_timestamp < now_ms)
2761 .filter_map(|candle| {
2762 candle_to_bar(candle, bar_type, price_precision, size_precision)
2763 .map_err(|e| {
2764 log::warn!("Failed to convert candle to bar: {e}");
2765 e
2766 })
2767 .ok()
2768 })
2769 .collect();
2770
2771 if let Some(limit) = limit
2772 && bars.len() > limit as usize
2773 {
2774 bars = bars.into_iter().take(limit as usize).collect();
2775 }
2776
2777 log::debug!("Fetched {} bars for {}", bars.len(), bar_type);
2778 Ok(bars)
2779}
2780
2781#[cfg(test)]
2782#[path = "../tests/integration/data_client.rs"]
2783mod data_client_tests;
2784
2785#[cfg(test)]
2786mod tests {
2787 use futures_util::StreamExt;
2788 use nautilus_common::live::runner::set_data_event_sender;
2789 use nautilus_model::{
2790 data::{
2791 QuoteTick,
2792 stubs::{stub_deltas, stub_depth10},
2793 },
2794 enums::{AggressorSide, CurrencyType},
2795 identifiers::{Symbol, TradeId},
2796 instruments::CryptoPerpetual,
2797 types::Currency,
2798 };
2799 use rstest::rstest;
2800 use rust_decimal_macros::dec;
2801 use ustr::Ustr;
2802
2803 use super::*;
2804 use crate::common::{
2805 consts::{HYPERLIQUID_CLIENT_ID, HYPERLIQUID_WS_URL},
2806 testing::load_test_data,
2807 };
2808
2809 fn btc_perp_id() -> InstrumentId {
2810 InstrumentId::from("BTC-PERP.HYPERLIQUID")
2811 }
2812
2813 #[rstest]
2814 fn test_candle_to_bar_uses_causal_initialization_timestamp() {
2815 let candle = HyperliquidCandle {
2816 timestamp: 1_700_000_000_000,
2817 end_timestamp: 1_700_000_059_999,
2818 open: dec!(100.0),
2819 high: dec!(101.0),
2820 low: dec!(99.0),
2821 close: dec!(100.5),
2822 volume: dec!(10.0),
2823 num_trades: Some(42),
2824 };
2825 let bar_type = BarType::from("BTC-USD-PERP.HYPERLIQUID-1-MINUTE-LAST-EXTERNAL");
2826
2827 let bar = candle_to_bar(&candle, bar_type, 1, 1).unwrap();
2828
2829 assert_eq!(candle.end_timestamp - candle.timestamp, 59_999);
2830 assert_eq!(bar.ts_event, millis_to_nanos(candle.timestamp).unwrap());
2831 assert_eq!(
2832 bar.ts_init,
2833 millis_to_nanos(candle.end_timestamp + 1).unwrap()
2834 );
2835 assert!(bar.ts_init > bar.ts_event);
2836 }
2837
2838 #[rstest]
2839 fn test_candle_to_bar_rejects_close_boundary_overflow() {
2840 let candle = HyperliquidCandle {
2841 timestamp: 1_700_000_000_000,
2842 end_timestamp: u64::MAX,
2843 open: dec!(100.0),
2844 high: dec!(101.0),
2845 low: dec!(99.0),
2846 close: dec!(100.5),
2847 volume: dec!(10.0),
2848 num_trades: Some(42),
2849 };
2850 let bar_type = BarType::from("BTC-USD-PERP.HYPERLIQUID-1-MINUTE-LAST-EXTERNAL");
2851
2852 let err = candle_to_bar(&candle, bar_type, 1, 1).unwrap_err();
2853
2854 assert!(err.to_string().contains("close boundary overflow"));
2855 }
2856
2857 #[rstest]
2858 fn test_stream_health_monitor_fresh_stream_does_not_warn() {
2859 let mut monitor =
2860 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2861 let instrument_id = btc_perp_id();
2862 let start = Instant::now();
2863
2864 monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2865
2866 let warnings = monitor.check_stale(
2867 start + Duration::from_secs(4),
2868 UnixNanos::from(4_000_000_000),
2869 );
2870 assert!(warnings.is_empty());
2871 }
2872
2873 #[rstest]
2874 fn test_stream_health_monitor_warns_once_after_threshold() {
2875 let mut monitor =
2876 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2877 let instrument_id = btc_perp_id();
2878 let start = Instant::now();
2879
2880 monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2881 monitor.record_receive(
2882 MarketDataChannel::Quote,
2883 instrument_id,
2884 start + Duration::from_secs(1),
2885 UnixNanos::from(1_000_000_000),
2886 );
2887
2888 let warnings = monitor.check_stale(
2889 start + Duration::from_secs(7),
2890 UnixNanos::from(9_000_000_000),
2891 );
2892
2893 assert_eq!(
2894 warnings,
2895 vec![MarketDataStaleEvent {
2896 channel: MarketDataChannel::Quote,
2897 instrument_id,
2898 receive_age: Duration::from_secs(6),
2899 venue_age: Some(Duration::from_secs(8)),
2900 stale_count: 1,
2901 action: StaleStreamAction::Warn,
2902 cooldown: Duration::from_secs(30),
2903 quote_is_fresh: false,
2904 }]
2905 );
2906 }
2907
2908 #[rstest]
2909 fn test_stream_health_monitor_warns_at_receive_threshold() {
2910 let mut monitor =
2911 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2912 let instrument_id = btc_perp_id();
2913 let start = Instant::now();
2914
2915 monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2916
2917 let warnings = monitor.check_stale(
2918 start + Duration::from_secs(5),
2919 UnixNanos::from(5_000_000_000),
2920 );
2921
2922 assert_eq!(warnings.len(), 1);
2923 assert_eq!(warnings[0].receive_age, Duration::from_secs(5));
2924 assert_eq!(warnings[0].stale_count, 1);
2925 }
2926
2927 #[rstest]
2928 fn test_stream_health_monitor_new_update_resets_age_and_stale_count() {
2929 let mut monitor =
2930 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2931 let instrument_id = btc_perp_id();
2932 let start = Instant::now();
2933
2934 monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
2935 assert_eq!(
2936 monitor
2937 .check_stale(
2938 start + Duration::from_secs(6),
2939 UnixNanos::from(6_000_000_000),
2940 )
2941 .len(),
2942 1,
2943 );
2944
2945 monitor.record_receive(
2946 MarketDataChannel::Depth,
2947 instrument_id,
2948 start + Duration::from_secs(7),
2949 UnixNanos::from(7_000_000_000),
2950 );
2951
2952 assert!(
2953 monitor
2954 .check_stale(
2955 start + Duration::from_secs(11),
2956 UnixNanos::from(11_000_000_000),
2957 )
2958 .is_empty()
2959 );
2960
2961 let warnings = monitor.check_stale(
2962 start + Duration::from_secs(13),
2963 UnixNanos::from(13_000_000_000),
2964 );
2965 assert_eq!(warnings.len(), 1);
2966 assert_eq!(warnings[0].stale_count, 1);
2967 assert_eq!(warnings[0].receive_age, Duration::from_secs(6));
2968 }
2969
2970 #[rstest]
2971 fn test_stream_health_monitor_unsubscribe_removes_stream() {
2972 let mut monitor =
2973 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2974 let instrument_id = btc_perp_id();
2975 let start = Instant::now();
2976
2977 monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2978 monitor.unsubscribe(MarketDataChannel::Deltas, instrument_id);
2979
2980 let warnings = monitor.check_stale(
2981 start + Duration::from_secs(6),
2982 UnixNanos::from(6_000_000_000),
2983 );
2984
2985 assert!(warnings.is_empty());
2986 }
2987
2988 #[rstest]
2989 #[case(0, 15)]
2990 #[case(120, 0)]
2991 fn test_data_client_stream_health_config_zero_disables_monitor(
2992 #[case] stale_receive_timeout_secs: u64,
2993 #[case] check_interval_secs: u64,
2994 ) {
2995 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
2996 set_data_event_sender(tx);
2997 let client = HyperliquidDataClient::new(
2998 *crate::common::consts::HYPERLIQUID_CLIENT_ID,
2999 HyperliquidDataClientConfig {
3000 stale_stream_receive_timeout_secs: stale_receive_timeout_secs,
3001 stream_health_check_interval_secs: check_interval_secs,
3002 ..HyperliquidDataClientConfig::default()
3003 },
3004 )
3005 .unwrap();
3006 let instrument_id = btc_perp_id();
3007 let start = Instant::now();
3008
3009 assert!(!client.stream_health_monitor_enabled());
3010 client.register_stream_health(MarketDataChannel::Deltas, instrument_id);
3011
3012 let warnings = client.stream_health.lock().check_stale(
3013 start + Duration::from_secs(121),
3014 UnixNanos::from(121_000_000_000),
3015 );
3016
3017 assert!(warnings.is_empty());
3018 }
3019
3020 #[rstest]
3021 fn test_data_client_recovery_requires_positive_cooldown() {
3022 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
3023 set_data_event_sender(tx);
3024 let client = HyperliquidDataClient::new(
3025 *crate::common::consts::HYPERLIQUID_CLIENT_ID,
3026 HyperliquidDataClientConfig {
3027 stale_stream_recovery_enabled: true,
3028 stale_stream_recovery_cooldown_secs: 0,
3029 ..HyperliquidDataClientConfig::default()
3030 },
3031 )
3032 .unwrap();
3033
3034 assert!(
3035 client.stream_health.lock().recovery.is_none(),
3036 "a zero recovery cooldown must leave the monitor observability-only",
3037 );
3038 }
3039
3040 #[rstest]
3041 fn test_stream_health_monitor_warning_cooldown_prevents_repeated_logs() {
3042 let mut monitor =
3043 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10));
3044 let instrument_id = btc_perp_id();
3045 let start = Instant::now();
3046
3047 monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
3048
3049 let first = monitor.check_stale(
3050 start + Duration::from_secs(6),
3051 UnixNanos::from(6_000_000_000),
3052 );
3053 let inside_cooldown = monitor.check_stale(
3054 start + Duration::from_secs(7),
3055 UnixNanos::from(7_000_000_000),
3056 );
3057 let second = monitor.check_stale(
3058 start + Duration::from_secs(16),
3059 UnixNanos::from(16_000_000_000),
3060 );
3061
3062 assert_eq!(first.len(), 1);
3063 assert!(inside_cooldown.is_empty());
3064 assert_eq!(second.len(), 1);
3065 assert_eq!(second[0].stale_count, 3);
3066 }
3067
3068 fn check_at(
3069 monitor: &mut MarketDataStreamHealthMonitor,
3070 start: Instant,
3071 secs: u64,
3072 ) -> Vec<MarketDataStaleEvent> {
3073 monitor.check_stale(
3074 start + Duration::from_secs(secs),
3075 UnixNanos::from(secs * 1_000_000_000),
3076 )
3077 }
3078
3079 #[rstest]
3080 fn test_stream_health_recovery_ladder_escalates_and_resets() {
3081 let mut monitor =
3082 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
3083 .with_recovery(Duration::from_secs(30), 2);
3084 let instrument_id = btc_perp_id();
3085 let start = Instant::now();
3086
3087 monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
3088
3089 let events = check_at(&mut monitor, start, 5);
3090 assert_eq!(events.len(), 1);
3091 assert_eq!(events[0].action, StaleStreamAction::Warn);
3092
3093 let events = check_at(&mut monitor, start, 20);
3094 assert_eq!(events[0].action, StaleStreamAction::Warn);
3095
3096 let events = check_at(&mut monitor, start, 35);
3097 assert_eq!(
3098 events,
3099 vec![MarketDataStaleEvent {
3100 channel: MarketDataChannel::Depth,
3101 instrument_id,
3102 receive_age: Duration::from_secs(35),
3103 venue_age: None,
3104 stale_count: 3,
3105 action: StaleStreamAction::Resubscribe,
3106 cooldown: Duration::from_secs(30),
3107 quote_is_fresh: false,
3108 }],
3109 );
3110
3111 let events = check_at(&mut monitor, start, 50);
3112 assert_eq!(events[0].action, StaleStreamAction::Warn);
3113
3114 let events = check_at(&mut monitor, start, 65);
3115 assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
3116
3117 let events = check_at(&mut monitor, start, 95);
3118 assert_eq!(events[0].action, StaleStreamAction::Reconnect);
3119
3120 let events = check_at(&mut monitor, start, 125);
3121 assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
3122 }
3123
3124 #[rstest]
3125 fn test_stream_health_recovery_hands_deltas_to_book_recovery_without_reconnect() {
3126 let mut monitor =
3127 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
3128 .with_recovery(Duration::from_secs(30), 1);
3129 let instrument_id = btc_perp_id();
3130 let start = Instant::now();
3131
3132 monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
3133
3134 let first = check_at(&mut monitor, start, 5);
3135 let second = check_at(&mut monitor, start, 35);
3136 let third = check_at(&mut monitor, start, 65);
3137 let fourth = check_at(&mut monitor, start, 95);
3138
3139 assert_eq!(first[0].action, StaleStreamAction::Warn);
3140 assert_eq!(
3141 second,
3142 vec![MarketDataStaleEvent {
3143 channel: MarketDataChannel::Deltas,
3144 instrument_id,
3145 receive_age: Duration::from_secs(35),
3146 venue_age: None,
3147 stale_count: 2,
3148 action: StaleStreamAction::Recover,
3149 cooldown: Duration::from_secs(30),
3150 quote_is_fresh: false,
3151 }],
3152 );
3153 assert_eq!(third[0].action, StaleStreamAction::Recover);
3154 assert_eq!(fourth[0].action, StaleStreamAction::Recover);
3155 }
3156
3157 #[rstest]
3158 fn test_stream_health_recovery_hands_shared_depth_to_book_recovery() {
3159 let mut monitor =
3160 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
3161 .with_recovery(Duration::from_secs(30), 1);
3162 let shared = btc_perp_id();
3163 let depth_only = InstrumentId::from("ETH-PERP.HYPERLIQUID");
3164 let start = Instant::now();
3165
3166 monitor.subscribe(MarketDataChannel::Deltas, shared, start);
3167 monitor.subscribe(MarketDataChannel::Depth, shared, start);
3168 monitor.subscribe(MarketDataChannel::Depth, depth_only, start);
3169
3170 let actions = |events: Vec<MarketDataStaleEvent>| {
3171 let mut actions = events
3172 .into_iter()
3173 .map(|event| (event.channel.as_str(), event.instrument_id, event.action))
3174 .collect::<Vec<_>>();
3175 actions.sort_by_key(|(channel, instrument_id, _)| (*channel, *instrument_id));
3176 actions
3177 };
3178
3179 check_at(&mut monitor, start, 5);
3180 let first = actions(check_at(&mut monitor, start, 35));
3181 let second = actions(check_at(&mut monitor, start, 65));
3182
3183 assert_eq!(
3184 first,
3185 vec![
3186 ("deltas", shared, StaleStreamAction::Recover),
3187 ("depth", shared, StaleStreamAction::Recover),
3188 ("depth", depth_only, StaleStreamAction::Resubscribe),
3189 ],
3190 );
3191 assert_eq!(
3192 second,
3193 vec![
3194 ("deltas", shared, StaleStreamAction::Recover),
3195 ("depth", shared, StaleStreamAction::Recover),
3196 ("depth", depth_only, StaleStreamAction::Reconnect),
3197 ],
3198 );
3199 }
3200
3201 #[rstest]
3202 fn test_stream_health_recovery_first_breach_warns_even_past_cooldown() {
3203 let mut monitor =
3204 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
3205 .with_recovery(Duration::from_secs(1), 1);
3206 let instrument_id = btc_perp_id();
3207 let start = Instant::now();
3208
3209 monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
3210
3211 let events = check_at(&mut monitor, start, 40);
3213 assert_eq!(events.len(), 1);
3214 assert_eq!(events[0].action, StaleStreamAction::Warn);
3215
3216 let events = check_at(&mut monitor, start, 41);
3217 assert_eq!(events.len(), 1);
3218 assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
3219 }
3220
3221 #[rstest]
3222 fn test_stream_health_receive_resets_recovery_state() {
3223 let mut monitor =
3224 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
3225 .with_recovery(Duration::from_secs(10), 1);
3226 let instrument_id = btc_perp_id();
3227 let start = Instant::now();
3228
3229 monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
3230 assert_eq!(
3231 check_at(&mut monitor, start, 5)[0].action,
3232 StaleStreamAction::Warn
3233 );
3234 assert_eq!(
3235 check_at(&mut monitor, start, 15)[0].action,
3236 StaleStreamAction::Resubscribe,
3237 );
3238
3239 monitor.record_receive(
3240 MarketDataChannel::Depth,
3241 instrument_id,
3242 start + Duration::from_secs(16),
3243 UnixNanos::from(16_000_000_000),
3244 );
3245
3246 assert!(check_at(&mut monitor, start, 20).is_empty());
3247
3248 let events = check_at(&mut monitor, start, 21);
3249 assert_eq!(events[0].action, StaleStreamAction::Warn);
3250 assert_eq!(events[0].stale_count, 1);
3251
3252 assert_eq!(
3253 check_at(&mut monitor, start, 31)[0].action,
3254 StaleStreamAction::Resubscribe,
3255 );
3256 assert_eq!(
3257 check_at(&mut monitor, start, 41)[0].action,
3258 StaleStreamAction::Reconnect,
3259 );
3260 }
3261
3262 #[rstest]
3263 fn test_check_stale_book_with_fresh_quote_flags_relative_staleness() {
3264 let mut monitor =
3265 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
3266 let instrument_id = btc_perp_id();
3267 let start = Instant::now();
3268
3269 monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
3270 monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
3271 monitor.record_receive(
3272 MarketDataChannel::Quote,
3273 instrument_id,
3274 start + Duration::from_secs(8),
3275 UnixNanos::from(8_000_000_000),
3276 );
3277
3278 let events = check_at(&mut monitor, start, 10);
3279
3280 assert_eq!(events.len(), 1, "fresh quote stream must not be reported");
3281 assert_eq!(events[0].channel, MarketDataChannel::Deltas);
3282 assert!(events[0].quote_is_fresh);
3283 }
3284
3285 #[rstest]
3286 #[case(true)]
3287 #[case(false)]
3288 fn test_check_stale_book_without_fresh_quote_is_not_flagged(#[case] quote_subscribed: bool) {
3289 let mut monitor =
3290 MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
3291 let instrument_id = btc_perp_id();
3292 let start = Instant::now();
3293
3294 monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
3295 if quote_subscribed {
3296 monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
3297 }
3298
3299 let events = check_at(&mut monitor, start, 10);
3300
3301 let deltas_event = events
3302 .iter()
3303 .find(|event| event.channel == MarketDataChannel::Deltas)
3304 .expect("deltas event");
3305 assert!(
3306 !deltas_event.quote_is_fresh,
3307 "a stale or absent quote stream must not flag relative staleness",
3308 );
3309
3310 if quote_subscribed {
3311 let quote_event = events
3312 .iter()
3313 .find(|event| event.channel == MarketDataChannel::Quote)
3314 .expect("quote event");
3315 assert!(!quote_event.quote_is_fresh);
3316 }
3317 }
3318
3319 #[rstest]
3320 fn test_stream_health_update_extracts_tracked_market_data_messages() {
3321 let quote = QuoteTick {
3322 instrument_id: btc_perp_id(),
3323 ts_event: UnixNanos::from(1),
3324 ..QuoteTick::default()
3325 };
3326 let deltas = stub_deltas();
3327 let depth = stub_depth10();
3328
3329 assert_eq!(
3330 stream_health_update(&NautilusWsMessage::Quote(quote)),
3331 Some((
3332 MarketDataChannel::Quote,
3333 quote.instrument_id,
3334 quote.ts_event
3335 )),
3336 );
3337 assert_eq!(
3338 stream_health_update(&NautilusWsMessage::Deltas(deltas.clone())),
3339 Some((
3340 MarketDataChannel::Deltas,
3341 deltas.instrument_id,
3342 deltas.ts_event
3343 )),
3344 );
3345 assert_eq!(
3346 stream_health_update(&NautilusWsMessage::Depth(Box::new(depth.clone()))),
3347 Some((
3348 MarketDataChannel::Depth,
3349 depth.instrument_id,
3350 depth.ts_event
3351 )),
3352 );
3353 assert_eq!(stream_health_update(&NautilusWsMessage::Reconnected), None,);
3354 }
3355
3356 #[rstest]
3357 fn test_funding_entry_to_update_parses_positive_rate() {
3358 let entry = HyperliquidFundingHistoryEntry {
3359 coin: Ustr::from("BTC"),
3360 funding_rate: dec!(0.0000125),
3361 premium: Some(dec!(0.00029005)),
3362 time: 1769908800000,
3363 };
3364 let instrument_id = btc_perp_id();
3365
3366 let update = funding_entry_to_update(&entry, instrument_id);
3367
3368 assert_eq!(update.instrument_id, instrument_id);
3369 assert_eq!(update.rate, dec!(0.0000125));
3370 assert_eq!(update.interval, Some(60));
3371 assert!(update.next_funding_ns.is_none());
3372 assert_eq!(update.ts_event, UnixNanos::from(1769908800000 * 1_000_000));
3373 assert_eq!(update.ts_init, update.ts_event);
3374 }
3375
3376 #[rstest]
3377 fn test_funding_entry_to_update_handles_negative_rate() {
3378 let entry = HyperliquidFundingHistoryEntry {
3379 coin: Ustr::from("BTC"),
3380 funding_rate: dec!(-0.0000081),
3381 premium: None,
3382 time: 1769912400000,
3383 };
3384 let update = funding_entry_to_update(&entry, btc_perp_id());
3385 assert_eq!(update.rate, dec!(-0.0000081));
3386 }
3387
3388 #[rstest]
3389 fn test_funding_history_entry_rejects_invalid_rate() {
3390 let json = r#"{"coin":"BTC","fundingRate":"not-a-number","time":1769912400000}"#;
3393 assert!(serde_json::from_str::<HyperliquidFundingHistoryEntry>(json).is_err());
3394 }
3395
3396 #[rstest]
3397 fn test_parse_book_precision_params_none() {
3398 let (n, m) = parse_book_precision_params(None).unwrap();
3399 assert_eq!(n, None);
3400 assert_eq!(m, None);
3401 }
3402
3403 fn make_params(json: serde_json::Value) -> Params {
3404 serde_json::from_value(json).expect("valid params payload")
3405 }
3406
3407 #[rstest]
3408 fn test_parse_book_precision_params_only_n_sig_figs() {
3409 let params = make_params(serde_json::json!({"n_sig_figs": 4}));
3410 let (n, m) = parse_book_precision_params(Some(¶ms)).unwrap();
3411 assert_eq!(n, Some(4));
3412 assert_eq!(m, None);
3413 }
3414
3415 #[rstest]
3416 fn test_parse_book_precision_params_both() {
3417 let params = make_params(serde_json::json!({"n_sig_figs": 5, "mantissa": 2}));
3418 let (n, m) = parse_book_precision_params(Some(¶ms)).unwrap();
3419 assert_eq!(n, Some(5));
3420 assert_eq!(m, Some(2));
3421 }
3422
3423 #[rstest]
3424 fn test_parse_book_precision_params_rejects_negative() {
3425 let params = make_params(serde_json::json!({"n_sig_figs": -1}));
3426 let err = parse_book_precision_params(Some(¶ms)).unwrap_err();
3427 assert!(err.to_string().contains("n_sig_figs"));
3428 }
3429
3430 #[rstest]
3431 fn test_funding_history_fixture_parses() {
3432 let entries: Vec<HyperliquidFundingHistoryEntry> =
3433 load_test_data("http_funding_history.json");
3434 assert_eq!(entries.len(), 3);
3435 assert_eq!(entries[0].coin, "BTC");
3436 assert_eq!(entries[0].funding_rate, dec!(0.0000125));
3437 assert_eq!(entries[0].premium, Some(dec!(0.00029005)));
3438 assert!(entries[2].premium.is_none());
3439
3440 let updates: Vec<FundingRateUpdate> = entries
3441 .iter()
3442 .map(|e| funding_entry_to_update(e, btc_perp_id()))
3443 .collect();
3444 assert_eq!(updates.len(), 3);
3445 assert_eq!(updates[0].rate, dec!(0.0000125));
3446 assert_eq!(updates[1].rate, dec!(-0.0000081));
3447 assert_eq!(updates[2].rate, dec!(0.0000033));
3448 }
3449
3450 fn level(px: &str, sz: &str) -> crate::http::models::HyperliquidLevel {
3451 crate::http::models::HyperliquidLevel {
3452 px: px.parse().unwrap(),
3453 sz: sz.parse().unwrap(),
3454 }
3455 }
3456
3457 fn sample_l2_book() -> HyperliquidL2Book {
3458 HyperliquidL2Book {
3459 coin: Ustr::from("BTC"),
3460 levels: vec![
3461 vec![
3462 level("98450.50", "2.5"),
3463 level("98449.00", "1.2"),
3464 level("98448.00", "0.8"),
3465 ],
3466 vec![
3467 level("98451.00", "1.5"),
3468 level("98452.00", "2.0"),
3469 level("98453.00", "0.5"),
3470 ],
3471 ],
3472 time: 1769908800000,
3473 }
3474 }
3475
3476 #[rstest]
3477 fn test_parse_l2_book_snapshot_populates_both_sides() {
3478 let book_data = sample_l2_book();
3479 let instrument_id = btc_perp_id();
3480 let book = parse_l2_book_snapshot(&book_data, instrument_id, 2, 4, None);
3481
3482 assert_eq!(book.instrument_id, instrument_id);
3483 assert_eq!(book.book_type, BookType::L2_MBP);
3484 assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3485 assert_eq!(book.best_ask_price(), Some(Price::new(98451.00, 2)));
3486 assert_eq!(book.best_bid_size(), Some(Quantity::new(2.5, 4)));
3487 assert_eq!(book.best_ask_size(), Some(Quantity::new(1.5, 4)));
3488 assert_eq!(book.update_count, 6);
3489 }
3490
3491 #[rstest]
3492 fn test_parse_l2_book_snapshot_truncates_to_depth() {
3493 let book_data = sample_l2_book();
3494 let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, Some(1));
3495
3496 assert_eq!(book.update_count, 2);
3498 assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3499 assert_eq!(book.best_ask_price(), Some(Price::new(98451.00, 2)));
3500 }
3501
3502 #[rstest]
3503 fn test_parse_l2_book_snapshot_uses_venue_time_as_ts_event() {
3504 let book_data = sample_l2_book();
3505 let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3506 let expected_ts = UnixNanos::from(1769908800000_u64 * 1_000_000);
3507
3508 assert_eq!(book.ts_last, expected_ts);
3511 }
3512
3513 #[rstest]
3514 fn test_parse_l2_book_snapshot_skips_non_positive_size() {
3515 let book_data = HyperliquidL2Book {
3516 coin: Ustr::from("BTC"),
3517 levels: vec![
3518 vec![level("98450.50", "2.5"), level("98449.00", "0")],
3519 vec![level("98451.00", "0"), level("98452.00", "1.5")],
3520 ],
3521 time: 1769908800000,
3522 };
3523 let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3524
3525 assert_eq!(book.update_count, 2, "zero-sized levels must be skipped");
3526 assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3527 assert_eq!(book.best_ask_price(), Some(Price::new(98452.00, 2)));
3528 }
3529
3530 #[rstest]
3531 fn test_parse_l2_book_snapshot_skips_zero_size_levels() {
3532 let book_data = HyperliquidL2Book {
3533 coin: Ustr::from("BTC"),
3534 levels: vec![
3535 vec![level("98448.00", "0.0"), level("98449.00", "1.2")],
3536 vec![level("98451.00", "0.0"), level("98452.00", "1.5")],
3537 ],
3538 time: 1769908800000,
3539 };
3540 let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3541
3542 assert_eq!(book.update_count, 2);
3544 assert_eq!(book.best_bid_price(), Some(Price::new(98449.00, 2)));
3545 assert_eq!(book.best_ask_price(), Some(Price::new(98452.00, 2)));
3546 }
3547
3548 #[rstest]
3549 fn test_parse_l2_book_snapshot_empty_levels_yields_empty_book() {
3550 let book_data = HyperliquidL2Book {
3551 coin: Ustr::from("BTC"),
3552 levels: vec![],
3553 time: 1769908800000,
3554 };
3555 let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3556
3557 assert_eq!(book.update_count, 0);
3558 assert!(book.best_bid_price().is_none());
3559 assert!(book.best_ask_price().is_none());
3560 }
3561
3562 fn trade_at(ts_ns: u64, tid: u64) -> TradeTick {
3563 TradeTick::new(
3564 btc_perp_id(),
3565 Price::from("104300.0"),
3566 Quantity::from("0.01000"),
3567 AggressorSide::Buy,
3568 TradeId::new(tid.to_string()),
3569 UnixNanos::from(ts_ns),
3570 UnixNanos::from(ts_ns),
3571 )
3572 }
3573
3574 fn sample_trades() -> Vec<TradeTick> {
3577 vec![trade_at(1000, 1), trade_at(2000, 2), trade_at(3000, 3)]
3578 }
3579
3580 #[rstest]
3581 fn test_recent_trades_fixture_parses_and_sorts() {
3582 let raw: Vec<crate::http::models::HyperliquidRecentTrade> =
3583 load_test_data("http_recent_trades_btc.json");
3584 assert_eq!(raw.len(), 3);
3585 assert_eq!(raw[0].tid, 300003);
3587
3588 let meta: crate::http::models::PerpMeta = load_test_data("http_meta_perp_sample.json");
3589 let defs = crate::http::parse::parse_perp_instruments(&meta, 0).unwrap();
3590 let instrument =
3591 crate::http::parse::create_instrument_from_def(&defs[0], UnixNanos::default()).unwrap();
3592
3593 let mut trades: Vec<TradeTick> = raw
3594 .iter()
3595 .map(|t| parse_recent_trade(t, &instrument).unwrap())
3596 .collect();
3597 trades.sort_by_key(|trade| trade.ts_event);
3598
3599 assert_eq!(trades[0].trade_id.to_string(), "300001");
3601 assert_eq!(trades[2].trade_id.to_string(), "300003");
3602 assert!(trades[0].ts_event <= trades[2].ts_event);
3603 assert_eq!(trades[0].ts_init, trades[0].ts_event);
3605 }
3606
3607 #[rstest]
3608 fn test_filter_recent_trades_full_window_returns_all() {
3609 let filtered = filter_recent_trades(sample_trades(), None, None, None, btc_perp_id());
3610
3611 assert_eq!(filtered.len(), 3);
3612 }
3613
3614 #[rstest]
3615 fn test_filter_recent_trades_empty_snapshot_returns_empty() {
3616 let filtered = filter_recent_trades(
3617 Vec::new(),
3618 Some(UnixNanos::from(500)),
3619 Some(UnixNanos::from(2500)),
3620 None,
3621 btc_perp_id(),
3622 );
3623
3624 assert!(filtered.is_empty());
3625 }
3626
3627 #[rstest]
3628 fn test_filter_recent_trades_entirely_older_returns_empty() {
3629 let filtered = filter_recent_trades(
3631 sample_trades(),
3632 Some(UnixNanos::from(100)),
3633 Some(UnixNanos::from(500)),
3634 None,
3635 btc_perp_id(),
3636 );
3637
3638 assert!(filtered.is_empty());
3639 }
3640
3641 #[rstest]
3642 fn test_filter_recent_trades_partial_keeps_in_range_subset() {
3643 let filtered = filter_recent_trades(
3645 sample_trades(),
3646 Some(UnixNanos::from(500)),
3647 Some(UnixNanos::from(2500)),
3648 None,
3649 btc_perp_id(),
3650 );
3651
3652 let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3653 assert_eq!(ts, vec![1000, 2000]);
3654 }
3655
3656 #[rstest]
3657 fn test_filter_recent_trades_within_window_filters_bounds() {
3658 let filtered = filter_recent_trades(
3659 sample_trades(),
3660 Some(UnixNanos::from(1500)),
3661 Some(UnixNanos::from(3000)),
3662 None,
3663 btc_perp_id(),
3664 );
3665
3666 let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3667 assert_eq!(ts, vec![2000, 3000]);
3668 }
3669
3670 #[rstest]
3671 fn test_filter_recent_trades_limit_keeps_most_recent() {
3672 let filtered = filter_recent_trades(sample_trades(), None, None, Some(2), btc_perp_id());
3673
3674 let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3675 assert_eq!(ts, vec![2000, 3000]);
3676 }
3677
3678 #[rstest]
3679 fn test_filter_recent_trades_end_equal_to_floor_keeps_floor_trade() {
3680 let filtered = filter_recent_trades(
3683 sample_trades(),
3684 None,
3685 Some(UnixNanos::from(1000)),
3686 None,
3687 btc_perp_id(),
3688 );
3689
3690 let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3691 assert_eq!(ts, vec![1000]);
3692 }
3693
3694 #[rstest]
3695 fn test_filter_recent_trades_bounds_are_inclusive() {
3696 let filtered = filter_recent_trades(
3699 sample_trades(),
3700 Some(UnixNanos::from(2000)),
3701 Some(UnixNanos::from(3000)),
3702 None,
3703 btc_perp_id(),
3704 );
3705
3706 let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3707 assert_eq!(ts, vec![2000, 3000]);
3708 }
3709
3710 fn perp_instrument(symbol: &str, tick_size: &str, ts_init: UnixNanos) -> InstrumentAny {
3711 let base = Currency::new("BTC", 8, 0, "BTC", CurrencyType::Crypto);
3712 let usd = Currency::new("USD", 8, 0, "USD", CurrencyType::Crypto);
3713 let usdc = Currency::new("USDC", 6, 0, "USDC", CurrencyType::Crypto);
3714
3715 InstrumentAny::CryptoPerpetual(
3716 CryptoPerpetual::builder()
3717 .instrument_id(InstrumentId::new(Symbol::new(symbol), *HYPERLIQUID_VENUE))
3718 .raw_symbol(Symbol::new("BTC"))
3719 .base_currency(base)
3720 .quote_currency(usd)
3721 .settlement_currency(usdc)
3722 .is_inverse(false)
3723 .price_precision(1)
3724 .size_precision(3)
3725 .price_increment(Price::from(tick_size))
3726 .size_increment(Quantity::from("0.001"))
3727 .ts_event(ts_init)
3728 .ts_init(ts_init)
3729 .build()
3730 .unwrap(),
3731 )
3732 }
3733
3734 fn cached(instruments: &[InstrumentAny]) -> Arc<AtomicMap<InstrumentId, InstrumentAny>> {
3735 let map = AtomicMap::new();
3736 map.rcu(|m| {
3737 for instrument in instruments {
3738 m.insert(instrument.id(), instrument.clone());
3739 }
3740 });
3741 Arc::new(map)
3742 }
3743
3744 fn data_client_with_refresh_interval(minutes: u64) -> HyperliquidDataClient {
3745 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
3746 set_data_event_sender(tx);
3747
3748 HyperliquidDataClient::new(
3749 *HYPERLIQUID_CLIENT_ID,
3750 HyperliquidDataClientConfig {
3751 update_instruments_interval_mins: minutes,
3752 ..HyperliquidDataClientConfig::default()
3753 },
3754 )
3755 .unwrap()
3756 }
3757
3758 #[rstest]
3759 #[case::disconnect(false, None)]
3760 #[case::reset_then_disconnect(true, None)]
3761 #[ignore = "connects to Hyperliquid mainnet public market data"]
3762 #[case::mainnet(false, Some(HYPERLIQUID_WS_URL))]
3763 #[tokio::test]
3764 async fn test_ws_consumer_exits_when_handler_output_closes(
3765 #[case] reset: bool,
3766 #[case] url: Option<&str>,
3767 ) {
3768 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3769 let addr = listener.local_addr().unwrap();
3770 let server = tokio::spawn(async move {
3771 let (socket, _) = listener.accept().await.unwrap();
3772 let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap();
3773 while let Some(Ok(_)) = socket.next().await {}
3774 });
3775
3776 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
3777 set_data_event_sender(tx);
3778
3779 let url = url.map_or_else(|| format!("ws://{addr}"), str::to_owned);
3780
3781 let mut client = HyperliquidDataClient::new(
3782 *HYPERLIQUID_CLIENT_ID,
3783 HyperliquidDataClientConfig {
3784 base_url_ws: Some(url),
3785 ..HyperliquidDataClientConfig::default()
3786 },
3787 )
3788 .unwrap();
3789
3790 client.spawn_ws().await.unwrap();
3791 assert_eq!(client.session_tasks.len(), 1);
3792
3793 client.ws_client.disconnect().await.unwrap();
3794 tokio::time::timeout(Duration::from_secs(2), async {
3795 while !client.session_tasks.all_finished() {
3796 tokio::task::yield_now().await;
3797 }
3798 })
3799 .await
3800 .expect("consumer must exit when handler output closes without cancellation");
3801
3802 assert!(!client.cancellation_token.is_cancelled());
3803 assert!(client.session_tasks.all_finished());
3804 assert!(client.pending_tasks.is_empty());
3805
3806 if reset {
3807 client.reset().unwrap();
3808 }
3809
3810 client.disconnect().await.unwrap();
3811
3812 assert!(client.session_tasks.is_empty());
3813 assert!(client.pending_tasks.is_empty());
3814 assert!(client.is_disconnected());
3815 server.abort();
3816 let _ = server.await;
3817 }
3818
3819 #[tokio::test]
3820 async fn test_spawn_instrument_refresh_skipped_when_interval_zero() {
3821 let client = data_client_with_refresh_interval(0);
3822
3823 client.spawn_instrument_refresh().unwrap();
3824
3825 assert!(client.session_tasks.is_empty());
3826 }
3827
3828 #[tokio::test]
3829 async fn test_spawn_instrument_refresh_registers_task() {
3830 let client = data_client_with_refresh_interval(60);
3831
3832 client.spawn_instrument_refresh().unwrap();
3833
3834 assert_eq!(client.session_tasks.len(), 1);
3835
3836 client.cancellation_token.cancel();
3837 client.await_session_tasks().await.unwrap();
3838 }
3839
3840 #[rstest]
3841 #[case::disconnect(false)]
3842 #[case::reset(true)]
3843 #[tokio::test]
3844 async fn test_instrument_refresh_shutdown_drains_generation(#[case] reset: bool) {
3845 let mut client = data_client_with_refresh_interval(1);
3846 client.spawn_instrument_refresh().unwrap();
3847 if reset {
3848 client.reset().unwrap();
3849 client.await_session_tasks().await.unwrap();
3850 } else {
3851 client.disconnect().await.unwrap();
3852 }
3853
3854 assert!(client.cancellation_token.is_cancelled());
3855 assert!(client.session_tasks.is_empty());
3856 assert!(!client.session_tasks.is_open());
3857 assert!(client.spawn_instrument_refresh().is_err());
3858 assert!(client.is_disconnected());
3859 }
3860
3861 #[tokio::test]
3862 async fn test_instrument_refresh_connected_client_keeps_one_task() {
3863 let mut client = data_client_with_refresh_interval(1);
3864 client.spawn_instrument_refresh().unwrap();
3865 client.is_connected.store(true, Ordering::Release);
3866
3867 client.connect().await.unwrap();
3868 client.connect().await.unwrap();
3869
3870 assert_eq!(client.session_tasks.len(), 1);
3871 client.disconnect().await.unwrap();
3872 assert!(client.session_tasks.is_empty());
3873 }
3874
3875 #[rstest]
3876 fn test_cache_instruments_replaces_complete_snapshots() {
3877 let client = data_client_with_refresh_interval(0);
3878 let mut original = vec![
3879 perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1)),
3880 perp_instrument("ETH-USD-PERP", "0.2", UnixNanos::from(1)),
3881 ];
3882 let mut refreshed = vec![
3883 perp_instrument("BTC-USD-PERP", "0.3", UnixNanos::from(2)),
3884 perp_instrument("ETH-USD-PERP", "0.4", UnixNanos::from(2)),
3885 ];
3886
3887 for instruments in [&mut original, &mut refreshed] {
3888 let InstrumentAny::CryptoPerpetual(eth) = &mut instruments[1] else {
3889 unreachable!();
3890 };
3891
3892 eth.raw_symbol = Symbol::new("ETH");
3893 }
3894
3895 cache_instruments(
3896 &original,
3897 &client.instruments,
3898 &client.coin_to_instrument_id,
3899 &client.http_client,
3900 &client.ws_client,
3901 );
3902 let previous = client.instruments.load();
3903 cache_instruments(
3904 &refreshed,
3905 &client.instruments,
3906 &client.coin_to_instrument_id,
3907 &client.http_client,
3908 &client.ws_client,
3909 );
3910 let current = client.instruments.load();
3911
3912 assert_eq!(previous.len(), 2);
3913 assert_eq!(current.len(), 2);
3914
3915 for (old, new) in original.iter().zip(&refreshed) {
3916 assert_eq!(
3917 serde_json::to_value(previous.get(&old.id()).unwrap()).unwrap(),
3918 serde_json::to_value(old).unwrap(),
3919 );
3920 assert_eq!(
3921 serde_json::to_value(current.get(&new.id()).unwrap()).unwrap(),
3922 serde_json::to_value(new).unwrap(),
3923 );
3924 assert_eq!(
3925 client
3926 .coin_to_instrument_id
3927 .get_cloned(&new.raw_symbol().inner()),
3928 Some(new.id()),
3929 );
3930 assert_eq!(
3931 serde_json::to_value(
3932 client
3933 .ws_client
3934 .instruments_cache()
3935 .load()
3936 .get(&new.raw_symbol().inner())
3937 .unwrap(),
3938 )
3939 .unwrap(),
3940 serde_json::to_value(new).unwrap(),
3941 );
3942 }
3943 }
3944
3945 #[rstest]
3946 fn test_changed_definitions_reports_a_newly_listed_market() {
3947 let cached_instruments =
3948 cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3949 let fetched = vec![
3950 perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1)),
3951 perp_instrument("NEW-USD-PERP", "0.1", UnixNanos::from(1)),
3952 ];
3953
3954 let changed = changed_definitions(&fetched, &cached_instruments);
3955
3956 assert_eq!(changed.len(), 1);
3957 assert_eq!(changed[0].id().symbol.as_str(), "NEW-USD-PERP");
3958 }
3959
3960 #[rstest]
3961 fn test_added_symbols_names_only_the_market_the_cache_never_held() {
3962 let cached_instruments =
3963 cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3964 let changed = vec![
3966 perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1)),
3967 perp_instrument("NEW-USD-PERP", "0.1", UnixNanos::from(1)),
3968 ];
3969
3970 let added = added_symbols(&changed, &cached_instruments);
3971
3972 assert_eq!(added, vec![Ustr::from("NEW-USD-PERP")]);
3973 }
3974
3975 #[rstest]
3976 fn test_added_symbols_is_empty_when_every_change_is_a_known_market() {
3977 let cached_instruments =
3978 cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3979 let changed = vec![perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1))];
3980
3981 assert!(added_symbols(&changed, &cached_instruments).is_empty());
3982 }
3983
3984 #[rstest]
3985 fn test_changed_definitions_ignores_a_later_ts_init_alone() {
3986 let cached_instruments =
3989 cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3990 let fetched = vec![perp_instrument(
3991 "BTC-USD-PERP",
3992 "0.1",
3993 UnixNanos::from(2_000_000_000),
3994 )];
3995
3996 assert!(changed_definitions(&fetched, &cached_instruments).is_empty());
3997 }
3998
3999 #[rstest]
4000 fn test_changed_definitions_reports_a_changed_tick_size() {
4001 let cached_instruments =
4002 cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
4003 let fetched = vec![perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1))];
4004
4005 let changed = changed_definitions(&fetched, &cached_instruments);
4006
4007 assert_eq!(changed.len(), 1);
4008 assert_eq!(changed[0].price_increment(), Price::from("0.5"));
4009 }
4010}