Skip to main content

nautilus_hyperliquid/
data.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    str::FromStr,
18    sync::{
19        Arc,
20        atomic::{AtomicBool, Ordering},
21    },
22    time::Duration,
23};
24
25use ahash::{AHashMap, AHashSet};
26use anyhow::Context;
27use jiff::Timestamp;
28use nautilus_common::{
29    cache::InstrumentLookupError,
30    clients::{DataClient, execute_bar_subscriptions, execute_instrument_subscriptions},
31    live::{dst::time::Instant, runner::get_data_event_sender, sender::EventSender},
32    messages::{
33        DataEvent,
34        data::{
35            BarsResponse, BookResponse, CustomDataResponse, DataResponse, FundingRatesResponse,
36            InstrumentResponse, InstrumentsResponse, RequestBars, RequestBookSnapshot,
37            RequestCustomData, RequestFundingRates, RequestInstrument, RequestInstruments,
38            RequestTrades, SubscribeBars, SubscribeBookDeltas, SubscribeBookDepth,
39            SubscribeCustomData, SubscribeFundingRates, SubscribeIndexPrices, SubscribeInstrument,
40            SubscribeInstrumentStatus, SubscribeMarkPrices, SubscribeQuotes, SubscribeTrades,
41            TradesResponse, UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth,
42            UnsubscribeCustomData, UnsubscribeFundingRates, UnsubscribeIndexPrices,
43            UnsubscribeInstrument, UnsubscribeInstrumentStatus, UnsubscribeInstruments,
44            UnsubscribeMarkPrices, UnsubscribeQuotes, UnsubscribeTrades,
45        },
46    },
47};
48use nautilus_core::{
49    AtomicMap, Params, UnixNanos,
50    datetime::{datetime_to_unix_nanos, unix_nanos_to_iso8601},
51    time::{AtomicTime, get_atomic_clock_realtime},
52};
53use nautilus_live::{
54    SocketControl,
55    task::{TaskGroup, TaskGroupGuard, TaskSpawner},
56};
57use nautilus_model::{
58    data::{Bar, BarType, BookOrder, CustomData, Data, DataType, FundingRateUpdate, TradeTick},
59    enums::{BarAggregation, BookType, OrderSide},
60    identifiers::{ClientId, InstrumentId, Venue},
61    instruments::{Instrument, InstrumentAny},
62    orderbook::OrderBook,
63    types::{Price, Quantity},
64};
65use parking_lot::Mutex;
66use rust_decimal::Decimal;
67use tokio_util::sync::CancellationToken;
68use ustr::Ustr;
69
70use crate::{
71    book::{self, sync::BookSyncTracker},
72    common::{
73        consts::HYPERLIQUID_VENUE,
74        credential::{Secrets, credential_env_vars},
75        parse::{bar_type_to_interval, millis_to_nanos},
76        status::InstrumentStatusState,
77    },
78    config::HyperliquidDataClientConfig,
79    data_types::register_hyperliquid_custom_data,
80    http::{
81        client::HyperliquidHttpClient,
82        models::{HyperliquidCandle, HyperliquidFundingHistoryEntry, HyperliquidL2Book},
83        parse::parse_recent_trade,
84    },
85    websocket::{
86        DATA_STREAMS_ENDPOINT, client::HyperliquidWebSocketClient, messages::NautilusWsMessage,
87    },
88};
89
90#[derive(Debug)]
91pub struct HyperliquidDataClient {
92    clock: &'static AtomicTime,
93    client_id: ClientId,
94    config: HyperliquidDataClientConfig,
95    http_client: HyperliquidHttpClient,
96    ws_client: HyperliquidWebSocketClient,
97    is_connected: AtomicBool,
98    cancellation_token: CancellationToken,
99    session_tasks: TaskGroup,
100    pending_tasks: TaskGroup,
101    shutdown_errors: Vec<String>,
102    data_sender: EventSender<DataEvent>,
103    instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
104    coin_to_instrument_id: Arc<AtomicMap<Ustr, InstrumentId>>,
105    // serializes instrument fetch-and-apply passes, see `refresh_instruments`
106    instrument_update_lock: Arc<tokio::sync::Mutex<()>>,
107    instrument_status: Arc<Mutex<InstrumentStatusState>>,
108    stream_health: Arc<Mutex<MarketDataStreamHealthMonitor>>,
109    book_sync: BookSyncTracker,
110}
111
112impl HyperliquidDataClient {
113    /// Creates a new [`HyperliquidDataClient`] instance.
114    ///
115    /// # Errors
116    ///
117    /// Returns an error if the HTTP client fails to initialize.
118    pub fn new(client_id: ClientId, config: HyperliquidDataClientConfig) -> anyhow::Result<Self> {
119        let clock = get_atomic_clock_realtime();
120        let data_sender = get_data_event_sender();
121
122        // Only fall back to unauthenticated when credentials are absent,
123        // not when they're invalid (fail fast on malformed keys)
124        let (pk_var, _) = credential_env_vars(config.environment);
125        let has_credentials = config.has_credentials() || std::env::var(pk_var).is_ok();
126        let proxy_url = config
127            .proxy_url
128            .as_ref()
129            .map(|value| value.expose_secret().to_owned());
130
131        let mut http_client = if has_credentials {
132            let secrets = Secrets::resolve(
133                config
134                    .private_key
135                    .as_ref()
136                    .map(|value| value.expose_secret()),
137                None,
138                config.environment,
139            )?;
140            HyperliquidHttpClient::with_secrets(
141                &secrets,
142                config.http_timeout_secs,
143                proxy_url.clone(),
144            )?
145        } else {
146            HyperliquidHttpClient::new(
147                config.environment,
148                config.http_timeout_secs,
149                proxy_url.clone(),
150            )?
151        };
152
153        if let Some(url) = &config.base_url_http {
154            http_client.set_base_info_url(url.clone());
155        }
156
157        let ws_url = config.base_url_ws.clone();
158        let ws_client = HyperliquidWebSocketClient::new(
159            ws_url,
160            config.environment,
161            None,
162            config.transport_backend,
163            proxy_url,
164        );
165        let ws_client = ws_client.with_socket_control(SocketControl::new(
166            client_id,
167            Some(*HYPERLIQUID_VENUE),
168            DATA_STREAMS_ENDPOINT,
169        ));
170        let mut stream_health_monitor = MarketDataStreamHealthMonitor::new(
171            Duration::from_secs(config.stale_stream_receive_timeout_secs),
172            Duration::from_secs(config.stale_stream_warning_cooldown_secs),
173        );
174
175        if config.stale_stream_recovery_enabled {
176            if config.stale_stream_recovery_cooldown_secs > 0 {
177                stream_health_monitor = stream_health_monitor.with_recovery(
178                    Duration::from_secs(config.stale_stream_recovery_cooldown_secs),
179                    config.stale_stream_max_targeted_resubscribes,
180                );
181            } else {
182                log::warn!(
183                    "Hyperliquid stale stream recovery disabled: \
184                     stale_stream_recovery_cooldown_secs must be positive"
185                );
186            }
187        }
188
189        let stream_health = Arc::new(Mutex::new(stream_health_monitor));
190
191        let session_tasks = TaskGroup::new();
192        let pending_tasks = TaskGroup::new();
193
194        Ok(Self {
195            clock,
196            client_id,
197            config,
198            http_client,
199            ws_client,
200            is_connected: AtomicBool::new(false),
201            cancellation_token: CancellationToken::new(),
202            session_tasks,
203            pending_tasks,
204            shutdown_errors: Vec::new(),
205            data_sender,
206            instruments: Arc::new(AtomicMap::new()),
207            coin_to_instrument_id: Arc::new(AtomicMap::new()),
208            instrument_update_lock: Arc::new(tokio::sync::Mutex::new(())),
209            instrument_status: Arc::new(Mutex::new(InstrumentStatusState::default())),
210            stream_health,
211            book_sync: BookSyncTracker::default(),
212        })
213    }
214
215    fn spawn_task<F>(&self, description: &'static str, fut: F)
216    where
217        F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
218    {
219        let future = async move {
220            if let Err(e) = fut.await {
221                log::warn!("{description} failed: {e:?}");
222            }
223        };
224
225        if let Err(e) = self.pending_tasks.spawn(future) {
226            log::warn!("Skipping Hyperliquid {description} after shutdown began: {e}");
227        }
228    }
229
230    fn abort_pending_tasks(&self) {
231        self.pending_tasks.begin_shutdown();
232    }
233
234    fn abort_session_tasks(&self) {
235        self.session_tasks.begin_shutdown();
236        self.ws_client.begin_shutdown();
237    }
238
239    async fn teardown_partial_connect(&mut self) -> anyhow::Result<()> {
240        self.cancellation_token.cancel();
241        self.abort_session_tasks();
242        self.abort_pending_tasks();
243
244        if let Err(e) = self.ws_client.disconnect().await {
245            self.shutdown_errors
246                .push(format!("Hyperliquid WebSocket shutdown failed: {e}"));
247        }
248
249        if let Err(e) = self.await_session_tasks().await {
250            self.shutdown_errors.push(e.to_string());
251        }
252
253        if let Err(e) = self.await_pending_tasks().await {
254            self.shutdown_errors.push(e.to_string());
255        }
256        self.clear_stream_health();
257        self.book_sync.clear();
258        self.is_connected.store(false, Ordering::Release);
259
260        if !self.shutdown_errors.is_empty() {
261            anyhow::bail!(std::mem::take(&mut self.shutdown_errors).join("; "));
262        }
263        Ok(())
264    }
265
266    async fn await_pending_tasks(&self) -> anyhow::Result<()> {
267        self.pending_tasks.begin_shutdown();
268        self.pending_tasks
269            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
270            .await
271            .map_err(|e| anyhow::anyhow!("Failed to terminate Hyperliquid data tasks: {e}"))?;
272        Ok(())
273    }
274
275    async fn await_session_tasks(&self) -> anyhow::Result<()> {
276        self.session_tasks.begin_shutdown();
277        self.session_tasks
278            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
279            .await
280            .map_err(|e| {
281                anyhow::anyhow!("Failed to terminate Hyperliquid data session tasks: {e}")
282            })?;
283        Ok(())
284    }
285
286    fn clear_stream_health(&self) {
287        self.stream_health.lock().clear();
288    }
289
290    fn register_stream_health(&self, channel: MarketDataChannel, instrument_id: InstrumentId) {
291        if !self.stream_health_monitor_enabled() {
292            return;
293        }
294
295        self.stream_health
296            .lock()
297            .subscribe(channel, instrument_id, Instant::now());
298    }
299
300    fn remove_stream_health(&self, channel: MarketDataChannel, instrument_id: InstrumentId) {
301        self.stream_health
302            .lock()
303            .unsubscribe(channel, instrument_id);
304    }
305
306    fn stream_health_monitor_enabled(&self) -> bool {
307        self.config.stale_stream_receive_timeout_secs > 0
308            && self.config.stream_health_check_interval_secs > 0
309    }
310
311    fn spawn_stream_health_monitor(&self) -> anyhow::Result<()> {
312        if !self.stream_health_monitor_enabled() {
313            return Ok(());
314        }
315
316        let stream_health = Arc::clone(&self.stream_health);
317        let cancellation_token = self.cancellation_token.clone();
318        let interval = Duration::from_secs(self.config.stream_health_check_interval_secs);
319        let clock = self.clock;
320        let ws_client = self.ws_client.clone();
321        let book_sync = self.book_sync.clone();
322        let snapshot_timeout = self.book_snapshot_timeout();
323        let book_tasks = self.pending_tasks.spawner()?;
324
325        self.session_tasks.spawn(async move {
326            log::debug!("Hyperliquid stream health monitor started");
327
328            loop {
329                tokio::select! {
330                    () = cancellation_token.cancelled() => {
331                        log::debug!("Hyperliquid stream health monitor cancelled");
332                        break;
333                    }
334                    () = tokio::time::sleep(interval) => {
335                        let events = stream_health
336                            .lock()
337                            .check_stale(Instant::now(), clock.get_time_ns());
338
339                        handle_stream_health_events(
340                            &ws_client,
341                            &events,
342                            &book_sync,
343                            snapshot_timeout,
344                            &book_tasks,
345                        )
346                        .await;
347                    }
348                }
349            }
350
351            log::debug!("Hyperliquid stream health monitor stopped");
352        })?;
353
354        Ok(())
355    }
356
357    fn book_snapshot_timeout(&self) -> Duration {
358        Duration::from_secs(self.config.book_snapshot_timeout_secs)
359    }
360
361    fn venue(&self) -> Venue {
362        *HYPERLIQUID_VENUE
363    }
364
365    fn custom_instrument_id(data_type: &DataType) -> anyhow::Result<Option<InstrumentId>> {
366        let Some(raw_instrument_id) = data_type
367            .metadata()
368            .and_then(|m| m.get("instrument_id"))
369            .and_then(|v| v.as_str())
370            .map(str::trim)
371            .filter(|value| !value.is_empty())
372        else {
373            return Ok(None);
374        };
375
376        let instrument_id = InstrumentId::from_str(raw_instrument_id)
377            .with_context(|| format!("invalid instrument_id metadata `{raw_instrument_id}`"))?;
378
379        Ok(Some(instrument_id))
380    }
381
382    fn custom_user(data_type: &DataType) -> anyhow::Result<Option<String>> {
383        let Some(user) = data_type
384            .metadata()
385            .and_then(|m| m.get("user"))
386            .and_then(|v| v.as_str())
387            .filter(|value| !value.is_empty())
388        else {
389            return Ok(None);
390        };
391
392        anyhow::ensure!(
393            user == user.trim(),
394            "metadata['user'] must not contain surrounding whitespace",
395        );
396
397        Ok(Some(user.to_string()))
398    }
399
400    async fn bootstrap_instruments(&self) -> anyhow::Result<Vec<InstrumentAny>> {
401        // a pass that fetched before the bootstrap must not apply over its universe
402        let _update_guard = self.instrument_update_lock.lock().await;
403
404        let snapshot = self
405            .http_client
406            .request_instruments_with_statuses()
407            .await
408            .context("failed to fetch instruments during bootstrap")?;
409        let instruments = snapshot.instruments;
410
411        cache_instruments(
412            &instruments,
413            &self.instruments,
414            &self.coin_to_instrument_id,
415            &self.http_client,
416            &self.ws_client,
417        );
418        self.instrument_status.lock().store(snapshot.statuses);
419        rebuild_all_dex_asset_ctxs_mapping(&self.http_client, &self.ws_client).await;
420
421        log::debug!(
422            "Bootstrapped {} instruments with {} coin mappings",
423            self.instruments.len(),
424            self.coin_to_instrument_id.len()
425        );
426        Ok(instruments)
427    }
428
429    /// Spawns the periodic instrument refresh task, disabled when the configured
430    /// interval is zero. The task is tracked in `self.session_tasks`, so the
431    /// cancellation token stops it on disconnect, failed connect, and dispose.
432    fn spawn_instrument_refresh(&self) -> anyhow::Result<()> {
433        if let Some(future) = self.instrument_refresh() {
434            self.session_tasks.spawn(future)?;
435        }
436
437        Ok(())
438    }
439
440    fn instrument_refresh(&self) -> Option<impl Future<Output = ()> + Send + 'static> {
441        let minutes = self.config.update_instruments_interval_mins;
442
443        if minutes == 0 {
444            log::debug!(
445                "Hyperliquid instrument refresh disabled (update_instruments_interval_mins=0)"
446            );
447            return None;
448        }
449
450        let interval = Duration::from_secs(minutes.saturating_mul(60));
451        let cancellation_token = self.cancellation_token.clone();
452        let http_client = self.http_client.clone();
453        let ws_client = self.ws_client.clone();
454        let instruments = Arc::clone(&self.instruments);
455        let coin_to_instrument_id = Arc::clone(&self.coin_to_instrument_id);
456        let instrument_update_lock = Arc::clone(&self.instrument_update_lock);
457        let instrument_status = Arc::clone(&self.instrument_status);
458        let data_sender = self.data_sender.clone();
459        let client_id = self.client_id;
460
461        Some(async move {
462            log::info!("Hyperliquid instrument refresh started, interval={interval:?}");
463
464            loop {
465                tokio::select! {
466                    () = cancellation_token.cancelled() => {
467                        log::debug!("Hyperliquid instrument refresh cancelled");
468                        break;
469                    }
470                    () = tokio::time::sleep(interval) => {}
471                }
472
473                // The refresh performs several REST calls, so cancellation is
474                // also raced against the pass itself rather than only the sleep
475                let result = tokio::select! {
476                    () = cancellation_token.cancelled() => {
477                        log::debug!("Hyperliquid instrument refresh cancelled");
478                        break;
479                    }
480                    result = refresh_instruments(
481                        &instrument_update_lock,
482                        &http_client,
483                        &ws_client,
484                        &instruments,
485                        &coin_to_instrument_id,
486                        &instrument_status,
487                        &data_sender,
488                    ) => result,
489                };
490
491                match result {
492                    Ok(summary) => summary.log(client_id),
493                    Err(e) => log::warn!(
494                        "Failed to refresh Hyperliquid instruments: client_id={client_id}, error={e:?}"
495                    ),
496                }
497            }
498
499            log::debug!("Hyperliquid instrument refresh stopped");
500        })
501    }
502
503    async fn spawn_ws(&self) -> anyhow::Result<()> {
504        // Clone client before connecting so the clone can have out_rx set
505        let mut ws_client = self.ws_client.clone();
506
507        ws_client
508            .connect()
509            .await
510            .context("failed to connect to Hyperliquid WebSocket")?;
511
512        let data_sender = self.data_sender.clone();
513        let cancellation_token = self.cancellation_token.clone();
514        let stream_health = Arc::clone(&self.stream_health);
515        let book_sync = self.book_sync.clone();
516        let snapshot_timeout = self.book_snapshot_timeout();
517        let book_tasks = self.pending_tasks.spawner()?;
518
519        self.session_tasks.spawn(async move {
520            log::debug!("Hyperliquid WebSocket consumption loop started");
521
522            loop {
523                tokio::select! {
524                    () = cancellation_token.cancelled() => {
525                        log::debug!("WebSocket consumption loop cancelled");
526                        break;
527                    }
528                    msg_opt = ws_client.next_event() => {
529                        if let Some(msg) = msg_opt {
530                            if let Some((channel, instrument_id, ts_event)) =
531                                stream_health_update(&msg)
532                            {
533                                record_stream_receive(
534                                    &stream_health,
535                                    channel,
536                                    instrument_id,
537                                    ts_event,
538                                );
539                            }
540
541                            match msg {
542                                NautilusWsMessage::Trades(trades) => {
543                                    for trade in trades {
544                                        if let Err(e) = data_sender
545                                            .send(DataEvent::Data(Data::Trade(trade)))
546                                        {
547                                            log::error!("Failed to send trade tick: {e}");
548                                        }
549                                    }
550                                }
551                                NautilusWsMessage::Quote(quote) => {
552                                    if let Err(e) = data_sender
553                                        .send(DataEvent::Data(Data::Quote(quote)))
554                                    {
555                                        log::error!("Failed to send quote tick: {e}");
556                                    }
557                                }
558                                NautilusWsMessage::Deltas(deltas) => {
559                                    if book_sync
560                                        .record_snapshot(deltas.instrument_id, Instant::now())
561                                        && let Err(e) = data_sender.send(DataEvent::Data(
562                                            Data::BookDeltas(Box::new(deltas)),
563                                        ))
564                                    {
565                                        log::error!("Failed to send order book deltas: {e}");
566                                    }
567                                }
568                                NautilusWsMessage::Depth(depth) => {
569                                    if let Err(e) =
570                                        data_sender.send(DataEvent::Data(Data::BookDepth(depth)))
571                                    {
572                                        log::error!("Failed to send order book depth: {e}");
573                                    }
574                                }
575                                NautilusWsMessage::BookInvalid(instrument_id) => {
576                                    book::recovery::reject_snapshot(
577                                        instrument_id,
578                                        &book_sync,
579                                        &ws_client,
580                                        snapshot_timeout,
581                                        &book_tasks,
582                                    );
583                                }
584                                NautilusWsMessage::Candle(bar) => {
585                                    if let Err(e) = data_sender
586                                        .send(DataEvent::Data(Data::Bar(bar)))
587                                    {
588                                        log::error!("Failed to send bar: {e}");
589                                    }
590                                }
591                                NautilusWsMessage::MarkPrice(update) => {
592                                    if let Err(e) = data_sender
593                                        .send(DataEvent::Data(Data::MarkPrice(update)))
594                                    {
595                                        log::error!("Failed to send mark price update: {e}");
596                                    }
597                                }
598                                NautilusWsMessage::IndexPrice(update) => {
599                                    if let Err(e) = data_sender
600                                        .send(DataEvent::Data(Data::IndexPrice(update)))
601                                    {
602                                        log::error!("Failed to send index price update: {e}");
603                                    }
604                                }
605                                NautilusWsMessage::FundingRate(update) => {
606                                    if let Err(e) = data_sender
607                                        .send(DataEvent::Data(Data::FundingRate(update)))
608                                    {
609                                        log::error!("Failed to send funding rate update: {e}");
610                                    }
611                                }
612                                NautilusWsMessage::CustomData(data) => {
613                                    if let Err(e) = data_sender.send(DataEvent::Data(data)) {
614                                        log::error!("Failed to send custom data: {e}");
615                                    }
616                                }
617                                NautilusWsMessage::Reconnected => {
618                                    log::info!("WebSocket reconnected");
619                                    book::recovery::reset_on_reconnect(
620                                        &book_sync,
621                                        &ws_client,
622                                        snapshot_timeout,
623                                        &book_tasks,
624                                    );
625                                }
626                                NautilusWsMessage::Error(e) => {
627                                    log::warn!("WebSocket error: {e}");
628                                }
629                                NautilusWsMessage::ExecutionReports(_)
630                                | NautilusWsMessage::ClearinghouseState(_)
631                                | NautilusWsMessage::SpotState(_) => {
632                                    // Handled by execution client
633                                }
634                            }
635                        } else {
636                            log::debug!("WebSocket next_event returned None, stream closed");
637                            break;
638                        }
639                    }
640                }
641            }
642
643            log::debug!("Hyperliquid WebSocket consumption loop finished");
644        })?;
645
646        log::debug!("WebSocket consumption task spawned");
647
648        Ok(())
649    }
650}
651
652#[async_trait::async_trait(?Send)]
653impl DataClient for HyperliquidDataClient {
654    fn client_id(&self) -> ClientId {
655        self.client_id
656    }
657
658    fn venue(&self) -> Option<Venue> {
659        Some(self.venue())
660    }
661
662    fn start(&mut self) -> anyhow::Result<()> {
663        log::info!(
664            "Starting Hyperliquid data client: client_id={}, environment={:?}, proxy_url={:?}",
665            self.client_id,
666            self.config.environment,
667            self.config.proxy_url,
668        );
669        Ok(())
670    }
671
672    fn stop(&mut self) -> anyhow::Result<()> {
673        log::info!("Stopping Hyperliquid data client {}", self.client_id);
674        self.cancellation_token.cancel();
675        self.abort_session_tasks();
676        self.abort_pending_tasks();
677        self.is_connected.store(false, Ordering::Relaxed);
678        Ok(())
679    }
680
681    fn reset(&mut self) -> anyhow::Result<()> {
682        log::debug!("Resetting Hyperliquid data client {}", self.client_id);
683        // Keep this generation cancelled until `connect()` has torn down the
684        // inner WebSocket client. Replacing it here would allow the next
685        // connection to reuse an active old-generation handler.
686        self.cancellation_token.cancel();
687        self.abort_session_tasks();
688        self.abort_pending_tasks();
689        self.is_connected.store(false, Ordering::Relaxed);
690        self.instruments.store(AHashMap::new());
691        self.coin_to_instrument_id.store(AHashMap::new());
692        self.instrument_status.lock().clear();
693        Ok(())
694    }
695
696    fn dispose(&mut self) -> anyhow::Result<()> {
697        log::debug!("Disposing Hyperliquid data client {}", self.client_id);
698        self.stop()
699    }
700
701    fn is_connected(&self) -> bool {
702        self.is_connected.load(Ordering::Acquire)
703    }
704
705    fn is_disconnected(&self) -> bool {
706        !self.is_connected()
707    }
708
709    async fn connect(&mut self) -> anyhow::Result<()> {
710        if self.is_connected()
711            && !self.cancellation_token.is_cancelled()
712            && self.session_tasks.is_open()
713            && self.pending_tasks.is_open()
714        {
715            return Ok(());
716        }
717
718        if self.cancellation_token.is_cancelled()
719            || !self.session_tasks.is_open()
720            || !self.pending_tasks.is_open()
721        {
722            // `reset()` is synchronous, while shutting down the inner socket
723            // is async. Complete that teardown before creating any new stream
724            // task so its receiver and subscription registries cannot belong
725            // to the previous generation.
726            self.ws_client.begin_shutdown();
727            self.ws_client
728                .disconnect()
729                .await
730                .context("failed to tear down Hyperliquid WebSocket before reconnect")?;
731            self.ws_client.reset_runtime_state();
732            self.book_sync.clear();
733            self.abort_session_tasks();
734            self.abort_pending_tasks();
735            let (session_result, pending_result) =
736                tokio::join!(self.await_session_tasks(), self.await_pending_tasks());
737            session_result?;
738            pending_result?;
739            self.session_tasks.start_generation().map_err(|e| {
740                anyhow::anyhow!("Failed to start Hyperliquid data session generation: {e}")
741            })?;
742            self.pending_tasks.start_generation().map_err(|e| {
743                anyhow::anyhow!("Failed to start Hyperliquid data task generation: {e}")
744            })?;
745            self.cancellation_token = CancellationToken::new();
746        }
747        let cancellation_token = self.cancellation_token.clone();
748        let ws_client = self.ws_client.clone();
749        let setup_guard =
750            TaskGroupGuard::new(&[&self.session_tasks, &self.pending_tasks], move || {
751                cancellation_token.cancel();
752                ws_client.begin_shutdown();
753            });
754
755        register_hyperliquid_custom_data();
756
757        let instruments = self
758            .bootstrap_instruments()
759            .await
760            .context("failed to bootstrap instruments")?;
761
762        for instrument in instruments {
763            if let Err(e) = self.data_sender.send(DataEvent::Data(instrument.into())) {
764                log::warn!("Failed to send instrument: {e}");
765            }
766        }
767
768        let session_result = async {
769            self.spawn_ws()
770                .await
771                .context("failed to spawn WebSocket client")?;
772            self.spawn_stream_health_monitor()?;
773            self.spawn_instrument_refresh()?;
774            Ok::<(), anyhow::Error>(())
775        }
776        .await;
777
778        if let Err(e) = session_result {
779            if let Err(teardown_error) = self.teardown_partial_connect().await {
780                return Err(e.context(format!(
781                    "Hyperliquid data startup teardown failed: {teardown_error}"
782                )));
783            }
784            return Err(e);
785        }
786
787        self.is_connected.store(true, Ordering::Relaxed);
788        setup_guard.disarm();
789        log::info!("Connected: client_id={}", self.client_id);
790
791        Ok(())
792    }
793
794    async fn disconnect(&mut self) -> anyhow::Result<()> {
795        self.teardown_partial_connect().await?;
796        self.instruments.store(AHashMap::new());
797        self.instrument_status.lock().clear();
798        log::info!("Disconnected: client_id={}", self.client_id);
799
800        Ok(())
801    }
802
803    fn subscribe(&mut self, cmd: SubscribeCustomData) -> anyhow::Result<()> {
804        let data_type = cmd.data_type.type_name();
805
806        if data_type == "HyperliquidAllMids" {
807            let ws = self.ws_client.clone();
808            let dex = cmd
809                .data_type
810                .metadata()
811                .as_ref()
812                .and_then(|m| m.get("dex"))
813                .and_then(|v| v.as_str())
814                .map(str::trim)
815                .filter(|value| !value.is_empty())
816                .map(ToString::to_string);
817
818            log::debug!("Subscribing to all mids (dex: {:?})", dex.as_deref());
819
820            self.spawn_task("subscribe_all_mids", async move {
821                ws.subscribe_all_mids_with_dex(dex.as_deref()).await
822            });
823
824            return Ok(());
825        }
826
827        if data_type == "HyperliquidAllDexsAssetCtxs" {
828            let ws = self.ws_client.clone();
829
830            self.spawn_task("subscribe_all_dexs_asset_ctxs", async move {
831                ws.subscribe_all_dexs_asset_ctxs().await
832            });
833
834            return Ok(());
835        }
836
837        if data_type == "HyperliquidOpenInterest" {
838            let ws = self.ws_client.clone();
839            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
840                "HyperliquidOpenInterest subscriptions require metadata['instrument_id']",
841            )?;
842
843            self.spawn_task("subscribe_open_interest", async move {
844                ws.subscribe_open_interest(instrument_id).await
845            });
846
847            return Ok(());
848        }
849
850        if data_type == "HyperliquidPublicTrade" {
851            let ws = self.ws_client.clone();
852            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
853                "HyperliquidPublicTrade subscriptions require metadata['instrument_id']",
854            )?;
855
856            self.spawn_task("subscribe_public_trades", async move {
857                ws.subscribe_public_trades(instrument_id).await
858            });
859
860            return Ok(());
861        }
862
863        if data_type == "HyperliquidTwapHistory" {
864            let ws = self.ws_client.clone();
865            let user = Self::custom_user(&cmd.data_type)?
866                .context("HyperliquidTwapHistory subscriptions require metadata['user']")?;
867
868            self.spawn_task("subscribe_user_twap_history", async move {
869                ws.subscribe_user_twap_history(&user).await
870            });
871
872            return Ok(());
873        }
874
875        if data_type == "HyperliquidTwapSliceFill" {
876            let ws = self.ws_client.clone();
877            let user = Self::custom_user(&cmd.data_type)?
878                .context("HyperliquidTwapSliceFill subscriptions require metadata['user']")?;
879
880            self.spawn_task("subscribe_user_twap_slice_fills", async move {
881                ws.subscribe_user_twap_slice_fills(&user).await
882            });
883
884            return Ok(());
885        }
886
887        log::warn!("Unsupported custom data subscription: {data_type}");
888        Ok(())
889    }
890
891    fn unsubscribe(&mut self, cmd: &UnsubscribeCustomData) -> anyhow::Result<()> {
892        let data_type = cmd.data_type.type_name();
893
894        if data_type == "HyperliquidAllMids" {
895            let ws = self.ws_client.clone();
896            let dex = cmd
897                .data_type
898                .metadata()
899                .as_ref()
900                .and_then(|m| m.get("dex"))
901                .and_then(|v| v.as_str())
902                .map(str::trim)
903                .filter(|value| !value.is_empty())
904                .map(ToString::to_string);
905
906            log::debug!("Unsubscribing from all mids (dex: {:?})", dex.as_deref());
907
908            self.spawn_task("unsubscribe_all_mids", async move {
909                ws.unsubscribe_all_mids_with_dex(dex.as_deref()).await
910            });
911
912            return Ok(());
913        }
914
915        if data_type == "HyperliquidAllDexsAssetCtxs" {
916            let ws = self.ws_client.clone();
917
918            self.spawn_task("unsubscribe_all_dexs_asset_ctxs", async move {
919                ws.unsubscribe_all_dexs_asset_ctxs().await
920            });
921
922            return Ok(());
923        }
924
925        if data_type == "HyperliquidOpenInterest" {
926            let ws = self.ws_client.clone();
927            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
928                "HyperliquidOpenInterest unsubscriptions require metadata['instrument_id']",
929            )?;
930
931            self.spawn_task("unsubscribe_open_interest", async move {
932                ws.unsubscribe_open_interest(instrument_id).await
933            });
934
935            return Ok(());
936        }
937
938        if data_type == "HyperliquidPublicTrade" {
939            let ws = self.ws_client.clone();
940            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
941                "HyperliquidPublicTrade unsubscriptions require metadata['instrument_id']",
942            )?;
943
944            self.spawn_task("unsubscribe_public_trades", async move {
945                ws.unsubscribe_public_trades(instrument_id).await
946            });
947
948            return Ok(());
949        }
950
951        if data_type == "HyperliquidTwapHistory" {
952            let ws = self.ws_client.clone();
953            let user = Self::custom_user(&cmd.data_type)?
954                .context("HyperliquidTwapHistory unsubscriptions require metadata['user']")?;
955
956            self.spawn_task("unsubscribe_user_twap_history", async move {
957                ws.unsubscribe_user_twap_history(&user).await
958            });
959
960            return Ok(());
961        }
962
963        if data_type == "HyperliquidTwapSliceFill" {
964            let ws = self.ws_client.clone();
965            let user = Self::custom_user(&cmd.data_type)?
966                .context("HyperliquidTwapSliceFill unsubscriptions require metadata['user']")?;
967
968            self.spawn_task("unsubscribe_user_twap_slice_fills", async move {
969                ws.unsubscribe_user_twap_slice_fills(&user).await
970            });
971
972            return Ok(());
973        }
974
975        log::warn!("Unsupported custom data unsubscription: {data_type}");
976        Ok(())
977    }
978
979    fn subscribe_instrument(&mut self, cmd: SubscribeInstrument) -> anyhow::Result<()> {
980        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
981            let instruments = self.instruments.load();
982            if let Some(instrument) = instruments.get(&instrument_id) {
983                if let Err(e) = self
984                    .data_sender
985                    .send(DataEvent::Data(instrument.clone().into()))
986                {
987                    log::error!("Failed to send instrument {instrument_id}: {e}");
988                }
989            } else {
990                log::warn!("Instrument {instrument_id} not found in cache");
991            }
992
993            Ok(())
994        })
995    }
996
997    fn subscribe_book_deltas(&mut self, subscription: SubscribeBookDeltas) -> anyhow::Result<()> {
998        execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
999            let subscription = subscription.with_instrument_ids(instrument_id);
1000            if subscription.book_type != BookType::L2_MBP {
1001                anyhow::bail!("Hyperliquid only supports L2_MBP order book deltas");
1002            }
1003
1004            let (n_sig_figs, mantissa) = parse_book_precision_params(subscription.params.as_ref())?;
1005            self.register_stream_health(MarketDataChannel::Deltas, instrument_id);
1006
1007            match self.pending_tasks.spawner() {
1008                Ok(tasks) => book::recovery::spawn_subscription_task(
1009                    instrument_id,
1010                    n_sig_figs,
1011                    mantissa,
1012                    self.book_sync.clone(),
1013                    self.ws_client.clone(),
1014                    self.book_snapshot_timeout(),
1015                    &tasks,
1016                ),
1017                Err(e) => {
1018                    log::warn!(
1019                        "Skipping Hyperliquid subscribe_book_deltas after shutdown began: {e}"
1020                    );
1021                }
1022            }
1023
1024            Ok(())
1025        })
1026    }
1027
1028    fn subscribe_book_depth(&mut self, subscription: SubscribeBookDepth) -> anyhow::Result<()> {
1029        execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1030            let subscription = subscription.with_instrument_ids(instrument_id);
1031            log::debug!("Subscribing to book depth: {instrument_id}");
1032
1033            if subscription.book_type != BookType::L2_MBP {
1034                anyhow::bail!("Hyperliquid only supports L2_MBP order book depth");
1035            }
1036
1037            let ws = self.ws_client.clone();
1038            let (n_sig_figs, mantissa) = parse_book_precision_params(subscription.params.as_ref())?;
1039            self.register_stream_health(MarketDataChannel::Depth, instrument_id);
1040            self.spawn_task("subscribe_book_depth", async move {
1041                ws.subscribe_book_depth_with_options(instrument_id, n_sig_figs, mantissa)
1042                    .await
1043            });
1044
1045            Ok(())
1046        })
1047    }
1048
1049    fn subscribe_quotes(&mut self, subscription: SubscribeQuotes) -> anyhow::Result<()> {
1050        execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1051            let ws = self.ws_client.clone();
1052            self.register_stream_health(MarketDataChannel::Quote, instrument_id);
1053            self.spawn_task("subscribe_quotes", async move {
1054                ws.subscribe_quotes(instrument_id).await
1055            });
1056
1057            Ok(())
1058        })
1059    }
1060
1061    fn subscribe_trades(&mut self, subscription: SubscribeTrades) -> anyhow::Result<()> {
1062        execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1063            let ws = self.ws_client.clone();
1064            self.spawn_task("subscribe_trades", async move {
1065                ws.subscribe_trades(instrument_id).await
1066            });
1067
1068            Ok(())
1069        })
1070    }
1071
1072    fn subscribe_mark_prices(&mut self, cmd: SubscribeMarkPrices) -> anyhow::Result<()> {
1073        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1074            let ws = self.ws_client.clone();
1075            self.spawn_task("subscribe_mark_prices", async move {
1076                ws.subscribe_mark_prices(instrument_id).await
1077            });
1078
1079            Ok(())
1080        })
1081    }
1082
1083    fn subscribe_index_prices(&mut self, cmd: SubscribeIndexPrices) -> anyhow::Result<()> {
1084        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1085            let ws = self.ws_client.clone();
1086            self.spawn_task("subscribe_index_prices", async move {
1087                ws.subscribe_index_prices(instrument_id).await
1088            });
1089
1090            Ok(())
1091        })
1092    }
1093
1094    fn subscribe_funding_rates(&mut self, cmd: SubscribeFundingRates) -> anyhow::Result<()> {
1095        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1096            let ws = self.ws_client.clone();
1097            self.spawn_task("subscribe_funding_rates", async move {
1098                ws.subscribe_funding_rates(instrument_id).await
1099            });
1100
1101            Ok(())
1102        })
1103    }
1104
1105    fn subscribe_bars(&mut self, subscription: SubscribeBars) -> anyhow::Result<()> {
1106        execute_bar_subscriptions(&subscription.bar_types, |bar_type| {
1107            let instrument_id = bar_type.instrument_id();
1108            if !self.instruments.contains_key(&instrument_id) {
1109                anyhow::bail!(InstrumentLookupError::not_found(instrument_id));
1110            }
1111
1112            let ws = self.ws_client.clone();
1113            self.spawn_task("subscribe_bars", async move {
1114                ws.subscribe_bars(bar_type).await
1115            });
1116
1117            Ok(())
1118        })
1119    }
1120
1121    fn subscribe_instrument_status(
1122        &mut self,
1123        cmd: SubscribeInstrumentStatus,
1124    ) -> anyhow::Result<()> {
1125        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1126            log::debug!(
1127                "Subscribing to instrument status: {instrument_id} (detected by the instrument refresh)"
1128            );
1129            let ts = self.clock.get_time_ns();
1130            self.instrument_status
1131                .lock()
1132                .subscribe(instrument_id, &self.data_sender, ts);
1133
1134            Ok(())
1135        })
1136    }
1137
1138    fn unsubscribe_instrument_status(
1139        &mut self,
1140        cmd: &UnsubscribeInstrumentStatus,
1141    ) -> anyhow::Result<()> {
1142        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1143            log::debug!("Unsubscribing from instrument status: {instrument_id}");
1144            self.instrument_status.lock().unsubscribe(&instrument_id);
1145            Ok(())
1146        })
1147    }
1148
1149    fn unsubscribe_instrument(&mut self, _cmd: &UnsubscribeInstrument) -> anyhow::Result<()> {
1150        // `subscribe_instrument` only emits the cached instrument; it opens no
1151        // venue channel, so there is nothing to tear down here.
1152        Ok(())
1153    }
1154
1155    fn unsubscribe_instruments(&mut self, _cmd: &UnsubscribeInstruments) -> anyhow::Result<()> {
1156        // See `unsubscribe_instrument`: instrument subscriptions carry no
1157        // venue-side state to unsubscribe from.
1158        Ok(())
1159    }
1160
1161    fn unsubscribe_book_deltas(
1162        &mut self,
1163        unsubscription: &UnsubscribeBookDeltas,
1164    ) -> anyhow::Result<()> {
1165        execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1166            log::debug!("Unsubscribing from book deltas: {instrument_id}");
1167
1168            let ws = self.ws_client.clone();
1169            self.remove_stream_health(MarketDataChannel::Deltas, instrument_id);
1170            self.book_sync.remove(instrument_id);
1171            self.spawn_task("unsubscribe_book_deltas", async move {
1172                ws.unsubscribe_book(instrument_id).await
1173            });
1174
1175            Ok(())
1176        })
1177    }
1178
1179    fn unsubscribe_book_depth(
1180        &mut self,
1181        unsubscription: &UnsubscribeBookDepth,
1182    ) -> anyhow::Result<()> {
1183        execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1184            log::debug!("Unsubscribing from book depth: {instrument_id}");
1185
1186            let ws = self.ws_client.clone();
1187            self.remove_stream_health(MarketDataChannel::Depth, instrument_id);
1188            self.spawn_task("unsubscribe_book_depth", async move {
1189                ws.unsubscribe_book_depth(instrument_id).await
1190            });
1191
1192            Ok(())
1193        })
1194    }
1195
1196    fn unsubscribe_quotes(&mut self, unsubscription: &UnsubscribeQuotes) -> anyhow::Result<()> {
1197        execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1198            log::debug!("Unsubscribing from quotes: {instrument_id}");
1199
1200            let ws = self.ws_client.clone();
1201            self.remove_stream_health(MarketDataChannel::Quote, instrument_id);
1202            self.spawn_task("unsubscribe_quotes", async move {
1203                ws.unsubscribe_quotes(instrument_id).await
1204            });
1205
1206            Ok(())
1207        })
1208    }
1209
1210    fn unsubscribe_trades(&mut self, unsubscription: &UnsubscribeTrades) -> anyhow::Result<()> {
1211        execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1212            log::debug!("Unsubscribing from trades: {instrument_id}");
1213
1214            let ws = self.ws_client.clone();
1215            self.spawn_task("unsubscribe_trades", async move {
1216                ws.unsubscribe_trades(instrument_id).await
1217            });
1218
1219            Ok(())
1220        })
1221    }
1222
1223    fn unsubscribe_mark_prices(&mut self, cmd: &UnsubscribeMarkPrices) -> anyhow::Result<()> {
1224        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1225            let ws = self.ws_client.clone();
1226            self.spawn_task("unsubscribe_mark_prices", async move {
1227                ws.unsubscribe_mark_prices(instrument_id).await
1228            });
1229
1230            Ok(())
1231        })
1232    }
1233
1234    fn unsubscribe_index_prices(&mut self, cmd: &UnsubscribeIndexPrices) -> anyhow::Result<()> {
1235        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1236            let ws = self.ws_client.clone();
1237            self.spawn_task("unsubscribe_index_prices", async move {
1238                ws.unsubscribe_index_prices(instrument_id).await
1239            });
1240
1241            Ok(())
1242        })
1243    }
1244
1245    fn unsubscribe_funding_rates(&mut self, cmd: &UnsubscribeFundingRates) -> anyhow::Result<()> {
1246        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1247            let ws = self.ws_client.clone();
1248            self.spawn_task("unsubscribe_funding_rates", async move {
1249                ws.unsubscribe_funding_rates(instrument_id).await
1250            });
1251
1252            Ok(())
1253        })
1254    }
1255
1256    fn unsubscribe_bars(&mut self, unsubscription: &UnsubscribeBars) -> anyhow::Result<()> {
1257        execute_bar_subscriptions(&unsubscription.bar_types, |bar_type| {
1258            let ws = self.ws_client.clone();
1259            self.spawn_task("unsubscribe_bars", async move {
1260                ws.unsubscribe_bars(bar_type).await
1261            });
1262
1263            Ok(())
1264        })
1265    }
1266
1267    fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
1268        log::debug!("Requesting all instruments");
1269
1270        let http = self.http_client.clone();
1271        let ws = self.ws_client.clone();
1272        let sender = self.data_sender.clone();
1273        let instruments_cache = self.instruments.clone();
1274        let coin_map = self.coin_to_instrument_id.clone();
1275        let update_lock = Arc::clone(&self.instrument_update_lock);
1276        let instrument_status = Arc::clone(&self.instrument_status);
1277        let request_id = request.request_id;
1278        let client_id = request.client_id.unwrap_or(self.client_id);
1279        let venue = self.venue();
1280        let start_nanos = datetime_to_unix_nanos(request.start);
1281        let end_nanos = datetime_to_unix_nanos(request.end);
1282        let params = request.params;
1283        let clock = self.clock;
1284
1285        self.spawn_task("request_instruments", async move {
1286            // the fetched universe also feeds the cache the periodic refresh diffs
1287            // against, so it must be published like a refresh pass or a market first
1288            // seen here would never reach execution
1289            let refresh = refresh_instruments(
1290                &update_lock,
1291                &http,
1292                &ws,
1293                &instruments_cache,
1294                &coin_map,
1295                &instrument_status,
1296                &sender,
1297            )
1298            .await?;
1299            refresh.log(client_id);
1300            let instruments = refresh.fetched;
1301
1302            let response = DataResponse::Instruments(InstrumentsResponse::new(
1303                request_id,
1304                client_id,
1305                venue,
1306                instruments,
1307                start_nanos,
1308                end_nanos,
1309                clock.get_time_ns(),
1310                params,
1311            ));
1312
1313            if let Err(e) = sender.send(DataEvent::Response(response)) {
1314                log::error!("Failed to send instruments response: {e}");
1315            }
1316            Ok(())
1317        });
1318
1319        Ok(())
1320    }
1321
1322    fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
1323        let instrument_id = request.single_instrument_id()?;
1324
1325        log::debug!("Requesting instrument: {instrument_id}");
1326
1327        let http = self.http_client.clone();
1328        let ws = self.ws_client.clone();
1329        let sender = self.data_sender.clone();
1330        let instruments_cache = self.instruments.clone();
1331        let coin_map = self.coin_to_instrument_id.clone();
1332        let update_lock = Arc::clone(&self.instrument_update_lock);
1333        let instrument_status = Arc::clone(&self.instrument_status);
1334        let request_id = request.request_id;
1335        let client_id = request.client_id.unwrap_or(self.client_id);
1336        let start_nanos = datetime_to_unix_nanos(request.start);
1337        let end_nanos = datetime_to_unix_nanos(request.end);
1338        let params = request.params;
1339        let clock = self.clock;
1340
1341        self.spawn_task("request_instrument", async move {
1342            // the venue only serves the whole universe, so a single-instrument request
1343            // can discover other new markets and must publish them like a refresh pass
1344            let refresh = refresh_instruments(
1345                &update_lock,
1346                &http,
1347                &ws,
1348                &instruments_cache,
1349                &coin_map,
1350                &instrument_status,
1351                &sender,
1352            )
1353            .await?;
1354            refresh.log(client_id);
1355            let all_instruments = refresh.fetched;
1356
1357            if let Some(instrument) = all_instruments
1358                .into_iter()
1359                .find(|i| i.id() == instrument_id)
1360            {
1361                let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
1362                    request_id,
1363                    client_id,
1364                    instrument.id(),
1365                    instrument,
1366                    start_nanos,
1367                    end_nanos,
1368                    clock.get_time_ns(),
1369                    params,
1370                )));
1371
1372                if let Err(e) = sender.send(DataEvent::Response(response)) {
1373                    log::error!("Failed to send instrument response: {e}");
1374                }
1375            } else {
1376                log::error!("Instrument not found: {instrument_id}");
1377            }
1378            Ok(())
1379        });
1380
1381        Ok(())
1382    }
1383
1384    fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
1385        let bar_type = request.single_bar_type()?;
1386
1387        log::debug!("Requesting bars for {bar_type}");
1388
1389        let http = self.http_client.clone();
1390        let sender = self.data_sender.clone();
1391        let start = request.start;
1392        let end = request.end;
1393        let limit = request.limit.map(|n| n.get() as u32);
1394        let request_id = request.request_id;
1395        let client_id = request.client_id.unwrap_or(self.client_id);
1396        let params = request.params;
1397        let clock = self.clock;
1398        let start_nanos = datetime_to_unix_nanos(start);
1399        let end_nanos = datetime_to_unix_nanos(end);
1400        let instruments = Arc::clone(&self.instruments);
1401
1402        self.spawn_task("request_bars", async move {
1403            let bars = request_bars_from_http(http, bar_type, start, end, limit, instruments)
1404                .await
1405                .context("bar request failed")?;
1406
1407            let response = DataResponse::Bars(BarsResponse::new(
1408                request_id,
1409                client_id,
1410                bar_type,
1411                bars,
1412                start_nanos,
1413                end_nanos,
1414                clock.get_time_ns(),
1415                params,
1416            ));
1417
1418            if let Err(e) = sender.send(DataEvent::Response(response)) {
1419                log::error!("Failed to send bars response: {e}");
1420            }
1421            Ok(())
1422        });
1423
1424        Ok(())
1425    }
1426
1427    fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
1428        let instrument_id = request.single_instrument_id()?;
1429
1430        log::debug!("Requesting trades for {instrument_id}");
1431
1432        let instruments = self.instruments.load();
1433        let instrument = instruments
1434            .get(&instrument_id)
1435            .cloned()
1436            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1437
1438        let coin = instrument.raw_symbol().to_string();
1439        let http = self.http_client.clone();
1440        let sender = self.data_sender.clone();
1441        let client_id = request.client_id.unwrap_or(self.client_id);
1442        let request_id = request.request_id;
1443        let params = request.params;
1444        let clock = self.clock;
1445        let limit = request.limit.map(|n| n.get());
1446        let start_nanos = datetime_to_unix_nanos(request.start);
1447        let end_nanos = datetime_to_unix_nanos(request.end);
1448
1449        self.spawn_task("request_trades", async move {
1450            // `recentTrades` depends on the Hyperliquid indexer; nodes without it
1451            // return HTTP 422. Treat that as "no coverage" and serve an empty
1452            // response so the awaiting caller still completes.
1453            let raw_trades = match http.info_recent_trades(&coin).await {
1454                Ok(trades) => trades,
1455                Err(e) if e.is_unprocessable_entity() => {
1456                    log::warn!(
1457                        "Recent trades endpoint unavailable for {instrument_id} \
1458                         (requires the Hyperliquid indexer); sending empty response"
1459                    );
1460                    Vec::new()
1461                }
1462                Err(e) => {
1463                    return Err(anyhow::Error::new(e))
1464                        .with_context(|| format!("trades request failed for {instrument_id}"));
1465                }
1466            };
1467
1468            let mut trades: Vec<TradeTick> = Vec::with_capacity(raw_trades.len());
1469            for raw in &raw_trades {
1470                match parse_recent_trade(raw, &instrument) {
1471                    Ok(trade) => trades.push(trade),
1472                    Err(e) => log::warn!("Skipping recent trade for {instrument_id}: {e}"),
1473                }
1474            }
1475            trades.sort_by_key(|trade| trade.ts_event);
1476
1477            let trades = filter_recent_trades(trades, start_nanos, end_nanos, limit, instrument_id);
1478
1479            log::debug!("Fetched {} trades for {instrument_id}", trades.len());
1480
1481            let response = DataResponse::Trades(TradesResponse::new(
1482                request_id,
1483                client_id,
1484                instrument_id,
1485                trades,
1486                start_nanos,
1487                end_nanos,
1488                clock.get_time_ns(),
1489                params,
1490            ));
1491
1492            if let Err(e) = sender.send(DataEvent::Response(response)) {
1493                log::error!("Failed to send trades response: {e}");
1494            }
1495            Ok(())
1496        });
1497
1498        Ok(())
1499    }
1500
1501    fn request_data(&self, request: RequestCustomData) -> anyhow::Result<()> {
1502        if request.data_type.type_name() != "HyperliquidPublicTrade" {
1503            log::warn!(
1504                "Unsupported custom data request: {}",
1505                request.data_type.type_name()
1506            );
1507            return Ok(());
1508        }
1509
1510        let instrument_id = Self::custom_instrument_id(&request.data_type)?
1511            .context("HyperliquidPublicTrade requests require metadata['instrument_id']")?;
1512        let data_type = DataType::new(
1513            request.data_type.type_name(),
1514            request.data_type.metadata().cloned(),
1515            Some(instrument_id.to_string()),
1516        );
1517        let http = self.http_client.clone();
1518        let sender = self.data_sender.clone();
1519        let request_id = request.request_id;
1520        let client_id = request.client_id;
1521        let params = request.params;
1522        let clock = self.clock;
1523        let limit = request.limit.map(|limit| limit.get());
1524        let start = request.start;
1525        let end = request.end;
1526        let start_nanos = datetime_to_unix_nanos(start);
1527        let end_nanos = datetime_to_unix_nanos(end);
1528        let venue = self.venue();
1529
1530        self.spawn_task("request_public_trades", async move {
1531            let trades = http
1532                .request_public_trades(instrument_id, start, end, limit)
1533                .await
1534                .map_err(anyhow::Error::new)
1535                .with_context(|| format!("public trades request failed for {instrument_id}"))?;
1536            let data: Vec<CustomData> = trades
1537                .into_iter()
1538                .map(|trade| CustomData::new(Arc::new(trade), data_type.clone()))
1539                .collect();
1540
1541            let response = DataResponse::Data(CustomDataResponse::new(
1542                request_id,
1543                client_id,
1544                Some(venue),
1545                data_type,
1546                data,
1547                start_nanos,
1548                end_nanos,
1549                clock.get_time_ns(),
1550                params,
1551            ));
1552
1553            if let Err(e) = sender.send(DataEvent::Response(response)) {
1554                log::error!("Failed to send public trades response: {e}");
1555            }
1556            Ok(())
1557        });
1558
1559        Ok(())
1560    }
1561
1562    fn request_funding_rates(&self, request: RequestFundingRates) -> anyhow::Result<()> {
1563        let instrument_id = request.single_instrument_id()?;
1564
1565        log::debug!("Requesting funding rates for {instrument_id}");
1566
1567        let instruments = self.instruments.load();
1568        let instrument = instruments
1569            .get(&instrument_id)
1570            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1571
1572        if !matches!(instrument, InstrumentAny::CryptoPerpetual(_)) {
1573            anyhow::bail!("Funding rates are only available for perpetual instruments");
1574        }
1575
1576        let instrument = instrument.clone();
1577        let http = self.http_client.clone();
1578        let sender = self.data_sender.clone();
1579        let client_id = request.client_id.unwrap_or(self.client_id);
1580        let request_id = request.request_id;
1581        let params = request.params;
1582        let clock = self.clock;
1583        let limit = request.limit.map(|n| n.get());
1584        let start_dt = request.start;
1585        let end_dt = request.end;
1586        let start_nanos = datetime_to_unix_nanos(start_dt);
1587        let end_nanos = datetime_to_unix_nanos(end_dt);
1588
1589        self.spawn_task("request_funding_rates", async move {
1590            let funding_rates = http
1591                .request_funding_rates(&instrument, start_dt, end_dt, limit)
1592                .await
1593                .with_context(|| format!("funding rates request failed for {instrument_id}"))?;
1594
1595            log::debug!(
1596                "Fetched {} funding rates for {instrument_id}",
1597                funding_rates.len(),
1598            );
1599
1600            let response = DataResponse::FundingRates(FundingRatesResponse::new(
1601                request_id,
1602                client_id,
1603                instrument_id,
1604                funding_rates,
1605                start_nanos,
1606                end_nanos,
1607                clock.get_time_ns(),
1608                params,
1609            ));
1610
1611            if let Err(e) = sender.send(DataEvent::Response(response)) {
1612                log::error!("Failed to send funding rates response: {e}");
1613            }
1614            Ok(())
1615        });
1616
1617        Ok(())
1618    }
1619
1620    fn request_book_snapshot(&self, request: RequestBookSnapshot) -> anyhow::Result<()> {
1621        let instrument_id = request.single_instrument_id()?;
1622
1623        let instruments = self.instruments.load();
1624        let instrument = instruments
1625            .get(&instrument_id)
1626            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1627
1628        let raw_symbol = instrument.raw_symbol().to_string();
1629        let price_precision = instrument.price_precision();
1630        let size_precision = instrument.size_precision();
1631        let depth = request.depth.map(|d| d.get());
1632
1633        let http = self.http_client.clone();
1634        let sender = self.data_sender.clone();
1635        let client_id = request.client_id.unwrap_or(self.client_id);
1636        let request_id = request.request_id;
1637        let params = request.params;
1638        let clock = self.clock;
1639
1640        self.spawn_task("request_book_snapshot", async move {
1641            let l2_book = http
1642                .info_l2_book(&raw_symbol)
1643                .await
1644                .with_context(|| format!("book snapshot request failed for {instrument_id}"))?;
1645
1646            let book = parse_l2_book_snapshot(
1647                &l2_book,
1648                instrument_id,
1649                price_precision,
1650                size_precision,
1651                depth,
1652            );
1653
1654            let response = DataResponse::Book(BookResponse::new(
1655                request_id,
1656                client_id,
1657                instrument_id,
1658                book,
1659                None,
1660                None,
1661                clock.get_time_ns(),
1662                params,
1663            ));
1664
1665            if let Err(e) = sender.send(DataEvent::Response(response)) {
1666                log::error!("Failed to send book snapshot response: {e}");
1667            }
1668            Ok(())
1669        });
1670
1671        Ok(())
1672    }
1673}
1674
1675/// Applies fetched instruments to the client caches and both transports.
1676fn cache_instruments(
1677    instruments: &[InstrumentAny],
1678    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1679    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1680    http_client: &HyperliquidHttpClient,
1681    ws_client: &HyperliquidWebSocketClient,
1682) {
1683    instruments_by_id.rcu(|m| {
1684        for instrument in instruments {
1685            m.insert(instrument.id(), instrument.clone());
1686        }
1687    });
1688
1689    coin_to_instrument_id.rcu(|m| {
1690        for instrument in instruments {
1691            m.insert(instrument.raw_symbol().inner(), instrument.id());
1692        }
1693    });
1694
1695    http_client.merge_instruments(instruments);
1696    ws_client.merge_instruments(instruments);
1697}
1698
1699/// Rebuilds the `allDexsAssetCtxs` mapping from the HTTP client's cached instruments.
1700///
1701/// Callers rebuild on every pass rather than only when a definition changed. The
1702/// mapping is positional over each perp dex universe, so a listing or delisting
1703/// shifts the entries of coins whose own definitions are unchanged, and a stale
1704/// mapping would misattribute incoming `ctxs` arrays. Rebuilding unconditionally
1705/// also retries a build that failed or fell back on an earlier pass.
1706async fn rebuild_all_dex_asset_ctxs_mapping(
1707    http_client: &HyperliquidHttpClient,
1708    ws_client: &HyperliquidWebSocketClient,
1709) {
1710    match http_client.build_all_dex_asset_ctxs_instrument_ids().await {
1711        Ok(mapping) => {
1712            let mapping = mapping
1713                .into_iter()
1714                .map(|(dex, instrument_ids)| (Ustr::from(dex.as_str()), instrument_ids))
1715                .collect();
1716            ws_client.cache_all_dex_asset_ctxs_instrument_ids(mapping);
1717        }
1718        Err(e) => {
1719            log::warn!("Failed to build Hyperliquid allDexsAssetCtxs mapping: {e}");
1720        }
1721    }
1722}
1723
1724/// Summary of a single instrument refresh pass.
1725#[derive(Debug)]
1726struct InstrumentRefresh {
1727    /// Instruments returned by the venue.
1728    fetched: Vec<InstrumentAny>,
1729    /// Symbols of the definitions that were new to the cache.
1730    added: Vec<Ustr>,
1731    /// New or materially changed definitions published downstream.
1732    changed: usize,
1733}
1734
1735impl InstrumentRefresh {
1736    fn log(&self, client_id: ClientId) {
1737        // a quiet pass every interval would be noise, but a market becoming
1738        // tradable mid-session is the event an operator needs to see
1739        if self.added.is_empty() {
1740            log::debug!(
1741                "Hyperliquid instruments refreshed: client_id={client_id}, fetched={}, changed={}",
1742                self.fetched.len(),
1743                self.changed,
1744            );
1745        } else {
1746            log::info!(
1747                "Hyperliquid instruments refreshed: client_id={client_id}, fetched={}, changed={}, added={:?}",
1748                self.fetched.len(),
1749                self.changed,
1750                self.added,
1751            );
1752        }
1753    }
1754}
1755
1756/// Fetches the instrument universe, reconciles it against the caches, and emits
1757/// instrument status changes for subscribed instruments.
1758///
1759/// Holds `update_lock` across the fetch and the reconcile so concurrent passes
1760/// apply in the order they fetched. The diff only asks whether a definition
1761/// differs from the cached one, not whether it is newer, so without the lock a
1762/// pass that fetched before another but finished after it would republish the
1763/// older definition.
1764async fn refresh_instruments(
1765    update_lock: &tokio::sync::Mutex<()>,
1766    http_client: &HyperliquidHttpClient,
1767    ws_client: &HyperliquidWebSocketClient,
1768    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1769    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1770    instrument_status: &Mutex<InstrumentStatusState>,
1771    data_sender: &EventSender<DataEvent>,
1772) -> anyhow::Result<InstrumentRefresh> {
1773    let _update_guard = update_lock.lock().await;
1774
1775    let snapshot = http_client
1776        .request_instruments_with_statuses()
1777        .await
1778        .context("failed to fetch Hyperliquid instruments")?;
1779
1780    // emit statuses after definitions, so a subscriber never sees a status for an
1781    // instrument it has not received
1782    let refresh = reconcile_instruments(
1783        snapshot.instruments,
1784        http_client,
1785        ws_client,
1786        instruments_by_id,
1787        coin_to_instrument_id,
1788        data_sender,
1789    )
1790    .await;
1791
1792    let ts = get_atomic_clock_realtime().get_time_ns();
1793    instrument_status
1794        .lock()
1795        .apply(&snapshot.statuses, &snapshot.loaded, data_sender, ts);
1796
1797    Ok(refresh)
1798}
1799
1800/// Reconciles the instrument caches against a fetched instrument universe.
1801///
1802/// Caches and publishes new or materially changed definitions as
1803/// [`DataEvent::Data`]. Unchanged definitions are not republished. Cached
1804/// instruments absent from `fetched` are retained because they may still back
1805/// open subscriptions.
1806///
1807/// `instruments_by_id` is the baseline later passes diff against, so every path
1808/// that writes fetched instruments into it must come through here, under the
1809/// update lock taken by [`refresh_instruments`]. A market cached without being
1810/// published would compare as unchanged on every later pass, and the execution
1811/// client would never receive its asset index.
1812async fn reconcile_instruments(
1813    fetched: Vec<InstrumentAny>,
1814    http_client: &HyperliquidHttpClient,
1815    ws_client: &HyperliquidWebSocketClient,
1816    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1817    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1818    data_sender: &EventSender<DataEvent>,
1819) -> InstrumentRefresh {
1820    let changed = changed_definitions(&fetched, instruments_by_id);
1821    let added = added_symbols(&changed, instruments_by_id);
1822
1823    cache_instruments(
1824        &changed,
1825        instruments_by_id,
1826        coin_to_instrument_id,
1827        http_client,
1828        ws_client,
1829    );
1830
1831    // publish before awaiting the mapping rebuild, so a pass cancelled during the
1832    // rebuild cannot leave these definitions cached but unpublished
1833    for instrument in &changed {
1834        if let Err(e) = data_sender.send(DataEvent::Data(instrument.clone().into())) {
1835            log::warn!("Failed to send instrument: {e}");
1836        }
1837    }
1838
1839    rebuild_all_dex_asset_ctxs_mapping(http_client, ws_client).await;
1840
1841    InstrumentRefresh {
1842        added,
1843        changed: changed.len(),
1844        fetched,
1845    }
1846}
1847
1848/// Returns the fetched instruments that are new or materially changed.
1849fn changed_definitions(
1850    fetched: &[InstrumentAny],
1851    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1852) -> Vec<InstrumentAny> {
1853    fetched
1854        .iter()
1855        .filter(|instrument| {
1856            instruments_by_id
1857                .get_cloned(&instrument.id())
1858                .is_none_or(|cached| !instrument_definitions_match(&cached, instrument))
1859        })
1860        .cloned()
1861        .collect()
1862}
1863
1864/// Returns the symbols of the changed definitions the cache has never held.
1865///
1866/// A market listed after startup is the case the refresh exists for, so a pass
1867/// reports which symbols became tradable rather than only how many definitions
1868/// moved. Callers pass the changed set, so a definition that merely moved its
1869/// tick size is not reported as new.
1870fn added_symbols(
1871    changed: &[InstrumentAny],
1872    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1873) -> Vec<Ustr> {
1874    changed
1875        .iter()
1876        .filter(|instrument| instruments_by_id.get_cloned(&instrument.id()).is_none())
1877        .map(|instrument| instrument.symbol().inner())
1878        .collect()
1879}
1880
1881/// Returns `true` when two instruments carry the same tradable definition,
1882/// ignoring event timestamps.
1883///
1884/// Comparison runs on the serialized form so every venue field, including the
1885/// free-form `info` metadata, participates without listing each field.
1886fn instrument_definitions_match(a: &InstrumentAny, b: &InstrumentAny) -> bool {
1887    fn normalized(instrument: &InstrumentAny) -> Option<serde_json::Value> {
1888        let mut value = serde_json::to_value(instrument).ok()?;
1889
1890        if let Some(definition) = value
1891            .as_object_mut()
1892            .and_then(|obj| obj.values_mut().next())
1893            .and_then(serde_json::Value::as_object_mut)
1894        {
1895            definition.remove("ts_event");
1896            definition.remove("ts_init");
1897        }
1898
1899        Some(value)
1900    }
1901
1902    // A serialization failure compares as changed so updates are never suppressed
1903    match (normalized(a), normalized(b)) {
1904        (Some(a), Some(b)) => a == b,
1905        _ => false,
1906    }
1907}
1908
1909#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1910enum MarketDataChannel {
1911    Deltas,
1912    Depth,
1913    Quote,
1914}
1915
1916impl MarketDataChannel {
1917    const fn as_str(self) -> &'static str {
1918        match self {
1919            Self::Deltas => "deltas",
1920            Self::Depth => "depth",
1921            Self::Quote => "quote",
1922        }
1923    }
1924}
1925
1926type MarketDataStreamKey = (MarketDataChannel, InstrumentId);
1927
1928#[derive(Debug, Clone)]
1929struct MarketDataStreamHealth {
1930    last_receive_at: Instant,
1931    last_venue_ts_event: Option<UnixNanos>,
1932    consecutive_stale_count: u32,
1933    last_warning_at: Option<Instant>,
1934    last_recovery_at: Option<Instant>,
1935    resubscribe_attempts: u32,
1936}
1937
1938impl MarketDataStreamHealth {
1939    fn new(receive_at: Instant) -> Self {
1940        Self {
1941            last_receive_at: receive_at,
1942            last_venue_ts_event: None,
1943            consecutive_stale_count: 0,
1944            last_warning_at: None,
1945            last_recovery_at: None,
1946            resubscribe_attempts: 0,
1947        }
1948    }
1949
1950    fn record_receive(&mut self, receive_at: Instant, venue_ts_event: UnixNanos) {
1951        self.last_receive_at = receive_at;
1952        self.last_venue_ts_event = Some(venue_ts_event);
1953        self.consecutive_stale_count = 0;
1954        self.last_warning_at = None;
1955        self.last_recovery_at = None;
1956        self.resubscribe_attempts = 0;
1957    }
1958
1959    // Records a recovery action at `recovery_at` and returns the next step of the ladder
1960    fn record_recovery(
1961        &mut self,
1962        recovery_at: Instant,
1963        book_recovery: bool,
1964        max_targeted_resubscribes: u32,
1965    ) -> StaleStreamAction {
1966        self.last_recovery_at = Some(recovery_at);
1967        self.last_warning_at = Some(recovery_at);
1968
1969        // Shared book recovery owns retries and backoff for a delta book's stream
1970        if book_recovery {
1971            return StaleStreamAction::Recover;
1972        }
1973
1974        if self.resubscribe_attempts < max_targeted_resubscribes {
1975            self.resubscribe_attempts += 1;
1976            return StaleStreamAction::Resubscribe;
1977        }
1978
1979        // Reconnect replays all active subscriptions
1980        self.resubscribe_attempts = 0;
1981        StaleStreamAction::Reconnect
1982    }
1983}
1984
1985#[derive(Debug, Clone, Copy)]
1986struct StreamRecoveryConfig {
1987    cooldown: Duration,
1988    max_targeted_resubscribes: u32,
1989}
1990
1991#[derive(Debug)]
1992struct MarketDataStreamHealthMonitor {
1993    stale_receive_threshold: Duration,
1994    warning_cooldown: Duration,
1995    recovery: Option<StreamRecoveryConfig>,
1996    streams: AHashMap<MarketDataStreamKey, MarketDataStreamHealth>,
1997}
1998
1999impl MarketDataStreamHealthMonitor {
2000    fn new(stale_receive_threshold: Duration, warning_cooldown: Duration) -> Self {
2001        Self {
2002            stale_receive_threshold,
2003            warning_cooldown,
2004            recovery: None,
2005            streams: AHashMap::new(),
2006        }
2007    }
2008
2009    fn with_recovery(mut self, cooldown: Duration, max_targeted_resubscribes: u32) -> Self {
2010        self.recovery = Some(StreamRecoveryConfig {
2011            cooldown,
2012            max_targeted_resubscribes,
2013        });
2014        self
2015    }
2016
2017    fn subscribe(
2018        &mut self,
2019        channel: MarketDataChannel,
2020        instrument_id: InstrumentId,
2021        receive_at: Instant,
2022    ) {
2023        self.streams.insert(
2024            (channel, instrument_id),
2025            MarketDataStreamHealth::new(receive_at),
2026        );
2027    }
2028
2029    fn unsubscribe(&mut self, channel: MarketDataChannel, instrument_id: InstrumentId) {
2030        self.streams.remove(&(channel, instrument_id));
2031    }
2032
2033    fn clear(&mut self) {
2034        self.streams.clear();
2035    }
2036
2037    fn record_receive(
2038        &mut self,
2039        channel: MarketDataChannel,
2040        instrument_id: InstrumentId,
2041        receive_at: Instant,
2042        venue_ts_event: UnixNanos,
2043    ) {
2044        if let Some(stream) = self.streams.get_mut(&(channel, instrument_id)) {
2045            stream.record_receive(receive_at, venue_ts_event);
2046        }
2047    }
2048
2049    fn check_stale(
2050        &mut self,
2051        now: Instant,
2052        wall_clock_now: UnixNanos,
2053    ) -> Vec<MarketDataStaleEvent> {
2054        // Fresh BBO makes stale book streams relative-stale, not transport-stale
2055        let fresh_quote_instruments: AHashSet<InstrumentId> = self
2056            .streams
2057            .iter()
2058            .filter(|((channel, _), stream)| {
2059                *channel == MarketDataChannel::Quote
2060                    && now.saturating_duration_since(stream.last_receive_at)
2061                        < self.stale_receive_threshold
2062            })
2063            .map(|((_, instrument_id), _)| *instrument_id)
2064            .collect();
2065
2066        // Depth shares the instrument's `l2Book` stream with its delta book
2067        let delta_book_instruments: AHashSet<InstrumentId> = self
2068            .streams
2069            .keys()
2070            .filter(|(channel, _)| *channel == MarketDataChannel::Deltas)
2071            .map(|(_, instrument_id)| *instrument_id)
2072            .collect();
2073
2074        let mut events = Vec::new();
2075
2076        for ((channel, instrument_id), stream) in &mut self.streams {
2077            let receive_age = now.saturating_duration_since(stream.last_receive_at);
2078            if receive_age < self.stale_receive_threshold {
2079                stream.consecutive_stale_count = 0;
2080                continue;
2081            }
2082
2083            stream.consecutive_stale_count = stream.consecutive_stale_count.saturating_add(1);
2084
2085            let quote_is_fresh = matches!(
2086                channel,
2087                MarketDataChannel::Deltas | MarketDataChannel::Depth
2088            ) && fresh_quote_instruments.contains(instrument_id);
2089
2090            let venue_age = stream.last_venue_ts_event.map(|ts_event| {
2091                Duration::from_nanos(wall_clock_now.as_u64().saturating_sub(ts_event.as_u64()))
2092            });
2093
2094            if let Some(recovery) = self.recovery {
2095                // Recovery requires one prior warning, even after long check lag
2096                let stale_since = stream.last_receive_at + self.stale_receive_threshold;
2097                let anchor = stream.last_recovery_at.unwrap_or(stale_since);
2098
2099                if stream.last_warning_at.is_some()
2100                    && now.saturating_duration_since(anchor) >= recovery.cooldown
2101                {
2102                    let book_recovery = matches!(
2103                        channel,
2104                        MarketDataChannel::Deltas | MarketDataChannel::Depth
2105                    ) && delta_book_instruments.contains(instrument_id);
2106                    let action = stream.record_recovery(
2107                        now,
2108                        book_recovery,
2109                        recovery.max_targeted_resubscribes,
2110                    );
2111
2112                    events.push(MarketDataStaleEvent {
2113                        channel: *channel,
2114                        instrument_id: *instrument_id,
2115                        receive_age,
2116                        venue_age,
2117                        stale_count: stream.consecutive_stale_count,
2118                        action,
2119                        cooldown: recovery.cooldown,
2120                        quote_is_fresh,
2121                    });
2122                    continue;
2123                }
2124            }
2125
2126            let should_warn = stream.last_warning_at.is_none_or(|last_warning_at| {
2127                now.saturating_duration_since(last_warning_at) >= self.warning_cooldown
2128            });
2129
2130            if !should_warn {
2131                continue;
2132            }
2133
2134            stream.last_warning_at = Some(now);
2135            events.push(MarketDataStaleEvent {
2136                channel: *channel,
2137                instrument_id: *instrument_id,
2138                receive_age,
2139                venue_age,
2140                stale_count: stream.consecutive_stale_count,
2141                action: StaleStreamAction::Warn,
2142                cooldown: self.warning_cooldown,
2143                quote_is_fresh,
2144            });
2145        }
2146
2147        events
2148    }
2149}
2150
2151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2152enum StaleStreamAction {
2153    Warn,
2154    Recover,
2155    Resubscribe,
2156    Reconnect,
2157}
2158
2159impl StaleStreamAction {
2160    const fn as_str(self) -> &'static str {
2161        match self {
2162            Self::Warn => "warn",
2163            Self::Recover => "recover",
2164            Self::Resubscribe => "resubscribe",
2165            Self::Reconnect => "reconnect",
2166        }
2167    }
2168}
2169
2170#[derive(Debug, Clone, PartialEq, Eq)]
2171struct MarketDataStaleEvent {
2172    channel: MarketDataChannel,
2173    instrument_id: InstrumentId,
2174    receive_age: Duration,
2175    venue_age: Option<Duration>,
2176    stale_count: u32,
2177    action: StaleStreamAction,
2178    cooldown: Duration,
2179    quote_is_fresh: bool,
2180}
2181
2182fn stream_health_update(
2183    msg: &NautilusWsMessage,
2184) -> Option<(MarketDataChannel, InstrumentId, UnixNanos)> {
2185    match msg {
2186        NautilusWsMessage::Quote(quote) => Some((
2187            MarketDataChannel::Quote,
2188            quote.instrument_id,
2189            quote.ts_event,
2190        )),
2191        NautilusWsMessage::Deltas(deltas) => Some((
2192            MarketDataChannel::Deltas,
2193            deltas.instrument_id,
2194            deltas.ts_event,
2195        )),
2196        NautilusWsMessage::Depth(depth) => Some((
2197            MarketDataChannel::Depth,
2198            depth.instrument_id,
2199            depth.ts_event,
2200        )),
2201        _ => None,
2202    }
2203}
2204
2205fn record_stream_receive(
2206    stream_health: &Arc<Mutex<MarketDataStreamHealthMonitor>>,
2207    channel: MarketDataChannel,
2208    instrument_id: InstrumentId,
2209    venue_ts_event: UnixNanos,
2210) {
2211    stream_health
2212        .lock()
2213        .record_receive(channel, instrument_id, Instant::now(), venue_ts_event);
2214}
2215
2216fn log_stream_health_event(event: &MarketDataStaleEvent) {
2217    let venue_age_ms = event
2218        .venue_age
2219        .map_or_else(|| "n/a".to_string(), |age| age.as_millis().to_string());
2220    let prefix = if event.quote_is_fresh {
2221        "Hyperliquid book stream stale while bbo advances"
2222    } else {
2223        "Hyperliquid market data stream stale"
2224    };
2225
2226    log::warn!(
2227        "{prefix}: channel={}, instrument_id={}, receive_age_ms={}, venue_age_ms={}, \
2228         stale_count={}, action={}, cooldown_secs={}",
2229        event.channel.as_str(),
2230        event.instrument_id,
2231        event.receive_age.as_millis(),
2232        venue_age_ms,
2233        event.stale_count,
2234        event.action.as_str(),
2235        event.cooldown.as_secs(),
2236    );
2237}
2238
2239async fn handle_stream_health_events(
2240    ws_client: &HyperliquidWebSocketClient,
2241    events: &[MarketDataStaleEvent],
2242    book_sync: &BookSyncTracker,
2243    snapshot_timeout: Duration,
2244    book_tasks: &TaskSpawner,
2245) {
2246    // Deltas and depth share one venue `l2Book` stream
2247    let mut resubscribed_books: AHashSet<InstrumentId> = AHashSet::new();
2248    let mut reconnect_requested = false;
2249
2250    for event in events {
2251        log_stream_health_event(event);
2252
2253        match event.action {
2254            StaleStreamAction::Warn => {}
2255            StaleStreamAction::Recover => book::recovery::start_recovery(
2256                event.instrument_id,
2257                book_sync,
2258                ws_client,
2259                snapshot_timeout,
2260                book_tasks,
2261            ),
2262            StaleStreamAction::Resubscribe => match event.channel {
2263                MarketDataChannel::Deltas | MarketDataChannel::Depth => {
2264                    if resubscribed_books.insert(event.instrument_id)
2265                        && let Err(e) = ws_client.resubscribe_book(event.instrument_id).await
2266                    {
2267                        log::warn!(
2268                            "Failed targeted l2Book resubscribe for {}: {e}",
2269                            event.instrument_id,
2270                        );
2271                    }
2272                }
2273                MarketDataChannel::Quote => {
2274                    if let Err(e) = ws_client.resubscribe_quotes(event.instrument_id).await {
2275                        log::warn!(
2276                            "Failed targeted bbo resubscribe for {}: {e}",
2277                            event.instrument_id,
2278                        );
2279                    }
2280                }
2281            },
2282            StaleStreamAction::Reconnect => reconnect_requested = true,
2283        }
2284    }
2285
2286    if reconnect_requested {
2287        if ws_client.request_reconnect() {
2288            log::warn!("Requested full WebSocket reconnect after failed targeted stream recovery");
2289        } else {
2290            log::debug!("Skipping reconnect request: connection not active");
2291        }
2292    }
2293}
2294
2295// Applies the request window and limit to a snapshot of recent trades. `trades`
2296// must be sorted ascending by `ts_event`. Returns the subset within `[start, end]`
2297// (each bound unbounded when `None`), keeping at most the most recent `limit`
2298// trades. Because `recentTrades` exposes only a recent snapshot with no historical
2299// depth, a warning is logged when the request reaches below the snapshot's
2300// coverage floor (its oldest trade).
2301fn filter_recent_trades(
2302    trades: Vec<TradeTick>,
2303    start: Option<UnixNanos>,
2304    end: Option<UnixNanos>,
2305    limit: Option<usize>,
2306    instrument_id: InstrumentId,
2307) -> Vec<TradeTick> {
2308    let Some(floor) = trades.first().map(|trade| trade.ts_event) else {
2309        return Vec::new();
2310    };
2311
2312    if let Some(end) = end
2313        && end < floor
2314    {
2315        log::warn!(
2316            "Recent trades for {instrument_id} are entirely older than the requested window; \
2317             snapshot only covers back to {}",
2318            unix_nanos_to_iso8601(floor),
2319        );
2320        return Vec::new();
2321    }
2322
2323    if let Some(start) = start
2324        && start < floor
2325    {
2326        log::warn!(
2327            "Recent trades for {instrument_id} only cover back to {}; \
2328             the requested start is earlier and cannot be served",
2329            unix_nanos_to_iso8601(floor),
2330        );
2331    }
2332
2333    let mut filtered: Vec<TradeTick> = trades
2334        .into_iter()
2335        .filter(|trade| start.is_none_or(|s| trade.ts_event >= s))
2336        .filter(|trade| end.is_none_or(|e| trade.ts_event <= e))
2337        .collect();
2338
2339    if let Some(limit) = limit
2340        && filtered.len() > limit
2341    {
2342        // Keep the most recent `limit` trades; ascending order is preserved
2343        filtered.drain(0..filtered.len() - limit);
2344    }
2345
2346    filtered
2347}
2348
2349// Levels with unparsable px/sz or non-positive size are skipped rather than
2350// erroring; the snapshot's `time` field (ms) becomes `ts_event` after the
2351// ms->ns conversion.
2352pub(crate) fn parse_l2_book_snapshot(
2353    l2_book: &HyperliquidL2Book,
2354    instrument_id: InstrumentId,
2355    price_precision: u8,
2356    size_precision: u8,
2357    depth: Option<usize>,
2358) -> OrderBook {
2359    let mut book = OrderBook::new(instrument_id, BookType::L2_MBP);
2360    let ts_event = UnixNanos::from(l2_book.time * 1_000_000);
2361
2362    let all_bids = l2_book
2363        .levels
2364        .first()
2365        .map_or([].as_slice(), |v| v.as_slice());
2366    let all_asks = l2_book
2367        .levels
2368        .get(1)
2369        .map_or([].as_slice(), |v| v.as_slice());
2370
2371    let bids = match depth {
2372        Some(d) if d < all_bids.len() => &all_bids[..d],
2373        _ => all_bids,
2374    };
2375    let asks = match depth {
2376        Some(d) if d < all_asks.len() => &all_asks[..d],
2377        _ => all_asks,
2378    };
2379
2380    for (i, level) in bids.iter().enumerate() {
2381        if level.sz <= Decimal::ZERO {
2382            continue;
2383        }
2384        let Ok(price) = Price::from_decimal_dp(level.px, price_precision) else {
2385            continue;
2386        };
2387        let Ok(size) = Quantity::from_decimal_dp(level.sz, size_precision) else {
2388            continue;
2389        };
2390
2391        let order = BookOrder::new(OrderSide::Buy, price, size, i as u64);
2392        book.add(order, 0, i as u64, ts_event);
2393    }
2394
2395    let bids_len = bids.len();
2396
2397    for (i, level) in asks.iter().enumerate() {
2398        if level.sz <= Decimal::ZERO {
2399            continue;
2400        }
2401        let Ok(price) = Price::from_decimal_dp(level.px, price_precision) else {
2402            continue;
2403        };
2404        let Ok(size) = Quantity::from_decimal_dp(level.sz, size_precision) else {
2405            continue;
2406        };
2407
2408        let order = BookOrder::new(OrderSide::Sell, price, size, (bids_len + i) as u64);
2409        book.add(order, 0, (bids_len + i) as u64, ts_event);
2410    }
2411
2412    log::debug!(
2413        "Built order book for {instrument_id} with {} bids and {} asks",
2414        bids.len(),
2415        asks.len(),
2416    );
2417
2418    book
2419}
2420
2421// Reads optional `nSigFigs` / `mantissa` L2 precision controls from
2422// `subscribe_params`; bails on non-positive integer values.
2423pub(crate) fn parse_book_precision_params(
2424    params: Option<&Params>,
2425) -> anyhow::Result<(Option<u32>, Option<u32>)> {
2426    let Some(params) = params else {
2427        return Ok((None, None));
2428    };
2429
2430    let read_u32 = |key: &str| -> anyhow::Result<Option<u32>> {
2431        match params.get(key) {
2432            None => Ok(None),
2433            Some(v) => v
2434                .as_u64()
2435                .and_then(|n| u32::try_from(n).ok())
2436                .ok_or_else(|| anyhow::anyhow!("`{key}` must be a positive u32"))
2437                .map(Some),
2438        }
2439    };
2440
2441    Ok((read_u32("n_sig_figs")?, read_u32("mantissa")?))
2442}
2443
2444// Hyperliquid funds perpetuals hourly, so `interval` is fixed at 60 mins;
2445// `time` from the venue marks the end of the funding interval in ms.
2446pub(crate) fn funding_entry_to_update(
2447    entry: &HyperliquidFundingHistoryEntry,
2448    instrument_id: InstrumentId,
2449) -> FundingRateUpdate {
2450    let rate = entry.funding_rate;
2451    let ts = UnixNanos::from(entry.time * 1_000_000);
2452    FundingRateUpdate::new(instrument_id, rate, Some(60), None, ts, ts)
2453}
2454
2455pub(crate) fn candle_to_bar(
2456    candle: &HyperliquidCandle,
2457    bar_type: BarType,
2458    price_precision: u8,
2459    size_precision: u8,
2460) -> anyhow::Result<Bar> {
2461    let ts_event = millis_to_nanos(candle.timestamp)?;
2462    let close_boundary = candle
2463        .end_timestamp
2464        .checked_add(1)
2465        .context("candle close boundary overflow")?;
2466    let ts_init = millis_to_nanos(close_boundary)?;
2467
2468    let open = Price::from_decimal_dp(candle.open, price_precision)
2469        .map_err(|e| anyhow::anyhow!("invalid open price: {e}"))?;
2470    let high = Price::from_decimal_dp(candle.high, price_precision)
2471        .map_err(|e| anyhow::anyhow!("invalid high price: {e}"))?;
2472    let low = Price::from_decimal_dp(candle.low, price_precision)
2473        .map_err(|e| anyhow::anyhow!("invalid low price: {e}"))?;
2474    let close = Price::from_decimal_dp(candle.close, price_precision)
2475        .map_err(|e| anyhow::anyhow!("invalid close price: {e}"))?;
2476    let volume = Quantity::from_decimal_dp(candle.volume, size_precision)
2477        .map_err(|e| anyhow::anyhow!("invalid volume: {e}"))?;
2478
2479    Ok(Bar::new(
2480        bar_type, open, high, low, close, volume, ts_event, ts_init,
2481    ))
2482}
2483
2484/// Request bars from HTTP API.
2485async fn request_bars_from_http(
2486    http_client: HyperliquidHttpClient,
2487    bar_type: BarType,
2488    start: Option<Timestamp>,
2489    end: Option<Timestamp>,
2490    limit: Option<u32>,
2491    instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
2492) -> anyhow::Result<Vec<Bar>> {
2493    // Get instrument details for precision
2494    let instrument_id = bar_type.instrument_id();
2495    let instrument = instruments
2496        .load()
2497        .get(&instrument_id)
2498        .cloned()
2499        .context("instrument not found in cache")?;
2500
2501    let price_precision = instrument.price_precision();
2502    let size_precision = instrument.size_precision();
2503    let raw_symbol = instrument.raw_symbol();
2504    let coin = raw_symbol.as_str();
2505
2506    let interval = bar_type_to_interval(&bar_type)?;
2507
2508    // Hyperliquid uses millisecond timestamps
2509    let now = Timestamp::now();
2510    let end_time = end.unwrap_or(now).as_millisecond() as u64;
2511    let start_time = if let Some(start) = start {
2512        start.as_millisecond() as u64
2513    } else {
2514        // Default to 1000 bars before end_time
2515        let spec = bar_type.spec();
2516        let step_ms = match spec.aggregation {
2517            BarAggregation::Minute => spec.step.get() as u64 * 60_000,
2518            BarAggregation::Hour => spec.step.get() as u64 * 3_600_000,
2519            BarAggregation::Day => spec.step.get() as u64 * 86_400_000,
2520            _ => 60_000,
2521        };
2522        end_time.saturating_sub(1000 * step_ms)
2523    };
2524
2525    let candles = http_client
2526        .info_candle_snapshot(coin, interval, start_time, end_time)
2527        .await
2528        .context("failed to fetch candle snapshot from Hyperliquid")?;
2529
2530    let now_ms = now.as_millisecond() as u64;
2531    let mut bars: Vec<Bar> = candles
2532        .iter()
2533        .filter(|candle| candle.end_timestamp < now_ms)
2534        .filter_map(|candle| {
2535            candle_to_bar(candle, bar_type, price_precision, size_precision)
2536                .map_err(|e| {
2537                    log::warn!("Failed to convert candle to bar: {e}");
2538                    e
2539                })
2540                .ok()
2541        })
2542        .collect();
2543
2544    if let Some(limit) = limit
2545        && bars.len() > limit as usize
2546    {
2547        bars = bars.into_iter().take(limit as usize).collect();
2548    }
2549
2550    log::debug!("Fetched {} bars for {}", bars.len(), bar_type);
2551    Ok(bars)
2552}
2553
2554#[cfg(test)]
2555#[path = "../tests/integration/data_client.rs"]
2556mod data_client_tests;
2557
2558#[cfg(test)]
2559mod tests {
2560    use futures_util::StreamExt;
2561    use nautilus_common::live::runner::set_data_event_sender;
2562    use nautilus_model::{
2563        data::{
2564            QuoteTick,
2565            stubs::{stub_deltas, stub_depth10},
2566        },
2567        enums::{AggressorSide, CurrencyType},
2568        identifiers::{Symbol, TradeId},
2569        instruments::CryptoPerpetual,
2570        types::Currency,
2571    };
2572    use rstest::rstest;
2573    use rust_decimal_macros::dec;
2574    use ustr::Ustr;
2575
2576    use super::*;
2577    use crate::common::{
2578        consts::{HYPERLIQUID_CLIENT_ID, HYPERLIQUID_WS_URL},
2579        testing::load_test_data,
2580    };
2581
2582    fn btc_perp_id() -> InstrumentId {
2583        InstrumentId::from("BTC-PERP.HYPERLIQUID")
2584    }
2585
2586    #[rstest]
2587    fn test_candle_to_bar_uses_causal_initialization_timestamp() {
2588        let candle = HyperliquidCandle {
2589            timestamp: 1_700_000_000_000,
2590            end_timestamp: 1_700_000_059_999,
2591            open: dec!(100.0),
2592            high: dec!(101.0),
2593            low: dec!(99.0),
2594            close: dec!(100.5),
2595            volume: dec!(10.0),
2596            num_trades: Some(42),
2597        };
2598        let bar_type = BarType::from("BTC-USD-PERP.HYPERLIQUID-1-MINUTE-LAST-EXTERNAL");
2599
2600        let bar = candle_to_bar(&candle, bar_type, 1, 1).unwrap();
2601
2602        assert_eq!(candle.end_timestamp - candle.timestamp, 59_999);
2603        assert_eq!(bar.ts_event, millis_to_nanos(candle.timestamp).unwrap());
2604        assert_eq!(
2605            bar.ts_init,
2606            millis_to_nanos(candle.end_timestamp + 1).unwrap()
2607        );
2608        assert!(bar.ts_init > bar.ts_event);
2609    }
2610
2611    #[rstest]
2612    fn test_candle_to_bar_rejects_close_boundary_overflow() {
2613        let candle = HyperliquidCandle {
2614            timestamp: 1_700_000_000_000,
2615            end_timestamp: u64::MAX,
2616            open: dec!(100.0),
2617            high: dec!(101.0),
2618            low: dec!(99.0),
2619            close: dec!(100.5),
2620            volume: dec!(10.0),
2621            num_trades: Some(42),
2622        };
2623        let bar_type = BarType::from("BTC-USD-PERP.HYPERLIQUID-1-MINUTE-LAST-EXTERNAL");
2624
2625        let err = candle_to_bar(&candle, bar_type, 1, 1).unwrap_err();
2626
2627        assert!(err.to_string().contains("close boundary overflow"));
2628    }
2629
2630    #[rstest]
2631    fn test_stream_health_monitor_fresh_stream_does_not_warn() {
2632        let mut monitor =
2633            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2634        let instrument_id = btc_perp_id();
2635        let start = Instant::now();
2636
2637        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2638
2639        let warnings = monitor.check_stale(
2640            start + Duration::from_secs(4),
2641            UnixNanos::from(4_000_000_000),
2642        );
2643        assert!(warnings.is_empty());
2644    }
2645
2646    #[rstest]
2647    fn test_stream_health_monitor_warns_once_after_threshold() {
2648        let mut monitor =
2649            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2650        let instrument_id = btc_perp_id();
2651        let start = Instant::now();
2652
2653        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2654        monitor.record_receive(
2655            MarketDataChannel::Quote,
2656            instrument_id,
2657            start + Duration::from_secs(1),
2658            UnixNanos::from(1_000_000_000),
2659        );
2660
2661        let warnings = monitor.check_stale(
2662            start + Duration::from_secs(7),
2663            UnixNanos::from(9_000_000_000),
2664        );
2665
2666        assert_eq!(
2667            warnings,
2668            vec![MarketDataStaleEvent {
2669                channel: MarketDataChannel::Quote,
2670                instrument_id,
2671                receive_age: Duration::from_secs(6),
2672                venue_age: Some(Duration::from_secs(8)),
2673                stale_count: 1,
2674                action: StaleStreamAction::Warn,
2675                cooldown: Duration::from_secs(30),
2676                quote_is_fresh: false,
2677            }]
2678        );
2679    }
2680
2681    #[rstest]
2682    fn test_stream_health_monitor_warns_at_receive_threshold() {
2683        let mut monitor =
2684            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2685        let instrument_id = btc_perp_id();
2686        let start = Instant::now();
2687
2688        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2689
2690        let warnings = monitor.check_stale(
2691            start + Duration::from_secs(5),
2692            UnixNanos::from(5_000_000_000),
2693        );
2694
2695        assert_eq!(warnings.len(), 1);
2696        assert_eq!(warnings[0].receive_age, Duration::from_secs(5));
2697        assert_eq!(warnings[0].stale_count, 1);
2698    }
2699
2700    #[rstest]
2701    fn test_stream_health_monitor_new_update_resets_age_and_stale_count() {
2702        let mut monitor =
2703            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2704        let instrument_id = btc_perp_id();
2705        let start = Instant::now();
2706
2707        monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
2708        assert_eq!(
2709            monitor
2710                .check_stale(
2711                    start + Duration::from_secs(6),
2712                    UnixNanos::from(6_000_000_000),
2713                )
2714                .len(),
2715            1,
2716        );
2717
2718        monitor.record_receive(
2719            MarketDataChannel::Depth,
2720            instrument_id,
2721            start + Duration::from_secs(7),
2722            UnixNanos::from(7_000_000_000),
2723        );
2724
2725        assert!(
2726            monitor
2727                .check_stale(
2728                    start + Duration::from_secs(11),
2729                    UnixNanos::from(11_000_000_000),
2730                )
2731                .is_empty()
2732        );
2733
2734        let warnings = monitor.check_stale(
2735            start + Duration::from_secs(13),
2736            UnixNanos::from(13_000_000_000),
2737        );
2738        assert_eq!(warnings.len(), 1);
2739        assert_eq!(warnings[0].stale_count, 1);
2740        assert_eq!(warnings[0].receive_age, Duration::from_secs(6));
2741    }
2742
2743    #[rstest]
2744    fn test_stream_health_monitor_unsubscribe_removes_stream() {
2745        let mut monitor =
2746            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2747        let instrument_id = btc_perp_id();
2748        let start = Instant::now();
2749
2750        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2751        monitor.unsubscribe(MarketDataChannel::Deltas, instrument_id);
2752
2753        let warnings = monitor.check_stale(
2754            start + Duration::from_secs(6),
2755            UnixNanos::from(6_000_000_000),
2756        );
2757
2758        assert!(warnings.is_empty());
2759    }
2760
2761    #[rstest]
2762    #[case(0, 15)]
2763    #[case(120, 0)]
2764    fn test_data_client_stream_health_config_zero_disables_monitor(
2765        #[case] stale_receive_timeout_secs: u64,
2766        #[case] check_interval_secs: u64,
2767    ) {
2768        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
2769        set_data_event_sender(tx);
2770        let client = HyperliquidDataClient::new(
2771            *crate::common::consts::HYPERLIQUID_CLIENT_ID,
2772            HyperliquidDataClientConfig {
2773                stale_stream_receive_timeout_secs: stale_receive_timeout_secs,
2774                stream_health_check_interval_secs: check_interval_secs,
2775                ..HyperliquidDataClientConfig::default()
2776            },
2777        )
2778        .unwrap();
2779        let instrument_id = btc_perp_id();
2780        let start = Instant::now();
2781
2782        assert!(!client.stream_health_monitor_enabled());
2783        client.register_stream_health(MarketDataChannel::Deltas, instrument_id);
2784
2785        let warnings = client.stream_health.lock().check_stale(
2786            start + Duration::from_secs(121),
2787            UnixNanos::from(121_000_000_000),
2788        );
2789
2790        assert!(warnings.is_empty());
2791    }
2792
2793    #[rstest]
2794    fn test_data_client_recovery_requires_positive_cooldown() {
2795        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
2796        set_data_event_sender(tx);
2797        let client = HyperliquidDataClient::new(
2798            *crate::common::consts::HYPERLIQUID_CLIENT_ID,
2799            HyperliquidDataClientConfig {
2800                stale_stream_recovery_enabled: true,
2801                stale_stream_recovery_cooldown_secs: 0,
2802                ..HyperliquidDataClientConfig::default()
2803            },
2804        )
2805        .unwrap();
2806
2807        assert!(
2808            client.stream_health.lock().recovery.is_none(),
2809            "a zero recovery cooldown must leave the monitor observability-only",
2810        );
2811    }
2812
2813    #[rstest]
2814    fn test_stream_health_monitor_warning_cooldown_prevents_repeated_logs() {
2815        let mut monitor =
2816            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10));
2817        let instrument_id = btc_perp_id();
2818        let start = Instant::now();
2819
2820        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2821
2822        let first = monitor.check_stale(
2823            start + Duration::from_secs(6),
2824            UnixNanos::from(6_000_000_000),
2825        );
2826        let inside_cooldown = monitor.check_stale(
2827            start + Duration::from_secs(7),
2828            UnixNanos::from(7_000_000_000),
2829        );
2830        let second = monitor.check_stale(
2831            start + Duration::from_secs(16),
2832            UnixNanos::from(16_000_000_000),
2833        );
2834
2835        assert_eq!(first.len(), 1);
2836        assert!(inside_cooldown.is_empty());
2837        assert_eq!(second.len(), 1);
2838        assert_eq!(second[0].stale_count, 3);
2839    }
2840
2841    fn check_at(
2842        monitor: &mut MarketDataStreamHealthMonitor,
2843        start: Instant,
2844        secs: u64,
2845    ) -> Vec<MarketDataStaleEvent> {
2846        monitor.check_stale(
2847            start + Duration::from_secs(secs),
2848            UnixNanos::from(secs * 1_000_000_000),
2849        )
2850    }
2851
2852    #[rstest]
2853    fn test_stream_health_recovery_ladder_escalates_and_resets() {
2854        let mut monitor =
2855            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2856                .with_recovery(Duration::from_secs(30), 2);
2857        let instrument_id = btc_perp_id();
2858        let start = Instant::now();
2859
2860        monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
2861
2862        let events = check_at(&mut monitor, start, 5);
2863        assert_eq!(events.len(), 1);
2864        assert_eq!(events[0].action, StaleStreamAction::Warn);
2865
2866        let events = check_at(&mut monitor, start, 20);
2867        assert_eq!(events[0].action, StaleStreamAction::Warn);
2868
2869        let events = check_at(&mut monitor, start, 35);
2870        assert_eq!(
2871            events,
2872            vec![MarketDataStaleEvent {
2873                channel: MarketDataChannel::Depth,
2874                instrument_id,
2875                receive_age: Duration::from_secs(35),
2876                venue_age: None,
2877                stale_count: 3,
2878                action: StaleStreamAction::Resubscribe,
2879                cooldown: Duration::from_secs(30),
2880                quote_is_fresh: false,
2881            }],
2882        );
2883
2884        let events = check_at(&mut monitor, start, 50);
2885        assert_eq!(events[0].action, StaleStreamAction::Warn);
2886
2887        let events = check_at(&mut monitor, start, 65);
2888        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2889
2890        let events = check_at(&mut monitor, start, 95);
2891        assert_eq!(events[0].action, StaleStreamAction::Reconnect);
2892
2893        let events = check_at(&mut monitor, start, 125);
2894        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2895    }
2896
2897    #[rstest]
2898    fn test_stream_health_recovery_hands_deltas_to_book_recovery_without_reconnect() {
2899        let mut monitor =
2900            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2901                .with_recovery(Duration::from_secs(30), 1);
2902        let instrument_id = btc_perp_id();
2903        let start = Instant::now();
2904
2905        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2906
2907        let first = check_at(&mut monitor, start, 5);
2908        let second = check_at(&mut monitor, start, 35);
2909        let third = check_at(&mut monitor, start, 65);
2910        let fourth = check_at(&mut monitor, start, 95);
2911
2912        assert_eq!(first[0].action, StaleStreamAction::Warn);
2913        assert_eq!(
2914            second,
2915            vec![MarketDataStaleEvent {
2916                channel: MarketDataChannel::Deltas,
2917                instrument_id,
2918                receive_age: Duration::from_secs(35),
2919                venue_age: None,
2920                stale_count: 2,
2921                action: StaleStreamAction::Recover,
2922                cooldown: Duration::from_secs(30),
2923                quote_is_fresh: false,
2924            }],
2925        );
2926        assert_eq!(third[0].action, StaleStreamAction::Recover);
2927        assert_eq!(fourth[0].action, StaleStreamAction::Recover);
2928    }
2929
2930    #[rstest]
2931    fn test_stream_health_recovery_hands_shared_depth_to_book_recovery() {
2932        let mut monitor =
2933            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2934                .with_recovery(Duration::from_secs(30), 1);
2935        let shared = btc_perp_id();
2936        let depth_only = InstrumentId::from("ETH-PERP.HYPERLIQUID");
2937        let start = Instant::now();
2938
2939        monitor.subscribe(MarketDataChannel::Deltas, shared, start);
2940        monitor.subscribe(MarketDataChannel::Depth, shared, start);
2941        monitor.subscribe(MarketDataChannel::Depth, depth_only, start);
2942
2943        let actions = |events: Vec<MarketDataStaleEvent>| {
2944            let mut actions = events
2945                .into_iter()
2946                .map(|event| (event.channel.as_str(), event.instrument_id, event.action))
2947                .collect::<Vec<_>>();
2948            actions.sort_by_key(|(channel, instrument_id, _)| (*channel, *instrument_id));
2949            actions
2950        };
2951
2952        check_at(&mut monitor, start, 5);
2953        let first = actions(check_at(&mut monitor, start, 35));
2954        let second = actions(check_at(&mut monitor, start, 65));
2955
2956        assert_eq!(
2957            first,
2958            vec![
2959                ("deltas", shared, StaleStreamAction::Recover),
2960                ("depth", shared, StaleStreamAction::Recover),
2961                ("depth", depth_only, StaleStreamAction::Resubscribe),
2962            ],
2963        );
2964        assert_eq!(
2965            second,
2966            vec![
2967                ("deltas", shared, StaleStreamAction::Recover),
2968                ("depth", shared, StaleStreamAction::Recover),
2969                ("depth", depth_only, StaleStreamAction::Reconnect),
2970            ],
2971        );
2972    }
2973
2974    #[rstest]
2975    fn test_stream_health_recovery_first_breach_warns_even_past_cooldown() {
2976        let mut monitor =
2977            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2978                .with_recovery(Duration::from_secs(1), 1);
2979        let instrument_id = btc_perp_id();
2980        let start = Instant::now();
2981
2982        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2983
2984        // First stale checks must stay observability-only, even after cooldown
2985        let events = check_at(&mut monitor, start, 40);
2986        assert_eq!(events.len(), 1);
2987        assert_eq!(events[0].action, StaleStreamAction::Warn);
2988
2989        let events = check_at(&mut monitor, start, 41);
2990        assert_eq!(events.len(), 1);
2991        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2992    }
2993
2994    #[rstest]
2995    fn test_stream_health_receive_resets_recovery_state() {
2996        let mut monitor =
2997            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2998                .with_recovery(Duration::from_secs(10), 1);
2999        let instrument_id = btc_perp_id();
3000        let start = Instant::now();
3001
3002        monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
3003        assert_eq!(
3004            check_at(&mut monitor, start, 5)[0].action,
3005            StaleStreamAction::Warn
3006        );
3007        assert_eq!(
3008            check_at(&mut monitor, start, 15)[0].action,
3009            StaleStreamAction::Resubscribe,
3010        );
3011
3012        monitor.record_receive(
3013            MarketDataChannel::Depth,
3014            instrument_id,
3015            start + Duration::from_secs(16),
3016            UnixNanos::from(16_000_000_000),
3017        );
3018
3019        assert!(check_at(&mut monitor, start, 20).is_empty());
3020
3021        let events = check_at(&mut monitor, start, 21);
3022        assert_eq!(events[0].action, StaleStreamAction::Warn);
3023        assert_eq!(events[0].stale_count, 1);
3024
3025        assert_eq!(
3026            check_at(&mut monitor, start, 31)[0].action,
3027            StaleStreamAction::Resubscribe,
3028        );
3029        assert_eq!(
3030            check_at(&mut monitor, start, 41)[0].action,
3031            StaleStreamAction::Reconnect,
3032        );
3033    }
3034
3035    #[rstest]
3036    fn test_check_stale_book_with_fresh_quote_flags_relative_staleness() {
3037        let mut monitor =
3038            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
3039        let instrument_id = btc_perp_id();
3040        let start = Instant::now();
3041
3042        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
3043        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
3044        monitor.record_receive(
3045            MarketDataChannel::Quote,
3046            instrument_id,
3047            start + Duration::from_secs(8),
3048            UnixNanos::from(8_000_000_000),
3049        );
3050
3051        let events = check_at(&mut monitor, start, 10);
3052
3053        assert_eq!(events.len(), 1, "fresh quote stream must not be reported");
3054        assert_eq!(events[0].channel, MarketDataChannel::Deltas);
3055        assert!(events[0].quote_is_fresh);
3056    }
3057
3058    #[rstest]
3059    #[case(true)]
3060    #[case(false)]
3061    fn test_check_stale_book_without_fresh_quote_is_not_flagged(#[case] quote_subscribed: bool) {
3062        let mut monitor =
3063            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
3064        let instrument_id = btc_perp_id();
3065        let start = Instant::now();
3066
3067        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
3068        if quote_subscribed {
3069            monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
3070        }
3071
3072        let events = check_at(&mut monitor, start, 10);
3073
3074        let deltas_event = events
3075            .iter()
3076            .find(|event| event.channel == MarketDataChannel::Deltas)
3077            .expect("deltas event");
3078        assert!(
3079            !deltas_event.quote_is_fresh,
3080            "a stale or absent quote stream must not flag relative staleness",
3081        );
3082
3083        if quote_subscribed {
3084            let quote_event = events
3085                .iter()
3086                .find(|event| event.channel == MarketDataChannel::Quote)
3087                .expect("quote event");
3088            assert!(!quote_event.quote_is_fresh);
3089        }
3090    }
3091
3092    #[rstest]
3093    fn test_stream_health_update_extracts_tracked_market_data_messages() {
3094        let quote = QuoteTick {
3095            instrument_id: btc_perp_id(),
3096            ts_event: UnixNanos::from(1),
3097            ..QuoteTick::default()
3098        };
3099        let deltas = stub_deltas();
3100        let depth = stub_depth10();
3101
3102        assert_eq!(
3103            stream_health_update(&NautilusWsMessage::Quote(quote)),
3104            Some((
3105                MarketDataChannel::Quote,
3106                quote.instrument_id,
3107                quote.ts_event
3108            )),
3109        );
3110        assert_eq!(
3111            stream_health_update(&NautilusWsMessage::Deltas(deltas.clone())),
3112            Some((
3113                MarketDataChannel::Deltas,
3114                deltas.instrument_id,
3115                deltas.ts_event
3116            )),
3117        );
3118        assert_eq!(
3119            stream_health_update(&NautilusWsMessage::Depth(Box::new(depth.clone()))),
3120            Some((
3121                MarketDataChannel::Depth,
3122                depth.instrument_id,
3123                depth.ts_event
3124            )),
3125        );
3126        assert_eq!(stream_health_update(&NautilusWsMessage::Reconnected), None,);
3127    }
3128
3129    #[rstest]
3130    fn test_funding_entry_to_update_parses_positive_rate() {
3131        let entry = HyperliquidFundingHistoryEntry {
3132            coin: Ustr::from("BTC"),
3133            funding_rate: dec!(0.0000125),
3134            premium: Some(dec!(0.00029005)),
3135            time: 1769908800000,
3136        };
3137        let instrument_id = btc_perp_id();
3138
3139        let update = funding_entry_to_update(&entry, instrument_id);
3140
3141        assert_eq!(update.instrument_id, instrument_id);
3142        assert_eq!(update.rate, dec!(0.0000125));
3143        assert_eq!(update.interval, Some(60));
3144        assert!(update.next_funding_ns.is_none());
3145        assert_eq!(update.ts_event, UnixNanos::from(1769908800000 * 1_000_000));
3146        assert_eq!(update.ts_init, update.ts_event);
3147    }
3148
3149    #[rstest]
3150    fn test_funding_entry_to_update_handles_negative_rate() {
3151        let entry = HyperliquidFundingHistoryEntry {
3152            coin: Ustr::from("BTC"),
3153            funding_rate: dec!(-0.0000081),
3154            premium: None,
3155            time: 1769912400000,
3156        };
3157        let update = funding_entry_to_update(&entry, btc_perp_id());
3158        assert_eq!(update.rate, dec!(-0.0000081));
3159    }
3160
3161    #[rstest]
3162    fn test_funding_history_entry_rejects_invalid_rate() {
3163        // The funding rate is now a Decimal field, so an invalid value is
3164        // rejected at deserialization rather than by funding_entry_to_update.
3165        let json = r#"{"coin":"BTC","fundingRate":"not-a-number","time":1769912400000}"#;
3166        assert!(serde_json::from_str::<HyperliquidFundingHistoryEntry>(json).is_err());
3167    }
3168
3169    #[rstest]
3170    fn test_parse_book_precision_params_none() {
3171        let (n, m) = parse_book_precision_params(None).unwrap();
3172        assert_eq!(n, None);
3173        assert_eq!(m, None);
3174    }
3175
3176    fn make_params(json: serde_json::Value) -> Params {
3177        serde_json::from_value(json).expect("valid params payload")
3178    }
3179
3180    #[rstest]
3181    fn test_parse_book_precision_params_only_n_sig_figs() {
3182        let params = make_params(serde_json::json!({"n_sig_figs": 4}));
3183        let (n, m) = parse_book_precision_params(Some(&params)).unwrap();
3184        assert_eq!(n, Some(4));
3185        assert_eq!(m, None);
3186    }
3187
3188    #[rstest]
3189    fn test_parse_book_precision_params_both() {
3190        let params = make_params(serde_json::json!({"n_sig_figs": 5, "mantissa": 2}));
3191        let (n, m) = parse_book_precision_params(Some(&params)).unwrap();
3192        assert_eq!(n, Some(5));
3193        assert_eq!(m, Some(2));
3194    }
3195
3196    #[rstest]
3197    fn test_parse_book_precision_params_rejects_negative() {
3198        let params = make_params(serde_json::json!({"n_sig_figs": -1}));
3199        let err = parse_book_precision_params(Some(&params)).unwrap_err();
3200        assert!(err.to_string().contains("n_sig_figs"));
3201    }
3202
3203    #[rstest]
3204    fn test_funding_history_fixture_parses() {
3205        let entries: Vec<HyperliquidFundingHistoryEntry> =
3206            load_test_data("http_funding_history.json");
3207        assert_eq!(entries.len(), 3);
3208        assert_eq!(entries[0].coin, "BTC");
3209        assert_eq!(entries[0].funding_rate, dec!(0.0000125));
3210        assert_eq!(entries[0].premium, Some(dec!(0.00029005)));
3211        assert!(entries[2].premium.is_none());
3212
3213        let updates: Vec<FundingRateUpdate> = entries
3214            .iter()
3215            .map(|e| funding_entry_to_update(e, btc_perp_id()))
3216            .collect();
3217        assert_eq!(updates.len(), 3);
3218        assert_eq!(updates[0].rate, dec!(0.0000125));
3219        assert_eq!(updates[1].rate, dec!(-0.0000081));
3220        assert_eq!(updates[2].rate, dec!(0.0000033));
3221    }
3222
3223    fn level(px: &str, sz: &str) -> crate::http::models::HyperliquidLevel {
3224        crate::http::models::HyperliquidLevel {
3225            px: px.parse().unwrap(),
3226            sz: sz.parse().unwrap(),
3227        }
3228    }
3229
3230    fn sample_l2_book() -> HyperliquidL2Book {
3231        HyperliquidL2Book {
3232            coin: Ustr::from("BTC"),
3233            levels: vec![
3234                vec![
3235                    level("98450.50", "2.5"),
3236                    level("98449.00", "1.2"),
3237                    level("98448.00", "0.8"),
3238                ],
3239                vec![
3240                    level("98451.00", "1.5"),
3241                    level("98452.00", "2.0"),
3242                    level("98453.00", "0.5"),
3243                ],
3244            ],
3245            time: 1769908800000,
3246        }
3247    }
3248
3249    #[rstest]
3250    fn test_parse_l2_book_snapshot_populates_both_sides() {
3251        let book_data = sample_l2_book();
3252        let instrument_id = btc_perp_id();
3253        let book = parse_l2_book_snapshot(&book_data, instrument_id, 2, 4, None);
3254
3255        assert_eq!(book.instrument_id, instrument_id);
3256        assert_eq!(book.book_type, BookType::L2_MBP);
3257        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3258        assert_eq!(book.best_ask_price(), Some(Price::new(98451.00, 2)));
3259        assert_eq!(book.best_bid_size(), Some(Quantity::new(2.5, 4)));
3260        assert_eq!(book.best_ask_size(), Some(Quantity::new(1.5, 4)));
3261        assert_eq!(book.update_count, 6);
3262    }
3263
3264    #[rstest]
3265    fn test_parse_l2_book_snapshot_truncates_to_depth() {
3266        let book_data = sample_l2_book();
3267        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, Some(1));
3268
3269        // depth=1 keeps the top of book on each side, drops the rest.
3270        assert_eq!(book.update_count, 2);
3271        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3272        assert_eq!(book.best_ask_price(), Some(Price::new(98451.00, 2)));
3273    }
3274
3275    #[rstest]
3276    fn test_parse_l2_book_snapshot_uses_venue_time_as_ts_event() {
3277        let book_data = sample_l2_book();
3278        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3279        let expected_ts = UnixNanos::from(1769908800000_u64 * 1_000_000);
3280
3281        // ts_last reflects the last applied delta; every added order
3282        // carries the venue time after the ms->ns conversion.
3283        assert_eq!(book.ts_last, expected_ts);
3284    }
3285
3286    #[rstest]
3287    fn test_parse_l2_book_snapshot_skips_non_positive_size() {
3288        let book_data = HyperliquidL2Book {
3289            coin: Ustr::from("BTC"),
3290            levels: vec![
3291                vec![level("98450.50", "2.5"), level("98449.00", "0")],
3292                vec![level("98451.00", "0"), level("98452.00", "1.5")],
3293            ],
3294            time: 1769908800000,
3295        };
3296        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3297
3298        assert_eq!(book.update_count, 2, "zero-sized levels must be skipped");
3299        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3300        assert_eq!(book.best_ask_price(), Some(Price::new(98452.00, 2)));
3301    }
3302
3303    #[rstest]
3304    fn test_parse_l2_book_snapshot_skips_zero_size_levels() {
3305        let book_data = HyperliquidL2Book {
3306            coin: Ustr::from("BTC"),
3307            levels: vec![
3308                vec![level("98448.00", "0.0"), level("98449.00", "1.2")],
3309                vec![level("98451.00", "0.0"), level("98452.00", "1.5")],
3310            ],
3311            time: 1769908800000,
3312        };
3313        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3314
3315        // Zero-size levels are skipped; one priced level remains per side.
3316        assert_eq!(book.update_count, 2);
3317        assert_eq!(book.best_bid_price(), Some(Price::new(98449.00, 2)));
3318        assert_eq!(book.best_ask_price(), Some(Price::new(98452.00, 2)));
3319    }
3320
3321    #[rstest]
3322    fn test_parse_l2_book_snapshot_empty_levels_yields_empty_book() {
3323        let book_data = HyperliquidL2Book {
3324            coin: Ustr::from("BTC"),
3325            levels: vec![],
3326            time: 1769908800000,
3327        };
3328        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3329
3330        assert_eq!(book.update_count, 0);
3331        assert!(book.best_bid_price().is_none());
3332        assert!(book.best_ask_price().is_none());
3333    }
3334
3335    fn trade_at(ts_ns: u64, tid: u64) -> TradeTick {
3336        TradeTick::new(
3337            btc_perp_id(),
3338            Price::from("104300.0"),
3339            Quantity::from("0.01000"),
3340            AggressorSide::Buy,
3341            TradeId::new(tid.to_string()),
3342            UnixNanos::from(ts_ns),
3343            UnixNanos::from(ts_ns),
3344        )
3345    }
3346
3347    // A snapshot of three trades at 1000/2000/3000ns, sorted ascending. The
3348    // coverage floor (oldest) is 1000ns.
3349    fn sample_trades() -> Vec<TradeTick> {
3350        vec![trade_at(1000, 1), trade_at(2000, 2), trade_at(3000, 3)]
3351    }
3352
3353    #[rstest]
3354    fn test_recent_trades_fixture_parses_and_sorts() {
3355        let raw: Vec<crate::http::models::HyperliquidRecentTrade> =
3356            load_test_data("http_recent_trades_btc.json");
3357        assert_eq!(raw.len(), 3);
3358        // Fixture is newest-first as the venue returns it.
3359        assert_eq!(raw[0].tid, 300003);
3360
3361        let meta: crate::http::models::PerpMeta = load_test_data("http_meta_perp_sample.json");
3362        let defs = crate::http::parse::parse_perp_instruments(&meta, 0).unwrap();
3363        let instrument =
3364            crate::http::parse::create_instrument_from_def(&defs[0], UnixNanos::default()).unwrap();
3365
3366        let mut trades: Vec<TradeTick> = raw
3367            .iter()
3368            .map(|t| parse_recent_trade(t, &instrument).unwrap())
3369            .collect();
3370        trades.sort_by_key(|trade| trade.ts_event);
3371
3372        // Ascending after sort: oldest tid first.
3373        assert_eq!(trades[0].trade_id.to_string(), "300001");
3374        assert_eq!(trades[2].trade_id.to_string(), "300003");
3375        assert!(trades[0].ts_event <= trades[2].ts_event);
3376        // Historical ticks carry ts_init == ts_event.
3377        assert_eq!(trades[0].ts_init, trades[0].ts_event);
3378    }
3379
3380    #[rstest]
3381    fn test_filter_recent_trades_full_window_returns_all() {
3382        let filtered = filter_recent_trades(sample_trades(), None, None, None, btc_perp_id());
3383
3384        assert_eq!(filtered.len(), 3);
3385    }
3386
3387    #[rstest]
3388    fn test_filter_recent_trades_empty_snapshot_returns_empty() {
3389        let filtered = filter_recent_trades(
3390            Vec::new(),
3391            Some(UnixNanos::from(500)),
3392            Some(UnixNanos::from(2500)),
3393            None,
3394            btc_perp_id(),
3395        );
3396
3397        assert!(filtered.is_empty());
3398    }
3399
3400    #[rstest]
3401    fn test_filter_recent_trades_entirely_older_returns_empty() {
3402        // Requested window ends before the snapshot floor (1000ns).
3403        let filtered = filter_recent_trades(
3404            sample_trades(),
3405            Some(UnixNanos::from(100)),
3406            Some(UnixNanos::from(500)),
3407            None,
3408            btc_perp_id(),
3409        );
3410
3411        assert!(filtered.is_empty());
3412    }
3413
3414    #[rstest]
3415    fn test_filter_recent_trades_partial_keeps_in_range_subset() {
3416        // Start (500ns) is below the floor; end (2500ns) drops the 3000ns trade.
3417        let filtered = filter_recent_trades(
3418            sample_trades(),
3419            Some(UnixNanos::from(500)),
3420            Some(UnixNanos::from(2500)),
3421            None,
3422            btc_perp_id(),
3423        );
3424
3425        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3426        assert_eq!(ts, vec![1000, 2000]);
3427    }
3428
3429    #[rstest]
3430    fn test_filter_recent_trades_within_window_filters_bounds() {
3431        let filtered = filter_recent_trades(
3432            sample_trades(),
3433            Some(UnixNanos::from(1500)),
3434            Some(UnixNanos::from(3000)),
3435            None,
3436            btc_perp_id(),
3437        );
3438
3439        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3440        assert_eq!(ts, vec![2000, 3000]);
3441    }
3442
3443    #[rstest]
3444    fn test_filter_recent_trades_limit_keeps_most_recent() {
3445        let filtered = filter_recent_trades(sample_trades(), None, None, Some(2), btc_perp_id());
3446
3447        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3448        assert_eq!(ts, vec![2000, 3000]);
3449    }
3450
3451    #[rstest]
3452    fn test_filter_recent_trades_end_equal_to_floor_keeps_floor_trade() {
3453        // `end` exactly on the floor (1000ns) is inclusive: not "entirely
3454        // older". Distinguishes `end < floor` from `end <= floor`.
3455        let filtered = filter_recent_trades(
3456            sample_trades(),
3457            None,
3458            Some(UnixNanos::from(1000)),
3459            None,
3460            btc_perp_id(),
3461        );
3462
3463        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3464        assert_eq!(ts, vec![1000]);
3465    }
3466
3467    #[rstest]
3468    fn test_filter_recent_trades_bounds_are_inclusive() {
3469        // `start`/`end` landing exactly on a trade's ts_event keep that trade.
3470        // Distinguishes `>=`/`<=` from strict `>`/`<`.
3471        let filtered = filter_recent_trades(
3472            sample_trades(),
3473            Some(UnixNanos::from(2000)),
3474            Some(UnixNanos::from(3000)),
3475            None,
3476            btc_perp_id(),
3477        );
3478
3479        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3480        assert_eq!(ts, vec![2000, 3000]);
3481    }
3482
3483    fn perp_instrument(symbol: &str, tick_size: &str, ts_init: UnixNanos) -> InstrumentAny {
3484        let base = Currency::new("BTC", 8, 0, "BTC", CurrencyType::Crypto);
3485        let usd = Currency::new("USD", 8, 0, "USD", CurrencyType::Crypto);
3486        let usdc = Currency::new("USDC", 6, 0, "USDC", CurrencyType::Crypto);
3487
3488        InstrumentAny::CryptoPerpetual(
3489            CryptoPerpetual::builder()
3490                .instrument_id(InstrumentId::new(Symbol::new(symbol), *HYPERLIQUID_VENUE))
3491                .raw_symbol(Symbol::new("BTC"))
3492                .base_currency(base)
3493                .quote_currency(usd)
3494                .settlement_currency(usdc)
3495                .is_inverse(false)
3496                .price_precision(1)
3497                .size_precision(3)
3498                .price_increment(Price::from(tick_size))
3499                .size_increment(Quantity::from("0.001"))
3500                .ts_event(ts_init)
3501                .ts_init(ts_init)
3502                .build()
3503                .unwrap(),
3504        )
3505    }
3506
3507    fn cached(instruments: &[InstrumentAny]) -> Arc<AtomicMap<InstrumentId, InstrumentAny>> {
3508        let map = AtomicMap::new();
3509        map.rcu(|m| {
3510            for instrument in instruments {
3511                m.insert(instrument.id(), instrument.clone());
3512            }
3513        });
3514        Arc::new(map)
3515    }
3516
3517    fn data_client_with_refresh_interval(minutes: u64) -> HyperliquidDataClient {
3518        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
3519        set_data_event_sender(tx);
3520
3521        HyperliquidDataClient::new(
3522            *HYPERLIQUID_CLIENT_ID,
3523            HyperliquidDataClientConfig {
3524                update_instruments_interval_mins: minutes,
3525                ..HyperliquidDataClientConfig::default()
3526            },
3527        )
3528        .unwrap()
3529    }
3530
3531    #[rstest]
3532    #[case::disconnect(false, None)]
3533    #[case::reset_then_disconnect(true, None)]
3534    #[ignore = "connects to Hyperliquid mainnet public market data"]
3535    #[case::mainnet(false, Some(HYPERLIQUID_WS_URL))]
3536    #[tokio::test]
3537    async fn test_ws_consumer_exits_when_handler_output_closes(
3538        #[case] reset: bool,
3539        #[case] url: Option<&str>,
3540    ) {
3541        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3542        let addr = listener.local_addr().unwrap();
3543        let server = tokio::spawn(async move {
3544            let (socket, _) = listener.accept().await.unwrap();
3545            let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap();
3546            while let Some(Ok(_)) = socket.next().await {}
3547        });
3548
3549        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
3550        set_data_event_sender(tx);
3551
3552        let url = url.map_or_else(|| format!("ws://{addr}"), str::to_owned);
3553
3554        let mut client = HyperliquidDataClient::new(
3555            *HYPERLIQUID_CLIENT_ID,
3556            HyperliquidDataClientConfig {
3557                base_url_ws: Some(url),
3558                ..HyperliquidDataClientConfig::default()
3559            },
3560        )
3561        .unwrap();
3562
3563        client.spawn_ws().await.unwrap();
3564        assert_eq!(client.session_tasks.len(), 1);
3565
3566        client.ws_client.disconnect().await.unwrap();
3567        tokio::time::timeout(Duration::from_secs(2), async {
3568            while !client.session_tasks.all_finished() {
3569                tokio::task::yield_now().await;
3570            }
3571        })
3572        .await
3573        .expect("consumer must exit when handler output closes without cancellation");
3574
3575        assert!(!client.cancellation_token.is_cancelled());
3576        assert!(client.session_tasks.all_finished());
3577        assert!(client.pending_tasks.is_empty());
3578
3579        if reset {
3580            client.reset().unwrap();
3581        }
3582
3583        client.disconnect().await.unwrap();
3584
3585        assert!(client.session_tasks.is_empty());
3586        assert!(client.pending_tasks.is_empty());
3587        assert!(client.is_disconnected());
3588        server.abort();
3589        let _ = server.await;
3590    }
3591
3592    #[tokio::test]
3593    async fn test_spawn_instrument_refresh_skipped_when_interval_zero() {
3594        let client = data_client_with_refresh_interval(0);
3595
3596        client.spawn_instrument_refresh().unwrap();
3597
3598        assert!(client.session_tasks.is_empty());
3599    }
3600
3601    #[tokio::test]
3602    async fn test_spawn_instrument_refresh_registers_task() {
3603        let client = data_client_with_refresh_interval(60);
3604
3605        client.spawn_instrument_refresh().unwrap();
3606
3607        assert_eq!(client.session_tasks.len(), 1);
3608
3609        client.cancellation_token.cancel();
3610        client.await_session_tasks().await.unwrap();
3611    }
3612
3613    #[rstest]
3614    #[case::disconnect(false)]
3615    #[case::reset(true)]
3616    #[tokio::test]
3617    async fn test_instrument_refresh_shutdown_drains_generation(#[case] reset: bool) {
3618        let mut client = data_client_with_refresh_interval(1);
3619        client.spawn_instrument_refresh().unwrap();
3620        if reset {
3621            client.reset().unwrap();
3622            client.await_session_tasks().await.unwrap();
3623        } else {
3624            client.disconnect().await.unwrap();
3625        }
3626
3627        assert!(client.cancellation_token.is_cancelled());
3628        assert!(client.session_tasks.is_empty());
3629        assert!(!client.session_tasks.is_open());
3630        assert!(client.spawn_instrument_refresh().is_err());
3631        assert!(client.is_disconnected());
3632    }
3633
3634    #[tokio::test]
3635    async fn test_instrument_refresh_connected_client_keeps_one_task() {
3636        let mut client = data_client_with_refresh_interval(1);
3637        client.spawn_instrument_refresh().unwrap();
3638        client.is_connected.store(true, Ordering::Release);
3639
3640        client.connect().await.unwrap();
3641        client.connect().await.unwrap();
3642
3643        assert_eq!(client.session_tasks.len(), 1);
3644        client.disconnect().await.unwrap();
3645        assert!(client.session_tasks.is_empty());
3646    }
3647
3648    #[rstest]
3649    fn test_cache_instruments_replaces_complete_snapshots() {
3650        let client = data_client_with_refresh_interval(0);
3651        let mut original = vec![
3652            perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1)),
3653            perp_instrument("ETH-USD-PERP", "0.2", UnixNanos::from(1)),
3654        ];
3655        let mut refreshed = vec![
3656            perp_instrument("BTC-USD-PERP", "0.3", UnixNanos::from(2)),
3657            perp_instrument("ETH-USD-PERP", "0.4", UnixNanos::from(2)),
3658        ];
3659
3660        for instruments in [&mut original, &mut refreshed] {
3661            let InstrumentAny::CryptoPerpetual(eth) = &mut instruments[1] else {
3662                unreachable!();
3663            };
3664
3665            eth.raw_symbol = Symbol::new("ETH");
3666        }
3667
3668        cache_instruments(
3669            &original,
3670            &client.instruments,
3671            &client.coin_to_instrument_id,
3672            &client.http_client,
3673            &client.ws_client,
3674        );
3675        let previous = client.instruments.load();
3676        cache_instruments(
3677            &refreshed,
3678            &client.instruments,
3679            &client.coin_to_instrument_id,
3680            &client.http_client,
3681            &client.ws_client,
3682        );
3683        let current = client.instruments.load();
3684
3685        assert_eq!(previous.len(), 2);
3686        assert_eq!(current.len(), 2);
3687
3688        for (old, new) in original.iter().zip(&refreshed) {
3689            assert_eq!(
3690                serde_json::to_value(previous.get(&old.id()).unwrap()).unwrap(),
3691                serde_json::to_value(old).unwrap(),
3692            );
3693            assert_eq!(
3694                serde_json::to_value(current.get(&new.id()).unwrap()).unwrap(),
3695                serde_json::to_value(new).unwrap(),
3696            );
3697            assert_eq!(
3698                client
3699                    .coin_to_instrument_id
3700                    .get_cloned(&new.raw_symbol().inner()),
3701                Some(new.id()),
3702            );
3703            assert_eq!(
3704                serde_json::to_value(
3705                    client
3706                        .ws_client
3707                        .instruments_cache()
3708                        .load()
3709                        .get(&new.raw_symbol().inner())
3710                        .unwrap(),
3711                )
3712                .unwrap(),
3713                serde_json::to_value(new).unwrap(),
3714            );
3715        }
3716    }
3717
3718    #[rstest]
3719    fn test_changed_definitions_reports_a_newly_listed_market() {
3720        let cached_instruments =
3721            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3722        let fetched = vec![
3723            perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1)),
3724            perp_instrument("NEW-USD-PERP", "0.1", UnixNanos::from(1)),
3725        ];
3726
3727        let changed = changed_definitions(&fetched, &cached_instruments);
3728
3729        assert_eq!(changed.len(), 1);
3730        assert_eq!(changed[0].id().symbol.as_str(), "NEW-USD-PERP");
3731    }
3732
3733    #[rstest]
3734    fn test_added_symbols_names_only_the_market_the_cache_never_held() {
3735        let cached_instruments =
3736            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3737        // BTC moved its tick size, so it is changed but not new
3738        let changed = vec![
3739            perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1)),
3740            perp_instrument("NEW-USD-PERP", "0.1", UnixNanos::from(1)),
3741        ];
3742
3743        let added = added_symbols(&changed, &cached_instruments);
3744
3745        assert_eq!(added, vec![Ustr::from("NEW-USD-PERP")]);
3746    }
3747
3748    #[rstest]
3749    fn test_added_symbols_is_empty_when_every_change_is_a_known_market() {
3750        let cached_instruments =
3751            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3752        let changed = vec![perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1))];
3753
3754        assert!(added_symbols(&changed, &cached_instruments).is_empty());
3755    }
3756
3757    #[rstest]
3758    fn test_changed_definitions_ignores_a_later_ts_init_alone() {
3759        // Every pass restamps `ts_init` from the clock, so comparing it would
3760        // republish the whole universe on every tick.
3761        let cached_instruments =
3762            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3763        let fetched = vec![perp_instrument(
3764            "BTC-USD-PERP",
3765            "0.1",
3766            UnixNanos::from(2_000_000_000),
3767        )];
3768
3769        assert!(changed_definitions(&fetched, &cached_instruments).is_empty());
3770    }
3771
3772    #[rstest]
3773    fn test_changed_definitions_reports_a_changed_tick_size() {
3774        let cached_instruments =
3775            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3776        let fetched = vec![perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1))];
3777
3778        let changed = changed_definitions(&fetched, &cached_instruments);
3779
3780        assert_eq!(changed.len(), 1);
3781        assert_eq!(changed[0].price_increment(), Price::from("0.5"));
3782    }
3783}