Skip to main content

nautilus_hyperliquid/
data.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    str::FromStr,
18    sync::{
19        Arc,
20        atomic::{AtomicBool, Ordering},
21    },
22    time::Duration,
23};
24
25use ahash::{AHashMap, AHashSet};
26use anyhow::Context;
27use jiff::Timestamp;
28use nautilus_common::{
29    cache::InstrumentLookupError,
30    clients::{DataClient, execute_bar_subscriptions, execute_instrument_subscriptions},
31    live::{dst::time::Instant, runner::get_data_event_sender, sender::EventSender},
32    messages::{
33        DataEvent,
34        data::{
35            BarsResponse, BookResponse, CustomDataResponse, DataResponse, FundingRatesResponse,
36            InstrumentResponse, InstrumentsResponse, RequestBars, RequestBookSnapshot,
37            RequestCustomData, RequestFundingRates, RequestInstrument, RequestInstruments,
38            RequestTrades, SubscribeBars, SubscribeBookDeltas, SubscribeBookDepth,
39            SubscribeCustomData, SubscribeFundingRates, SubscribeIndexPrices, SubscribeInstrument,
40            SubscribeInstrumentStatus, SubscribeMarkPrices, SubscribeQuotes, SubscribeTrades,
41            TradesResponse, UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth,
42            UnsubscribeCustomData, UnsubscribeFundingRates, UnsubscribeIndexPrices,
43            UnsubscribeInstrument, UnsubscribeInstrumentStatus, UnsubscribeInstruments,
44            UnsubscribeMarkPrices, UnsubscribeQuotes, UnsubscribeTrades,
45        },
46    },
47};
48use nautilus_core::{
49    AtomicMap, Params, UnixNanos,
50    datetime::{datetime_to_unix_nanos, unix_nanos_to_iso8601},
51    time::{AtomicTime, get_atomic_clock_realtime},
52};
53use nautilus_live::{
54    SocketControl,
55    task::{TaskGroup, TaskGroupGuard, TaskSpawner},
56};
57use nautilus_model::{
58    data::{Bar, BarType, BookOrder, CustomData, Data, DataType, FundingRateUpdate, TradeTick},
59    enums::{BarAggregation, BookType, OrderSide},
60    identifiers::{ClientId, InstrumentId, Venue},
61    instruments::{Instrument, InstrumentAny},
62    orderbook::OrderBook,
63    types::{Price, Quantity},
64};
65use parking_lot::Mutex;
66use rust_decimal::Decimal;
67use tokio_util::sync::CancellationToken;
68use ustr::Ustr;
69
70use crate::{
71    book::{self, sync::BookSyncTracker},
72    common::{
73        consts::HYPERLIQUID_VENUE,
74        credential::{Secrets, credential_env_vars},
75        parse::{bar_type_to_interval, millis_to_nanos},
76        status::InstrumentStatusState,
77    },
78    config::HyperliquidDataClientConfig,
79    data_types::register_hyperliquid_custom_data,
80    http::{
81        client::HyperliquidHttpClient,
82        models::{HyperliquidCandle, HyperliquidFundingHistoryEntry, HyperliquidL2Book},
83        parse::parse_recent_trade,
84    },
85    websocket::{
86        DATA_STREAMS_ENDPOINT, client::HyperliquidWebSocketClient, messages::NautilusWsMessage,
87    },
88};
89
90#[derive(Debug)]
91pub struct HyperliquidDataClient {
92    clock: &'static AtomicTime,
93    client_id: ClientId,
94    config: HyperliquidDataClientConfig,
95    http_client: HyperliquidHttpClient,
96    ws_client: HyperliquidWebSocketClient,
97    is_connected: AtomicBool,
98    cancellation_token: CancellationToken,
99    session_tasks: TaskGroup,
100    pending_tasks: TaskGroup,
101    shutdown_errors: Vec<String>,
102    data_sender: EventSender<DataEvent>,
103    instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
104    coin_to_instrument_id: Arc<AtomicMap<Ustr, InstrumentId>>,
105    // serializes instrument fetch-and-apply passes, see `refresh_instruments`
106    instrument_update_lock: Arc<tokio::sync::Mutex<()>>,
107    instrument_status: Arc<Mutex<InstrumentStatusState>>,
108    stream_health: Arc<Mutex<MarketDataStreamHealthMonitor>>,
109    book_sync: BookSyncTracker,
110}
111
112impl HyperliquidDataClient {
113    /// Creates a new [`HyperliquidDataClient`] instance.
114    ///
115    /// # Errors
116    ///
117    /// Returns an error if the HTTP client fails to initialize.
118    pub fn new(client_id: ClientId, config: HyperliquidDataClientConfig) -> anyhow::Result<Self> {
119        let clock = get_atomic_clock_realtime();
120        let data_sender = get_data_event_sender();
121
122        // Only fall back to unauthenticated when credentials are absent,
123        // not when they're invalid (fail fast on malformed keys)
124        let (pk_var, _) = credential_env_vars(config.environment);
125        let has_credentials = config.has_credentials() || std::env::var(pk_var).is_ok();
126        let proxy_url = config
127            .proxy_url
128            .as_ref()
129            .map(|value| value.expose_secret().to_owned());
130
131        let mut http_client = if has_credentials {
132            let secrets = Secrets::resolve(
133                config
134                    .private_key
135                    .as_ref()
136                    .map(|value| value.expose_secret()),
137                None,
138                config.environment,
139            )?;
140            HyperliquidHttpClient::with_secrets(
141                &secrets,
142                config.http_timeout_secs,
143                proxy_url.clone(),
144            )?
145        } else {
146            HyperliquidHttpClient::new(
147                config.environment,
148                config.http_timeout_secs,
149                proxy_url.clone(),
150            )?
151        };
152
153        if let Some(url) = &config.base_url_http {
154            http_client.set_base_info_url(url.clone());
155        }
156
157        let ws_url = config.base_url_ws.clone();
158        let ws_client = HyperliquidWebSocketClient::new(
159            ws_url,
160            config.environment,
161            None,
162            config.transport_backend,
163            proxy_url,
164        );
165        let ws_client = ws_client.with_socket_control(SocketControl::new(
166            client_id,
167            Some(*HYPERLIQUID_VENUE),
168            DATA_STREAMS_ENDPOINT,
169        ));
170        let mut stream_health_monitor = MarketDataStreamHealthMonitor::new(
171            Duration::from_secs(config.stale_stream_receive_timeout_secs),
172            Duration::from_secs(config.stale_stream_warning_cooldown_secs),
173        );
174
175        if config.stale_stream_recovery_enabled {
176            if config.stale_stream_recovery_cooldown_secs > 0 {
177                stream_health_monitor = stream_health_monitor.with_recovery(
178                    Duration::from_secs(config.stale_stream_recovery_cooldown_secs),
179                    config.stale_stream_max_targeted_resubscribes,
180                );
181            } else {
182                log::warn!(
183                    "Hyperliquid stale stream recovery disabled: \
184                     stale_stream_recovery_cooldown_secs must be positive"
185                );
186            }
187        }
188
189        let stream_health = Arc::new(Mutex::new(stream_health_monitor));
190
191        let session_tasks = TaskGroup::new();
192        let pending_tasks = TaskGroup::new();
193
194        Ok(Self {
195            clock,
196            client_id,
197            config,
198            http_client,
199            ws_client,
200            is_connected: AtomicBool::new(false),
201            cancellation_token: CancellationToken::new(),
202            session_tasks,
203            pending_tasks,
204            shutdown_errors: Vec::new(),
205            data_sender,
206            instruments: Arc::new(AtomicMap::new()),
207            coin_to_instrument_id: Arc::new(AtomicMap::new()),
208            instrument_update_lock: Arc::new(tokio::sync::Mutex::new(())),
209            instrument_status: Arc::new(Mutex::new(InstrumentStatusState::default())),
210            stream_health,
211            book_sync: BookSyncTracker::default(),
212        })
213    }
214
215    fn spawn_task<F>(&self, description: &'static str, fut: F)
216    where
217        F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
218    {
219        let future = async move {
220            if let Err(e) = fut.await {
221                log::warn!("{description} failed: {e:?}");
222            }
223        };
224
225        if let Err(e) = self.pending_tasks.spawn(future) {
226            log::warn!("Skipping Hyperliquid {description} after shutdown began: {e}");
227        }
228    }
229
230    fn abort_pending_tasks(&self) {
231        self.pending_tasks.begin_shutdown();
232    }
233
234    fn abort_session_tasks(&self) {
235        self.session_tasks.begin_shutdown();
236        self.ws_client.begin_shutdown();
237    }
238
239    async fn teardown_partial_connect(&mut self) -> anyhow::Result<()> {
240        self.cancellation_token.cancel();
241        self.abort_session_tasks();
242        self.abort_pending_tasks();
243
244        if let Err(e) = self.ws_client.disconnect().await {
245            self.shutdown_errors
246                .push(format!("Hyperliquid WebSocket shutdown failed: {e}"));
247        }
248
249        if let Err(e) = self.await_session_tasks().await {
250            self.shutdown_errors.push(e.to_string());
251        }
252
253        if let Err(e) = self.await_pending_tasks().await {
254            self.shutdown_errors.push(e.to_string());
255        }
256        self.clear_stream_health();
257        self.book_sync.clear();
258        self.is_connected.store(false, Ordering::Release);
259
260        if !self.shutdown_errors.is_empty() {
261            anyhow::bail!(std::mem::take(&mut self.shutdown_errors).join("; "));
262        }
263        Ok(())
264    }
265
266    async fn await_pending_tasks(&self) -> anyhow::Result<()> {
267        self.pending_tasks.begin_shutdown();
268        self.pending_tasks
269            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
270            .await
271            .map_err(|e| anyhow::anyhow!("Failed to terminate Hyperliquid data tasks: {e}"))?;
272        Ok(())
273    }
274
275    async fn await_session_tasks(&self) -> anyhow::Result<()> {
276        self.session_tasks.begin_shutdown();
277        self.session_tasks
278            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
279            .await
280            .map_err(|e| {
281                anyhow::anyhow!("Failed to terminate Hyperliquid data session tasks: {e}")
282            })?;
283        Ok(())
284    }
285
286    fn clear_stream_health(&self) {
287        self.stream_health.lock().clear();
288    }
289
290    fn register_stream_health(&self, channel: MarketDataChannel, instrument_id: InstrumentId) {
291        if !self.stream_health_monitor_enabled() {
292            return;
293        }
294
295        self.stream_health
296            .lock()
297            .subscribe(channel, instrument_id, Instant::now());
298    }
299
300    fn remove_stream_health(&self, channel: MarketDataChannel, instrument_id: InstrumentId) {
301        self.stream_health
302            .lock()
303            .unsubscribe(channel, instrument_id);
304    }
305
306    fn stream_health_monitor_enabled(&self) -> bool {
307        self.config.stale_stream_receive_timeout_secs > 0
308            && self.config.stream_health_check_interval_secs > 0
309    }
310
311    fn spawn_stream_health_monitor(&self) -> anyhow::Result<()> {
312        if !self.stream_health_monitor_enabled() {
313            return Ok(());
314        }
315
316        let stream_health = Arc::clone(&self.stream_health);
317        let cancellation_token = self.cancellation_token.clone();
318        let interval = Duration::from_secs(self.config.stream_health_check_interval_secs);
319        let clock = self.clock;
320        let ws_client = self.ws_client.clone();
321        let book_sync = self.book_sync.clone();
322        let snapshot_timeout = self.book_snapshot_timeout();
323        let book_tasks = self.pending_tasks.spawner()?;
324
325        self.session_tasks.spawn(async move {
326            log::debug!("Hyperliquid stream health monitor started");
327
328            loop {
329                tokio::select! {
330                    () = cancellation_token.cancelled() => {
331                        log::debug!("Hyperliquid stream health monitor cancelled");
332                        break;
333                    }
334                    () = tokio::time::sleep(interval) => {
335                        let events = stream_health
336                            .lock()
337                            .check_stale(Instant::now(), clock.get_time_ns());
338
339                        handle_stream_health_events(
340                            &ws_client,
341                            &events,
342                            &book_sync,
343                            snapshot_timeout,
344                            &book_tasks,
345                        )
346                        .await;
347                    }
348                }
349            }
350
351            log::debug!("Hyperliquid stream health monitor stopped");
352        })?;
353
354        Ok(())
355    }
356
357    fn book_snapshot_timeout(&self) -> Duration {
358        Duration::from_secs(self.config.book_snapshot_timeout_secs)
359    }
360
361    fn venue(&self) -> Venue {
362        *HYPERLIQUID_VENUE
363    }
364
365    fn custom_instrument_id(data_type: &DataType) -> anyhow::Result<Option<InstrumentId>> {
366        let Some(raw_instrument_id) = data_type
367            .metadata()
368            .and_then(|m| m.get("instrument_id"))
369            .and_then(|v| v.as_str())
370            .map(str::trim)
371            .filter(|value| !value.is_empty())
372        else {
373            return Ok(None);
374        };
375
376        let instrument_id = InstrumentId::from_str(raw_instrument_id)
377            .with_context(|| format!("invalid instrument_id metadata `{raw_instrument_id}`"))?;
378
379        Ok(Some(instrument_id))
380    }
381
382    fn custom_user(data_type: &DataType) -> anyhow::Result<Option<String>> {
383        let Some(user) = data_type
384            .metadata()
385            .and_then(|m| m.get("user"))
386            .and_then(|v| v.as_str())
387            .filter(|value| !value.is_empty())
388        else {
389            return Ok(None);
390        };
391
392        anyhow::ensure!(
393            user == user.trim(),
394            "metadata['user'] must not contain surrounding whitespace",
395        );
396
397        Ok(Some(user.to_string()))
398    }
399
400    async fn bootstrap_instruments(&self) -> anyhow::Result<Vec<InstrumentAny>> {
401        // a pass that fetched before the bootstrap must not apply over its universe
402        let _update_guard = self.instrument_update_lock.lock().await;
403
404        let snapshot = self
405            .http_client
406            .request_instruments_with_statuses()
407            .await
408            .context("failed to fetch instruments during bootstrap")?;
409        let instruments = snapshot.instruments;
410
411        cache_instruments(
412            &instruments,
413            &self.instruments,
414            &self.coin_to_instrument_id,
415            &self.http_client,
416            &self.ws_client,
417        );
418        self.instrument_status.lock().store(snapshot.statuses);
419        rebuild_all_dex_asset_ctxs_mapping(&self.http_client, &self.ws_client).await;
420
421        log::debug!(
422            "Bootstrapped {} instruments with {} coin mappings",
423            self.instruments.len(),
424            self.coin_to_instrument_id.len()
425        );
426        Ok(instruments)
427    }
428
429    /// Spawns the periodic instrument refresh task, disabled when the configured
430    /// interval is zero. The task is tracked in `self.session_tasks`, so the
431    /// cancellation token stops it on disconnect, failed connect, and dispose.
432    fn spawn_instrument_refresh(&self) -> anyhow::Result<()> {
433        let minutes = self.config.update_instruments_interval_mins;
434
435        if minutes == 0 {
436            log::debug!(
437                "Hyperliquid instrument refresh disabled (update_instruments_interval_mins=0)"
438            );
439            return Ok(());
440        }
441
442        let interval = Duration::from_secs(minutes.saturating_mul(60));
443        let cancellation_token = self.cancellation_token.clone();
444        let http_client = self.http_client.clone();
445        let ws_client = self.ws_client.clone();
446        let instruments = Arc::clone(&self.instruments);
447        let coin_to_instrument_id = Arc::clone(&self.coin_to_instrument_id);
448        let instrument_update_lock = Arc::clone(&self.instrument_update_lock);
449        let instrument_status = Arc::clone(&self.instrument_status);
450        let data_sender = self.data_sender.clone();
451        let client_id = self.client_id;
452
453        self.session_tasks.spawn(async move {
454            log::info!("Hyperliquid instrument refresh started, interval={interval:?}");
455
456            loop {
457                tokio::select! {
458                    () = cancellation_token.cancelled() => {
459                        log::debug!("Hyperliquid instrument refresh cancelled");
460                        break;
461                    }
462                    () = tokio::time::sleep(interval) => {}
463                }
464
465                // The refresh performs several REST calls, so cancellation is
466                // also raced against the pass itself rather than only the sleep
467                let result = tokio::select! {
468                    () = cancellation_token.cancelled() => {
469                        log::debug!("Hyperliquid instrument refresh cancelled");
470                        break;
471                    }
472                    result = refresh_instruments(
473                        &instrument_update_lock,
474                        &http_client,
475                        &ws_client,
476                        &instruments,
477                        &coin_to_instrument_id,
478                        &instrument_status,
479                        &data_sender,
480                    ) => result,
481                };
482
483                match result {
484                    Ok(summary) => summary.log(client_id),
485                    Err(e) => log::warn!(
486                        "Failed to refresh Hyperliquid instruments: client_id={client_id}, error={e:?}"
487                    ),
488                }
489            }
490
491            log::debug!("Hyperliquid instrument refresh stopped");
492        })?;
493
494        Ok(())
495    }
496
497    async fn spawn_ws(&self) -> anyhow::Result<()> {
498        // Clone client before connecting so the clone can have out_rx set
499        let mut ws_client = self.ws_client.clone();
500
501        ws_client
502            .connect()
503            .await
504            .context("failed to connect to Hyperliquid WebSocket")?;
505
506        let data_sender = self.data_sender.clone();
507        let cancellation_token = self.cancellation_token.clone();
508        let stream_health = Arc::clone(&self.stream_health);
509        let book_sync = self.book_sync.clone();
510        let snapshot_timeout = self.book_snapshot_timeout();
511        let book_tasks = self.pending_tasks.spawner()?;
512
513        self.session_tasks.spawn(async move {
514            log::debug!("Hyperliquid WebSocket consumption loop started");
515
516            loop {
517                tokio::select! {
518                    () = cancellation_token.cancelled() => {
519                        log::debug!("WebSocket consumption loop cancelled");
520                        break;
521                    }
522                    msg_opt = ws_client.next_event() => {
523                        if let Some(msg) = msg_opt {
524                            if let Some((channel, instrument_id, ts_event)) =
525                                stream_health_update(&msg)
526                            {
527                                record_stream_receive(
528                                    &stream_health,
529                                    channel,
530                                    instrument_id,
531                                    ts_event,
532                                );
533                            }
534
535                            match msg {
536                                NautilusWsMessage::Trades(trades) => {
537                                    for trade in trades {
538                                        if let Err(e) = data_sender
539                                            .send(DataEvent::Data(Data::Trade(trade)))
540                                        {
541                                            log::error!("Failed to send trade tick: {e}");
542                                        }
543                                    }
544                                }
545                                NautilusWsMessage::Quote(quote) => {
546                                    if let Err(e) = data_sender
547                                        .send(DataEvent::Data(Data::Quote(quote)))
548                                    {
549                                        log::error!("Failed to send quote tick: {e}");
550                                    }
551                                }
552                                NautilusWsMessage::Deltas(deltas) => {
553                                    if book_sync
554                                        .record_snapshot(deltas.instrument_id, Instant::now())
555                                        && let Err(e) = data_sender.send(DataEvent::Data(
556                                            Data::BookDeltas(Box::new(deltas)),
557                                        ))
558                                    {
559                                        log::error!("Failed to send order book deltas: {e}");
560                                    }
561                                }
562                                NautilusWsMessage::Depth(depth) => {
563                                    if let Err(e) =
564                                        data_sender.send(DataEvent::Data(Data::BookDepth(depth)))
565                                    {
566                                        log::error!("Failed to send order book depth: {e}");
567                                    }
568                                }
569                                NautilusWsMessage::BookInvalid(instrument_id) => {
570                                    book::recovery::reject_snapshot(
571                                        instrument_id,
572                                        &book_sync,
573                                        &ws_client,
574                                        snapshot_timeout,
575                                        &book_tasks,
576                                    );
577                                }
578                                NautilusWsMessage::Candle(bar) => {
579                                    if let Err(e) = data_sender
580                                        .send(DataEvent::Data(Data::Bar(bar)))
581                                    {
582                                        log::error!("Failed to send bar: {e}");
583                                    }
584                                }
585                                NautilusWsMessage::MarkPrice(update) => {
586                                    if let Err(e) = data_sender
587                                        .send(DataEvent::Data(Data::MarkPrice(update)))
588                                    {
589                                        log::error!("Failed to send mark price update: {e}");
590                                    }
591                                }
592                                NautilusWsMessage::IndexPrice(update) => {
593                                    if let Err(e) = data_sender
594                                        .send(DataEvent::Data(Data::IndexPrice(update)))
595                                    {
596                                        log::error!("Failed to send index price update: {e}");
597                                    }
598                                }
599                                NautilusWsMessage::FundingRate(update) => {
600                                    if let Err(e) = data_sender
601                                        .send(DataEvent::Data(Data::FundingRate(update)))
602                                    {
603                                        log::error!("Failed to send funding rate update: {e}");
604                                    }
605                                }
606                                NautilusWsMessage::CustomData(data) => {
607                                    if let Err(e) = data_sender.send(DataEvent::Data(data)) {
608                                        log::error!("Failed to send custom data: {e}");
609                                    }
610                                }
611                                NautilusWsMessage::Reconnected => {
612                                    log::info!("WebSocket reconnected");
613                                    book::recovery::reset_on_reconnect(
614                                        &book_sync,
615                                        &ws_client,
616                                        snapshot_timeout,
617                                        &book_tasks,
618                                    );
619                                }
620                                NautilusWsMessage::Error(e) => {
621                                    log::warn!("WebSocket error: {e}");
622                                }
623                                NautilusWsMessage::ExecutionReports(_) => {
624                                    // Handled by execution client
625                                }
626                            }
627                        } else {
628                            log::debug!("WebSocket next_event returned None, stream closed");
629                            break;
630                        }
631                    }
632                }
633            }
634
635            log::debug!("Hyperliquid WebSocket consumption loop finished");
636        })?;
637
638        log::debug!("WebSocket consumption task spawned");
639
640        Ok(())
641    }
642}
643
644#[async_trait::async_trait(?Send)]
645impl DataClient for HyperliquidDataClient {
646    fn client_id(&self) -> ClientId {
647        self.client_id
648    }
649
650    fn venue(&self) -> Option<Venue> {
651        Some(self.venue())
652    }
653
654    fn start(&mut self) -> anyhow::Result<()> {
655        log::info!(
656            "Starting Hyperliquid data client: client_id={}, environment={:?}, proxy_url={:?}",
657            self.client_id,
658            self.config.environment,
659            self.config.proxy_url,
660        );
661        Ok(())
662    }
663
664    fn stop(&mut self) -> anyhow::Result<()> {
665        log::info!("Stopping Hyperliquid data client {}", self.client_id);
666        self.cancellation_token.cancel();
667        self.abort_session_tasks();
668        self.abort_pending_tasks();
669        self.is_connected.store(false, Ordering::Relaxed);
670        Ok(())
671    }
672
673    fn reset(&mut self) -> anyhow::Result<()> {
674        log::debug!("Resetting Hyperliquid data client {}", self.client_id);
675        // Keep this generation cancelled until `connect()` has torn down the
676        // inner WebSocket client. Replacing it here would allow the next
677        // connection to reuse an active old-generation handler.
678        self.cancellation_token.cancel();
679        self.abort_session_tasks();
680        self.abort_pending_tasks();
681        self.is_connected.store(false, Ordering::Relaxed);
682        self.instruments.store(AHashMap::new());
683        self.coin_to_instrument_id.store(AHashMap::new());
684        self.instrument_status.lock().clear();
685        Ok(())
686    }
687
688    fn dispose(&mut self) -> anyhow::Result<()> {
689        log::debug!("Disposing Hyperliquid data client {}", self.client_id);
690        self.stop()
691    }
692
693    fn is_connected(&self) -> bool {
694        self.is_connected.load(Ordering::Acquire)
695    }
696
697    fn is_disconnected(&self) -> bool {
698        !self.is_connected()
699    }
700
701    async fn connect(&mut self) -> anyhow::Result<()> {
702        if self.is_connected()
703            && !self.cancellation_token.is_cancelled()
704            && self.session_tasks.is_open()
705            && self.pending_tasks.is_open()
706        {
707            return Ok(());
708        }
709
710        if self.cancellation_token.is_cancelled()
711            || !self.session_tasks.is_open()
712            || !self.pending_tasks.is_open()
713        {
714            // `reset()` is synchronous, while shutting down the inner socket
715            // is async. Complete that teardown before creating any new stream
716            // task so its receiver and subscription registries cannot belong
717            // to the previous generation.
718            self.ws_client.begin_shutdown();
719            self.ws_client
720                .disconnect()
721                .await
722                .context("failed to tear down Hyperliquid WebSocket before reconnect")?;
723            self.ws_client.reset_runtime_state();
724            self.book_sync.clear();
725            self.abort_session_tasks();
726            self.abort_pending_tasks();
727            let (session_result, pending_result) =
728                tokio::join!(self.await_session_tasks(), self.await_pending_tasks());
729            session_result?;
730            pending_result?;
731            self.session_tasks.start_generation().map_err(|e| {
732                anyhow::anyhow!("Failed to start Hyperliquid data session generation: {e}")
733            })?;
734            self.pending_tasks.start_generation().map_err(|e| {
735                anyhow::anyhow!("Failed to start Hyperliquid data task generation: {e}")
736            })?;
737            self.cancellation_token = CancellationToken::new();
738        }
739        let cancellation_token = self.cancellation_token.clone();
740        let ws_client = self.ws_client.clone();
741        let setup_guard =
742            TaskGroupGuard::new(&[&self.session_tasks, &self.pending_tasks], move || {
743                cancellation_token.cancel();
744                ws_client.begin_shutdown();
745            });
746
747        register_hyperliquid_custom_data();
748
749        let instruments = self
750            .bootstrap_instruments()
751            .await
752            .context("failed to bootstrap instruments")?;
753
754        for instrument in instruments {
755            if let Err(e) = self.data_sender.send(DataEvent::Data(instrument.into())) {
756                log::warn!("Failed to send instrument: {e}");
757            }
758        }
759
760        let session_result = async {
761            self.spawn_ws()
762                .await
763                .context("failed to spawn WebSocket client")?;
764            self.spawn_stream_health_monitor()?;
765            self.spawn_instrument_refresh()?;
766            Ok::<(), anyhow::Error>(())
767        }
768        .await;
769
770        if let Err(e) = session_result {
771            if let Err(teardown_error) = self.teardown_partial_connect().await {
772                return Err(e.context(format!(
773                    "Hyperliquid data startup teardown failed: {teardown_error}"
774                )));
775            }
776            return Err(e);
777        }
778
779        self.is_connected.store(true, Ordering::Relaxed);
780        setup_guard.disarm();
781        log::info!("Connected: client_id={}", self.client_id);
782
783        Ok(())
784    }
785
786    async fn disconnect(&mut self) -> anyhow::Result<()> {
787        self.teardown_partial_connect().await?;
788        self.instruments.store(AHashMap::new());
789        self.instrument_status.lock().clear();
790        log::info!("Disconnected: client_id={}", self.client_id);
791
792        Ok(())
793    }
794
795    fn subscribe(&mut self, cmd: SubscribeCustomData) -> anyhow::Result<()> {
796        let data_type = cmd.data_type.type_name();
797
798        if data_type == "HyperliquidAllMids" {
799            let ws = self.ws_client.clone();
800            let dex = cmd
801                .data_type
802                .metadata()
803                .as_ref()
804                .and_then(|m| m.get("dex"))
805                .and_then(|v| v.as_str())
806                .map(str::trim)
807                .filter(|value| !value.is_empty())
808                .map(ToString::to_string);
809
810            log::debug!("Subscribing to all mids (dex: {:?})", dex.as_deref());
811
812            self.spawn_task("subscribe_all_mids", async move {
813                ws.subscribe_all_mids_with_dex(dex.as_deref()).await
814            });
815
816            return Ok(());
817        }
818
819        if data_type == "HyperliquidAllDexsAssetCtxs" {
820            let ws = self.ws_client.clone();
821
822            self.spawn_task("subscribe_all_dexs_asset_ctxs", async move {
823                ws.subscribe_all_dexs_asset_ctxs().await
824            });
825
826            return Ok(());
827        }
828
829        if data_type == "HyperliquidOpenInterest" {
830            let ws = self.ws_client.clone();
831            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
832                "HyperliquidOpenInterest subscriptions require metadata['instrument_id']",
833            )?;
834
835            self.spawn_task("subscribe_open_interest", async move {
836                ws.subscribe_open_interest(instrument_id).await
837            });
838
839            return Ok(());
840        }
841
842        if data_type == "HyperliquidPublicTrade" {
843            let ws = self.ws_client.clone();
844            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
845                "HyperliquidPublicTrade subscriptions require metadata['instrument_id']",
846            )?;
847
848            self.spawn_task("subscribe_public_trades", async move {
849                ws.subscribe_public_trades(instrument_id).await
850            });
851
852            return Ok(());
853        }
854
855        if data_type == "HyperliquidTwapHistory" {
856            let ws = self.ws_client.clone();
857            let user = Self::custom_user(&cmd.data_type)?
858                .context("HyperliquidTwapHistory subscriptions require metadata['user']")?;
859
860            self.spawn_task("subscribe_user_twap_history", async move {
861                ws.subscribe_user_twap_history(&user).await
862            });
863
864            return Ok(());
865        }
866
867        if data_type == "HyperliquidTwapSliceFill" {
868            let ws = self.ws_client.clone();
869            let user = Self::custom_user(&cmd.data_type)?
870                .context("HyperliquidTwapSliceFill subscriptions require metadata['user']")?;
871
872            self.spawn_task("subscribe_user_twap_slice_fills", async move {
873                ws.subscribe_user_twap_slice_fills(&user).await
874            });
875
876            return Ok(());
877        }
878
879        log::warn!("Unsupported custom data subscription: {data_type}");
880        Ok(())
881    }
882
883    fn unsubscribe(&mut self, cmd: &UnsubscribeCustomData) -> anyhow::Result<()> {
884        let data_type = cmd.data_type.type_name();
885
886        if data_type == "HyperliquidAllMids" {
887            let ws = self.ws_client.clone();
888            let dex = cmd
889                .data_type
890                .metadata()
891                .as_ref()
892                .and_then(|m| m.get("dex"))
893                .and_then(|v| v.as_str())
894                .map(str::trim)
895                .filter(|value| !value.is_empty())
896                .map(ToString::to_string);
897
898            log::debug!("Unsubscribing from all mids (dex: {:?})", dex.as_deref());
899
900            self.spawn_task("unsubscribe_all_mids", async move {
901                ws.unsubscribe_all_mids_with_dex(dex.as_deref()).await
902            });
903
904            return Ok(());
905        }
906
907        if data_type == "HyperliquidAllDexsAssetCtxs" {
908            let ws = self.ws_client.clone();
909
910            self.spawn_task("unsubscribe_all_dexs_asset_ctxs", async move {
911                ws.unsubscribe_all_dexs_asset_ctxs().await
912            });
913
914            return Ok(());
915        }
916
917        if data_type == "HyperliquidOpenInterest" {
918            let ws = self.ws_client.clone();
919            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
920                "HyperliquidOpenInterest unsubscriptions require metadata['instrument_id']",
921            )?;
922
923            self.spawn_task("unsubscribe_open_interest", async move {
924                ws.unsubscribe_open_interest(instrument_id).await
925            });
926
927            return Ok(());
928        }
929
930        if data_type == "HyperliquidPublicTrade" {
931            let ws = self.ws_client.clone();
932            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
933                "HyperliquidPublicTrade unsubscriptions require metadata['instrument_id']",
934            )?;
935
936            self.spawn_task("unsubscribe_public_trades", async move {
937                ws.unsubscribe_public_trades(instrument_id).await
938            });
939
940            return Ok(());
941        }
942
943        if data_type == "HyperliquidTwapHistory" {
944            let ws = self.ws_client.clone();
945            let user = Self::custom_user(&cmd.data_type)?
946                .context("HyperliquidTwapHistory unsubscriptions require metadata['user']")?;
947
948            self.spawn_task("unsubscribe_user_twap_history", async move {
949                ws.unsubscribe_user_twap_history(&user).await
950            });
951
952            return Ok(());
953        }
954
955        if data_type == "HyperliquidTwapSliceFill" {
956            let ws = self.ws_client.clone();
957            let user = Self::custom_user(&cmd.data_type)?
958                .context("HyperliquidTwapSliceFill unsubscriptions require metadata['user']")?;
959
960            self.spawn_task("unsubscribe_user_twap_slice_fills", async move {
961                ws.unsubscribe_user_twap_slice_fills(&user).await
962            });
963
964            return Ok(());
965        }
966
967        log::warn!("Unsupported custom data unsubscription: {data_type}");
968        Ok(())
969    }
970
971    fn subscribe_instrument(&mut self, cmd: SubscribeInstrument) -> anyhow::Result<()> {
972        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
973            let instruments = self.instruments.load();
974            if let Some(instrument) = instruments.get(&instrument_id) {
975                if let Err(e) = self
976                    .data_sender
977                    .send(DataEvent::Data(instrument.clone().into()))
978                {
979                    log::error!("Failed to send instrument {instrument_id}: {e}");
980                }
981            } else {
982                log::warn!("Instrument {instrument_id} not found in cache");
983            }
984
985            Ok(())
986        })
987    }
988
989    fn subscribe_book_deltas(&mut self, subscription: SubscribeBookDeltas) -> anyhow::Result<()> {
990        execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
991            let subscription = subscription.with_instrument_ids(instrument_id);
992            if subscription.book_type != BookType::L2_MBP {
993                anyhow::bail!("Hyperliquid only supports L2_MBP order book deltas");
994            }
995
996            let (n_sig_figs, mantissa) = parse_book_precision_params(subscription.params.as_ref())?;
997            self.register_stream_health(MarketDataChannel::Deltas, instrument_id);
998
999            match self.pending_tasks.spawner() {
1000                Ok(tasks) => book::recovery::spawn_subscription_task(
1001                    instrument_id,
1002                    n_sig_figs,
1003                    mantissa,
1004                    self.book_sync.clone(),
1005                    self.ws_client.clone(),
1006                    self.book_snapshot_timeout(),
1007                    &tasks,
1008                ),
1009                Err(e) => {
1010                    log::warn!(
1011                        "Skipping Hyperliquid subscribe_book_deltas after shutdown began: {e}"
1012                    );
1013                }
1014            }
1015
1016            Ok(())
1017        })
1018    }
1019
1020    fn subscribe_book_depth(&mut self, subscription: SubscribeBookDepth) -> anyhow::Result<()> {
1021        execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1022            let subscription = subscription.with_instrument_ids(instrument_id);
1023            log::debug!("Subscribing to book depth: {instrument_id}");
1024
1025            if subscription.book_type != BookType::L2_MBP {
1026                anyhow::bail!("Hyperliquid only supports L2_MBP order book depth");
1027            }
1028
1029            let ws = self.ws_client.clone();
1030            let (n_sig_figs, mantissa) = parse_book_precision_params(subscription.params.as_ref())?;
1031            self.register_stream_health(MarketDataChannel::Depth, instrument_id);
1032            self.spawn_task("subscribe_book_depth", async move {
1033                ws.subscribe_book_depth_with_options(instrument_id, n_sig_figs, mantissa)
1034                    .await
1035            });
1036
1037            Ok(())
1038        })
1039    }
1040
1041    fn subscribe_quotes(&mut self, subscription: SubscribeQuotes) -> anyhow::Result<()> {
1042        execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1043            let ws = self.ws_client.clone();
1044            self.register_stream_health(MarketDataChannel::Quote, instrument_id);
1045            self.spawn_task("subscribe_quotes", async move {
1046                ws.subscribe_quotes(instrument_id).await
1047            });
1048
1049            Ok(())
1050        })
1051    }
1052
1053    fn subscribe_trades(&mut self, subscription: SubscribeTrades) -> anyhow::Result<()> {
1054        execute_instrument_subscriptions(&subscription.instrument_ids, |instrument_id| {
1055            let ws = self.ws_client.clone();
1056            self.spawn_task("subscribe_trades", async move {
1057                ws.subscribe_trades(instrument_id).await
1058            });
1059
1060            Ok(())
1061        })
1062    }
1063
1064    fn subscribe_mark_prices(&mut self, cmd: SubscribeMarkPrices) -> anyhow::Result<()> {
1065        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1066            let ws = self.ws_client.clone();
1067            self.spawn_task("subscribe_mark_prices", async move {
1068                ws.subscribe_mark_prices(instrument_id).await
1069            });
1070
1071            Ok(())
1072        })
1073    }
1074
1075    fn subscribe_index_prices(&mut self, cmd: SubscribeIndexPrices) -> anyhow::Result<()> {
1076        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1077            let ws = self.ws_client.clone();
1078            self.spawn_task("subscribe_index_prices", async move {
1079                ws.subscribe_index_prices(instrument_id).await
1080            });
1081
1082            Ok(())
1083        })
1084    }
1085
1086    fn subscribe_funding_rates(&mut self, cmd: SubscribeFundingRates) -> anyhow::Result<()> {
1087        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1088            let ws = self.ws_client.clone();
1089            self.spawn_task("subscribe_funding_rates", async move {
1090                ws.subscribe_funding_rates(instrument_id).await
1091            });
1092
1093            Ok(())
1094        })
1095    }
1096
1097    fn subscribe_bars(&mut self, subscription: SubscribeBars) -> anyhow::Result<()> {
1098        execute_bar_subscriptions(&subscription.bar_types, |bar_type| {
1099            let instrument_id = bar_type.instrument_id();
1100            if !self.instruments.contains_key(&instrument_id) {
1101                anyhow::bail!(InstrumentLookupError::not_found(instrument_id));
1102            }
1103
1104            let ws = self.ws_client.clone();
1105            self.spawn_task("subscribe_bars", async move {
1106                ws.subscribe_bars(bar_type).await
1107            });
1108
1109            Ok(())
1110        })
1111    }
1112
1113    fn subscribe_instrument_status(
1114        &mut self,
1115        cmd: SubscribeInstrumentStatus,
1116    ) -> anyhow::Result<()> {
1117        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1118            log::debug!(
1119                "Subscribing to instrument status: {instrument_id} (detected by the instrument refresh)"
1120            );
1121            let ts = self.clock.get_time_ns();
1122            self.instrument_status
1123                .lock()
1124                .subscribe(instrument_id, &self.data_sender, ts);
1125
1126            Ok(())
1127        })
1128    }
1129
1130    fn unsubscribe_instrument_status(
1131        &mut self,
1132        cmd: &UnsubscribeInstrumentStatus,
1133    ) -> anyhow::Result<()> {
1134        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1135            log::debug!("Unsubscribing from instrument status: {instrument_id}");
1136            self.instrument_status.lock().unsubscribe(&instrument_id);
1137            Ok(())
1138        })
1139    }
1140
1141    fn unsubscribe_instrument(&mut self, _cmd: &UnsubscribeInstrument) -> anyhow::Result<()> {
1142        // `subscribe_instrument` only emits the cached instrument; it opens no
1143        // venue channel, so there is nothing to tear down here.
1144        Ok(())
1145    }
1146
1147    fn unsubscribe_instruments(&mut self, _cmd: &UnsubscribeInstruments) -> anyhow::Result<()> {
1148        // See `unsubscribe_instrument`: instrument subscriptions carry no
1149        // venue-side state to unsubscribe from.
1150        Ok(())
1151    }
1152
1153    fn unsubscribe_book_deltas(
1154        &mut self,
1155        unsubscription: &UnsubscribeBookDeltas,
1156    ) -> anyhow::Result<()> {
1157        execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1158            log::debug!("Unsubscribing from book deltas: {instrument_id}");
1159
1160            let ws = self.ws_client.clone();
1161            self.remove_stream_health(MarketDataChannel::Deltas, instrument_id);
1162            self.book_sync.remove(instrument_id);
1163            self.spawn_task("unsubscribe_book_deltas", async move {
1164                ws.unsubscribe_book(instrument_id).await
1165            });
1166
1167            Ok(())
1168        })
1169    }
1170
1171    fn unsubscribe_book_depth(
1172        &mut self,
1173        unsubscription: &UnsubscribeBookDepth,
1174    ) -> anyhow::Result<()> {
1175        execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1176            log::debug!("Unsubscribing from book depth: {instrument_id}");
1177
1178            let ws = self.ws_client.clone();
1179            self.remove_stream_health(MarketDataChannel::Depth, instrument_id);
1180            self.spawn_task("unsubscribe_book_depth", async move {
1181                ws.unsubscribe_book_depth(instrument_id).await
1182            });
1183
1184            Ok(())
1185        })
1186    }
1187
1188    fn unsubscribe_quotes(&mut self, unsubscription: &UnsubscribeQuotes) -> anyhow::Result<()> {
1189        execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1190            log::debug!("Unsubscribing from quotes: {instrument_id}");
1191
1192            let ws = self.ws_client.clone();
1193            self.remove_stream_health(MarketDataChannel::Quote, instrument_id);
1194            self.spawn_task("unsubscribe_quotes", async move {
1195                ws.unsubscribe_quotes(instrument_id).await
1196            });
1197
1198            Ok(())
1199        })
1200    }
1201
1202    fn unsubscribe_trades(&mut self, unsubscription: &UnsubscribeTrades) -> anyhow::Result<()> {
1203        execute_instrument_subscriptions(&unsubscription.instrument_ids, |instrument_id| {
1204            log::debug!("Unsubscribing from trades: {instrument_id}");
1205
1206            let ws = self.ws_client.clone();
1207            self.spawn_task("unsubscribe_trades", async move {
1208                ws.unsubscribe_trades(instrument_id).await
1209            });
1210
1211            Ok(())
1212        })
1213    }
1214
1215    fn unsubscribe_mark_prices(&mut self, cmd: &UnsubscribeMarkPrices) -> anyhow::Result<()> {
1216        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1217            let ws = self.ws_client.clone();
1218            self.spawn_task("unsubscribe_mark_prices", async move {
1219                ws.unsubscribe_mark_prices(instrument_id).await
1220            });
1221
1222            Ok(())
1223        })
1224    }
1225
1226    fn unsubscribe_index_prices(&mut self, cmd: &UnsubscribeIndexPrices) -> anyhow::Result<()> {
1227        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1228            let ws = self.ws_client.clone();
1229            self.spawn_task("unsubscribe_index_prices", async move {
1230                ws.unsubscribe_index_prices(instrument_id).await
1231            });
1232
1233            Ok(())
1234        })
1235    }
1236
1237    fn unsubscribe_funding_rates(&mut self, cmd: &UnsubscribeFundingRates) -> anyhow::Result<()> {
1238        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1239            let ws = self.ws_client.clone();
1240            self.spawn_task("unsubscribe_funding_rates", async move {
1241                ws.unsubscribe_funding_rates(instrument_id).await
1242            });
1243
1244            Ok(())
1245        })
1246    }
1247
1248    fn unsubscribe_bars(&mut self, unsubscription: &UnsubscribeBars) -> anyhow::Result<()> {
1249        execute_bar_subscriptions(&unsubscription.bar_types, |bar_type| {
1250            let ws = self.ws_client.clone();
1251            self.spawn_task("unsubscribe_bars", async move {
1252                ws.unsubscribe_bars(bar_type).await
1253            });
1254
1255            Ok(())
1256        })
1257    }
1258
1259    fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
1260        log::debug!("Requesting all instruments");
1261
1262        let http = self.http_client.clone();
1263        let ws = self.ws_client.clone();
1264        let sender = self.data_sender.clone();
1265        let instruments_cache = self.instruments.clone();
1266        let coin_map = self.coin_to_instrument_id.clone();
1267        let update_lock = Arc::clone(&self.instrument_update_lock);
1268        let instrument_status = Arc::clone(&self.instrument_status);
1269        let request_id = request.request_id;
1270        let client_id = request.client_id.unwrap_or(self.client_id);
1271        let venue = self.venue();
1272        let start_nanos = datetime_to_unix_nanos(request.start);
1273        let end_nanos = datetime_to_unix_nanos(request.end);
1274        let params = request.params;
1275        let clock = self.clock;
1276
1277        self.spawn_task("request_instruments", async move {
1278            // the fetched universe also feeds the cache the periodic refresh diffs
1279            // against, so it must be published like a refresh pass or a market first
1280            // seen here would never reach execution
1281            let refresh = refresh_instruments(
1282                &update_lock,
1283                &http,
1284                &ws,
1285                &instruments_cache,
1286                &coin_map,
1287                &instrument_status,
1288                &sender,
1289            )
1290            .await?;
1291            refresh.log(client_id);
1292            let instruments = refresh.fetched;
1293
1294            let response = DataResponse::Instruments(InstrumentsResponse::new(
1295                request_id,
1296                client_id,
1297                venue,
1298                instruments,
1299                start_nanos,
1300                end_nanos,
1301                clock.get_time_ns(),
1302                params,
1303            ));
1304
1305            if let Err(e) = sender.send(DataEvent::Response(response)) {
1306                log::error!("Failed to send instruments response: {e}");
1307            }
1308            Ok(())
1309        });
1310
1311        Ok(())
1312    }
1313
1314    fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
1315        let instrument_id = request.single_instrument_id()?;
1316
1317        log::debug!("Requesting instrument: {instrument_id}");
1318
1319        let http = self.http_client.clone();
1320        let ws = self.ws_client.clone();
1321        let sender = self.data_sender.clone();
1322        let instruments_cache = self.instruments.clone();
1323        let coin_map = self.coin_to_instrument_id.clone();
1324        let update_lock = Arc::clone(&self.instrument_update_lock);
1325        let instrument_status = Arc::clone(&self.instrument_status);
1326        let request_id = request.request_id;
1327        let client_id = request.client_id.unwrap_or(self.client_id);
1328        let start_nanos = datetime_to_unix_nanos(request.start);
1329        let end_nanos = datetime_to_unix_nanos(request.end);
1330        let params = request.params;
1331        let clock = self.clock;
1332
1333        self.spawn_task("request_instrument", async move {
1334            // the venue only serves the whole universe, so a single-instrument request
1335            // can discover other new markets and must publish them like a refresh pass
1336            let refresh = refresh_instruments(
1337                &update_lock,
1338                &http,
1339                &ws,
1340                &instruments_cache,
1341                &coin_map,
1342                &instrument_status,
1343                &sender,
1344            )
1345            .await?;
1346            refresh.log(client_id);
1347            let all_instruments = refresh.fetched;
1348
1349            if let Some(instrument) = all_instruments
1350                .into_iter()
1351                .find(|i| i.id() == instrument_id)
1352            {
1353                let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
1354                    request_id,
1355                    client_id,
1356                    instrument.id(),
1357                    instrument,
1358                    start_nanos,
1359                    end_nanos,
1360                    clock.get_time_ns(),
1361                    params,
1362                )));
1363
1364                if let Err(e) = sender.send(DataEvent::Response(response)) {
1365                    log::error!("Failed to send instrument response: {e}");
1366                }
1367            } else {
1368                log::error!("Instrument not found: {instrument_id}");
1369            }
1370            Ok(())
1371        });
1372
1373        Ok(())
1374    }
1375
1376    fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
1377        let bar_type = request.single_bar_type()?;
1378
1379        log::debug!("Requesting bars for {bar_type}");
1380
1381        let http = self.http_client.clone();
1382        let sender = self.data_sender.clone();
1383        let start = request.start;
1384        let end = request.end;
1385        let limit = request.limit.map(|n| n.get() as u32);
1386        let request_id = request.request_id;
1387        let client_id = request.client_id.unwrap_or(self.client_id);
1388        let params = request.params;
1389        let clock = self.clock;
1390        let start_nanos = datetime_to_unix_nanos(start);
1391        let end_nanos = datetime_to_unix_nanos(end);
1392        let instruments = Arc::clone(&self.instruments);
1393
1394        self.spawn_task("request_bars", async move {
1395            let bars = request_bars_from_http(http, bar_type, start, end, limit, instruments)
1396                .await
1397                .context("bar request failed")?;
1398
1399            let response = DataResponse::Bars(BarsResponse::new(
1400                request_id,
1401                client_id,
1402                bar_type,
1403                bars,
1404                start_nanos,
1405                end_nanos,
1406                clock.get_time_ns(),
1407                params,
1408            ));
1409
1410            if let Err(e) = sender.send(DataEvent::Response(response)) {
1411                log::error!("Failed to send bars response: {e}");
1412            }
1413            Ok(())
1414        });
1415
1416        Ok(())
1417    }
1418
1419    fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
1420        let instrument_id = request.single_instrument_id()?;
1421
1422        log::debug!("Requesting trades for {instrument_id}");
1423
1424        let instruments = self.instruments.load();
1425        let instrument = instruments
1426            .get(&instrument_id)
1427            .cloned()
1428            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1429
1430        let coin = instrument.raw_symbol().to_string();
1431        let http = self.http_client.clone();
1432        let sender = self.data_sender.clone();
1433        let client_id = request.client_id.unwrap_or(self.client_id);
1434        let request_id = request.request_id;
1435        let params = request.params;
1436        let clock = self.clock;
1437        let limit = request.limit.map(|n| n.get());
1438        let start_nanos = datetime_to_unix_nanos(request.start);
1439        let end_nanos = datetime_to_unix_nanos(request.end);
1440
1441        self.spawn_task("request_trades", async move {
1442            // `recentTrades` depends on the Hyperliquid indexer; nodes without it
1443            // return HTTP 422. Treat that as "no coverage" and serve an empty
1444            // response so the awaiting caller still completes.
1445            let raw_trades = match http.info_recent_trades(&coin).await {
1446                Ok(trades) => trades,
1447                Err(e) if e.is_unprocessable_entity() => {
1448                    log::warn!(
1449                        "Recent trades endpoint unavailable for {instrument_id} \
1450                         (requires the Hyperliquid indexer); sending empty response"
1451                    );
1452                    Vec::new()
1453                }
1454                Err(e) => {
1455                    return Err(anyhow::Error::new(e))
1456                        .with_context(|| format!("trades request failed for {instrument_id}"));
1457                }
1458            };
1459
1460            let mut trades: Vec<TradeTick> = Vec::with_capacity(raw_trades.len());
1461            for raw in &raw_trades {
1462                match parse_recent_trade(raw, &instrument) {
1463                    Ok(trade) => trades.push(trade),
1464                    Err(e) => log::warn!("Skipping recent trade for {instrument_id}: {e}"),
1465                }
1466            }
1467            trades.sort_by_key(|trade| trade.ts_event);
1468
1469            let trades = filter_recent_trades(trades, start_nanos, end_nanos, limit, instrument_id);
1470
1471            log::debug!("Fetched {} trades for {instrument_id}", trades.len());
1472
1473            let response = DataResponse::Trades(TradesResponse::new(
1474                request_id,
1475                client_id,
1476                instrument_id,
1477                trades,
1478                start_nanos,
1479                end_nanos,
1480                clock.get_time_ns(),
1481                params,
1482            ));
1483
1484            if let Err(e) = sender.send(DataEvent::Response(response)) {
1485                log::error!("Failed to send trades response: {e}");
1486            }
1487            Ok(())
1488        });
1489
1490        Ok(())
1491    }
1492
1493    fn request_data(&self, request: RequestCustomData) -> anyhow::Result<()> {
1494        if request.data_type.type_name() != "HyperliquidPublicTrade" {
1495            log::warn!(
1496                "Unsupported custom data request: {}",
1497                request.data_type.type_name()
1498            );
1499            return Ok(());
1500        }
1501
1502        let instrument_id = Self::custom_instrument_id(&request.data_type)?
1503            .context("HyperliquidPublicTrade requests require metadata['instrument_id']")?;
1504        let data_type = DataType::new(
1505            request.data_type.type_name(),
1506            request.data_type.metadata().cloned(),
1507            Some(instrument_id.to_string()),
1508        );
1509        let http = self.http_client.clone();
1510        let sender = self.data_sender.clone();
1511        let request_id = request.request_id;
1512        let client_id = request.client_id;
1513        let params = request.params;
1514        let clock = self.clock;
1515        let limit = request.limit.map(|limit| limit.get());
1516        let start = request.start;
1517        let end = request.end;
1518        let start_nanos = datetime_to_unix_nanos(start);
1519        let end_nanos = datetime_to_unix_nanos(end);
1520        let venue = self.venue();
1521
1522        self.spawn_task("request_public_trades", async move {
1523            let trades = http
1524                .request_public_trades(instrument_id, start, end, limit)
1525                .await
1526                .map_err(anyhow::Error::new)
1527                .with_context(|| format!("public trades request failed for {instrument_id}"))?;
1528            let data: Vec<CustomData> = trades
1529                .into_iter()
1530                .map(|trade| CustomData::new(Arc::new(trade), data_type.clone()))
1531                .collect();
1532
1533            let response = DataResponse::Data(CustomDataResponse::new(
1534                request_id,
1535                client_id,
1536                Some(venue),
1537                data_type,
1538                data,
1539                start_nanos,
1540                end_nanos,
1541                clock.get_time_ns(),
1542                params,
1543            ));
1544
1545            if let Err(e) = sender.send(DataEvent::Response(response)) {
1546                log::error!("Failed to send public trades response: {e}");
1547            }
1548            Ok(())
1549        });
1550
1551        Ok(())
1552    }
1553
1554    fn request_funding_rates(&self, request: RequestFundingRates) -> anyhow::Result<()> {
1555        let instrument_id = request.single_instrument_id()?;
1556
1557        log::debug!("Requesting funding rates for {instrument_id}");
1558
1559        let instruments = self.instruments.load();
1560        let instrument = instruments
1561            .get(&instrument_id)
1562            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1563
1564        if !matches!(instrument, InstrumentAny::CryptoPerpetual(_)) {
1565            anyhow::bail!("Funding rates are only available for perpetual instruments");
1566        }
1567
1568        let instrument = instrument.clone();
1569        let http = self.http_client.clone();
1570        let sender = self.data_sender.clone();
1571        let client_id = request.client_id.unwrap_or(self.client_id);
1572        let request_id = request.request_id;
1573        let params = request.params;
1574        let clock = self.clock;
1575        let limit = request.limit.map(|n| n.get());
1576        let start_dt = request.start;
1577        let end_dt = request.end;
1578        let start_nanos = datetime_to_unix_nanos(start_dt);
1579        let end_nanos = datetime_to_unix_nanos(end_dt);
1580
1581        self.spawn_task("request_funding_rates", async move {
1582            let funding_rates = http
1583                .request_funding_rates(&instrument, start_dt, end_dt, limit)
1584                .await
1585                .with_context(|| format!("funding rates request failed for {instrument_id}"))?;
1586
1587            log::debug!(
1588                "Fetched {} funding rates for {instrument_id}",
1589                funding_rates.len(),
1590            );
1591
1592            let response = DataResponse::FundingRates(FundingRatesResponse::new(
1593                request_id,
1594                client_id,
1595                instrument_id,
1596                funding_rates,
1597                start_nanos,
1598                end_nanos,
1599                clock.get_time_ns(),
1600                params,
1601            ));
1602
1603            if let Err(e) = sender.send(DataEvent::Response(response)) {
1604                log::error!("Failed to send funding rates response: {e}");
1605            }
1606            Ok(())
1607        });
1608
1609        Ok(())
1610    }
1611
1612    fn request_book_snapshot(&self, request: RequestBookSnapshot) -> anyhow::Result<()> {
1613        let instrument_id = request.single_instrument_id()?;
1614
1615        let instruments = self.instruments.load();
1616        let instrument = instruments
1617            .get(&instrument_id)
1618            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1619
1620        let raw_symbol = instrument.raw_symbol().to_string();
1621        let price_precision = instrument.price_precision();
1622        let size_precision = instrument.size_precision();
1623        let depth = request.depth.map(|d| d.get());
1624
1625        let http = self.http_client.clone();
1626        let sender = self.data_sender.clone();
1627        let client_id = request.client_id.unwrap_or(self.client_id);
1628        let request_id = request.request_id;
1629        let params = request.params;
1630        let clock = self.clock;
1631
1632        self.spawn_task("request_book_snapshot", async move {
1633            let l2_book = http
1634                .info_l2_book(&raw_symbol)
1635                .await
1636                .with_context(|| format!("book snapshot request failed for {instrument_id}"))?;
1637
1638            let book = parse_l2_book_snapshot(
1639                &l2_book,
1640                instrument_id,
1641                price_precision,
1642                size_precision,
1643                depth,
1644            );
1645
1646            let response = DataResponse::Book(BookResponse::new(
1647                request_id,
1648                client_id,
1649                instrument_id,
1650                book,
1651                None,
1652                None,
1653                clock.get_time_ns(),
1654                params,
1655            ));
1656
1657            if let Err(e) = sender.send(DataEvent::Response(response)) {
1658                log::error!("Failed to send book snapshot response: {e}");
1659            }
1660            Ok(())
1661        });
1662
1663        Ok(())
1664    }
1665}
1666
1667/// Applies fetched instruments to the client caches and both transports.
1668fn cache_instruments(
1669    instruments: &[InstrumentAny],
1670    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1671    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1672    http_client: &HyperliquidHttpClient,
1673    ws_client: &HyperliquidWebSocketClient,
1674) {
1675    instruments_by_id.rcu(|m| {
1676        for instrument in instruments {
1677            m.insert(instrument.id(), instrument.clone());
1678        }
1679    });
1680
1681    coin_to_instrument_id.rcu(|m| {
1682        for instrument in instruments {
1683            m.insert(instrument.raw_symbol().inner(), instrument.id());
1684        }
1685    });
1686
1687    http_client.merge_instruments(instruments);
1688    ws_client.merge_instruments(instruments);
1689}
1690
1691/// Rebuilds the `allDexsAssetCtxs` mapping from the HTTP client's cached instruments.
1692///
1693/// Callers rebuild on every pass rather than only when a definition changed. The
1694/// mapping is positional over each perp dex universe, so a listing or delisting
1695/// shifts the entries of coins whose own definitions are unchanged, and a stale
1696/// mapping would misattribute incoming `ctxs` arrays. Rebuilding unconditionally
1697/// also retries a build that failed or fell back on an earlier pass.
1698async fn rebuild_all_dex_asset_ctxs_mapping(
1699    http_client: &HyperliquidHttpClient,
1700    ws_client: &HyperliquidWebSocketClient,
1701) {
1702    match http_client.build_all_dex_asset_ctxs_instrument_ids().await {
1703        Ok(mapping) => {
1704            let mapping = mapping
1705                .into_iter()
1706                .map(|(dex, instrument_ids)| (Ustr::from(dex.as_str()), instrument_ids))
1707                .collect();
1708            ws_client.cache_all_dex_asset_ctxs_instrument_ids(mapping);
1709        }
1710        Err(e) => {
1711            log::warn!("Failed to build Hyperliquid allDexsAssetCtxs mapping: {e}");
1712        }
1713    }
1714}
1715
1716/// Summary of a single instrument refresh pass.
1717#[derive(Debug)]
1718struct InstrumentRefresh {
1719    /// Instruments returned by the venue.
1720    fetched: Vec<InstrumentAny>,
1721    /// Symbols of the definitions that were new to the cache.
1722    added: Vec<Ustr>,
1723    /// New or materially changed definitions published downstream.
1724    changed: usize,
1725}
1726
1727impl InstrumentRefresh {
1728    fn log(&self, client_id: ClientId) {
1729        // a quiet pass every interval would be noise, but a market becoming
1730        // tradable mid-session is the event an operator needs to see
1731        if self.added.is_empty() {
1732            log::debug!(
1733                "Hyperliquid instruments refreshed: client_id={client_id}, fetched={}, changed={}",
1734                self.fetched.len(),
1735                self.changed,
1736            );
1737        } else {
1738            log::info!(
1739                "Hyperliquid instruments refreshed: client_id={client_id}, fetched={}, changed={}, added={:?}",
1740                self.fetched.len(),
1741                self.changed,
1742                self.added,
1743            );
1744        }
1745    }
1746}
1747
1748/// Fetches the instrument universe, reconciles it against the caches, and emits
1749/// instrument status changes for subscribed instruments.
1750///
1751/// Holds `update_lock` across the fetch and the reconcile so concurrent passes
1752/// apply in the order they fetched. The diff only asks whether a definition
1753/// differs from the cached one, not whether it is newer, so without the lock a
1754/// pass that fetched before another but finished after it would republish the
1755/// older definition.
1756async fn refresh_instruments(
1757    update_lock: &tokio::sync::Mutex<()>,
1758    http_client: &HyperliquidHttpClient,
1759    ws_client: &HyperliquidWebSocketClient,
1760    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1761    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1762    instrument_status: &Mutex<InstrumentStatusState>,
1763    data_sender: &EventSender<DataEvent>,
1764) -> anyhow::Result<InstrumentRefresh> {
1765    let _update_guard = update_lock.lock().await;
1766
1767    let snapshot = http_client
1768        .request_instruments_with_statuses()
1769        .await
1770        .context("failed to fetch Hyperliquid instruments")?;
1771
1772    // emit statuses after definitions, so a subscriber never sees a status for an
1773    // instrument it has not received
1774    let refresh = reconcile_instruments(
1775        snapshot.instruments,
1776        http_client,
1777        ws_client,
1778        instruments_by_id,
1779        coin_to_instrument_id,
1780        data_sender,
1781    )
1782    .await;
1783
1784    let ts = get_atomic_clock_realtime().get_time_ns();
1785    instrument_status
1786        .lock()
1787        .apply(&snapshot.statuses, &snapshot.loaded, data_sender, ts);
1788
1789    Ok(refresh)
1790}
1791
1792/// Reconciles the instrument caches against a fetched instrument universe.
1793///
1794/// Caches and publishes new or materially changed definitions as
1795/// [`DataEvent::Data`]. Unchanged definitions are not republished. Cached
1796/// instruments absent from `fetched` are retained because they may still back
1797/// open subscriptions.
1798///
1799/// `instruments_by_id` is the baseline later passes diff against, so every path
1800/// that writes fetched instruments into it must come through here, under the
1801/// update lock taken by [`refresh_instruments`]. A market cached without being
1802/// published would compare as unchanged on every later pass, and the execution
1803/// client would never receive its asset index.
1804async fn reconcile_instruments(
1805    fetched: Vec<InstrumentAny>,
1806    http_client: &HyperliquidHttpClient,
1807    ws_client: &HyperliquidWebSocketClient,
1808    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1809    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1810    data_sender: &EventSender<DataEvent>,
1811) -> InstrumentRefresh {
1812    let changed = changed_definitions(&fetched, instruments_by_id);
1813    let added = added_symbols(&changed, instruments_by_id);
1814
1815    cache_instruments(
1816        &changed,
1817        instruments_by_id,
1818        coin_to_instrument_id,
1819        http_client,
1820        ws_client,
1821    );
1822
1823    // publish before awaiting the mapping rebuild, so a pass cancelled during the
1824    // rebuild cannot leave these definitions cached but unpublished
1825    for instrument in &changed {
1826        if let Err(e) = data_sender.send(DataEvent::Data(instrument.clone().into())) {
1827            log::warn!("Failed to send instrument: {e}");
1828        }
1829    }
1830
1831    rebuild_all_dex_asset_ctxs_mapping(http_client, ws_client).await;
1832
1833    InstrumentRefresh {
1834        added,
1835        changed: changed.len(),
1836        fetched,
1837    }
1838}
1839
1840/// Returns the fetched instruments that are new or materially changed.
1841fn changed_definitions(
1842    fetched: &[InstrumentAny],
1843    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1844) -> Vec<InstrumentAny> {
1845    fetched
1846        .iter()
1847        .filter(|instrument| {
1848            instruments_by_id
1849                .get_cloned(&instrument.id())
1850                .is_none_or(|cached| !instrument_definitions_match(&cached, instrument))
1851        })
1852        .cloned()
1853        .collect()
1854}
1855
1856/// Returns the symbols of the changed definitions the cache has never held.
1857///
1858/// A market listed after startup is the case the refresh exists for, so a pass
1859/// reports which symbols became tradable rather than only how many definitions
1860/// moved. Callers pass the changed set, so a definition that merely moved its
1861/// tick size is not reported as new.
1862fn added_symbols(
1863    changed: &[InstrumentAny],
1864    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1865) -> Vec<Ustr> {
1866    changed
1867        .iter()
1868        .filter(|instrument| instruments_by_id.get_cloned(&instrument.id()).is_none())
1869        .map(|instrument| instrument.symbol().inner())
1870        .collect()
1871}
1872
1873/// Returns `true` when two instruments carry the same tradable definition,
1874/// ignoring event timestamps.
1875///
1876/// Comparison runs on the serialized form so every venue field, including the
1877/// free-form `info` metadata, participates without listing each field.
1878fn instrument_definitions_match(a: &InstrumentAny, b: &InstrumentAny) -> bool {
1879    fn normalized(instrument: &InstrumentAny) -> Option<serde_json::Value> {
1880        let mut value = serde_json::to_value(instrument).ok()?;
1881
1882        if let Some(definition) = value
1883            .as_object_mut()
1884            .and_then(|obj| obj.values_mut().next())
1885            .and_then(serde_json::Value::as_object_mut)
1886        {
1887            definition.remove("ts_event");
1888            definition.remove("ts_init");
1889        }
1890
1891        Some(value)
1892    }
1893
1894    // A serialization failure compares as changed so updates are never suppressed
1895    match (normalized(a), normalized(b)) {
1896        (Some(a), Some(b)) => a == b,
1897        _ => false,
1898    }
1899}
1900
1901#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1902enum MarketDataChannel {
1903    Deltas,
1904    Depth,
1905    Quote,
1906}
1907
1908impl MarketDataChannel {
1909    const fn as_str(self) -> &'static str {
1910        match self {
1911            Self::Deltas => "deltas",
1912            Self::Depth => "depth",
1913            Self::Quote => "quote",
1914        }
1915    }
1916}
1917
1918type MarketDataStreamKey = (MarketDataChannel, InstrumentId);
1919
1920#[derive(Debug, Clone)]
1921struct MarketDataStreamHealth {
1922    last_receive_at: Instant,
1923    last_venue_ts_event: Option<UnixNanos>,
1924    consecutive_stale_count: u32,
1925    last_warning_at: Option<Instant>,
1926    last_recovery_at: Option<Instant>,
1927    resubscribe_attempts: u32,
1928}
1929
1930impl MarketDataStreamHealth {
1931    fn new(receive_at: Instant) -> Self {
1932        Self {
1933            last_receive_at: receive_at,
1934            last_venue_ts_event: None,
1935            consecutive_stale_count: 0,
1936            last_warning_at: None,
1937            last_recovery_at: None,
1938            resubscribe_attempts: 0,
1939        }
1940    }
1941
1942    fn record_receive(&mut self, receive_at: Instant, venue_ts_event: UnixNanos) {
1943        self.last_receive_at = receive_at;
1944        self.last_venue_ts_event = Some(venue_ts_event);
1945        self.consecutive_stale_count = 0;
1946        self.last_warning_at = None;
1947        self.last_recovery_at = None;
1948        self.resubscribe_attempts = 0;
1949    }
1950
1951    // Records a recovery action at `recovery_at` and returns the next step of the ladder
1952    fn record_recovery(
1953        &mut self,
1954        recovery_at: Instant,
1955        book_recovery: bool,
1956        max_targeted_resubscribes: u32,
1957    ) -> StaleStreamAction {
1958        self.last_recovery_at = Some(recovery_at);
1959        self.last_warning_at = Some(recovery_at);
1960
1961        // Shared book recovery owns retries and backoff for a delta book's stream
1962        if book_recovery {
1963            return StaleStreamAction::Recover;
1964        }
1965
1966        if self.resubscribe_attempts < max_targeted_resubscribes {
1967            self.resubscribe_attempts += 1;
1968            return StaleStreamAction::Resubscribe;
1969        }
1970
1971        // Reconnect replays all active subscriptions
1972        self.resubscribe_attempts = 0;
1973        StaleStreamAction::Reconnect
1974    }
1975}
1976
1977#[derive(Debug, Clone, Copy)]
1978struct StreamRecoveryConfig {
1979    cooldown: Duration,
1980    max_targeted_resubscribes: u32,
1981}
1982
1983#[derive(Debug)]
1984struct MarketDataStreamHealthMonitor {
1985    stale_receive_threshold: Duration,
1986    warning_cooldown: Duration,
1987    recovery: Option<StreamRecoveryConfig>,
1988    streams: AHashMap<MarketDataStreamKey, MarketDataStreamHealth>,
1989}
1990
1991impl MarketDataStreamHealthMonitor {
1992    fn new(stale_receive_threshold: Duration, warning_cooldown: Duration) -> Self {
1993        Self {
1994            stale_receive_threshold,
1995            warning_cooldown,
1996            recovery: None,
1997            streams: AHashMap::new(),
1998        }
1999    }
2000
2001    fn with_recovery(mut self, cooldown: Duration, max_targeted_resubscribes: u32) -> Self {
2002        self.recovery = Some(StreamRecoveryConfig {
2003            cooldown,
2004            max_targeted_resubscribes,
2005        });
2006        self
2007    }
2008
2009    fn subscribe(
2010        &mut self,
2011        channel: MarketDataChannel,
2012        instrument_id: InstrumentId,
2013        receive_at: Instant,
2014    ) {
2015        self.streams.insert(
2016            (channel, instrument_id),
2017            MarketDataStreamHealth::new(receive_at),
2018        );
2019    }
2020
2021    fn unsubscribe(&mut self, channel: MarketDataChannel, instrument_id: InstrumentId) {
2022        self.streams.remove(&(channel, instrument_id));
2023    }
2024
2025    fn clear(&mut self) {
2026        self.streams.clear();
2027    }
2028
2029    fn record_receive(
2030        &mut self,
2031        channel: MarketDataChannel,
2032        instrument_id: InstrumentId,
2033        receive_at: Instant,
2034        venue_ts_event: UnixNanos,
2035    ) {
2036        if let Some(stream) = self.streams.get_mut(&(channel, instrument_id)) {
2037            stream.record_receive(receive_at, venue_ts_event);
2038        }
2039    }
2040
2041    fn check_stale(
2042        &mut self,
2043        now: Instant,
2044        wall_clock_now: UnixNanos,
2045    ) -> Vec<MarketDataStaleEvent> {
2046        // Fresh BBO makes stale book streams relative-stale, not transport-stale
2047        let fresh_quote_instruments: AHashSet<InstrumentId> = self
2048            .streams
2049            .iter()
2050            .filter(|((channel, _), stream)| {
2051                *channel == MarketDataChannel::Quote
2052                    && now.saturating_duration_since(stream.last_receive_at)
2053                        < self.stale_receive_threshold
2054            })
2055            .map(|((_, instrument_id), _)| *instrument_id)
2056            .collect();
2057
2058        // Depth shares the instrument's `l2Book` stream with its delta book
2059        let delta_book_instruments: AHashSet<InstrumentId> = self
2060            .streams
2061            .keys()
2062            .filter(|(channel, _)| *channel == MarketDataChannel::Deltas)
2063            .map(|(_, instrument_id)| *instrument_id)
2064            .collect();
2065
2066        let mut events = Vec::new();
2067
2068        for ((channel, instrument_id), stream) in &mut self.streams {
2069            let receive_age = now.saturating_duration_since(stream.last_receive_at);
2070            if receive_age < self.stale_receive_threshold {
2071                stream.consecutive_stale_count = 0;
2072                continue;
2073            }
2074
2075            stream.consecutive_stale_count = stream.consecutive_stale_count.saturating_add(1);
2076
2077            let quote_is_fresh = matches!(
2078                channel,
2079                MarketDataChannel::Deltas | MarketDataChannel::Depth
2080            ) && fresh_quote_instruments.contains(instrument_id);
2081
2082            let venue_age = stream.last_venue_ts_event.map(|ts_event| {
2083                Duration::from_nanos(wall_clock_now.as_u64().saturating_sub(ts_event.as_u64()))
2084            });
2085
2086            if let Some(recovery) = self.recovery {
2087                // Recovery requires one prior warning, even after long check lag
2088                let stale_since = stream.last_receive_at + self.stale_receive_threshold;
2089                let anchor = stream.last_recovery_at.unwrap_or(stale_since);
2090
2091                if stream.last_warning_at.is_some()
2092                    && now.saturating_duration_since(anchor) >= recovery.cooldown
2093                {
2094                    let book_recovery = matches!(
2095                        channel,
2096                        MarketDataChannel::Deltas | MarketDataChannel::Depth
2097                    ) && delta_book_instruments.contains(instrument_id);
2098                    let action = stream.record_recovery(
2099                        now,
2100                        book_recovery,
2101                        recovery.max_targeted_resubscribes,
2102                    );
2103
2104                    events.push(MarketDataStaleEvent {
2105                        channel: *channel,
2106                        instrument_id: *instrument_id,
2107                        receive_age,
2108                        venue_age,
2109                        stale_count: stream.consecutive_stale_count,
2110                        action,
2111                        cooldown: recovery.cooldown,
2112                        quote_is_fresh,
2113                    });
2114                    continue;
2115                }
2116            }
2117
2118            let should_warn = stream.last_warning_at.is_none_or(|last_warning_at| {
2119                now.saturating_duration_since(last_warning_at) >= self.warning_cooldown
2120            });
2121
2122            if !should_warn {
2123                continue;
2124            }
2125
2126            stream.last_warning_at = Some(now);
2127            events.push(MarketDataStaleEvent {
2128                channel: *channel,
2129                instrument_id: *instrument_id,
2130                receive_age,
2131                venue_age,
2132                stale_count: stream.consecutive_stale_count,
2133                action: StaleStreamAction::Warn,
2134                cooldown: self.warning_cooldown,
2135                quote_is_fresh,
2136            });
2137        }
2138
2139        events
2140    }
2141}
2142
2143#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2144enum StaleStreamAction {
2145    Warn,
2146    Recover,
2147    Resubscribe,
2148    Reconnect,
2149}
2150
2151impl StaleStreamAction {
2152    const fn as_str(self) -> &'static str {
2153        match self {
2154            Self::Warn => "warn",
2155            Self::Recover => "recover",
2156            Self::Resubscribe => "resubscribe",
2157            Self::Reconnect => "reconnect",
2158        }
2159    }
2160}
2161
2162#[derive(Debug, Clone, PartialEq, Eq)]
2163struct MarketDataStaleEvent {
2164    channel: MarketDataChannel,
2165    instrument_id: InstrumentId,
2166    receive_age: Duration,
2167    venue_age: Option<Duration>,
2168    stale_count: u32,
2169    action: StaleStreamAction,
2170    cooldown: Duration,
2171    quote_is_fresh: bool,
2172}
2173
2174fn stream_health_update(
2175    msg: &NautilusWsMessage,
2176) -> Option<(MarketDataChannel, InstrumentId, UnixNanos)> {
2177    match msg {
2178        NautilusWsMessage::Quote(quote) => Some((
2179            MarketDataChannel::Quote,
2180            quote.instrument_id,
2181            quote.ts_event,
2182        )),
2183        NautilusWsMessage::Deltas(deltas) => Some((
2184            MarketDataChannel::Deltas,
2185            deltas.instrument_id,
2186            deltas.ts_event,
2187        )),
2188        NautilusWsMessage::Depth(depth) => Some((
2189            MarketDataChannel::Depth,
2190            depth.instrument_id,
2191            depth.ts_event,
2192        )),
2193        _ => None,
2194    }
2195}
2196
2197fn record_stream_receive(
2198    stream_health: &Arc<Mutex<MarketDataStreamHealthMonitor>>,
2199    channel: MarketDataChannel,
2200    instrument_id: InstrumentId,
2201    venue_ts_event: UnixNanos,
2202) {
2203    stream_health
2204        .lock()
2205        .record_receive(channel, instrument_id, Instant::now(), venue_ts_event);
2206}
2207
2208fn log_stream_health_event(event: &MarketDataStaleEvent) {
2209    let venue_age_ms = event
2210        .venue_age
2211        .map_or_else(|| "n/a".to_string(), |age| age.as_millis().to_string());
2212    let prefix = if event.quote_is_fresh {
2213        "Hyperliquid book stream stale while bbo advances"
2214    } else {
2215        "Hyperliquid market data stream stale"
2216    };
2217
2218    log::warn!(
2219        "{prefix}: channel={}, instrument_id={}, receive_age_ms={}, venue_age_ms={}, \
2220         stale_count={}, action={}, cooldown_secs={}",
2221        event.channel.as_str(),
2222        event.instrument_id,
2223        event.receive_age.as_millis(),
2224        venue_age_ms,
2225        event.stale_count,
2226        event.action.as_str(),
2227        event.cooldown.as_secs(),
2228    );
2229}
2230
2231async fn handle_stream_health_events(
2232    ws_client: &HyperliquidWebSocketClient,
2233    events: &[MarketDataStaleEvent],
2234    book_sync: &BookSyncTracker,
2235    snapshot_timeout: Duration,
2236    book_tasks: &TaskSpawner,
2237) {
2238    // Deltas and depth share one venue `l2Book` stream
2239    let mut resubscribed_books: AHashSet<InstrumentId> = AHashSet::new();
2240    let mut reconnect_requested = false;
2241
2242    for event in events {
2243        log_stream_health_event(event);
2244
2245        match event.action {
2246            StaleStreamAction::Warn => {}
2247            StaleStreamAction::Recover => book::recovery::start_recovery(
2248                event.instrument_id,
2249                book_sync,
2250                ws_client,
2251                snapshot_timeout,
2252                book_tasks,
2253            ),
2254            StaleStreamAction::Resubscribe => match event.channel {
2255                MarketDataChannel::Deltas | MarketDataChannel::Depth => {
2256                    if resubscribed_books.insert(event.instrument_id)
2257                        && let Err(e) = ws_client.resubscribe_book(event.instrument_id).await
2258                    {
2259                        log::warn!(
2260                            "Failed targeted l2Book resubscribe for {}: {e}",
2261                            event.instrument_id,
2262                        );
2263                    }
2264                }
2265                MarketDataChannel::Quote => {
2266                    if let Err(e) = ws_client.resubscribe_quotes(event.instrument_id).await {
2267                        log::warn!(
2268                            "Failed targeted bbo resubscribe for {}: {e}",
2269                            event.instrument_id,
2270                        );
2271                    }
2272                }
2273            },
2274            StaleStreamAction::Reconnect => reconnect_requested = true,
2275        }
2276    }
2277
2278    if reconnect_requested {
2279        if ws_client.request_reconnect() {
2280            log::warn!("Requested full WebSocket reconnect after failed targeted stream recovery");
2281        } else {
2282            log::debug!("Skipping reconnect request: connection not active");
2283        }
2284    }
2285}
2286
2287// Applies the request window and limit to a snapshot of recent trades. `trades`
2288// must be sorted ascending by `ts_event`. Returns the subset within `[start, end]`
2289// (each bound unbounded when `None`), keeping at most the most recent `limit`
2290// trades. Because `recentTrades` exposes only a recent snapshot with no historical
2291// depth, a warning is logged when the request reaches below the snapshot's
2292// coverage floor (its oldest trade).
2293fn filter_recent_trades(
2294    trades: Vec<TradeTick>,
2295    start: Option<UnixNanos>,
2296    end: Option<UnixNanos>,
2297    limit: Option<usize>,
2298    instrument_id: InstrumentId,
2299) -> Vec<TradeTick> {
2300    let Some(floor) = trades.first().map(|trade| trade.ts_event) else {
2301        return Vec::new();
2302    };
2303
2304    if let Some(end) = end
2305        && end < floor
2306    {
2307        log::warn!(
2308            "Recent trades for {instrument_id} are entirely older than the requested window; \
2309             snapshot only covers back to {}",
2310            unix_nanos_to_iso8601(floor),
2311        );
2312        return Vec::new();
2313    }
2314
2315    if let Some(start) = start
2316        && start < floor
2317    {
2318        log::warn!(
2319            "Recent trades for {instrument_id} only cover back to {}; \
2320             the requested start is earlier and cannot be served",
2321            unix_nanos_to_iso8601(floor),
2322        );
2323    }
2324
2325    let mut filtered: Vec<TradeTick> = trades
2326        .into_iter()
2327        .filter(|trade| start.is_none_or(|s| trade.ts_event >= s))
2328        .filter(|trade| end.is_none_or(|e| trade.ts_event <= e))
2329        .collect();
2330
2331    if let Some(limit) = limit
2332        && filtered.len() > limit
2333    {
2334        // Keep the most recent `limit` trades; ascending order is preserved
2335        filtered.drain(0..filtered.len() - limit);
2336    }
2337
2338    filtered
2339}
2340
2341// Levels with unparsable px/sz or non-positive size are skipped rather than
2342// erroring; the snapshot's `time` field (ms) becomes `ts_event` after the
2343// ms->ns conversion.
2344pub(crate) fn parse_l2_book_snapshot(
2345    l2_book: &HyperliquidL2Book,
2346    instrument_id: InstrumentId,
2347    price_precision: u8,
2348    size_precision: u8,
2349    depth: Option<usize>,
2350) -> OrderBook {
2351    let mut book = OrderBook::new(instrument_id, BookType::L2_MBP);
2352    let ts_event = UnixNanos::from(l2_book.time * 1_000_000);
2353
2354    let all_bids = l2_book
2355        .levels
2356        .first()
2357        .map_or([].as_slice(), |v| v.as_slice());
2358    let all_asks = l2_book
2359        .levels
2360        .get(1)
2361        .map_or([].as_slice(), |v| v.as_slice());
2362
2363    let bids = match depth {
2364        Some(d) if d < all_bids.len() => &all_bids[..d],
2365        _ => all_bids,
2366    };
2367    let asks = match depth {
2368        Some(d) if d < all_asks.len() => &all_asks[..d],
2369        _ => all_asks,
2370    };
2371
2372    for (i, level) in bids.iter().enumerate() {
2373        if level.sz <= Decimal::ZERO {
2374            continue;
2375        }
2376        let Ok(price) = Price::from_decimal_dp(level.px, price_precision) else {
2377            continue;
2378        };
2379        let Ok(size) = Quantity::from_decimal_dp(level.sz, size_precision) else {
2380            continue;
2381        };
2382
2383        let order = BookOrder::new(OrderSide::Buy, price, size, i as u64);
2384        book.add(order, 0, i as u64, ts_event);
2385    }
2386
2387    let bids_len = bids.len();
2388
2389    for (i, level) in asks.iter().enumerate() {
2390        if level.sz <= Decimal::ZERO {
2391            continue;
2392        }
2393        let Ok(price) = Price::from_decimal_dp(level.px, price_precision) else {
2394            continue;
2395        };
2396        let Ok(size) = Quantity::from_decimal_dp(level.sz, size_precision) else {
2397            continue;
2398        };
2399
2400        let order = BookOrder::new(OrderSide::Sell, price, size, (bids_len + i) as u64);
2401        book.add(order, 0, (bids_len + i) as u64, ts_event);
2402    }
2403
2404    log::debug!(
2405        "Built order book for {instrument_id} with {} bids and {} asks",
2406        bids.len(),
2407        asks.len(),
2408    );
2409
2410    book
2411}
2412
2413// Reads optional `nSigFigs` / `mantissa` L2 precision controls from
2414// `subscribe_params`; bails on non-positive integer values.
2415pub(crate) fn parse_book_precision_params(
2416    params: Option<&Params>,
2417) -> anyhow::Result<(Option<u32>, Option<u32>)> {
2418    let Some(params) = params else {
2419        return Ok((None, None));
2420    };
2421
2422    let read_u32 = |key: &str| -> anyhow::Result<Option<u32>> {
2423        match params.get(key) {
2424            None => Ok(None),
2425            Some(v) => v
2426                .as_u64()
2427                .and_then(|n| u32::try_from(n).ok())
2428                .ok_or_else(|| anyhow::anyhow!("`{key}` must be a positive u32"))
2429                .map(Some),
2430        }
2431    };
2432
2433    Ok((read_u32("n_sig_figs")?, read_u32("mantissa")?))
2434}
2435
2436// Hyperliquid funds perpetuals hourly, so `interval` is fixed at 60 mins;
2437// `time` from the venue marks the end of the funding interval in ms.
2438pub(crate) fn funding_entry_to_update(
2439    entry: &HyperliquidFundingHistoryEntry,
2440    instrument_id: InstrumentId,
2441) -> FundingRateUpdate {
2442    let rate = entry.funding_rate;
2443    let ts = UnixNanos::from(entry.time * 1_000_000);
2444    FundingRateUpdate::new(instrument_id, rate, Some(60), None, ts, ts)
2445}
2446
2447pub(crate) fn candle_to_bar(
2448    candle: &HyperliquidCandle,
2449    bar_type: BarType,
2450    price_precision: u8,
2451    size_precision: u8,
2452) -> anyhow::Result<Bar> {
2453    let ts_event = millis_to_nanos(candle.timestamp)?;
2454    let close_boundary = candle
2455        .end_timestamp
2456        .checked_add(1)
2457        .context("candle close boundary overflow")?;
2458    let ts_init = millis_to_nanos(close_boundary)?;
2459
2460    let open = Price::from_decimal_dp(candle.open, price_precision)
2461        .map_err(|e| anyhow::anyhow!("invalid open price: {e}"))?;
2462    let high = Price::from_decimal_dp(candle.high, price_precision)
2463        .map_err(|e| anyhow::anyhow!("invalid high price: {e}"))?;
2464    let low = Price::from_decimal_dp(candle.low, price_precision)
2465        .map_err(|e| anyhow::anyhow!("invalid low price: {e}"))?;
2466    let close = Price::from_decimal_dp(candle.close, price_precision)
2467        .map_err(|e| anyhow::anyhow!("invalid close price: {e}"))?;
2468    let volume = Quantity::from_decimal_dp(candle.volume, size_precision)
2469        .map_err(|e| anyhow::anyhow!("invalid volume: {e}"))?;
2470
2471    Ok(Bar::new(
2472        bar_type, open, high, low, close, volume, ts_event, ts_init,
2473    ))
2474}
2475
2476/// Request bars from HTTP API.
2477async fn request_bars_from_http(
2478    http_client: HyperliquidHttpClient,
2479    bar_type: BarType,
2480    start: Option<Timestamp>,
2481    end: Option<Timestamp>,
2482    limit: Option<u32>,
2483    instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
2484) -> anyhow::Result<Vec<Bar>> {
2485    // Get instrument details for precision
2486    let instrument_id = bar_type.instrument_id();
2487    let instrument = instruments
2488        .load()
2489        .get(&instrument_id)
2490        .cloned()
2491        .context("instrument not found in cache")?;
2492
2493    let price_precision = instrument.price_precision();
2494    let size_precision = instrument.size_precision();
2495    let raw_symbol = instrument.raw_symbol();
2496    let coin = raw_symbol.as_str();
2497
2498    let interval = bar_type_to_interval(&bar_type)?;
2499
2500    // Hyperliquid uses millisecond timestamps
2501    let now = Timestamp::now();
2502    let end_time = end.unwrap_or(now).as_millisecond() as u64;
2503    let start_time = if let Some(start) = start {
2504        start.as_millisecond() as u64
2505    } else {
2506        // Default to 1000 bars before end_time
2507        let spec = bar_type.spec();
2508        let step_ms = match spec.aggregation {
2509            BarAggregation::Minute => spec.step.get() as u64 * 60_000,
2510            BarAggregation::Hour => spec.step.get() as u64 * 3_600_000,
2511            BarAggregation::Day => spec.step.get() as u64 * 86_400_000,
2512            _ => 60_000,
2513        };
2514        end_time.saturating_sub(1000 * step_ms)
2515    };
2516
2517    let candles = http_client
2518        .info_candle_snapshot(coin, interval, start_time, end_time)
2519        .await
2520        .context("failed to fetch candle snapshot from Hyperliquid")?;
2521
2522    let now_ms = now.as_millisecond() as u64;
2523    let mut bars: Vec<Bar> = candles
2524        .iter()
2525        .filter(|candle| candle.end_timestamp < now_ms)
2526        .filter_map(|candle| {
2527            candle_to_bar(candle, bar_type, price_precision, size_precision)
2528                .map_err(|e| {
2529                    log::warn!("Failed to convert candle to bar: {e}");
2530                    e
2531                })
2532                .ok()
2533        })
2534        .collect();
2535
2536    if let Some(limit) = limit
2537        && bars.len() > limit as usize
2538    {
2539        bars = bars.into_iter().take(limit as usize).collect();
2540    }
2541
2542    log::debug!("Fetched {} bars for {}", bars.len(), bar_type);
2543    Ok(bars)
2544}
2545
2546#[cfg(test)]
2547mod tests {
2548    use futures_util::StreamExt;
2549    use nautilus_common::live::runner::set_data_event_sender;
2550    use nautilus_model::{
2551        data::{
2552            QuoteTick,
2553            stubs::{stub_deltas, stub_depth10},
2554        },
2555        enums::{AggressorSide, CurrencyType},
2556        identifiers::{Symbol, TradeId},
2557        instruments::CryptoPerpetual,
2558        types::Currency,
2559    };
2560    use rstest::rstest;
2561    use rust_decimal_macros::dec;
2562    use ustr::Ustr;
2563
2564    use super::*;
2565    use crate::common::{
2566        consts::{HYPERLIQUID_CLIENT_ID, HYPERLIQUID_WS_URL},
2567        testing::load_test_data,
2568    };
2569
2570    fn btc_perp_id() -> InstrumentId {
2571        InstrumentId::from("BTC-PERP.HYPERLIQUID")
2572    }
2573
2574    #[rstest]
2575    fn test_candle_to_bar_uses_causal_initialization_timestamp() {
2576        let candle = HyperliquidCandle {
2577            timestamp: 1_700_000_000_000,
2578            end_timestamp: 1_700_000_059_999,
2579            open: dec!(100.0),
2580            high: dec!(101.0),
2581            low: dec!(99.0),
2582            close: dec!(100.5),
2583            volume: dec!(10.0),
2584            num_trades: Some(42),
2585        };
2586        let bar_type = BarType::from("BTC-USD-PERP.HYPERLIQUID-1-MINUTE-LAST-EXTERNAL");
2587
2588        let bar = candle_to_bar(&candle, bar_type, 1, 1).unwrap();
2589
2590        assert_eq!(candle.end_timestamp - candle.timestamp, 59_999);
2591        assert_eq!(bar.ts_event, millis_to_nanos(candle.timestamp).unwrap());
2592        assert_eq!(
2593            bar.ts_init,
2594            millis_to_nanos(candle.end_timestamp + 1).unwrap()
2595        );
2596        assert!(bar.ts_init > bar.ts_event);
2597    }
2598
2599    #[rstest]
2600    fn test_candle_to_bar_rejects_close_boundary_overflow() {
2601        let candle = HyperliquidCandle {
2602            timestamp: 1_700_000_000_000,
2603            end_timestamp: u64::MAX,
2604            open: dec!(100.0),
2605            high: dec!(101.0),
2606            low: dec!(99.0),
2607            close: dec!(100.5),
2608            volume: dec!(10.0),
2609            num_trades: Some(42),
2610        };
2611        let bar_type = BarType::from("BTC-USD-PERP.HYPERLIQUID-1-MINUTE-LAST-EXTERNAL");
2612
2613        let err = candle_to_bar(&candle, bar_type, 1, 1).unwrap_err();
2614
2615        assert!(err.to_string().contains("close boundary overflow"));
2616    }
2617
2618    #[rstest]
2619    fn test_stream_health_monitor_fresh_stream_does_not_warn() {
2620        let mut monitor =
2621            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2622        let instrument_id = btc_perp_id();
2623        let start = Instant::now();
2624
2625        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2626
2627        let warnings = monitor.check_stale(
2628            start + Duration::from_secs(4),
2629            UnixNanos::from(4_000_000_000),
2630        );
2631        assert!(warnings.is_empty());
2632    }
2633
2634    #[rstest]
2635    fn test_stream_health_monitor_warns_once_after_threshold() {
2636        let mut monitor =
2637            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2638        let instrument_id = btc_perp_id();
2639        let start = Instant::now();
2640
2641        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2642        monitor.record_receive(
2643            MarketDataChannel::Quote,
2644            instrument_id,
2645            start + Duration::from_secs(1),
2646            UnixNanos::from(1_000_000_000),
2647        );
2648
2649        let warnings = monitor.check_stale(
2650            start + Duration::from_secs(7),
2651            UnixNanos::from(9_000_000_000),
2652        );
2653
2654        assert_eq!(
2655            warnings,
2656            vec![MarketDataStaleEvent {
2657                channel: MarketDataChannel::Quote,
2658                instrument_id,
2659                receive_age: Duration::from_secs(6),
2660                venue_age: Some(Duration::from_secs(8)),
2661                stale_count: 1,
2662                action: StaleStreamAction::Warn,
2663                cooldown: Duration::from_secs(30),
2664                quote_is_fresh: false,
2665            }]
2666        );
2667    }
2668
2669    #[rstest]
2670    fn test_stream_health_monitor_warns_at_receive_threshold() {
2671        let mut monitor =
2672            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2673        let instrument_id = btc_perp_id();
2674        let start = Instant::now();
2675
2676        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2677
2678        let warnings = monitor.check_stale(
2679            start + Duration::from_secs(5),
2680            UnixNanos::from(5_000_000_000),
2681        );
2682
2683        assert_eq!(warnings.len(), 1);
2684        assert_eq!(warnings[0].receive_age, Duration::from_secs(5));
2685        assert_eq!(warnings[0].stale_count, 1);
2686    }
2687
2688    #[rstest]
2689    fn test_stream_health_monitor_new_update_resets_age_and_stale_count() {
2690        let mut monitor =
2691            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2692        let instrument_id = btc_perp_id();
2693        let start = Instant::now();
2694
2695        monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
2696        assert_eq!(
2697            monitor
2698                .check_stale(
2699                    start + Duration::from_secs(6),
2700                    UnixNanos::from(6_000_000_000),
2701                )
2702                .len(),
2703            1,
2704        );
2705
2706        monitor.record_receive(
2707            MarketDataChannel::Depth,
2708            instrument_id,
2709            start + Duration::from_secs(7),
2710            UnixNanos::from(7_000_000_000),
2711        );
2712
2713        assert!(
2714            monitor
2715                .check_stale(
2716                    start + Duration::from_secs(11),
2717                    UnixNanos::from(11_000_000_000),
2718                )
2719                .is_empty()
2720        );
2721
2722        let warnings = monitor.check_stale(
2723            start + Duration::from_secs(13),
2724            UnixNanos::from(13_000_000_000),
2725        );
2726        assert_eq!(warnings.len(), 1);
2727        assert_eq!(warnings[0].stale_count, 1);
2728        assert_eq!(warnings[0].receive_age, Duration::from_secs(6));
2729    }
2730
2731    #[rstest]
2732    fn test_stream_health_monitor_unsubscribe_removes_stream() {
2733        let mut monitor =
2734            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2735        let instrument_id = btc_perp_id();
2736        let start = Instant::now();
2737
2738        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2739        monitor.unsubscribe(MarketDataChannel::Deltas, instrument_id);
2740
2741        let warnings = monitor.check_stale(
2742            start + Duration::from_secs(6),
2743            UnixNanos::from(6_000_000_000),
2744        );
2745
2746        assert!(warnings.is_empty());
2747    }
2748
2749    #[rstest]
2750    #[case(0, 15)]
2751    #[case(120, 0)]
2752    fn test_data_client_stream_health_config_zero_disables_monitor(
2753        #[case] stale_receive_timeout_secs: u64,
2754        #[case] check_interval_secs: u64,
2755    ) {
2756        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
2757        set_data_event_sender(tx);
2758        let client = HyperliquidDataClient::new(
2759            *crate::common::consts::HYPERLIQUID_CLIENT_ID,
2760            HyperliquidDataClientConfig {
2761                stale_stream_receive_timeout_secs: stale_receive_timeout_secs,
2762                stream_health_check_interval_secs: check_interval_secs,
2763                ..HyperliquidDataClientConfig::default()
2764            },
2765        )
2766        .unwrap();
2767        let instrument_id = btc_perp_id();
2768        let start = Instant::now();
2769
2770        assert!(!client.stream_health_monitor_enabled());
2771        client.register_stream_health(MarketDataChannel::Deltas, instrument_id);
2772
2773        let warnings = client.stream_health.lock().check_stale(
2774            start + Duration::from_secs(121),
2775            UnixNanos::from(121_000_000_000),
2776        );
2777
2778        assert!(warnings.is_empty());
2779    }
2780
2781    #[rstest]
2782    fn test_data_client_recovery_requires_positive_cooldown() {
2783        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
2784        set_data_event_sender(tx);
2785        let client = HyperliquidDataClient::new(
2786            *crate::common::consts::HYPERLIQUID_CLIENT_ID,
2787            HyperliquidDataClientConfig {
2788                stale_stream_recovery_enabled: true,
2789                stale_stream_recovery_cooldown_secs: 0,
2790                ..HyperliquidDataClientConfig::default()
2791            },
2792        )
2793        .unwrap();
2794
2795        assert!(
2796            client.stream_health.lock().recovery.is_none(),
2797            "a zero recovery cooldown must leave the monitor observability-only",
2798        );
2799    }
2800
2801    #[rstest]
2802    fn test_stream_health_monitor_warning_cooldown_prevents_repeated_logs() {
2803        let mut monitor =
2804            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10));
2805        let instrument_id = btc_perp_id();
2806        let start = Instant::now();
2807
2808        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2809
2810        let first = monitor.check_stale(
2811            start + Duration::from_secs(6),
2812            UnixNanos::from(6_000_000_000),
2813        );
2814        let inside_cooldown = monitor.check_stale(
2815            start + Duration::from_secs(7),
2816            UnixNanos::from(7_000_000_000),
2817        );
2818        let second = monitor.check_stale(
2819            start + Duration::from_secs(16),
2820            UnixNanos::from(16_000_000_000),
2821        );
2822
2823        assert_eq!(first.len(), 1);
2824        assert!(inside_cooldown.is_empty());
2825        assert_eq!(second.len(), 1);
2826        assert_eq!(second[0].stale_count, 3);
2827    }
2828
2829    fn check_at(
2830        monitor: &mut MarketDataStreamHealthMonitor,
2831        start: Instant,
2832        secs: u64,
2833    ) -> Vec<MarketDataStaleEvent> {
2834        monitor.check_stale(
2835            start + Duration::from_secs(secs),
2836            UnixNanos::from(secs * 1_000_000_000),
2837        )
2838    }
2839
2840    #[rstest]
2841    fn test_stream_health_recovery_ladder_escalates_and_resets() {
2842        let mut monitor =
2843            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2844                .with_recovery(Duration::from_secs(30), 2);
2845        let instrument_id = btc_perp_id();
2846        let start = Instant::now();
2847
2848        monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
2849
2850        let events = check_at(&mut monitor, start, 5);
2851        assert_eq!(events.len(), 1);
2852        assert_eq!(events[0].action, StaleStreamAction::Warn);
2853
2854        let events = check_at(&mut monitor, start, 20);
2855        assert_eq!(events[0].action, StaleStreamAction::Warn);
2856
2857        let events = check_at(&mut monitor, start, 35);
2858        assert_eq!(
2859            events,
2860            vec![MarketDataStaleEvent {
2861                channel: MarketDataChannel::Depth,
2862                instrument_id,
2863                receive_age: Duration::from_secs(35),
2864                venue_age: None,
2865                stale_count: 3,
2866                action: StaleStreamAction::Resubscribe,
2867                cooldown: Duration::from_secs(30),
2868                quote_is_fresh: false,
2869            }],
2870        );
2871
2872        let events = check_at(&mut monitor, start, 50);
2873        assert_eq!(events[0].action, StaleStreamAction::Warn);
2874
2875        let events = check_at(&mut monitor, start, 65);
2876        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2877
2878        let events = check_at(&mut monitor, start, 95);
2879        assert_eq!(events[0].action, StaleStreamAction::Reconnect);
2880
2881        let events = check_at(&mut monitor, start, 125);
2882        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2883    }
2884
2885    #[rstest]
2886    fn test_stream_health_recovery_hands_deltas_to_book_recovery_without_reconnect() {
2887        let mut monitor =
2888            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2889                .with_recovery(Duration::from_secs(30), 1);
2890        let instrument_id = btc_perp_id();
2891        let start = Instant::now();
2892
2893        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2894
2895        let first = check_at(&mut monitor, start, 5);
2896        let second = check_at(&mut monitor, start, 35);
2897        let third = check_at(&mut monitor, start, 65);
2898        let fourth = check_at(&mut monitor, start, 95);
2899
2900        assert_eq!(first[0].action, StaleStreamAction::Warn);
2901        assert_eq!(
2902            second,
2903            vec![MarketDataStaleEvent {
2904                channel: MarketDataChannel::Deltas,
2905                instrument_id,
2906                receive_age: Duration::from_secs(35),
2907                venue_age: None,
2908                stale_count: 2,
2909                action: StaleStreamAction::Recover,
2910                cooldown: Duration::from_secs(30),
2911                quote_is_fresh: false,
2912            }],
2913        );
2914        assert_eq!(third[0].action, StaleStreamAction::Recover);
2915        assert_eq!(fourth[0].action, StaleStreamAction::Recover);
2916    }
2917
2918    #[rstest]
2919    fn test_stream_health_recovery_hands_shared_depth_to_book_recovery() {
2920        let mut monitor =
2921            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2922                .with_recovery(Duration::from_secs(30), 1);
2923        let shared = btc_perp_id();
2924        let depth_only = InstrumentId::from("ETH-PERP.HYPERLIQUID");
2925        let start = Instant::now();
2926
2927        monitor.subscribe(MarketDataChannel::Deltas, shared, start);
2928        monitor.subscribe(MarketDataChannel::Depth, shared, start);
2929        monitor.subscribe(MarketDataChannel::Depth, depth_only, start);
2930
2931        let actions = |events: Vec<MarketDataStaleEvent>| {
2932            let mut actions = events
2933                .into_iter()
2934                .map(|event| (event.channel.as_str(), event.instrument_id, event.action))
2935                .collect::<Vec<_>>();
2936            actions.sort_by_key(|(channel, instrument_id, _)| (*channel, *instrument_id));
2937            actions
2938        };
2939
2940        check_at(&mut monitor, start, 5);
2941        let first = actions(check_at(&mut monitor, start, 35));
2942        let second = actions(check_at(&mut monitor, start, 65));
2943
2944        assert_eq!(
2945            first,
2946            vec![
2947                ("deltas", shared, StaleStreamAction::Recover),
2948                ("depth", shared, StaleStreamAction::Recover),
2949                ("depth", depth_only, StaleStreamAction::Resubscribe),
2950            ],
2951        );
2952        assert_eq!(
2953            second,
2954            vec![
2955                ("deltas", shared, StaleStreamAction::Recover),
2956                ("depth", shared, StaleStreamAction::Recover),
2957                ("depth", depth_only, StaleStreamAction::Reconnect),
2958            ],
2959        );
2960    }
2961
2962    #[rstest]
2963    fn test_stream_health_recovery_first_breach_warns_even_past_cooldown() {
2964        let mut monitor =
2965            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2966                .with_recovery(Duration::from_secs(1), 1);
2967        let instrument_id = btc_perp_id();
2968        let start = Instant::now();
2969
2970        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2971
2972        // First stale checks must stay observability-only, even after cooldown
2973        let events = check_at(&mut monitor, start, 40);
2974        assert_eq!(events.len(), 1);
2975        assert_eq!(events[0].action, StaleStreamAction::Warn);
2976
2977        let events = check_at(&mut monitor, start, 41);
2978        assert_eq!(events.len(), 1);
2979        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2980    }
2981
2982    #[rstest]
2983    fn test_stream_health_receive_resets_recovery_state() {
2984        let mut monitor =
2985            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2986                .with_recovery(Duration::from_secs(10), 1);
2987        let instrument_id = btc_perp_id();
2988        let start = Instant::now();
2989
2990        monitor.subscribe(MarketDataChannel::Depth, instrument_id, start);
2991        assert_eq!(
2992            check_at(&mut monitor, start, 5)[0].action,
2993            StaleStreamAction::Warn
2994        );
2995        assert_eq!(
2996            check_at(&mut monitor, start, 15)[0].action,
2997            StaleStreamAction::Resubscribe,
2998        );
2999
3000        monitor.record_receive(
3001            MarketDataChannel::Depth,
3002            instrument_id,
3003            start + Duration::from_secs(16),
3004            UnixNanos::from(16_000_000_000),
3005        );
3006
3007        assert!(check_at(&mut monitor, start, 20).is_empty());
3008
3009        let events = check_at(&mut monitor, start, 21);
3010        assert_eq!(events[0].action, StaleStreamAction::Warn);
3011        assert_eq!(events[0].stale_count, 1);
3012
3013        assert_eq!(
3014            check_at(&mut monitor, start, 31)[0].action,
3015            StaleStreamAction::Resubscribe,
3016        );
3017        assert_eq!(
3018            check_at(&mut monitor, start, 41)[0].action,
3019            StaleStreamAction::Reconnect,
3020        );
3021    }
3022
3023    #[rstest]
3024    fn test_check_stale_book_with_fresh_quote_flags_relative_staleness() {
3025        let mut monitor =
3026            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
3027        let instrument_id = btc_perp_id();
3028        let start = Instant::now();
3029
3030        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
3031        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
3032        monitor.record_receive(
3033            MarketDataChannel::Quote,
3034            instrument_id,
3035            start + Duration::from_secs(8),
3036            UnixNanos::from(8_000_000_000),
3037        );
3038
3039        let events = check_at(&mut monitor, start, 10);
3040
3041        assert_eq!(events.len(), 1, "fresh quote stream must not be reported");
3042        assert_eq!(events[0].channel, MarketDataChannel::Deltas);
3043        assert!(events[0].quote_is_fresh);
3044    }
3045
3046    #[rstest]
3047    #[case(true)]
3048    #[case(false)]
3049    fn test_check_stale_book_without_fresh_quote_is_not_flagged(#[case] quote_subscribed: bool) {
3050        let mut monitor =
3051            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
3052        let instrument_id = btc_perp_id();
3053        let start = Instant::now();
3054
3055        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
3056        if quote_subscribed {
3057            monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
3058        }
3059
3060        let events = check_at(&mut monitor, start, 10);
3061
3062        let deltas_event = events
3063            .iter()
3064            .find(|event| event.channel == MarketDataChannel::Deltas)
3065            .expect("deltas event");
3066        assert!(
3067            !deltas_event.quote_is_fresh,
3068            "a stale or absent quote stream must not flag relative staleness",
3069        );
3070
3071        if quote_subscribed {
3072            let quote_event = events
3073                .iter()
3074                .find(|event| event.channel == MarketDataChannel::Quote)
3075                .expect("quote event");
3076            assert!(!quote_event.quote_is_fresh);
3077        }
3078    }
3079
3080    #[rstest]
3081    fn test_stream_health_update_extracts_tracked_market_data_messages() {
3082        let quote = QuoteTick {
3083            instrument_id: btc_perp_id(),
3084            ts_event: UnixNanos::from(1),
3085            ..QuoteTick::default()
3086        };
3087        let deltas = stub_deltas();
3088        let depth = stub_depth10();
3089
3090        assert_eq!(
3091            stream_health_update(&NautilusWsMessage::Quote(quote)),
3092            Some((
3093                MarketDataChannel::Quote,
3094                quote.instrument_id,
3095                quote.ts_event
3096            )),
3097        );
3098        assert_eq!(
3099            stream_health_update(&NautilusWsMessage::Deltas(deltas.clone())),
3100            Some((
3101                MarketDataChannel::Deltas,
3102                deltas.instrument_id,
3103                deltas.ts_event
3104            )),
3105        );
3106        assert_eq!(
3107            stream_health_update(&NautilusWsMessage::Depth(Box::new(depth.clone()))),
3108            Some((
3109                MarketDataChannel::Depth,
3110                depth.instrument_id,
3111                depth.ts_event
3112            )),
3113        );
3114        assert_eq!(stream_health_update(&NautilusWsMessage::Reconnected), None,);
3115    }
3116
3117    #[rstest]
3118    fn test_funding_entry_to_update_parses_positive_rate() {
3119        let entry = HyperliquidFundingHistoryEntry {
3120            coin: Ustr::from("BTC"),
3121            funding_rate: dec!(0.0000125),
3122            premium: Some(dec!(0.00029005)),
3123            time: 1769908800000,
3124        };
3125        let instrument_id = btc_perp_id();
3126
3127        let update = funding_entry_to_update(&entry, instrument_id);
3128
3129        assert_eq!(update.instrument_id, instrument_id);
3130        assert_eq!(update.rate, dec!(0.0000125));
3131        assert_eq!(update.interval, Some(60));
3132        assert!(update.next_funding_ns.is_none());
3133        assert_eq!(update.ts_event, UnixNanos::from(1769908800000 * 1_000_000));
3134        assert_eq!(update.ts_init, update.ts_event);
3135    }
3136
3137    #[rstest]
3138    fn test_funding_entry_to_update_handles_negative_rate() {
3139        let entry = HyperliquidFundingHistoryEntry {
3140            coin: Ustr::from("BTC"),
3141            funding_rate: dec!(-0.0000081),
3142            premium: None,
3143            time: 1769912400000,
3144        };
3145        let update = funding_entry_to_update(&entry, btc_perp_id());
3146        assert_eq!(update.rate, dec!(-0.0000081));
3147    }
3148
3149    #[rstest]
3150    fn test_funding_history_entry_rejects_invalid_rate() {
3151        // The funding rate is now a Decimal field, so an invalid value is
3152        // rejected at deserialization rather than by funding_entry_to_update.
3153        let json = r#"{"coin":"BTC","fundingRate":"not-a-number","time":1769912400000}"#;
3154        assert!(serde_json::from_str::<HyperliquidFundingHistoryEntry>(json).is_err());
3155    }
3156
3157    #[rstest]
3158    fn test_parse_book_precision_params_none() {
3159        let (n, m) = parse_book_precision_params(None).unwrap();
3160        assert_eq!(n, None);
3161        assert_eq!(m, None);
3162    }
3163
3164    fn make_params(json: serde_json::Value) -> Params {
3165        serde_json::from_value(json).expect("valid params payload")
3166    }
3167
3168    #[rstest]
3169    fn test_parse_book_precision_params_only_n_sig_figs() {
3170        let params = make_params(serde_json::json!({"n_sig_figs": 4}));
3171        let (n, m) = parse_book_precision_params(Some(&params)).unwrap();
3172        assert_eq!(n, Some(4));
3173        assert_eq!(m, None);
3174    }
3175
3176    #[rstest]
3177    fn test_parse_book_precision_params_both() {
3178        let params = make_params(serde_json::json!({"n_sig_figs": 5, "mantissa": 2}));
3179        let (n, m) = parse_book_precision_params(Some(&params)).unwrap();
3180        assert_eq!(n, Some(5));
3181        assert_eq!(m, Some(2));
3182    }
3183
3184    #[rstest]
3185    fn test_parse_book_precision_params_rejects_negative() {
3186        let params = make_params(serde_json::json!({"n_sig_figs": -1}));
3187        let err = parse_book_precision_params(Some(&params)).unwrap_err();
3188        assert!(err.to_string().contains("n_sig_figs"));
3189    }
3190
3191    #[rstest]
3192    fn test_funding_history_fixture_parses() {
3193        let entries: Vec<HyperliquidFundingHistoryEntry> =
3194            load_test_data("http_funding_history.json");
3195        assert_eq!(entries.len(), 3);
3196        assert_eq!(entries[0].coin, "BTC");
3197        assert_eq!(entries[0].funding_rate, dec!(0.0000125));
3198        assert_eq!(entries[0].premium, Some(dec!(0.00029005)));
3199        assert!(entries[2].premium.is_none());
3200
3201        let updates: Vec<FundingRateUpdate> = entries
3202            .iter()
3203            .map(|e| funding_entry_to_update(e, btc_perp_id()))
3204            .collect();
3205        assert_eq!(updates.len(), 3);
3206        assert_eq!(updates[0].rate, dec!(0.0000125));
3207        assert_eq!(updates[1].rate, dec!(-0.0000081));
3208        assert_eq!(updates[2].rate, dec!(0.0000033));
3209    }
3210
3211    fn level(px: &str, sz: &str) -> crate::http::models::HyperliquidLevel {
3212        crate::http::models::HyperliquidLevel {
3213            px: px.parse().unwrap(),
3214            sz: sz.parse().unwrap(),
3215        }
3216    }
3217
3218    fn sample_l2_book() -> HyperliquidL2Book {
3219        HyperliquidL2Book {
3220            coin: Ustr::from("BTC"),
3221            levels: vec![
3222                vec![
3223                    level("98450.50", "2.5"),
3224                    level("98449.00", "1.2"),
3225                    level("98448.00", "0.8"),
3226                ],
3227                vec![
3228                    level("98451.00", "1.5"),
3229                    level("98452.00", "2.0"),
3230                    level("98453.00", "0.5"),
3231                ],
3232            ],
3233            time: 1769908800000,
3234        }
3235    }
3236
3237    #[rstest]
3238    fn test_parse_l2_book_snapshot_populates_both_sides() {
3239        let book_data = sample_l2_book();
3240        let instrument_id = btc_perp_id();
3241        let book = parse_l2_book_snapshot(&book_data, instrument_id, 2, 4, None);
3242
3243        assert_eq!(book.instrument_id, instrument_id);
3244        assert_eq!(book.book_type, BookType::L2_MBP);
3245        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3246        assert_eq!(book.best_ask_price(), Some(Price::new(98451.00, 2)));
3247        assert_eq!(book.best_bid_size(), Some(Quantity::new(2.5, 4)));
3248        assert_eq!(book.best_ask_size(), Some(Quantity::new(1.5, 4)));
3249        assert_eq!(book.update_count, 6);
3250    }
3251
3252    #[rstest]
3253    fn test_parse_l2_book_snapshot_truncates_to_depth() {
3254        let book_data = sample_l2_book();
3255        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, Some(1));
3256
3257        // depth=1 keeps the top of book on each side, drops the rest.
3258        assert_eq!(book.update_count, 2);
3259        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3260        assert_eq!(book.best_ask_price(), Some(Price::new(98451.00, 2)));
3261    }
3262
3263    #[rstest]
3264    fn test_parse_l2_book_snapshot_uses_venue_time_as_ts_event() {
3265        let book_data = sample_l2_book();
3266        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3267        let expected_ts = UnixNanos::from(1769908800000_u64 * 1_000_000);
3268
3269        // ts_last reflects the last applied delta; every added order
3270        // carries the venue time after the ms->ns conversion.
3271        assert_eq!(book.ts_last, expected_ts);
3272    }
3273
3274    #[rstest]
3275    fn test_parse_l2_book_snapshot_skips_non_positive_size() {
3276        let book_data = HyperliquidL2Book {
3277            coin: Ustr::from("BTC"),
3278            levels: vec![
3279                vec![level("98450.50", "2.5"), level("98449.00", "0")],
3280                vec![level("98451.00", "0"), level("98452.00", "1.5")],
3281            ],
3282            time: 1769908800000,
3283        };
3284        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3285
3286        assert_eq!(book.update_count, 2, "zero-sized levels must be skipped");
3287        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3288        assert_eq!(book.best_ask_price(), Some(Price::new(98452.00, 2)));
3289    }
3290
3291    #[rstest]
3292    fn test_parse_l2_book_snapshot_skips_zero_size_levels() {
3293        let book_data = HyperliquidL2Book {
3294            coin: Ustr::from("BTC"),
3295            levels: vec![
3296                vec![level("98448.00", "0.0"), level("98449.00", "1.2")],
3297                vec![level("98451.00", "0.0"), level("98452.00", "1.5")],
3298            ],
3299            time: 1769908800000,
3300        };
3301        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3302
3303        // Zero-size levels are skipped; one priced level remains per side.
3304        assert_eq!(book.update_count, 2);
3305        assert_eq!(book.best_bid_price(), Some(Price::new(98449.00, 2)));
3306        assert_eq!(book.best_ask_price(), Some(Price::new(98452.00, 2)));
3307    }
3308
3309    #[rstest]
3310    fn test_parse_l2_book_snapshot_empty_levels_yields_empty_book() {
3311        let book_data = HyperliquidL2Book {
3312            coin: Ustr::from("BTC"),
3313            levels: vec![],
3314            time: 1769908800000,
3315        };
3316        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3317
3318        assert_eq!(book.update_count, 0);
3319        assert!(book.best_bid_price().is_none());
3320        assert!(book.best_ask_price().is_none());
3321    }
3322
3323    fn trade_at(ts_ns: u64, tid: u64) -> TradeTick {
3324        TradeTick::new(
3325            btc_perp_id(),
3326            Price::from("104300.0"),
3327            Quantity::from("0.01000"),
3328            AggressorSide::Buy,
3329            TradeId::new(tid.to_string()),
3330            UnixNanos::from(ts_ns),
3331            UnixNanos::from(ts_ns),
3332        )
3333    }
3334
3335    // A snapshot of three trades at 1000/2000/3000ns, sorted ascending. The
3336    // coverage floor (oldest) is 1000ns.
3337    fn sample_trades() -> Vec<TradeTick> {
3338        vec![trade_at(1000, 1), trade_at(2000, 2), trade_at(3000, 3)]
3339    }
3340
3341    #[rstest]
3342    fn test_recent_trades_fixture_parses_and_sorts() {
3343        let raw: Vec<crate::http::models::HyperliquidRecentTrade> =
3344            load_test_data("http_recent_trades_btc.json");
3345        assert_eq!(raw.len(), 3);
3346        // Fixture is newest-first as the venue returns it.
3347        assert_eq!(raw[0].tid, 300003);
3348
3349        let meta: crate::http::models::PerpMeta = load_test_data("http_meta_perp_sample.json");
3350        let defs = crate::http::parse::parse_perp_instruments(&meta, 0).unwrap();
3351        let instrument =
3352            crate::http::parse::create_instrument_from_def(&defs[0], UnixNanos::default()).unwrap();
3353
3354        let mut trades: Vec<TradeTick> = raw
3355            .iter()
3356            .map(|t| parse_recent_trade(t, &instrument).unwrap())
3357            .collect();
3358        trades.sort_by_key(|trade| trade.ts_event);
3359
3360        // Ascending after sort: oldest tid first.
3361        assert_eq!(trades[0].trade_id.to_string(), "300001");
3362        assert_eq!(trades[2].trade_id.to_string(), "300003");
3363        assert!(trades[0].ts_event <= trades[2].ts_event);
3364        // Historical ticks carry ts_init == ts_event.
3365        assert_eq!(trades[0].ts_init, trades[0].ts_event);
3366    }
3367
3368    #[rstest]
3369    fn test_filter_recent_trades_full_window_returns_all() {
3370        let filtered = filter_recent_trades(sample_trades(), None, None, None, btc_perp_id());
3371
3372        assert_eq!(filtered.len(), 3);
3373    }
3374
3375    #[rstest]
3376    fn test_filter_recent_trades_empty_snapshot_returns_empty() {
3377        let filtered = filter_recent_trades(
3378            Vec::new(),
3379            Some(UnixNanos::from(500)),
3380            Some(UnixNanos::from(2500)),
3381            None,
3382            btc_perp_id(),
3383        );
3384
3385        assert!(filtered.is_empty());
3386    }
3387
3388    #[rstest]
3389    fn test_filter_recent_trades_entirely_older_returns_empty() {
3390        // Requested window ends before the snapshot floor (1000ns).
3391        let filtered = filter_recent_trades(
3392            sample_trades(),
3393            Some(UnixNanos::from(100)),
3394            Some(UnixNanos::from(500)),
3395            None,
3396            btc_perp_id(),
3397        );
3398
3399        assert!(filtered.is_empty());
3400    }
3401
3402    #[rstest]
3403    fn test_filter_recent_trades_partial_keeps_in_range_subset() {
3404        // Start (500ns) is below the floor; end (2500ns) drops the 3000ns trade.
3405        let filtered = filter_recent_trades(
3406            sample_trades(),
3407            Some(UnixNanos::from(500)),
3408            Some(UnixNanos::from(2500)),
3409            None,
3410            btc_perp_id(),
3411        );
3412
3413        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3414        assert_eq!(ts, vec![1000, 2000]);
3415    }
3416
3417    #[rstest]
3418    fn test_filter_recent_trades_within_window_filters_bounds() {
3419        let filtered = filter_recent_trades(
3420            sample_trades(),
3421            Some(UnixNanos::from(1500)),
3422            Some(UnixNanos::from(3000)),
3423            None,
3424            btc_perp_id(),
3425        );
3426
3427        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3428        assert_eq!(ts, vec![2000, 3000]);
3429    }
3430
3431    #[rstest]
3432    fn test_filter_recent_trades_limit_keeps_most_recent() {
3433        let filtered = filter_recent_trades(sample_trades(), None, None, Some(2), btc_perp_id());
3434
3435        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3436        assert_eq!(ts, vec![2000, 3000]);
3437    }
3438
3439    #[rstest]
3440    fn test_filter_recent_trades_end_equal_to_floor_keeps_floor_trade() {
3441        // `end` exactly on the floor (1000ns) is inclusive: not "entirely
3442        // older". Distinguishes `end < floor` from `end <= floor`.
3443        let filtered = filter_recent_trades(
3444            sample_trades(),
3445            None,
3446            Some(UnixNanos::from(1000)),
3447            None,
3448            btc_perp_id(),
3449        );
3450
3451        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3452        assert_eq!(ts, vec![1000]);
3453    }
3454
3455    #[rstest]
3456    fn test_filter_recent_trades_bounds_are_inclusive() {
3457        // `start`/`end` landing exactly on a trade's ts_event keep that trade.
3458        // Distinguishes `>=`/`<=` from strict `>`/`<`.
3459        let filtered = filter_recent_trades(
3460            sample_trades(),
3461            Some(UnixNanos::from(2000)),
3462            Some(UnixNanos::from(3000)),
3463            None,
3464            btc_perp_id(),
3465        );
3466
3467        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3468        assert_eq!(ts, vec![2000, 3000]);
3469    }
3470
3471    fn perp_instrument(symbol: &str, tick_size: &str, ts_init: UnixNanos) -> InstrumentAny {
3472        let base = Currency::new("BTC", 8, 0, "BTC", CurrencyType::Crypto);
3473        let usd = Currency::new("USD", 8, 0, "USD", CurrencyType::Crypto);
3474        let usdc = Currency::new("USDC", 6, 0, "USDC", CurrencyType::Crypto);
3475
3476        InstrumentAny::CryptoPerpetual(
3477            CryptoPerpetual::builder()
3478                .instrument_id(InstrumentId::new(Symbol::new(symbol), *HYPERLIQUID_VENUE))
3479                .raw_symbol(Symbol::new("BTC"))
3480                .base_currency(base)
3481                .quote_currency(usd)
3482                .settlement_currency(usdc)
3483                .is_inverse(false)
3484                .price_precision(1)
3485                .size_precision(3)
3486                .price_increment(Price::from(tick_size))
3487                .size_increment(Quantity::from("0.001"))
3488                .ts_event(ts_init)
3489                .ts_init(ts_init)
3490                .build()
3491                .unwrap(),
3492        )
3493    }
3494
3495    fn cached(instruments: &[InstrumentAny]) -> Arc<AtomicMap<InstrumentId, InstrumentAny>> {
3496        let map = AtomicMap::new();
3497        map.rcu(|m| {
3498            for instrument in instruments {
3499                m.insert(instrument.id(), instrument.clone());
3500            }
3501        });
3502        Arc::new(map)
3503    }
3504
3505    fn data_client_with_refresh_interval(minutes: u64) -> HyperliquidDataClient {
3506        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
3507        set_data_event_sender(tx);
3508
3509        HyperliquidDataClient::new(
3510            *HYPERLIQUID_CLIENT_ID,
3511            HyperliquidDataClientConfig {
3512                update_instruments_interval_mins: minutes,
3513                ..HyperliquidDataClientConfig::default()
3514            },
3515        )
3516        .unwrap()
3517    }
3518
3519    #[rstest]
3520    #[case::disconnect(false, None)]
3521    #[case::reset_then_disconnect(true, None)]
3522    #[ignore = "connects to Hyperliquid mainnet public market data"]
3523    #[case::mainnet(false, Some(HYPERLIQUID_WS_URL))]
3524    #[tokio::test]
3525    async fn test_ws_consumer_exits_when_handler_output_closes(
3526        #[case] reset: bool,
3527        #[case] url: Option<&str>,
3528    ) {
3529        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3530        let addr = listener.local_addr().unwrap();
3531        let server = tokio::spawn(async move {
3532            let (socket, _) = listener.accept().await.unwrap();
3533            let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap();
3534            while let Some(Ok(_)) = socket.next().await {}
3535        });
3536
3537        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
3538        set_data_event_sender(tx);
3539
3540        let url = url.map_or_else(|| format!("ws://{addr}"), str::to_owned);
3541
3542        let mut client = HyperliquidDataClient::new(
3543            *HYPERLIQUID_CLIENT_ID,
3544            HyperliquidDataClientConfig {
3545                base_url_ws: Some(url),
3546                ..HyperliquidDataClientConfig::default()
3547            },
3548        )
3549        .unwrap();
3550
3551        client.spawn_ws().await.unwrap();
3552        assert_eq!(client.session_tasks.len(), 1);
3553
3554        client.ws_client.disconnect().await.unwrap();
3555        tokio::time::timeout(Duration::from_secs(2), async {
3556            while !client.session_tasks.all_finished() {
3557                tokio::task::yield_now().await;
3558            }
3559        })
3560        .await
3561        .expect("consumer must exit when handler output closes without cancellation");
3562
3563        assert!(!client.cancellation_token.is_cancelled());
3564        assert!(client.session_tasks.all_finished());
3565        assert!(client.pending_tasks.is_empty());
3566
3567        if reset {
3568            client.reset().unwrap();
3569        }
3570
3571        client.disconnect().await.unwrap();
3572
3573        assert!(client.session_tasks.is_empty());
3574        assert!(client.pending_tasks.is_empty());
3575        assert!(client.is_disconnected());
3576        server.abort();
3577        let _ = server.await;
3578    }
3579
3580    #[tokio::test]
3581    async fn test_spawn_instrument_refresh_skipped_when_interval_zero() {
3582        let client = data_client_with_refresh_interval(0);
3583
3584        client.spawn_instrument_refresh().unwrap();
3585
3586        assert!(client.session_tasks.is_empty());
3587    }
3588
3589    #[tokio::test]
3590    async fn test_spawn_instrument_refresh_registers_task() {
3591        let client = data_client_with_refresh_interval(60);
3592
3593        client.spawn_instrument_refresh().unwrap();
3594
3595        assert_eq!(client.session_tasks.len(), 1);
3596
3597        client.cancellation_token.cancel();
3598        client.await_session_tasks().await.unwrap();
3599    }
3600
3601    #[rstest]
3602    #[case::disconnect(false)]
3603    #[case::reset(true)]
3604    #[tokio::test]
3605    async fn test_instrument_refresh_shutdown_drains_generation(#[case] reset: bool) {
3606        let mut client = data_client_with_refresh_interval(1);
3607        client.spawn_instrument_refresh().unwrap();
3608        if reset {
3609            client.reset().unwrap();
3610            client.await_session_tasks().await.unwrap();
3611        } else {
3612            client.disconnect().await.unwrap();
3613        }
3614
3615        assert!(client.cancellation_token.is_cancelled());
3616        assert!(client.session_tasks.is_empty());
3617        assert!(!client.session_tasks.is_open());
3618        assert!(client.spawn_instrument_refresh().is_err());
3619        assert!(client.is_disconnected());
3620    }
3621
3622    #[tokio::test]
3623    async fn test_instrument_refresh_connected_client_keeps_one_task() {
3624        let mut client = data_client_with_refresh_interval(1);
3625        client.spawn_instrument_refresh().unwrap();
3626        client.is_connected.store(true, Ordering::Release);
3627
3628        client.connect().await.unwrap();
3629        client.connect().await.unwrap();
3630
3631        assert_eq!(client.session_tasks.len(), 1);
3632        client.disconnect().await.unwrap();
3633        assert!(client.session_tasks.is_empty());
3634    }
3635
3636    #[rstest]
3637    fn test_cache_instruments_replaces_complete_snapshots() {
3638        let client = data_client_with_refresh_interval(0);
3639        let mut original = vec![
3640            perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1)),
3641            perp_instrument("ETH-USD-PERP", "0.2", UnixNanos::from(1)),
3642        ];
3643        let mut refreshed = vec![
3644            perp_instrument("BTC-USD-PERP", "0.3", UnixNanos::from(2)),
3645            perp_instrument("ETH-USD-PERP", "0.4", UnixNanos::from(2)),
3646        ];
3647
3648        for instruments in [&mut original, &mut refreshed] {
3649            let InstrumentAny::CryptoPerpetual(eth) = &mut instruments[1] else {
3650                unreachable!();
3651            };
3652
3653            eth.raw_symbol = Symbol::new("ETH");
3654        }
3655
3656        cache_instruments(
3657            &original,
3658            &client.instruments,
3659            &client.coin_to_instrument_id,
3660            &client.http_client,
3661            &client.ws_client,
3662        );
3663        let previous = client.instruments.load();
3664        cache_instruments(
3665            &refreshed,
3666            &client.instruments,
3667            &client.coin_to_instrument_id,
3668            &client.http_client,
3669            &client.ws_client,
3670        );
3671        let current = client.instruments.load();
3672
3673        assert_eq!(previous.len(), 2);
3674        assert_eq!(current.len(), 2);
3675
3676        for (old, new) in original.iter().zip(&refreshed) {
3677            assert_eq!(
3678                serde_json::to_value(previous.get(&old.id()).unwrap()).unwrap(),
3679                serde_json::to_value(old).unwrap(),
3680            );
3681            assert_eq!(
3682                serde_json::to_value(current.get(&new.id()).unwrap()).unwrap(),
3683                serde_json::to_value(new).unwrap(),
3684            );
3685            assert_eq!(
3686                client
3687                    .coin_to_instrument_id
3688                    .get_cloned(&new.raw_symbol().inner()),
3689                Some(new.id()),
3690            );
3691            assert_eq!(
3692                serde_json::to_value(
3693                    client
3694                        .ws_client
3695                        .instruments_cache()
3696                        .load()
3697                        .get(&new.raw_symbol().inner())
3698                        .unwrap(),
3699                )
3700                .unwrap(),
3701                serde_json::to_value(new).unwrap(),
3702            );
3703        }
3704    }
3705
3706    #[rstest]
3707    fn test_changed_definitions_reports_a_newly_listed_market() {
3708        let cached_instruments =
3709            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3710        let fetched = vec![
3711            perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1)),
3712            perp_instrument("NEW-USD-PERP", "0.1", UnixNanos::from(1)),
3713        ];
3714
3715        let changed = changed_definitions(&fetched, &cached_instruments);
3716
3717        assert_eq!(changed.len(), 1);
3718        assert_eq!(changed[0].id().symbol.as_str(), "NEW-USD-PERP");
3719    }
3720
3721    #[rstest]
3722    fn test_added_symbols_names_only_the_market_the_cache_never_held() {
3723        let cached_instruments =
3724            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3725        // BTC moved its tick size, so it is changed but not new
3726        let changed = vec![
3727            perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1)),
3728            perp_instrument("NEW-USD-PERP", "0.1", UnixNanos::from(1)),
3729        ];
3730
3731        let added = added_symbols(&changed, &cached_instruments);
3732
3733        assert_eq!(added, vec![Ustr::from("NEW-USD-PERP")]);
3734    }
3735
3736    #[rstest]
3737    fn test_added_symbols_is_empty_when_every_change_is_a_known_market() {
3738        let cached_instruments =
3739            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3740        let changed = vec![perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1))];
3741
3742        assert!(added_symbols(&changed, &cached_instruments).is_empty());
3743    }
3744
3745    #[rstest]
3746    fn test_changed_definitions_ignores_a_later_ts_init_alone() {
3747        // Every pass restamps `ts_init` from the clock, so comparing it would
3748        // republish the whole universe on every tick.
3749        let cached_instruments =
3750            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3751        let fetched = vec![perp_instrument(
3752            "BTC-USD-PERP",
3753            "0.1",
3754            UnixNanos::from(2_000_000_000),
3755        )];
3756
3757        assert!(changed_definitions(&fetched, &cached_instruments).is_empty());
3758    }
3759
3760    #[rstest]
3761    fn test_changed_definitions_reports_a_changed_tick_size() {
3762        let cached_instruments =
3763            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3764        let fetched = vec![perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1))];
3765
3766        let changed = changed_definitions(&fetched, &cached_instruments);
3767
3768        assert_eq!(changed.len(), 1);
3769        assert_eq!(changed[0].price_increment(), Price::from("0.5"));
3770    }
3771}