Skip to main content

nautilus_databento/
data.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Provides a unified data client that combines Databento's live streaming and historical data capabilities.
17//!
18//! This module implements a data client that manages connections to multiple Databento datasets,
19//! handles live market data subscriptions, and provides access to historical data on demand.
20
21use std::{
22    path::PathBuf,
23    str::FromStr,
24    sync::{
25        Arc,
26        atomic::{AtomicBool, Ordering},
27    },
28    time::Duration,
29};
30
31use ahash::AHashMap;
32use databento::{dbn, live::Subscription};
33use indexmap::IndexMap;
34use nautilus_common::{
35    clients::{DataClient, execute_bar_subscriptions, execute_instrument_subscriptions},
36    live::{runner::get_data_event_sender, sender::EventSender},
37    messages::{
38        DataEvent, DataResponse,
39        data::{
40            BarsResponse, BookDeltasResponse, BookDepthResponse, DataRequestFailed,
41            DataResponseEnvelope, InstrumentResponse, InstrumentsResponse, IntoInstrumentIds,
42            QuotesResponse, RequestBars, RequestBookDeltas, RequestBookDepth, RequestInstrument,
43            RequestInstruments, RequestQuotes, RequestTrades, SubscribeBars, SubscribeBookDeltas,
44            SubscribeBookDepth, SubscribeCustomData, SubscribeInstrument,
45            SubscribeInstrumentStatus, SubscribeQuotes, SubscribeTrades, SubscriptionError,
46            TradesResponse, UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth,
47            UnsubscribeCustomData, UnsubscribeInstrumentStatus, UnsubscribeQuotes,
48            UnsubscribeTrades,
49        },
50    },
51};
52use nautilus_core::{
53    AtomicMap, DurationNanos, Params, UnixNanos,
54    datetime::datetime_to_unix_nanos,
55    time::{AtomicTime, get_atomic_clock_realtime},
56};
57use nautilus_live::task::TaskGroup;
58use nautilus_model::{
59    data::{BarSpecification, BarType, CustomData, Data, DataType, custom::CustomDataTrait},
60    enums::{BarAggregation, PriceType},
61    identifiers::{ClientId, InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
62    instruments::{Instrument, InstrumentAny},
63};
64use parking_lot::Mutex;
65use tokio_util::sync::CancellationToken;
66
67use crate::{
68    common::{Credential, DATABENTO_VENUE, ohlcv_schema_from_aggregation},
69    historical::{DatabentoHistoricalClient, RangeQueryParams},
70    live::{DatabentoFeedHandler, DatabentoMessage, HandlerCommand},
71    loader::DatabentoDataLoader,
72    symbology::{
73        check_consistent_symbology, infer_symbology_type, instrument_id_to_symbol_string,
74        raw_symbol,
75    },
76    types::{DatabentoImbalance, DatabentoStatistics, Dataset, PublisherId},
77};
78
79const PRICE_PRECISION_PARAM: &str = "price_precision";
80const SCHEMA_PARAM: &str = "schema";
81const MBP10_DEPTH: usize = 10;
82const QUOTE_SCHEMAS: &[dbn::Schema] = &[
83    dbn::Schema::Mbp1,
84    dbn::Schema::Bbo1S,
85    dbn::Schema::Bbo1M,
86    dbn::Schema::Cmbp1,
87    dbn::Schema::Cbbo1S,
88    dbn::Schema::Cbbo1M,
89    dbn::Schema::Tbbo,
90    dbn::Schema::Tcbbo,
91];
92const TRADE_SCHEMAS: &[dbn::Schema] = &[
93    dbn::Schema::Trades,
94    dbn::Schema::Tbbo,
95    dbn::Schema::Tcbbo,
96    dbn::Schema::Mbp1,
97    dbn::Schema::Cmbp1,
98];
99
100/// Configuration for the Databento data client.
101#[derive(Debug, Clone)]
102#[cfg_attr(
103    feature = "python",
104    pyo3::pyclass(module = "nautilus_trader.adapters.databento", from_py_object)
105)]
106#[cfg_attr(
107    feature = "python",
108    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
109)]
110pub struct DatabentoDataClientConfig {
111    /// Databento API credential.
112    pub(crate) credential: Credential,
113    /// Path to publishers.json file.
114    pub publishers_filepath: PathBuf,
115    /// Override for the Databento Historical API base URL.
116    pub historical_base_url: Option<String>,
117    /// Override for the Databento live gateway address.
118    pub live_gateway_addr: Option<String>,
119    /// Venue-to-dataset overrides applied on top of the publishers.json mappings.
120    pub venue_dataset_map: IndexMap<String, String>,
121    /// Whether to use exchange as venue for GLBX instruments.
122    pub use_exchange_as_venue: bool,
123    /// Whether to timestamp bars on close.
124    pub bars_timestamp_on_close: bool,
125    /// Reconnection timeout in minutes (None for infinite retries).
126    pub reconnect_timeout_mins: Option<u64>,
127    /// Year digits in futures IDs: 2 by default, or 1 for existing catalogs and caches.
128    pub futures_year_digits: FuturesYearDigits,
129    /// Whether to spell a share class with a space in instrument IDs (`BRK.B` -> `BRK B`).
130    pub share_class_space: bool,
131}
132
133#[cfg(feature = "python")]
134nautilus_core::impl_pyo3_config_getters!(DatabentoDataClientConfig {
135    publishers_filepath: PathBuf,
136    historical_base_url: Option<String>,
137    live_gateway_addr: Option<String>,
138    use_exchange_as_venue: bool,
139    bars_timestamp_on_close: bool,
140    venue_dataset_map: IndexMap<String, String>,
141    share_class_space: bool,
142});
143
144impl DatabentoDataClientConfig {
145    /// Creates a new [`DatabentoDataClientConfig`] instance.
146    #[must_use]
147    pub fn new(
148        api_key: impl Into<String>,
149        publishers_filepath: PathBuf,
150        use_exchange_as_venue: bool,
151        bars_timestamp_on_close: bool,
152    ) -> Self {
153        Self {
154            futures_year_digits: FuturesYearDigits::default(),
155            share_class_space: true,
156            credential: Credential::new(api_key),
157            publishers_filepath,
158            venue_dataset_map: IndexMap::new(),
159            use_exchange_as_venue,
160            bars_timestamp_on_close,
161            reconnect_timeout_mins: Some(10), // Default: 10 minutes
162            historical_base_url: None,
163            live_gateway_addr: None,
164        }
165    }
166
167    /// Returns the API key associated with this config.
168    #[must_use]
169    pub fn api_key(&self) -> &str {
170        self.credential.api_key()
171    }
172
173    /// Returns a masked version of the API key for logging purposes.
174    #[must_use]
175    pub fn api_key_masked(&self) -> String {
176        self.credential.api_key_masked()
177    }
178}
179
180/// A Databento data client that combines live streaming and historical data functionality.
181///
182/// This client uses the existing `DatabentoFeedHandler` for live data subscriptions
183/// and `DatabentoHistoricalClient` for historical data requests. It supports multiple
184/// datasets simultaneously, with separate feed handlers per dataset.
185#[cfg_attr(feature = "python", pyo3::pyclass)]
186#[cfg_attr(
187    feature = "python",
188    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
189)]
190#[derive(Debug)]
191pub struct DatabentoDataClient {
192    client_id: ClientId,
193    config: DatabentoDataClientConfig,
194    is_connected: AtomicBool,
195    historical: DatabentoHistoricalClient,
196    loader: DatabentoDataLoader,
197    cmd_channels: Arc<Mutex<AHashMap<String, tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>>,
198    task_handles: TaskGroup,
199    cancellation_token: CancellationToken,
200    publisher_venue_map: Arc<IndexMap<PublisherId, Venue>>,
201    symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
202    data_sender: EventSender<DataEvent>,
203}
204
205impl DatabentoDataClient {
206    /// Creates a new [`DatabentoDataClient`] instance.
207    ///
208    /// # Errors
209    ///
210    /// Returns an error if client creation or publisher configuration loading fails.
211    pub fn new(
212        client_id: ClientId,
213        config: DatabentoDataClientConfig,
214        clock: &'static AtomicTime,
215    ) -> anyhow::Result<Self> {
216        let historical = match &config.historical_base_url {
217            Some(base_url) => DatabentoHistoricalClient::new_with_base_url(
218                config.credential.clone(),
219                config.publishers_filepath.clone(),
220                clock,
221                config.use_exchange_as_venue,
222                base_url,
223            )?,
224            None => DatabentoHistoricalClient::new(
225                config.credential.clone(),
226                config.publishers_filepath.clone(),
227                clock,
228                config.use_exchange_as_venue,
229            )?,
230        }
231        .with_futures_year_digits(config.futures_year_digits)
232        .with_share_class_space(config.share_class_space);
233
234        // Create data loader for venue-to-dataset mapping
235        let mut loader = DatabentoDataLoader::new(Some(config.publishers_filepath.clone()))?
236            .with_futures_year_digits(config.futures_year_digits)
237            .with_share_class_space(config.share_class_space);
238        for (venue, dataset) in &config.venue_dataset_map {
239            loader.set_dataset_for_venue(
240                Dataset::from(dataset.as_str()),
241                Venue::from(venue.as_str()),
242            );
243        }
244
245        // Load publisher configuration
246        let file_content = std::fs::read_to_string(&config.publishers_filepath)?;
247        let publishers_vec: Vec<crate::types::DatabentoPublisher> =
248            serde_json::from_str(&file_content)?;
249
250        let publisher_venue_map = publishers_vec
251            .into_iter()
252            .map(|p| (p.publisher_id, Venue::from(p.venue.as_str())))
253            .collect::<IndexMap<u16, Venue>>();
254
255        let data_sender = get_data_event_sender();
256
257        let task_handles = TaskGroup::new();
258
259        Ok(Self {
260            client_id,
261            config,
262            is_connected: AtomicBool::new(false),
263            historical,
264            loader,
265            cmd_channels: Arc::new(Mutex::new(AHashMap::new())),
266            cancellation_token: task_handles.cancellation_token(),
267            task_handles,
268            publisher_venue_map: Arc::new(publisher_venue_map),
269            symbol_venue_map: Arc::new(AtomicMap::new()),
270            data_sender,
271        })
272    }
273
274    /// Returns the API key associated with this client.
275    #[must_use]
276    pub fn api_key(&self) -> &str {
277        self.config.api_key()
278    }
279
280    /// Returns a masked version of the API key for logging purposes.
281    #[must_use]
282    pub fn api_key_masked(&self) -> String {
283        self.config.api_key_masked()
284    }
285
286    /// Gets the dataset for a given venue using the data loader.
287    ///
288    /// # Errors
289    ///
290    /// Returns an error if the venue-to-dataset mapping cannot be found.
291    fn get_dataset_for_venue(&self, venue: Venue) -> anyhow::Result<String> {
292        self.loader
293            .get_dataset_for_venue(&venue)
294            .map(ToString::to_string)
295            .ok_or_else(|| anyhow::anyhow!("No dataset found for venue: {venue}"))
296    }
297
298    /// Gets or creates a feed handler for the specified dataset.
299    fn get_or_create_feed_handler(&self, dataset: &str) -> bool {
300        let mut channels = self.cmd_channels.lock();
301
302        if !channels.contains_key(dataset) {
303            log::debug!("Creating new feed handler for dataset: {dataset}");
304            let cmd_tx = self.initialize_live_feed(dataset.to_string());
305            channels.insert(dataset.to_string(), cmd_tx);
306
307            log::debug!("Feed handler created for dataset: {dataset}, channel stored");
308            return true;
309        }
310
311        false
312    }
313
314    fn send_subscription_to_dataset(
315        &self,
316        dataset: &str,
317        price_precisions: impl IntoIterator<Item = (Symbol, u8)>,
318        mut subscription: Subscription,
319        start_after_subscribe: bool,
320    ) -> anyhow::Result<()> {
321        if subscription.stype_in == dbn::SType::RawSymbol
322            && let databento::Symbols::Symbols(symbols) = &mut subscription.symbols
323        {
324            for symbol in symbols {
325                let raw = raw_symbol(
326                    symbol,
327                    dataset,
328                    self.config.futures_year_digits,
329                    self.config.share_class_space,
330                );
331
332                *symbol = raw;
333            }
334        }
335
336        let price_precisions = price_precisions.into_iter().map(|(symbol, precision)| {
337            let raw = raw_symbol(
338                symbol.as_str(),
339                dataset,
340                self.config.futures_year_digits,
341                self.config.share_class_space,
342            );
343            (Symbol::from(raw.as_str()), precision)
344        });
345
346        let tx = {
347            let channels = self.cmd_channels.lock();
348            channels
349                .get(dataset)
350                .cloned()
351                .ok_or_else(|| anyhow::anyhow!("No feed handler found for dataset: {dataset}"))?
352        };
353
354        send_subscription_commands(
355            &tx,
356            dataset,
357            price_precisions,
358            subscription,
359            start_after_subscribe,
360        )
361    }
362
363    fn subscribe_schema(
364        &self,
365        instrument_id: InstrumentId,
366        schema: dbn::Schema,
367    ) -> anyhow::Result<()> {
368        let symbol = instrument_id.symbol.to_string();
369        // Records resolve to the underlying contract, so a continuous or parent symbol would
370        // publish on topics that its subscriber does not listen on
371        anyhow::ensure!(
372            infer_symbology_type(&symbol) == dbn::SType::RawSymbol,
373            "Unsupported symbol {symbol} for a Databento {schema} subscription, only raw symbols \
374             are available (not continuous or parent symbols)",
375        );
376
377        let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
378        let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
379
380        self.symbol_venue_map
381            .insert(instrument_id.symbol, instrument_id.venue);
382
383        let subscription = Subscription::builder()
384            .schema(schema)
385            .symbols(symbol)
386            .build();
387
388        self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)
389    }
390
391    fn send_close_to_active_feeds(&self) {
392        let channels = self.cmd_channels.lock();
393        for (dataset, tx) in channels.iter() {
394            if let Err(e) = tx.send(HandlerCommand::Close) {
395                log::warn!("Failed to send close command to dataset {dataset}: {e}");
396            }
397        }
398    }
399
400    fn clear_feed_channels(&self) {
401        let mut channels = self.cmd_channels.lock();
402        channels.clear();
403    }
404
405    fn abort_active_tasks(&self) {
406        self.task_handles.begin_shutdown();
407    }
408
409    fn spawn_task<F>(&self, future: F)
410    where
411        F: std::future::Future<Output = ()> + Send + 'static,
412    {
413        if let Err(e) = self.task_handles.spawn(future) {
414            log::debug!("Skipping Databento task after shutdown began: {e}");
415        }
416    }
417
418    /// Initializes the live feed handler for streaming data.
419    fn initialize_live_feed(
420        &self,
421        dataset: String,
422    ) -> tokio::sync::mpsc::UnboundedSender<HandlerCommand> {
423        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
424        let (msg_tx, msg_rx) = tokio::sync::mpsc::unbounded_channel();
425        let feed_dataset = dataset.clone();
426        let feed_channels = self.cmd_channels.clone();
427
428        let mut feed_handler = DatabentoFeedHandler::new(
429            self.config.credential.clone(),
430            dataset,
431            cmd_rx,
432            msg_tx,
433            (*self.publisher_venue_map).clone(),
434            self.symbol_venue_map.clone(),
435            self.config.use_exchange_as_venue,
436            self.config.bars_timestamp_on_close,
437            self.config.reconnect_timeout_mins,
438        )
439        .with_futures_year_digits(self.config.futures_year_digits)
440        .with_share_class_space(self.config.share_class_space);
441
442        if let Some(addr) = &self.config.live_gateway_addr {
443            feed_handler = feed_handler.with_gateway_addr(addr.clone());
444        }
445
446        let feed_future = async move {
447            // run() logs its terminal error and forwards it as `DatabentoMessage::Error`
448            let _ = feed_handler.run().await;
449            feed_channels.lock().remove(&feed_dataset);
450        };
451
452        let cancellation_token = self.cancellation_token.clone();
453        let data_sender = self.data_sender.clone();
454
455        // Spawn message processing task with cancellation support
456        let msg_future = async move {
457            let mut msg_rx = msg_rx;
458
459            loop {
460                tokio::select! {
461                    msg = msg_rx.recv() => {
462                        match msg {
463                            Some(DatabentoMessage::Data(data)) => {
464                                log::debug!("Received data: {data:?}");
465                                if let Err(e) = data_sender.send(DataEvent::Data(data)) {
466                                    log::error!("Failed to send data event: {e}");
467                                }
468                            }
469                            Some(DatabentoMessage::Instrument(instrument)) => {
470                                log::debug!("Received instrument definition: {}", instrument.id());
471                                if let Err(e) = data_sender.send(DataEvent::Data(Data::Instrument(instrument))) {
472                                    log::error!("Failed to send instrument: {e}");
473                                }
474                            }
475                            Some(DatabentoMessage::Status(status)) => {
476                                log::debug!("Received status: {status:?}");
477                                if let Err(e) =
478                                    data_sender.send(DataEvent::Data(Data::InstrumentStatus(status)))
479                                {
480                                    log::error!("Failed to send status data event: {e}");
481                                }
482                            }
483                            Some(DatabentoMessage::Imbalance(imbalance)) => {
484                                log::debug!("Received imbalance: {imbalance:?}");
485                                let instrument_id = imbalance.instrument_id;
486                                let data = Data::Custom(custom_data_for_instrument(Arc::new(imbalance), instrument_id));
487                                if let Err(e) = data_sender.send(DataEvent::Data(data)) {
488                                    log::error!("Failed to send imbalance data event: {e}");
489                                }
490                            }
491                            Some(DatabentoMessage::Statistics(statistics)) => {
492                                log::debug!("Received statistics: {statistics:?}");
493                                let instrument_id = statistics.instrument_id;
494                                let data = Data::Custom(custom_data_for_instrument(Arc::new(statistics), instrument_id));
495                                if let Err(e) = data_sender.send(DataEvent::Data(data)) {
496                                    log::error!("Failed to send statistics data event: {e}");
497                                }
498                            }
499                            Some(DatabentoMessage::SubscriptionAck(ack)) => {
500                                log::debug!("Received subscription ack: {}", ack.message);
501                            }
502                            Some(DatabentoMessage::Error(error)) => {
503                                log::error!("Feed handler error: {error}");
504                            }
505                            Some(DatabentoMessage::Close) => {
506                                log::debug!("Feed handler closed");
507                                break;
508                            }
509                            None => {
510                                log::debug!("Message channel closed");
511                                break;
512                            }
513                        }
514                    }
515                    () = cancellation_token.cancelled() => {
516                        log::debug!("Message processing cancelled");
517                        break;
518                    }
519                }
520            }
521        };
522
523        if let Err(e) = self.task_handles.spawn(feed_future) {
524            log::warn!("Skipping Databento feed task after shutdown began: {e}");
525        }
526
527        if let Err(e) = self.task_handles.spawn(msg_future) {
528            log::warn!("Skipping Databento message task after shutdown began: {e}");
529        }
530
531        cmd_tx
532    }
533}
534
535#[async_trait::async_trait(?Send)]
536impl DataClient for DatabentoDataClient {
537    /// Returns the client identifier.
538    fn client_id(&self) -> ClientId {
539        self.client_id
540    }
541
542    /// Returns the venue associated with this client (None for multi-venue clients).
543    fn venue(&self) -> Option<Venue> {
544        None
545    }
546
547    /// Starts the data client.
548    ///
549    /// # Errors
550    ///
551    /// Returns an error if the client fails to start.
552    fn start(&mut self) -> anyhow::Result<()> {
553        log::debug!("Starting");
554        Ok(())
555    }
556
557    /// Stops the data client and cancels all active subscriptions.
558    ///
559    /// # Errors
560    ///
561    /// Returns an error if the client fails to stop cleanly.
562    fn stop(&mut self) -> anyhow::Result<()> {
563        log::debug!("Stopping");
564
565        self.send_close_to_active_feeds();
566        self.clear_feed_channels();
567        self.cancellation_token.cancel();
568        self.abort_active_tasks();
569        self.is_connected.store(false, Ordering::Relaxed);
570
571        Ok(())
572    }
573
574    fn reset(&mut self) -> anyhow::Result<()> {
575        log::debug!("Resetting");
576        self.send_close_to_active_feeds();
577        self.clear_feed_channels();
578        self.abort_active_tasks();
579        self.is_connected.store(false, Ordering::Relaxed);
580        Ok(())
581    }
582
583    fn dispose(&mut self) -> anyhow::Result<()> {
584        log::debug!("Disposing");
585        self.stop()
586    }
587
588    async fn connect(&mut self) -> anyhow::Result<()> {
589        log::debug!("Connecting...");
590
591        if !self.task_handles.is_open() {
592            self.task_handles
593                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
594                .await
595                .map_err(|e| anyhow::anyhow!("Failed to terminate Databento tasks: {e}"))?;
596            self.task_handles
597                .start_generation()
598                .map_err(|e| anyhow::anyhow!("Failed to start Databento task generation: {e}"))?;
599            self.cancellation_token = self.task_handles.cancellation_token();
600        }
601
602        self.is_connected.store(true, Ordering::Relaxed);
603
604        log::info!("Connected");
605        Ok(())
606    }
607
608    async fn disconnect(&mut self) -> anyhow::Result<()> {
609        log::debug!("Disconnecting...");
610
611        self.send_close_to_active_feeds();
612        self.clear_feed_channels();
613        self.task_handles.begin_shutdown();
614
615        let tasks_result = self
616            .task_handles
617            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
618            .await
619            .map_err(|e| anyhow::anyhow!("Failed to terminate Databento tasks: {e}"));
620
621        self.is_connected.store(false, Ordering::Relaxed);
622
623        log::info!("Disconnected");
624        tasks_result
625    }
626
627    /// Returns whether the client is currently connected.
628    fn is_connected(&self) -> bool {
629        self.is_connected.load(Ordering::Relaxed)
630    }
631
632    fn is_disconnected(&self) -> bool {
633        !self.is_connected()
634    }
635
636    /// Subscribes to instrument definition data for the specified instrument.
637    ///
638    /// # Errors
639    ///
640    /// Returns an error if the subscription request fails.
641    fn subscribe_instrument(&mut self, cmd: SubscribeInstrument) -> anyhow::Result<()> {
642        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
643            let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
644            let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
645
646            self.symbol_venue_map
647                .insert(instrument_id.symbol, instrument_id.venue);
648            let symbol = instrument_id.symbol.to_string();
649
650            let subscription = Subscription::builder()
651                .schema(databento::dbn::Schema::Definition)
652                .symbols(symbol)
653                .build();
654
655            self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
656
657            Ok(())
658        })
659    }
660
661    /// Subscribes to quote tick data for the specified instruments.
662    ///
663    /// # Errors
664    ///
665    /// Returns an error if the subscription request fails.
666    fn subscribe_quotes(&mut self, cmd: SubscribeQuotes) -> anyhow::Result<()> {
667        self.subscribe_batch(
668            cmd.instrument_ids,
669            cmd.params.as_ref(),
670            dbn::Schema::Mbp1,
671            QUOTE_SCHEMAS,
672        )
673    }
674
675    /// Subscribes to trade tick data for the specified instruments.
676    ///
677    /// # Errors
678    ///
679    /// Returns an error if the subscription request fails.
680    fn subscribe_trades(&mut self, cmd: SubscribeTrades) -> anyhow::Result<()> {
681        self.subscribe_batch(
682            cmd.instrument_ids,
683            cmd.params.as_ref(),
684            dbn::Schema::Trades,
685            TRADE_SCHEMAS,
686        )
687    }
688
689    /// Subscribes to order book delta updates for the specified instruments.
690    ///
691    /// # Errors
692    ///
693    /// Returns an error if the subscription request fails.
694    fn subscribe_book_deltas(&mut self, cmd: SubscribeBookDeltas) -> anyhow::Result<()> {
695        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
696            let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
697            let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
698
699            self.symbol_venue_map
700                .insert(instrument_id.symbol, instrument_id.venue);
701            let symbol = instrument_id.symbol.to_string();
702
703            let subscription = Subscription::builder()
704                .schema(databento::dbn::Schema::Mbo) // Market by order for book deltas
705                .symbols(symbol)
706                .build();
707
708            self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
709
710            Ok(())
711        })
712    }
713
714    /// Subscribes to `DatabentoStatistics` or `DatabentoImbalance` for the instrument given by the
715    /// identifier of the data type.
716    ///
717    /// # Errors
718    ///
719    /// Returns an error if the data type is not supported, has metadata, has a missing or invalid
720    /// instrument ID identifier, or the subscription request fails.
721    fn subscribe(&mut self, cmd: SubscribeCustomData) -> anyhow::Result<()> {
722        let schema = custom_data_schema(&cmd.data_type)?;
723        let instrument_id = custom_data_instrument_id(&cmd.data_type)?;
724        self.subscribe_schema(instrument_id, schema)
725    }
726
727    /// Subscribes to 10-level order book depth snapshots for the specified instrument.
728    ///
729    /// # Errors
730    ///
731    /// Returns an error if the requested depth is not 10, the symbol is not a raw symbol, or the
732    /// subscription request fails.
733    fn subscribe_book_depth(&mut self, cmd: SubscribeBookDepth) -> anyhow::Result<()> {
734        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
735            let cmd = cmd.with_instrument_ids(instrument_id);
736            if let Some(depth) = cmd.depth {
737                anyhow::ensure!(
738                    depth.get() == MBP10_DEPTH,
739                    "Unsupported book depth {depth} for Databento `mbp-10`, only a depth of {MBP10_DEPTH} is available",
740                );
741            }
742
743            self.subscribe_schema(instrument_id, dbn::Schema::Mbp10)
744        })
745    }
746
747    /// Subscribes to OHLCV bars for the specified bar type.
748    ///
749    /// # Errors
750    ///
751    /// Returns an error if the bar type has no matching OHLCV schema or the subscription request
752    /// fails.
753    fn subscribe_bars(&mut self, cmd: SubscribeBars) -> anyhow::Result<()> {
754        execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
755            let schema = ohlcv_schema(bar_type)?;
756            self.subscribe_schema(bar_type.instrument_id(), schema)
757        })
758    }
759
760    /// Subscribes to instrument status updates for the specified instruments.
761    ///
762    /// # Errors
763    ///
764    /// Returns an error if the subscription request fails.
765    fn subscribe_instrument_status(
766        &mut self,
767        cmd: SubscribeInstrumentStatus,
768    ) -> anyhow::Result<()> {
769        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
770            let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
771            let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
772
773            self.symbol_venue_map
774                .insert(instrument_id.symbol, instrument_id.venue);
775            let symbol = instrument_id.symbol.to_string();
776
777            let subscription = Subscription::builder()
778                .schema(databento::dbn::Schema::Status)
779                .symbols(symbol)
780                .build();
781
782            self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
783
784            Ok(())
785        })
786    }
787
788    // Unsubscribe methods
789    fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<()> {
790        log::debug!(
791            "Databento does not support granular unsubscribing, ignoring quote unsubscribe request for {:?}",
792            cmd.instrument_ids
793        );
794        Ok(())
795    }
796
797    fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> anyhow::Result<()> {
798        // Note: Databento live API doesn't support granular unsubscribing.
799        // The feed handler manages subscriptions and can handle reconnections
800        // with the appropriate subscription state.
801        log::debug!(
802            "Databento does not support granular unsubscribing - ignoring unsubscribe request for {:?}",
803            cmd.instrument_ids
804        );
805
806        Ok(())
807    }
808
809    fn unsubscribe_book_deltas(&mut self, cmd: &UnsubscribeBookDeltas) -> anyhow::Result<()> {
810        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
811            // Note: Databento live API doesn't support granular unsubscribing.
812            // The feed handler manages subscriptions and can handle reconnections
813            // with the appropriate subscription state.
814            log::debug!(
815                "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
816            );
817
818            Ok(())
819        })
820    }
821
822    fn unsubscribe(&mut self, cmd: &UnsubscribeCustomData) -> anyhow::Result<()> {
823        // Note: Databento live API doesn't support granular unsubscribing.
824        // The feed handler manages subscriptions and can handle reconnections
825        // with the appropriate subscription state.
826        log::debug!(
827            "Databento does not support granular unsubscribing - ignoring unsubscribe request for {}",
828            cmd.data_type
829        );
830
831        Ok(())
832    }
833
834    fn unsubscribe_book_depth(&mut self, cmd: &UnsubscribeBookDepth) -> anyhow::Result<()> {
835        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
836            // Note: Databento live API doesn't support granular unsubscribing.
837            // The feed handler manages subscriptions and can handle reconnections
838            // with the appropriate subscription state.
839            log::debug!(
840                "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
841            );
842
843            Ok(())
844        })
845    }
846
847    fn unsubscribe_bars(&mut self, cmd: &UnsubscribeBars) -> anyhow::Result<()> {
848        execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
849            // Note: Databento live API doesn't support granular unsubscribing.
850            // The feed handler manages subscriptions and can handle reconnections
851            // with the appropriate subscription state.
852            log::debug!(
853                "Databento does not support granular unsubscribing - ignoring unsubscribe request for {bar_type}"
854            );
855
856            Ok(())
857        })
858    }
859
860    fn unsubscribe_instrument_status(
861        &mut self,
862        cmd: &UnsubscribeInstrumentStatus,
863    ) -> anyhow::Result<()> {
864        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
865            // Note: Databento live API doesn't support granular unsubscribing.
866            // The feed handler manages subscriptions and can handle reconnections
867            // with the appropriate subscription state.
868            log::debug!(
869                "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
870            );
871
872            Ok(())
873        })
874    }
875
876    fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
877        log::debug!("Request instruments: {request:?}");
878
879        let historical_client = self.historical.clone();
880        let data_sender = self.data_sender.clone();
881        let dataset = request
882            .venue
883            .map(|venue| self.get_dataset_for_venue(venue))
884            .transpose()?
885            .unwrap_or_else(|| "GLBX.MDP3".to_string());
886        let request_id = request.request_id;
887        let client_id = request.client_id.unwrap_or(self.client_id);
888        let venue = request.venue.unwrap_or(*DATABENTO_VENUE);
889        let start_nanos = datetime_to_unix_nanos(request.start);
890        let end_nanos = datetime_to_unix_nanos(request.end);
891        let request_params = request.params;
892        let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
893
894        self.spawn_task(async move {
895            let query_params = instruments_query_params(dataset, query_start, query_end);
896
897            match historical_client.get_range_instruments(query_params).await {
898                Ok(instruments) => {
899                    log::debug!("Retrieved {} instruments", instruments.len());
900
901                    let response = DataResponse::Instruments(InstrumentsResponse::new(
902                        request_id,
903                        client_id,
904                        venue,
905                        instruments,
906                        start_nanos,
907                        end_nanos,
908                        get_atomic_clock_realtime().get_time_ns(),
909                        request_params,
910                    ));
911
912                    send_data_response(&data_sender, response, "instruments");
913                }
914                Err(e) => {
915                    send_request_result(&data_sender, request_id, None, &[e.to_string()]);
916                }
917            }
918        });
919
920        Ok(())
921    }
922
923    fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
924        let groups = self.group_request_instruments(&request.instrument_ids)?;
925        let historical_client = self.historical.clone();
926        let data_sender = self.data_sender.clone();
927        let client_id = request.client_id.unwrap_or(self.client_id);
928        let start = datetime_to_unix_nanos(request.start);
929        let end = datetime_to_unix_nanos(request.end);
930        let (query_start, query_end) = resolve_request_time_range(start, end);
931        self.spawn_task(async move {
932            let mut data = Vec::new();
933            let mut failures = Vec::new();
934
935            for (dataset, ids) in groups {
936                let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
937
938                let params = RangeQueryParams {
939                    dataset: dataset.clone(),
940                    symbols,
941                    start: query_start,
942                    end: query_end,
943                    limit: None,
944                    price_precision: None,
945                };
946
947                match historical_client.get_range_instruments(params).await {
948                    Ok(instruments) => {
949                        for id in ids {
950                            if let Some(instrument) = requested_instrument(&instruments, id) {
951                                data.push(instrument);
952                            } else {
953                                failures.push(format!("Instrument not found: {id}"));
954                            }
955                        }
956                    }
957                    Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
958                }
959            }
960
961            let ts_init = get_atomic_clock_realtime().get_time_ns();
962
963            let response = if request.instrument_ids.len() == 1 && data.len() == 1 {
964                let instrument = data.remove(0);
965                Some(DataResponse::Instrument(Box::new(InstrumentResponse::new(
966                    request.request_id,
967                    client_id,
968                    instrument.id(),
969                    instrument,
970                    start,
971                    end,
972                    ts_init,
973                    request.params,
974                ))))
975            } else if request.instrument_ids.len() > 1 {
976                Some(DataResponse::Instruments(InstrumentsResponse::new(
977                    request.request_id,
978                    client_id,
979                    *DATABENTO_VENUE,
980                    data,
981                    start,
982                    end,
983                    ts_init,
984                    request.params,
985                )))
986            } else {
987                None
988            };
989
990            send_request_result(&data_sender, request.request_id, response, &failures);
991        });
992
993        Ok(())
994    }
995
996    fn request_quotes(&self, request: RequestQuotes) -> anyhow::Result<()> {
997        anyhow::ensure!(
998            request.instrument_ids.len() == 1 || request.limit.is_none(),
999            "A record limit is unsupported for multi-instrument requests"
1000        );
1001        let groups = self.group_request_instruments(&request.instrument_ids)?;
1002        let historical_client = self.historical.clone();
1003        let data_sender = self.data_sender.clone();
1004        let client_id = request.client_id.unwrap_or(self.client_id);
1005        let start = datetime_to_unix_nanos(request.start);
1006        let end = datetime_to_unix_nanos(request.end);
1007        let limit = request.limit.map(|value| value.get() as u64);
1008        let precision = price_precision_from_params(request.params.as_ref())?;
1009        let schema = schema_from_params(request.params.as_ref(), dbn::Schema::Mbp1, QUOTE_SCHEMAS)?
1010            .to_string();
1011        let (query_start, query_end) = resolve_request_time_range(start, end);
1012        self.spawn_task(async move {
1013            let mut data = Vec::new();
1014            let mut failures = Vec::new();
1015
1016            for (dataset, ids) in groups {
1017                for &id in &ids {
1018                    seed_price_precision_if_needed(
1019                        &historical_client,
1020                        &dataset,
1021                        id,
1022                        query_start,
1023                        query_end,
1024                        precision,
1025                    )
1026                    .await;
1027                }
1028
1029                let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
1030
1031                let params = RangeQueryParams {
1032                    dataset: dataset.clone(),
1033                    symbols,
1034                    start: query_start,
1035                    end: query_end,
1036                    limit,
1037                    price_precision: precision,
1038                };
1039
1040                match historical_client
1041                    .get_range_quotes(params, Some(schema.clone()))
1042                    .await
1043                {
1044                    Ok(rows) => data.extend(rows),
1045                    Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
1046                }
1047            }
1048
1049            data.sort_by_key(|row| (row.ts_init, row.instrument_id));
1050            let response = DataResponse::Quotes(QuotesResponse::new(
1051                request.request_id,
1052                client_id,
1053                request.instrument_ids,
1054                data,
1055                start,
1056                end,
1057                get_atomic_clock_realtime().get_time_ns(),
1058                request.params,
1059            ));
1060            send_request_result(&data_sender, request.request_id, Some(response), &failures);
1061        });
1062
1063        Ok(())
1064    }
1065
1066    fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
1067        anyhow::ensure!(
1068            request.instrument_ids.len() == 1 || request.limit.is_none(),
1069            "A record limit is unsupported for multi-instrument requests"
1070        );
1071        let groups = self.group_request_instruments(&request.instrument_ids)?;
1072        let historical_client = self.historical.clone();
1073        let data_sender = self.data_sender.clone();
1074        let client_id = request.client_id.unwrap_or(self.client_id);
1075        let start = datetime_to_unix_nanos(request.start);
1076        let end = datetime_to_unix_nanos(request.end);
1077        let limit = request.limit.map(|value| value.get() as u64);
1078        let precision = price_precision_from_params(request.params.as_ref())?;
1079        let schema =
1080            schema_from_params(request.params.as_ref(), dbn::Schema::Trades, TRADE_SCHEMAS)?
1081                .to_string();
1082        let (query_start, query_end) = resolve_request_time_range(start, end);
1083        self.spawn_task(async move {
1084            let mut data = Vec::new();
1085            let mut failures = Vec::new();
1086
1087            for (dataset, ids) in groups {
1088                for &id in &ids {
1089                    seed_price_precision_if_needed(
1090                        &historical_client,
1091                        &dataset,
1092                        id,
1093                        query_start,
1094                        query_end,
1095                        precision,
1096                    )
1097                    .await;
1098                }
1099
1100                let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
1101
1102                let params = RangeQueryParams {
1103                    dataset: dataset.clone(),
1104                    symbols,
1105                    start: query_start,
1106                    end: query_end,
1107                    limit,
1108                    price_precision: precision,
1109                };
1110
1111                match historical_client
1112                    .get_range_trades(params, Some(schema.clone()))
1113                    .await
1114                {
1115                    Ok(rows) => data.extend(rows),
1116                    Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
1117                }
1118            }
1119
1120            data.sort_by_key(|row| (row.ts_init, row.instrument_id));
1121            let response = DataResponse::Trades(TradesResponse::new(
1122                request.request_id,
1123                client_id,
1124                request.instrument_ids,
1125                data,
1126                start,
1127                end,
1128                get_atomic_clock_realtime().get_time_ns(),
1129                request.params,
1130            ));
1131            send_request_result(&data_sender, request.request_id, Some(response), &failures);
1132        });
1133
1134        Ok(())
1135    }
1136
1137    fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
1138        let bar_type = request.single_bar_type()?;
1139        let aggregation = bar_type.spec().aggregation;
1140        anyhow::ensure!(
1141            matches!(
1142                aggregation,
1143                BarAggregation::Second
1144                    | BarAggregation::Minute
1145                    | BarAggregation::Hour
1146                    | BarAggregation::Day
1147            ),
1148            "Unsupported bar aggregation: {aggregation:?}"
1149        );
1150
1151        log::debug!("Request bars: {request:?}");
1152
1153        let historical_client = self.historical.clone();
1154        let data_sender = self.data_sender.clone();
1155        let instrument_id = bar_type.instrument_id();
1156        let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1157        let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1158        let request_id = request.request_id;
1159        let client_id = request.client_id.unwrap_or(self.client_id);
1160        let start_nanos = datetime_to_unix_nanos(request.start);
1161        let end_nanos = datetime_to_unix_nanos(request.end);
1162        let limit = request.limit.map(|limit| limit.get() as u64);
1163        let request_params = request.params;
1164        let price_precision = price_precision_from_params(request_params.as_ref())?;
1165        let timestamp_on_close = self.config.bars_timestamp_on_close;
1166        let spec = BarSpecification::new(1, aggregation, PriceType::Last);
1167        let interval = DurationNanos::try_from(spec.timedelta()).expect("supported OHLCV interval");
1168        let (start, end) = resolve_request_bounds(start_nanos, end_nanos);
1169
1170        // Databento indexes bars by open, while ts_init always records the close
1171        let (query_start, query_end) = resolve_request_time_range(
1172            Some(start.saturating_sub(interval)),
1173            Some(end.saturating_sub(interval)),
1174        );
1175
1176        self.spawn_task(async move {
1177            seed_price_precision_if_needed(
1178                &historical_client,
1179                dataset.as_str(),
1180                instrument_id,
1181                query_start,
1182                query_end,
1183                price_precision,
1184            )
1185            .await;
1186
1187            let params = RangeQueryParams {
1188                dataset,
1189                symbols,
1190                start: query_start,
1191                end: query_end,
1192                limit,
1193                price_precision,
1194            };
1195
1196            match historical_client
1197                .get_range_bars(params, aggregation, timestamp_on_close)
1198                .await
1199            {
1200                Ok(bars) => {
1201                    log::debug!("Retrieved {} bars", bars.len());
1202                    let response = DataResponse::Bars(BarsResponse::new(
1203                        request_id,
1204                        client_id,
1205                        bar_type,
1206                        bars,
1207                        start_nanos,
1208                        end_nanos,
1209                        get_atomic_clock_realtime().get_time_ns(),
1210                        request_params,
1211                    ));
1212
1213                    send_data_response(&data_sender, response, "bars");
1214                }
1215                Err(e) => {
1216                    send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1217                }
1218            }
1219        });
1220
1221        Ok(())
1222    }
1223
1224    fn request_book_depth(&self, request: RequestBookDepth) -> anyhow::Result<()> {
1225        let instrument_id = request.single_instrument_id()?;
1226
1227        log::debug!("Request book depth: {request:?}");
1228
1229        let historical_client = self.historical.clone();
1230        let data_sender = self.data_sender.clone();
1231        let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1232        let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1233        let request_id = request.request_id;
1234        let client_id = request.client_id.unwrap_or(self.client_id);
1235        let start_nanos = datetime_to_unix_nanos(request.start);
1236        let end_nanos = datetime_to_unix_nanos(request.end);
1237        let limit = request.limit.map(|limit| limit.get() as u64);
1238        let depth = request.depth.map(|depth| depth.get());
1239        let request_params = request.params;
1240        let price_precision = price_precision_from_params(request_params.as_ref())?;
1241        let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
1242
1243        self.spawn_task(async move {
1244            seed_price_precision_if_needed(
1245                &historical_client,
1246                dataset.as_str(),
1247                instrument_id,
1248                query_start,
1249                query_end,
1250                price_precision,
1251            )
1252            .await;
1253
1254            let params = RangeQueryParams {
1255                dataset,
1256                symbols,
1257                start: query_start,
1258                end: query_end,
1259                limit,
1260                price_precision,
1261            };
1262
1263            match historical_client
1264                .get_range_order_book_depth(params, depth)
1265                .await
1266            {
1267                Ok(depths) => {
1268                    log::debug!("Retrieved {} order book depths", depths.len());
1269                    let response = DataResponse::BookDepth(BookDepthResponse::new(
1270                        request_id,
1271                        client_id,
1272                        instrument_id,
1273                        depths,
1274                        start_nanos,
1275                        end_nanos,
1276                        get_atomic_clock_realtime().get_time_ns(),
1277                        request_params,
1278                    ));
1279
1280                    send_data_response(&data_sender, response, "book depth");
1281                }
1282                Err(e) => {
1283                    send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1284                }
1285            }
1286        });
1287
1288        Ok(())
1289    }
1290
1291    fn request_book_deltas(&self, request: RequestBookDeltas) -> anyhow::Result<()> {
1292        let instrument_id = request.single_instrument_id()?;
1293
1294        log::debug!("Request book deltas: {request:?}");
1295
1296        let historical_client = self.historical.clone();
1297        let data_sender = self.data_sender.clone();
1298        let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1299        let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1300        let request_id = request.request_id;
1301        let client_id = request.client_id.unwrap_or(self.client_id);
1302        let start_nanos = datetime_to_unix_nanos(request.start);
1303        let end_nanos = datetime_to_unix_nanos(request.end);
1304        let limit = request.limit.map(|limit| limit.get() as u64);
1305        let request_params = request.params;
1306        let price_precision = price_precision_from_params(request_params.as_ref())?;
1307        let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
1308
1309        self.spawn_task(async move {
1310            seed_price_precision_if_needed(
1311                &historical_client,
1312                dataset.as_str(),
1313                instrument_id,
1314                query_start,
1315                query_end,
1316                price_precision,
1317            )
1318            .await;
1319
1320            let params = RangeQueryParams {
1321                dataset,
1322                symbols,
1323                start: query_start,
1324                end: query_end,
1325                limit,
1326                price_precision,
1327            };
1328
1329            match historical_client.get_range_order_book_deltas(params).await {
1330                Ok(deltas) => {
1331                    log::debug!("Retrieved {} order book deltas", deltas.len());
1332                    let response = BookDeltasResponse::new(
1333                        request_id,
1334                        client_id,
1335                        instrument_id,
1336                        deltas,
1337                        start_nanos,
1338                        end_nanos,
1339                        get_atomic_clock_realtime().get_time_ns(),
1340                        request_params,
1341                    );
1342
1343                    send_book_deltas_response(&data_sender, response);
1344                }
1345                Err(e) => {
1346                    send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1347                }
1348            }
1349        });
1350
1351        Ok(())
1352    }
1353}
1354
1355impl DatabentoDataClient {
1356    fn subscribe_batch(
1357        &self,
1358        instrument_ids: Vec<InstrumentId>,
1359        params: Option<&Params>,
1360        default_schema: dbn::Schema,
1361        schemas: &[dbn::Schema],
1362    ) -> anyhow::Result<()> {
1363        let price_precision = price_precision_from_params(params)?;
1364        let schema = schema_from_params(params, default_schema, schemas)?;
1365        let ids = instrument_ids.into_instrument_ids()?;
1366        let groups = self.group_instruments(&ids)?;
1367
1368        let mut accepted = Vec::new();
1369        let mut failure = None;
1370
1371        for (dataset, ids) in &groups {
1372            match self.subscribe_dataset(dataset, ids, schema, price_precision) {
1373                Ok(()) => accepted.extend_from_slice(ids),
1374                Err(e) => {
1375                    log::error!(
1376                        "Databento subscription enqueue failed: dataset={dataset}, schema={schema}, instrument_ids={ids:?}"
1377                    );
1378
1379                    if failure.is_none() {
1380                        failure = Some(e);
1381                    }
1382                }
1383            }
1384        }
1385
1386        match failure {
1387            Some(e) => Err(SubscriptionError::instruments(accepted, e).into()),
1388            None => Ok(()),
1389        }
1390    }
1391
1392    fn group_request_instruments(
1393        &self,
1394        ids: &[InstrumentId],
1395    ) -> anyhow::Result<IndexMap<String, Vec<InstrumentId>>> {
1396        let groups = self.group_instruments(ids)?;
1397        for ids in groups.values() {
1398            let symbols = ids.iter().map(|id| id.symbol.as_str()).collect::<Vec<_>>();
1399            check_consistent_symbology(&symbols)?;
1400        }
1401
1402        Ok(groups)
1403    }
1404
1405    fn group_instruments(
1406        &self,
1407        ids: &[InstrumentId],
1408    ) -> anyhow::Result<IndexMap<String, Vec<InstrumentId>>> {
1409        let mut groups: IndexMap<String, Vec<InstrumentId>> = IndexMap::new();
1410        let mut symbol_venues = AHashMap::new();
1411
1412        for &id in ids {
1413            let dataset = self.get_dataset_for_venue(id.venue)?;
1414            if let Some(venue) = symbol_venues.insert(id.symbol, id.venue) {
1415                anyhow::ensure!(
1416                    venue == id.venue,
1417                    "Identifiers share raw symbol {} across different venues",
1418                    id.symbol
1419                );
1420            }
1421
1422            groups.entry(dataset).or_default().push(id);
1423        }
1424
1425        Ok(groups)
1426    }
1427
1428    fn subscribe_dataset(
1429        &self,
1430        dataset: &str,
1431        ids: &[InstrumentId],
1432        schema: dbn::Schema,
1433        price_precision: Option<u8>,
1434    ) -> anyhow::Result<()> {
1435        let symbols: Vec<String> = ids.iter().map(|id| id.symbol.to_string()).collect();
1436        let precisions: Vec<(Symbol, u8)> = ids
1437            .iter()
1438            .filter_map(|id| price_precision.map(|precision| (id.symbol, precision)))
1439            .collect();
1440        let subscription = Subscription::builder()
1441            .schema(schema)
1442            .symbols(symbols)
1443            .build();
1444        let start_after_subscribe = self.get_or_create_feed_handler(dataset);
1445        let previous_venues: Vec<_> = ids
1446            .iter()
1447            .map(|id| (id.symbol, self.symbol_venue_map.get_cloned(&id.symbol)))
1448            .collect();
1449
1450        for id in ids {
1451            self.symbol_venue_map.insert(id.symbol, id.venue);
1452        }
1453
1454        if let Err(e) = self.send_subscription_to_dataset(
1455            dataset,
1456            precisions,
1457            subscription,
1458            start_after_subscribe,
1459        ) {
1460            for (symbol, previous) in previous_venues {
1461                if let Some(venue) = previous {
1462                    self.symbol_venue_map.insert(symbol, venue);
1463                } else {
1464                    self.symbol_venue_map.remove(&symbol);
1465                }
1466            }
1467
1468            return Err(e);
1469        }
1470
1471        Ok(())
1472    }
1473}
1474
1475fn instruments_query_params(
1476    dataset: String,
1477    start_nanos: UnixNanos,
1478    end_nanos: Option<UnixNanos>,
1479) -> RangeQueryParams {
1480    RangeQueryParams {
1481        dataset,
1482        symbols: vec!["ALL_SYMBOLS".to_string()],
1483        start: start_nanos,
1484        end: end_nanos,
1485        limit: None,
1486        price_precision: None,
1487    }
1488}
1489
1490fn instrument_query_params(
1491    dataset: String,
1492    instrument_id: InstrumentId,
1493    start_nanos: UnixNanos,
1494    end_nanos: Option<UnixNanos>,
1495) -> RangeQueryParams {
1496    RangeQueryParams {
1497        dataset,
1498        symbols: vec![instrument_id_to_symbol_string(
1499            instrument_id,
1500            &mut AHashMap::new(),
1501        )],
1502        start: start_nanos,
1503        end: end_nanos,
1504        limit: None,
1505        price_precision: None,
1506    }
1507}
1508
1509fn resolve_request_time_range(
1510    start_nanos: Option<UnixNanos>,
1511    end_nanos: Option<UnixNanos>,
1512) -> (UnixNanos, Option<UnixNanos>) {
1513    let (start, end) = resolve_request_bounds(start_nanos, end_nanos);
1514
1515    // Engine bounds include the end; Databento's query excludes it
1516    (start, Some(end.saturating_add(DurationNanos::new(1))))
1517}
1518
1519fn resolve_request_bounds(
1520    start_nanos: Option<UnixNanos>,
1521    end_nanos: Option<UnixNanos>,
1522) -> (UnixNanos, UnixNanos) {
1523    let end = end_nanos.unwrap_or_else(|| get_atomic_clock_realtime().get_time_ns());
1524    let mut start = start_nanos.unwrap_or_else(|| start_of_utc_day(end));
1525
1526    if start > end {
1527        start = end;
1528    }
1529
1530    (start, end)
1531}
1532
1533fn start_of_utc_day(timestamp: UnixNanos) -> UnixNanos {
1534    timestamp.floor(DurationNanos::from_days(1))
1535}
1536
1537async fn seed_price_precision_if_needed(
1538    historical_client: &DatabentoHistoricalClient,
1539    dataset: &str,
1540    instrument_id: InstrumentId,
1541    start_nanos: UnixNanos,
1542    end_nanos: Option<UnixNanos>,
1543    price_precision: Option<u8>,
1544) {
1545    if price_precision.is_some()
1546        || historical_client
1547            .price_precision(instrument_id.symbol)
1548            .is_some()
1549    {
1550        return;
1551    }
1552
1553    let query_start = start_of_utc_day(start_nanos);
1554    let query_params =
1555        instrument_query_params(dataset.to_string(), instrument_id, query_start, end_nanos);
1556
1557    if let Err(e) = historical_client.get_range_instruments(query_params).await {
1558        log::warn!("Failed to seed price precision for {instrument_id}: {e}");
1559    }
1560}
1561
1562fn send_request_result(
1563    data_sender: &EventSender<DataEvent>,
1564    request_id: nautilus_core::UUID4,
1565    response: Option<DataResponse>,
1566    failures: &[String],
1567) {
1568    let event = if failures.is_empty() {
1569        let Some(response) = response else {
1570            return;
1571        };
1572
1573        let envelope = DataResponseEnvelope::new(response, true, None);
1574        DataEvent::ResponseEnvelope(envelope)
1575    } else {
1576        DataEvent::RequestFailed(DataRequestFailed {
1577            request_id,
1578            reason: failures.join("; "),
1579            response,
1580        })
1581    };
1582
1583    if let Err(e) = data_sender.send(event) {
1584        log::error!("Failed to send request result: {e}");
1585    }
1586}
1587
1588fn send_data_response(data_sender: &EventSender<DataEvent>, response: DataResponse, label: &str) {
1589    let envelope = DataResponseEnvelope::new(response, true, None);
1590    if let Err(e) = data_sender.send(DataEvent::ResponseEnvelope(envelope)) {
1591        log::error!("Failed to send {label} response: {e}");
1592    }
1593}
1594
1595fn send_book_deltas_response(sender: &EventSender<DataEvent>, response: BookDeltasResponse) {
1596    let responses = partition_book_deltas_response(response);
1597    let last = responses.len() - 1;
1598    for (index, response) in responses.into_iter().enumerate() {
1599        let envelope =
1600            DataResponseEnvelope::new(DataResponse::BookDeltas(response), index == last, None);
1601
1602        if let Err(e) = sender.send(DataEvent::ResponseEnvelope(envelope)) {
1603            log::error!("Failed to send book deltas response: {e}");
1604        }
1605    }
1606}
1607
1608fn partition_book_deltas_response(response: BookDeltasResponse) -> Vec<BookDeltasResponse> {
1609    if response.data.is_empty() {
1610        return vec![response];
1611    }
1612
1613    let mut partitions = IndexMap::new();
1614    for delta in response.data.iter().copied() {
1615        partitions
1616            .entry(delta.instrument_id)
1617            .or_insert_with(Vec::new)
1618            .push(delta);
1619    }
1620
1621    partitions
1622        .into_iter()
1623        .map(|(instrument_id, data)| {
1624            let mut child = response.clone();
1625            child.instrument_id = instrument_id;
1626            child.data = data.into();
1627            child
1628        })
1629        .collect()
1630}
1631
1632fn requested_instrument(
1633    instruments: &[InstrumentAny],
1634    instrument_id: InstrumentId,
1635) -> Option<InstrumentAny> {
1636    instruments
1637        .iter()
1638        .rev()
1639        .find(|instrument| instrument.id() == instrument_id)
1640        .cloned()
1641}
1642
1643fn price_precision_from_params(params: Option<&Params>) -> anyhow::Result<Option<u8>> {
1644    let Some(price_precision) = params.and_then(|params| params.get_u64(PRICE_PRECISION_PARAM))
1645    else {
1646        return Ok(None);
1647    };
1648
1649    Ok(Some(u8::try_from(price_precision).map_err(|_| {
1650        anyhow::anyhow!(
1651            "`{PRICE_PRECISION_PARAM}` must be less than or equal to {}",
1652            u8::MAX
1653        )
1654    })?))
1655}
1656
1657fn custom_data_for_instrument(
1658    data: Arc<dyn CustomDataTrait>,
1659    instrument_id: InstrumentId,
1660) -> CustomData {
1661    let data_type = DataType::new(data.type_name(), None, Some(instrument_id.to_string()));
1662    CustomData::new(data, data_type)
1663}
1664
1665fn custom_data_schema(data_type: &DataType) -> anyhow::Result<dbn::Schema> {
1666    match data_type.type_name() {
1667        name if name == DatabentoStatistics::type_name_static() => Ok(dbn::Schema::Statistics),
1668        name if name == DatabentoImbalance::type_name_static() => Ok(dbn::Schema::Imbalance),
1669        name => anyhow::bail!(
1670            "Unsupported custom data type for Databento live subscription: {name}, expected {} or {}",
1671            DatabentoStatistics::type_name_static(),
1672            DatabentoImbalance::type_name_static(),
1673        ),
1674    }
1675}
1676
1677fn custom_data_instrument_id(data_type: &DataType) -> anyhow::Result<InstrumentId> {
1678    // Records are published without metadata, so metadata would change the subscribed topic
1679    anyhow::ensure!(
1680        data_type
1681            .metadata()
1682            .is_none_or(|metadata| metadata.is_empty()),
1683        "{} subscriptions do not support metadata, received {}",
1684        data_type.type_name(),
1685        data_type.metadata_str(),
1686    );
1687
1688    let identifier = data_type.identifier().ok_or_else(|| {
1689        anyhow::anyhow!(
1690            "{} subscriptions require an instrument ID identifier",
1691            data_type.type_name()
1692        )
1693    })?;
1694
1695    InstrumentId::from_str(identifier)
1696        .map_err(|e| anyhow::anyhow!("Invalid instrument ID identifier `{identifier}`: {e}"))
1697}
1698
1699fn ohlcv_schema(bar_type: BarType) -> anyhow::Result<dbn::Schema> {
1700    let spec = bar_type.spec();
1701    anyhow::ensure!(
1702        spec.price_type == PriceType::Last,
1703        "Unsupported bar price type for Databento OHLCV: {spec}, only LAST is emitted",
1704    );
1705    anyhow::ensure!(
1706        spec.step.get() == 1,
1707        "Unsupported bar step for Databento OHLCV: {spec}, only a step of 1 is available",
1708    );
1709
1710    ohlcv_schema_from_aggregation(spec.aggregation)
1711}
1712
1713fn schema_from_params(
1714    params: Option<&Params>,
1715    default_schema: dbn::Schema,
1716    allowed_schemas: &[dbn::Schema],
1717) -> anyhow::Result<dbn::Schema> {
1718    let schema = if let Some(schema) = params.and_then(|params| params.get_str(SCHEMA_PARAM)) {
1719        dbn::Schema::from_str(schema)?
1720    } else {
1721        default_schema
1722    };
1723
1724    if allowed_schemas.contains(&schema) {
1725        return Ok(schema);
1726    }
1727
1728    let allowed = allowed_schemas
1729        .iter()
1730        .map(dbn::Schema::as_str)
1731        .collect::<Vec<_>>()
1732        .join(", ");
1733    anyhow::bail!(
1734        "Invalid `{SCHEMA_PARAM}` '{}'. Must be one of: {allowed}",
1735        schema.as_str()
1736    );
1737}
1738
1739fn send_subscription_commands(
1740    tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1741    dataset: &str,
1742    price_precisions: impl IntoIterator<Item = (Symbol, u8)>,
1743    subscription: Subscription,
1744    start_after_subscribe: bool,
1745) -> anyhow::Result<()> {
1746    for (symbol, precision) in price_precisions {
1747        tx.send(HandlerCommand::SetPricePrecision(symbol, precision))
1748            .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1749    }
1750
1751    tx.send(HandlerCommand::Subscribe(subscription))
1752        .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1753
1754    if start_after_subscribe {
1755        tx.send(HandlerCommand::Start)
1756            .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1757    }
1758
1759    Ok(())
1760}
1761
1762#[cfg(test)]
1763mod tests {
1764    use std::{num::NonZeroUsize, path::PathBuf};
1765
1766    use databento::dbn::{
1767        MappingInterval, SymbolMapping,
1768        decode::{DbnMetadata, DecodeRecord, dbn::Decoder},
1769        encode::EncodeRecord,
1770    };
1771    use nautilus_common::{
1772        live::runner::replace_data_event_sender,
1773        msgbus::{self, switchboard::get_custom_topic},
1774    };
1775    use nautilus_core::UUID4;
1776    use nautilus_model::{
1777        data::{BatchView, NautilusDataType, OrderBookDelta, QuoteTick},
1778        enums::BookType,
1779        identifiers::{ClientId, InstrumentId},
1780        instruments::{CurrencyPair, InstrumentAny},
1781        types::{Currency, Price, Quantity},
1782    };
1783    use rstest::rstest;
1784    use serde_json::json;
1785    use time::macros::date;
1786    use tokio::{
1787        io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
1788        net::TcpListener,
1789    };
1790
1791    use super::*;
1792    use crate::enums::{DatabentoStatisticType, DatabentoStatisticUpdateAction};
1793
1794    #[derive(Clone, Copy)]
1795    enum SubscribeKind {
1796        Quotes,
1797        Trades,
1798    }
1799
1800    fn currency_pair(instrument_id: &str) -> InstrumentAny {
1801        currency_pair_with_ts_init(instrument_id, UnixNanos::default())
1802    }
1803
1804    fn test_data_client() -> DatabentoDataClient {
1805        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
1806        replace_data_event_sender(sender);
1807
1808        let config = DatabentoDataClientConfig::new(
1809            "32-character-with-lots-of-filler",
1810            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1811            true,
1812            true,
1813        );
1814        DatabentoDataClient::new(
1815            ClientId::from("DATABENTO-TEST"),
1816            config,
1817            get_atomic_clock_realtime(),
1818        )
1819        .expect("test client should initialize")
1820    }
1821
1822    #[rstest]
1823    #[case::quotes(NautilusDataType::QuoteTick, true)]
1824    #[case::trades(NautilusDataType::TradeTick, true)]
1825    #[case::bars_close(NautilusDataType::Bar, true)]
1826    #[case::bars_open(NautilusDataType::Bar, false)]
1827    #[tokio::test]
1828    #[ignore = "Requires DATABENTO_API_KEY and GLBX.MDP3 historical access"]
1829    async fn test_historical_request_list_live(
1830        #[case] kind: NautilusDataType,
1831        #[case] timestamp_on_close: bool,
1832    ) {
1833        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
1834        replace_data_event_sender(sender);
1835        let api_key = std::env::var("DATABENTO_API_KEY").unwrap();
1836
1837        let config = DatabentoDataClientConfig::new(
1838            api_key,
1839            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1840            true,
1841            timestamp_on_close,
1842        );
1843        let client_id = ClientId::from("DATABENTO-HISTORY");
1844        let client =
1845            DatabentoDataClient::new(client_id, config, get_atomic_clock_realtime()).unwrap();
1846        let mut ids = vec![
1847            InstrumentId::from("ESM4.GLBX"),
1848            InstrumentId::from("NQM4.GLBX"),
1849        ];
1850        let start = UnixNanos::from(1_717_423_200_000_000_000);
1851
1852        let end = if kind == NautilusDataType::Bar {
1853            ids.truncate(1);
1854            start
1855        } else {
1856            start + DurationNanos::from_secs(1)
1857        };
1858
1859        let request_id = UUID4::new();
1860        let params = Params::new([(PRICE_PRECISION_PARAM.to_owned(), json!(2))].into());
1861        let bar_type = "ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"
1862            .parse::<BarType>()
1863            .unwrap();
1864
1865        match kind {
1866            NautilusDataType::QuoteTick => client
1867                .request_quotes(RequestQuotes::new(
1868                    ids.clone(),
1869                    Some(start.to_datetime_utc()),
1870                    Some(end.to_datetime_utc()),
1871                    None,
1872                    Some(client_id),
1873                    request_id,
1874                    UnixNanos::from(37),
1875                    Some(params.clone()),
1876                ))
1877                .unwrap(),
1878            NautilusDataType::TradeTick => client
1879                .request_trades(RequestTrades::new(
1880                    ids.clone(),
1881                    Some(start.to_datetime_utc()),
1882                    Some(end.to_datetime_utc()),
1883                    None,
1884                    Some(client_id),
1885                    request_id,
1886                    UnixNanos::from(37),
1887                    Some(params.clone()),
1888                ))
1889                .unwrap(),
1890            NautilusDataType::Bar => client
1891                .request_bars(RequestBars::new(
1892                    [bar_type],
1893                    Some(start.to_datetime_utc()),
1894                    Some(end.to_datetime_utc()),
1895                    None,
1896                    Some(client_id),
1897                    request_id,
1898                    UnixNanos::from(37),
1899                    Some(params.clone()),
1900                ))
1901                .unwrap(),
1902            _ => panic!("unexpected live request kind"),
1903        }
1904
1905        let event = tokio::time::timeout(Duration::from_secs(60), receiver.recv())
1906            .await
1907            .unwrap()
1908            .unwrap();
1909
1910        let DataEvent::ResponseEnvelope(envelope) = event else {
1911            panic!("expected successful historical envelope, received {event:?}");
1912        };
1913
1914        let response = envelope.response().unwrap();
1915
1916        let (response_ids, rows) = match (kind, response) {
1917            (NautilusDataType::QuoteTick, DataResponse::Quotes(response)) => (
1918                response.instrument_ids.clone(),
1919                response
1920                    .data
1921                    .iter()
1922                    .map(|row| (row.ts_init, row.instrument_id))
1923                    .collect::<Vec<_>>(),
1924            ),
1925            (NautilusDataType::TradeTick, DataResponse::Trades(response)) => (
1926                response.instrument_ids.clone(),
1927                response
1928                    .data
1929                    .iter()
1930                    .map(|row| (row.ts_init, row.instrument_id))
1931                    .collect::<Vec<_>>(),
1932            ),
1933            (NautilusDataType::Bar, DataResponse::Bars(response)) => {
1934                let expected_event = if timestamp_on_close {
1935                    start
1936                } else {
1937                    start - DurationNanos::from_mins(1)
1938                };
1939
1940                assert_eq!(response.bar_type, bar_type);
1941                assert_eq!(response.data.len(), 1);
1942                assert_eq!(response.data[0].bar_type, bar_type);
1943                assert_eq!(response.data[0].ts_init, start);
1944                assert_eq!(response.data[0].ts_event, expected_event);
1945                (
1946                    vec![response.bar_type.instrument_id()],
1947                    response
1948                        .data
1949                        .iter()
1950                        .map(|row| (row.ts_init, row.bar_type.instrument_id()))
1951                        .collect::<Vec<_>>(),
1952                )
1953            }
1954            _ => panic!("unexpected historical response variant"),
1955        };
1956
1957        let mut actual_ids = rows.iter().map(|(_, id)| *id).collect::<Vec<_>>();
1958        actual_ids.sort_unstable();
1959        actual_ids.dedup();
1960        let mut expected_ids = ids.clone();
1961        expected_ids.sort_unstable();
1962
1963        assert_eq!(envelope.correlation_id(), &request_id);
1964        assert!(envelope.completed());
1965        assert_eq!(envelope.sub_correlation_id(), None);
1966        assert_eq!(envelope.failure_reason(), None);
1967        assert_eq!(response.correlation_id(), &request_id);
1968        assert_eq!(response.client_id(), &client_id);
1969        assert_eq!(response.start(), Some(start));
1970        assert_eq!(response.end(), Some(end));
1971        assert_eq!(response.params(), Some(&params));
1972        assert_eq!(response_ids, ids);
1973        assert_eq!(actual_ids, expected_ids);
1974        assert!(rows.is_sorted());
1975        assert!(rows.iter().all(|(ts, _)| *ts >= start && *ts <= end));
1976        assert_eq!(
1977            receiver.try_recv().unwrap_err(),
1978            tokio::sync::mpsc::error::TryRecvError::Empty
1979        );
1980    }
1981
1982    #[rstest]
1983    #[case::raw("ESM4.GLBX", "NQM4.GLBX")]
1984    #[case::parent("ES.FUT.GLBX", "NQ.FUT.GLBX")]
1985    fn test_historical_request_groups_keep_dataset_and_input_order(
1986        #[case] first: &str,
1987        #[case] third: &str,
1988    ) {
1989        let client = test_data_client();
1990        let first = InstrumentId::from(first);
1991        let second = InstrumentId::from("AAPL.EQUS");
1992        let third = InstrumentId::from(third);
1993        let groups = client
1994            .group_request_instruments(&[first, second, third])
1995            .unwrap();
1996        assert_eq!(
1997            groups.into_iter().collect::<Vec<_>>(),
1998            vec![
1999                ("GLBX.MDP3".to_owned(), vec![first, third]),
2000                ("EQUS.MINI".to_owned(), vec![second])
2001            ]
2002        );
2003    }
2004
2005    #[rstest]
2006    #[case::instruments(NautilusDataType::Instrument)]
2007    #[case::quotes(NautilusDataType::QuoteTick)]
2008    #[case::trades(NautilusDataType::TradeTick)]
2009    #[tokio::test]
2010    async fn test_historical_requests_reject_mixed_symbology_before_dispatch(
2011        #[case] kind: NautilusDataType,
2012        #[values(false, true)] earlier_dataset: bool,
2013    ) {
2014        let client = test_data_client();
2015        let mut ids = Vec::new();
2016        if earlier_dataset {
2017            ids.push(InstrumentId::from("AAPL.EQUS"));
2018        }
2019
2020        ids.extend([
2021            InstrumentId::from("ES.FUT.GLBX"),
2022            InstrumentId::from("ESM4.GLBX"),
2023        ]);
2024        let request_id = UUID4::new();
2025
2026        let result = match kind {
2027            NautilusDataType::Instrument => client.request_instrument(RequestInstrument::new(
2028                ids,
2029                None,
2030                None,
2031                None,
2032                request_id,
2033                UnixNanos::from(37),
2034                None,
2035            )),
2036            NautilusDataType::QuoteTick => client.request_quotes(RequestQuotes::new(
2037                ids,
2038                None,
2039                None,
2040                None,
2041                None,
2042                request_id,
2043                UnixNanos::from(37),
2044                None,
2045            )),
2046            NautilusDataType::TradeTick => client.request_trades(RequestTrades::new(
2047                ids,
2048                None,
2049                None,
2050                None,
2051                None,
2052                request_id,
2053                UnixNanos::from(37),
2054                None,
2055            )),
2056            _ => panic!("unexpected historical request kind"),
2057        };
2058
2059        assert_eq!(
2060            result.unwrap_err().to_string(),
2061            "Inconsistent symbology types: 'parent' for ES.FUT vs 'raw_symbol' for ESM4"
2062        );
2063        assert!(client.task_handles.is_empty());
2064    }
2065
2066    #[rstest]
2067    #[case::quotes(true)]
2068    #[case::trades(false)]
2069    fn test_historical_request_lists_reject_limit_before_dispatch(#[case] quotes: bool) {
2070        let client = test_data_client();
2071        let ids = [
2072            InstrumentId::from("ESM4.GLBX"),
2073            InstrumentId::from("AAPL.EQUS"),
2074        ];
2075
2076        let result = if quotes {
2077            client.request_quotes(RequestQuotes::new(
2078                ids,
2079                None,
2080                None,
2081                NonZeroUsize::new(5),
2082                None,
2083                UUID4::new(),
2084                UnixNanos::from(37),
2085                None,
2086            ))
2087        } else {
2088            client.request_trades(RequestTrades::new(
2089                ids,
2090                None,
2091                None,
2092                NonZeroUsize::new(5),
2093                None,
2094                UUID4::new(),
2095                UnixNanos::from(37),
2096                None,
2097            ))
2098        };
2099
2100        assert_eq!(
2101            result.unwrap_err().to_string(),
2102            "A record limit is unsupported for multi-instrument requests"
2103        );
2104    }
2105
2106    #[rstest]
2107    fn test_historical_bars_reject_unsupported_aggregation_before_dispatch() {
2108        let client = test_data_client();
2109
2110        let request = RequestBars::new(
2111            BarType::from("ESM4.GLBX-100-TICK-LAST-EXTERNAL"),
2112            None,
2113            None,
2114            None,
2115            None,
2116            UUID4::new(),
2117            UnixNanos::from(37),
2118            None,
2119        );
2120        assert_eq!(
2121            client.request_bars(request).unwrap_err().to_string(),
2122            "Unsupported bar aggregation: Tick"
2123        );
2124        assert!(client.task_handles.is_empty());
2125    }
2126
2127    #[rstest]
2128    fn test_dataset_failure_preserves_successful_group_rows() {
2129        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2130        let sender: EventSender<DataEvent> = sender.into();
2131        let root = UUID4::new();
2132        let first = InstrumentId::from("ESM6.GLBX");
2133        let second = InstrumentId::from("AAPL.EQUS");
2134
2135        let quote = QuoteTick {
2136            instrument_id: first,
2137            ts_init: UnixNanos::from(17),
2138            ..Default::default()
2139        };
2140
2141        let response = QuotesResponse::new(
2142            root,
2143            ClientId::from("HISTORY"),
2144            [first, second],
2145            vec![quote],
2146            Some(UnixNanos::from(11)),
2147            Some(UnixNanos::from(29)),
2148            UnixNanos::from(31),
2149            None,
2150        );
2151
2152        send_request_result(
2153            &sender,
2154            root,
2155            Some(DataResponse::Quotes(response)),
2156            &["Dataset EQUS: source unavailable".to_owned()],
2157        );
2158
2159        let DataEvent::RequestFailed(failure) = receiver.try_recv().unwrap() else {
2160            panic!("Expected failed dataset result");
2161        };
2162
2163        assert_eq!(failure.request_id, root);
2164        assert_eq!(failure.reason, "Dataset EQUS: source unavailable");
2165
2166        let Some(DataResponse::Quotes(response)) = failure.response else {
2167            panic!("Expected successful sibling rows");
2168        };
2169
2170        assert_eq!(response.correlation_id, root);
2171        assert_eq!(response.client_id, ClientId::from("HISTORY"));
2172        assert_eq!(response.instrument_ids, vec![first, second]);
2173        assert_eq!(response.data.as_ref(), &[quote]);
2174        assert_eq!(response.start, Some(UnixNanos::from(11)));
2175        assert_eq!(response.end, Some(UnixNanos::from(29)));
2176        assert_eq!(response.ts_init, UnixNanos::from(31));
2177        assert_eq!(response.params, None);
2178        assert_eq!(
2179            receiver.try_recv().unwrap_err(),
2180            tokio::sync::mpsc::error::TryRecvError::Empty
2181        );
2182    }
2183
2184    #[rstest]
2185    fn test_book_reply_completes_after_all_partitioned_payloads() {
2186        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2187        let sender: EventSender<DataEvent> = sender.into();
2188        let root = UUID4::new();
2189        let first = InstrumentId::from("ESM6.GLBX");
2190        let second = InstrumentId::from("ESU6.GLBX");
2191        let a = OrderBookDelta::clear(first, 11, UnixNanos::from(13), UnixNanos::from(17));
2192        let b = OrderBookDelta::clear(second, 19, UnixNanos::from(23), UnixNanos::from(29));
2193        let c = OrderBookDelta::clear(first, 31, UnixNanos::from(37), UnixNanos::from(41));
2194
2195        let response = BookDeltasResponse::new(
2196            root,
2197            ClientId::from("HISTORY"),
2198            InstrumentId::from("ES.FUT.GLBX"),
2199            vec![a, b, c],
2200            None,
2201            None,
2202            UnixNanos::from(43),
2203            None,
2204        );
2205        send_book_deltas_response(&sender, response);
2206
2207        let DataEvent::ResponseEnvelope(left) = receiver.try_recv().unwrap() else {
2208            panic!("expected first envelope");
2209        };
2210
2211        let DataEvent::ResponseEnvelope(right) = receiver.try_recv().unwrap() else {
2212            panic!("expected final envelope");
2213        };
2214
2215        let Some(DataResponse::BookDeltas(left_payload)) = left.response() else {
2216            panic!("expected first book");
2217        };
2218
2219        let Some(DataResponse::BookDeltas(right_payload)) = right.response() else {
2220            panic!("expected second book");
2221        };
2222
2223        assert_eq!(
2224            (left.correlation_id(), right.correlation_id()),
2225            (&root, &root)
2226        );
2227        assert!(!left.completed());
2228        assert!(right.completed());
2229        assert_eq!(left_payload.instrument_id, first);
2230        assert_eq!(right_payload.instrument_id, second);
2231        assert_eq!(left_payload.data.as_ref(), &vec![a, c]);
2232        assert_eq!(right_payload.data.as_ref(), &vec![b]);
2233        assert_eq!(
2234            receiver.try_recv().unwrap_err(),
2235            tokio::sync::mpsc::error::TryRecvError::Empty
2236        );
2237    }
2238
2239    #[rstest]
2240    #[tokio::test]
2241    async fn test_stop_closes_task_admission_until_disconnect_drains() {
2242        let mut client = test_data_client();
2243
2244        client
2245            .task_handles
2246            .spawn(async { std::future::pending::<()>().await })
2247            .unwrap();
2248        client.is_connected.store(true, Ordering::Relaxed);
2249
2250        client.stop().unwrap();
2251
2252        assert!(!client.task_handles.is_open());
2253        assert!(client.is_disconnected());
2254
2255        client.disconnect().await.unwrap();
2256        assert!(client.task_handles.is_empty());
2257    }
2258
2259    #[rstest]
2260    #[case("EQUS", "EQUS.PLUS")] // overrides the apply_default EQUS -> EQUS.MINI mapping
2261    #[case("GLBX", "EQUS.MINI")] // overrides the apply_default GLBX -> GLBX.MDP3 mapping
2262    fn test_venue_dataset_map_overrides_default(#[case] venue: &str, #[case] dataset: &str) {
2263        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2264        replace_data_event_sender(sender);
2265
2266        let mut config = DatabentoDataClientConfig::new(
2267            "32-character-with-lots-of-filler",
2268            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2269            true,
2270            true,
2271        );
2272        config.venue_dataset_map = IndexMap::from([(venue.to_string(), dataset.to_string())]);
2273
2274        let client = DatabentoDataClient::new(
2275            ClientId::from("DATABENTO-TEST"),
2276            config,
2277            get_atomic_clock_realtime(),
2278        )
2279        .expect("test client should initialize");
2280
2281        assert_eq!(
2282            client.get_dataset_for_venue(Venue::from(venue)).unwrap(),
2283            dataset
2284        );
2285
2286        // The override is targeted: an unrelated venue keeps its default.
2287        assert_eq!(
2288            client.get_dataset_for_venue(Venue::from("XCBO")).unwrap(),
2289            "OPRA.PILLAR"
2290        );
2291    }
2292
2293    #[rstest]
2294    fn test_config_base_url_overrides_default_to_none() {
2295        let config = DatabentoDataClientConfig::new(
2296            "32-character-with-lots-of-filler",
2297            PathBuf::from("test_publishers.json"),
2298            true,
2299            true,
2300        );
2301
2302        assert!(config.historical_base_url.is_none());
2303        assert!(config.live_gateway_addr.is_none());
2304    }
2305
2306    #[rstest]
2307    fn test_historical_base_url_override_invalid_errors() {
2308        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2309        replace_data_event_sender(sender);
2310
2311        let mut config = DatabentoDataClientConfig::new(
2312            "32-character-with-lots-of-filler",
2313            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2314            true,
2315            true,
2316        );
2317        config.historical_base_url = Some("not a url".to_string());
2318
2319        let result = DatabentoDataClient::new(
2320            ClientId::from("DATABENTO-TEST"),
2321            config,
2322            get_atomic_clock_realtime(),
2323        );
2324
2325        assert!(result.is_err());
2326    }
2327
2328    fn subscribe_quotes_cmd(params: Option<Params>) -> SubscribeQuotes {
2329        SubscribeQuotes::new(
2330            InstrumentId::from("ESM4.GLBX"),
2331            Some(ClientId::from("DATABENTO-TEST")),
2332            None,
2333            UUID4::new(),
2334            UnixNanos::default(),
2335            None,
2336            params,
2337        )
2338    }
2339
2340    fn subscribe_trades_cmd(params: Option<Params>) -> SubscribeTrades {
2341        SubscribeTrades::new(
2342            InstrumentId::from("ESM4.GLBX"),
2343            Some(ClientId::from("DATABENTO-TEST")),
2344            None,
2345            UUID4::new(),
2346            UnixNanos::default(),
2347            None,
2348            params,
2349        )
2350    }
2351
2352    fn subscribe_book_depth_cmd(depth: Option<NonZeroUsize>) -> SubscribeBookDepth {
2353        SubscribeBookDepth::new(
2354            InstrumentId::from("ESM4.GLBX"),
2355            BookType::L2_MBP,
2356            Some(ClientId::from("DATABENTO-TEST")),
2357            None,
2358            UUID4::new(),
2359            UnixNanos::default(),
2360            depth,
2361            true,
2362            None,
2363            None,
2364        )
2365    }
2366
2367    fn currency_pair_with_ts_init(instrument_id: &str, ts_init: UnixNanos) -> InstrumentAny {
2368        let instrument_id = InstrumentId::from(instrument_id);
2369        InstrumentAny::CurrencyPair(
2370            CurrencyPair::builder()
2371                .instrument_id(instrument_id)
2372                .raw_symbol(instrument_id.symbol)
2373                .base_currency(Currency::from("BTC"))
2374                .quote_currency(Currency::from("USDT"))
2375                .price_precision(2)
2376                .size_precision(6)
2377                .price_increment(Price::from("0.01"))
2378                .size_increment(Quantity::from("0.000001"))
2379                .ts_event(UnixNanos::default())
2380                .ts_init(ts_init)
2381                .build()
2382                .unwrap(),
2383        )
2384    }
2385
2386    #[rstest]
2387    fn test_instruments_query_params_requests_all_symbols() {
2388        let start = UnixNanos::from(1_000_000_000);
2389        let end = UnixNanos::from(2_000_000_000);
2390
2391        let params = instruments_query_params("GLBX.MDP3".to_string(), start, Some(end));
2392
2393        assert_eq!(params.dataset, "GLBX.MDP3");
2394        assert_eq!(params.symbols, vec!["ALL_SYMBOLS"]);
2395        assert_eq!(params.start, start);
2396        assert_eq!(params.end, Some(end));
2397        assert_eq!(params.limit, None);
2398        assert_eq!(params.price_precision, None);
2399    }
2400
2401    #[rstest]
2402    fn test_instrument_query_params_requests_single_symbol() {
2403        let instrument_id = InstrumentId::from("ESM4.GLBX");
2404
2405        let start = UnixNanos::from(1_000_000_000);
2406        let end = UnixNanos::from(2_000_000_000);
2407
2408        let params =
2409            instrument_query_params("GLBX.MDP3".to_string(), instrument_id, start, Some(end));
2410
2411        assert_eq!(params.dataset, "GLBX.MDP3");
2412        assert_eq!(params.symbols, vec!["ESM4"]);
2413        assert_eq!(params.start, start);
2414        assert_eq!(params.end, Some(end));
2415        assert_eq!(params.limit, None);
2416        assert_eq!(params.price_precision, None);
2417    }
2418
2419    #[rstest]
2420    #[case::intraday(1_782_302_400_000_000_001)]
2421    #[case::midnight(1_782_259_200_000_000_000)]
2422    #[case::day_end(1_782_345_599_999_999_999)]
2423    #[tokio::test]
2424    async fn test_seed_price_precision_includes_midnight(#[case] start: u64) {
2425        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2426            .join("test_data/test_data.definition.futures_contract.dbn.zst");
2427        let mut decoder = Decoder::from_zstd_file(path).unwrap();
2428        let mut metadata = decoder.metadata().clone();
2429        let record = decoder
2430            .decode_record::<dbn::InstrumentDefMsg>()
2431            .unwrap()
2432            .unwrap();
2433        metadata.symbols = vec!["ESU6".to_string()];
2434
2435        metadata.mappings = vec![SymbolMapping {
2436            raw_symbol: "ESU6".to_string(),
2437            intervals: vec![MappingInterval {
2438                start_date: date!(2026 - 06 - 24),
2439                end_date: date!(2026 - 06 - 25),
2440                symbol: record.hd.instrument_id.to_string(),
2441            }],
2442        }];
2443
2444        let mut body = Vec::new();
2445        {
2446            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
2447            encoder.encode_record(record).unwrap();
2448            encoder.flush().unwrap();
2449        }
2450
2451        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2452        let base_url = format!("http://{}/", listener.local_addr().unwrap());
2453
2454        let server = tokio::spawn(serve_historical_response(listener, body));
2455
2456        let client = DatabentoHistoricalClient::new_with_base_url(
2457            Credential::new("32-character-with-lots-of-filler"),
2458            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2459            get_atomic_clock_realtime(),
2460            false,
2461            &base_url,
2462        )
2463        .unwrap();
2464        let instrument_id = InstrumentId::from("ESU26.GLBX");
2465        let end = UnixNanos::from(1_782_345_600_000_000_123);
2466
2467        tokio::time::timeout(
2468            std::time::Duration::from_secs(5),
2469            seed_price_precision_if_needed(
2470                &client,
2471                "GLBX.MDP3",
2472                instrument_id,
2473                start.into(),
2474                Some(end),
2475                None,
2476            ),
2477        )
2478        .await
2479        .unwrap();
2480        let request = server.await.unwrap();
2481        let fields: AHashMap<_, _> = request
2482            .split('&')
2483            .map(|field| field.split_once('=').unwrap())
2484            .collect();
2485
2486        assert_eq!(fields["dataset"], "GLBX.MDP3");
2487        assert_eq!(fields["schema"], "definition");
2488        assert_eq!(fields["symbols"], "ESU6");
2489        assert_eq!(fields["start"], "1782259200000000000");
2490        assert_eq!(fields["end"], end.to_string());
2491        assert_eq!(client.price_precision(instrument_id.symbol), Some(2));
2492    }
2493
2494    async fn serve_historical_response(listener: TcpListener, body: Vec<u8>) -> String {
2495        let (stream, _) = listener.accept().await.unwrap();
2496        let mut stream = BufReader::new(stream);
2497        let mut line = String::new();
2498        stream.read_line(&mut line).await.unwrap();
2499        assert_eq!(line, "POST /v0/timeseries.get_range HTTP/1.1\r\n");
2500        let mut content_length = None;
2501
2502        loop {
2503            line.clear();
2504            stream.read_line(&mut line).await.unwrap();
2505
2506            if line == "\r\n" {
2507                break;
2508            }
2509
2510            if let Some((name, value)) = line.split_once(':')
2511                && name.eq_ignore_ascii_case("content-length")
2512            {
2513                content_length = Some(value.trim().parse::<usize>().unwrap());
2514            }
2515        }
2516
2517        let mut request = vec![0; content_length.unwrap()];
2518        stream.read_exact(&mut request).await.unwrap();
2519        let headers = format!(
2520            "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
2521            body.len(),
2522        );
2523        stream.write_all(headers.as_bytes()).await.unwrap();
2524        stream.write_all(&body).await.unwrap();
2525        stream.shutdown().await.unwrap();
2526        String::from_utf8(request).unwrap()
2527    }
2528
2529    #[rstest]
2530    #[case::explicit(Some(5), None)]
2531    #[case::cached(None, Some(3))]
2532    #[tokio::test]
2533    async fn test_seed_price_precision_skips_known_precision(
2534        #[case] price_precision: Option<u8>,
2535        #[case] cached_precision: Option<u8>,
2536    ) {
2537        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2538        let base_url = format!("http://{}/", listener.local_addr().unwrap());
2539
2540        let client = DatabentoHistoricalClient::new_with_base_url(
2541            Credential::new("32-character-with-lots-of-filler"),
2542            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2543            get_atomic_clock_realtime(),
2544            false,
2545            &base_url,
2546        )
2547        .unwrap();
2548        let instrument_id = InstrumentId::from("ESU6.GLBX");
2549        if let Some(precision) = cached_precision {
2550            client.set_price_precision(instrument_id.symbol, precision);
2551        }
2552
2553        tokio::select! {
2554            () = seed_price_precision_if_needed(
2555                &client,
2556                "GLBX.MDP3",
2557                instrument_id,
2558                1_706_443_200_000_000_001.into(),
2559                Some(1_706_486_400_000_000_123.into()),
2560                price_precision,
2561            ) => {}
2562            result = listener.accept() => panic!("Unexpected definition request: {result:?}"),
2563            () = tokio::time::sleep(std::time::Duration::from_secs(5)) => {
2564                panic!("Precision seeding did not finish");
2565            }
2566        }
2567
2568        assert_eq!(
2569            client.price_precision(instrument_id.symbol),
2570            cached_precision
2571        );
2572    }
2573
2574    #[rstest]
2575    fn test_resolve_request_time_range_defaults_to_end_day() {
2576        let end = UnixNanos::from(1_706_443_200_000_000_001);
2577
2578        let (start, resolved_end) = resolve_request_time_range(None, Some(end));
2579
2580        assert_eq!(start, UnixNanos::from(1_706_400_000_000_000_000));
2581        assert_eq!(resolved_end, Some(end + DurationNanos::new(1)));
2582    }
2583
2584    #[rstest]
2585    #[case::range(17, 23, 17, 24)]
2586    #[case::point(17, 17, 17, 18)]
2587    #[case::zero(0, 0, 0, 1)]
2588    #[case::inverted(23, 17, 17, 18)]
2589    fn test_resolve_request_time_range_converts_inclusive_bounds(
2590        #[case] start: u64,
2591        #[case] end: u64,
2592        #[case] expected_start: u64,
2593        #[case] expected_end: u64,
2594    ) {
2595        let (start, end) = resolve_request_time_range(Some(start.into()), Some(end.into()));
2596
2597        assert_eq!(start, UnixNanos::from(expected_start));
2598        assert_eq!(end, Some(UnixNanos::from(expected_end)));
2599    }
2600
2601    #[rstest]
2602    #[case::quotes(false, true, false)]
2603    #[case::bars_close(true, true, false)]
2604    #[case::bars_open(true, false, false)]
2605    #[case::bars_default_start(true, true, true)]
2606    #[tokio::test]
2607    async fn test_historical_request_sends_inclusive_bounds(
2608        #[case] bars: bool,
2609        #[case] timestamp_on_close: bool,
2610        #[case] missing_start: bool,
2611    ) {
2612        let filename = if bars {
2613            "test_data.ohlcv-1m.dbn.zst"
2614        } else {
2615            "test_data.mbp-1.dbn.zst"
2616        };
2617
2618        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2619            .join("test_data")
2620            .join(filename);
2621        let decoder = Decoder::from_zstd_file(path).unwrap();
2622        let mut metadata = decoder.metadata().clone();
2623        metadata.dataset = "GLBX.MDP3".to_owned();
2624        let mut body = Vec::new();
2625        {
2626            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
2627            encoder.flush().unwrap();
2628        }
2629
2630        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2631        let base_url = format!("http://{}/", listener.local_addr().unwrap());
2632
2633        let server = tokio::spawn(serve_historical_response(listener, body));
2634        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
2635        replace_data_event_sender(sender);
2636
2637        let mut config = DatabentoDataClientConfig::new(
2638            "32-character-with-lots-of-filler",
2639            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2640            true,
2641            timestamp_on_close,
2642        );
2643        config.historical_base_url = Some(base_url);
2644        let client_id = ClientId::from("DATABENTO-BOUNDS");
2645        let client =
2646            DatabentoDataClient::new(client_id, config, get_atomic_clock_realtime()).unwrap();
2647        let id = InstrumentId::from("ESM4.GLBX");
2648        let start = UnixNanos::from(1_717_423_200_000_000_000);
2649        let request_start = (!missing_start).then_some(start);
2650        let end = start + DurationNanos::from_secs(60) + DurationNanos::new(123);
2651        let request_id = UUID4::new();
2652        let params = Params::new([(PRICE_PRECISION_PARAM.to_owned(), json!(2))].into());
2653
2654        if bars {
2655            let bar_type = "ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"
2656                .parse::<BarType>()
2657                .unwrap();
2658            client
2659                .request_bars(RequestBars::new(
2660                    bar_type,
2661                    request_start.map(|timestamp| timestamp.to_datetime_utc()),
2662                    Some(end.to_datetime_utc()),
2663                    None,
2664                    Some(client_id),
2665                    request_id,
2666                    UnixNanos::from(37),
2667                    Some(params.clone()),
2668                ))
2669                .unwrap();
2670        } else {
2671            client
2672                .request_quotes(RequestQuotes::new(
2673                    id,
2674                    request_start.map(|timestamp| timestamp.to_datetime_utc()),
2675                    Some(end.to_datetime_utc()),
2676                    None,
2677                    Some(client_id),
2678                    request_id,
2679                    UnixNanos::from(37),
2680                    Some(params.clone()),
2681                ))
2682                .unwrap();
2683        }
2684
2685        let event = tokio::time::timeout(Duration::from_secs(5), receiver.recv())
2686            .await
2687            .unwrap()
2688            .unwrap();
2689
2690        let DataEvent::ResponseEnvelope(envelope) = event else {
2691            panic!("expected completed response, received {event:?}");
2692        };
2693
2694        let response = envelope.response().unwrap();
2695        let request = server.await.unwrap();
2696        let fields: AHashMap<_, _> = request
2697            .split('&')
2698            .map(|field| field.split_once('=').unwrap())
2699            .collect();
2700
2701        let offset = if bars {
2702            DurationNanos::from_secs(60)
2703        } else {
2704            DurationNanos::new(0)
2705        };
2706
2707        let expected_start = if missing_start {
2708            UnixNanos::from(1_717_372_800_000_000_000)
2709        } else {
2710            start
2711        };
2712
2713        assert_eq!(fields["dataset"], "GLBX.MDP3");
2714        assert_eq!(fields["schema"], if bars { "ohlcv-1m" } else { "mbp-1" });
2715        assert_eq!(fields["symbols"], "ESM4");
2716        assert_eq!(fields["start"], (expected_start - offset).to_string());
2717        assert_eq!(
2718            fields["end"],
2719            (end - offset + DurationNanos::new(1)).to_string()
2720        );
2721        assert_eq!(envelope.correlation_id(), &request_id);
2722        assert!(envelope.completed());
2723        assert_eq!(envelope.failure_reason(), None);
2724        assert_eq!(response.start(), request_start);
2725        assert_eq!(response.end(), Some(end));
2726        assert_eq!(response.client_id(), &client_id);
2727        assert_eq!(response.params(), Some(&params));
2728        assert_eq!(response.record_count(), Some(0));
2729    }
2730
2731    #[rstest]
2732    fn test_requested_instrument_filters_exact_id() {
2733        let requested_id = InstrumentId::from("BTCUSDT.BINANCE");
2734        let instruments = vec![
2735            currency_pair("ETHUSDT.BINANCE"),
2736            currency_pair("BTCUSDT.BINANCE"),
2737        ];
2738
2739        let instrument = requested_instrument(&instruments, requested_id).expect("instrument");
2740
2741        assert_eq!(instrument.id(), requested_id);
2742    }
2743
2744    #[rstest]
2745    fn test_requested_instrument_returns_latest_matching_id() {
2746        let requested_id = InstrumentId::from("BTCUSDT.BINANCE");
2747        let instruments = vec![
2748            currency_pair_with_ts_init("BTCUSDT.BINANCE", UnixNanos::from(1)),
2749            currency_pair_with_ts_init("BTCUSDT.BINANCE", UnixNanos::from(2)),
2750        ];
2751
2752        let instrument = requested_instrument(&instruments, requested_id).expect("instrument");
2753
2754        assert_eq!(instrument.ts_init(), UnixNanos::from(2));
2755    }
2756
2757    #[rstest]
2758    fn test_requested_instrument_returns_none_on_miss() {
2759        let instruments = vec![currency_pair("ETHUSDT.BINANCE")];
2760
2761        let instrument = requested_instrument(&instruments, InstrumentId::from("BTCUSDT.BINANCE"));
2762
2763        assert!(instrument.is_none());
2764    }
2765
2766    #[rstest]
2767    fn test_price_precision_from_params() {
2768        let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
2769
2770        let price_precision = price_precision_from_params(Some(&params)).unwrap();
2771
2772        assert_eq!(price_precision, Some(5));
2773    }
2774
2775    #[rstest]
2776    fn test_price_precision_from_params_rejects_out_of_range_value() {
2777        let params = Params::new(
2778            [(
2779                PRICE_PRECISION_PARAM.to_string(),
2780                json!(u64::from(u8::MAX) + 1),
2781            )]
2782            .into(),
2783        );
2784
2785        let result = price_precision_from_params(Some(&params));
2786
2787        assert!(result.is_err());
2788    }
2789
2790    #[rstest]
2791    fn test_partition_book_deltas_response_by_child_instrument(
2792        #[values(false, true)] narrowed: bool,
2793    ) {
2794        let correlation_id = UUID4::new();
2795        let client_id = ClientId::from("DATABENTO-TEST");
2796        let parent = InstrumentId::from("ES.FUT.GLBX");
2797        let child_a = InstrumentId::from("ESM6.GLBX");
2798        let child_b = InstrumentId::from("ESU6.GLBX");
2799        let deltas = vec![
2800            OrderBookDelta::clear(child_a, 1, UnixNanos::from(1_000), UnixNanos::from(1_000)),
2801            OrderBookDelta::clear(child_b, 2, UnixNanos::from(2_000), UnixNanos::from(2_000)),
2802            OrderBookDelta::clear(child_a, 3, UnixNanos::from(3_000), UnixNanos::from(3_000)),
2803        ];
2804        let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
2805
2806        let rows = if narrowed {
2807            let hidden = OrderBookDelta::clear(
2808                InstrumentId::from("HIDDEN.GLBX"),
2809                4,
2810                UnixNanos::from(5_000),
2811                UnixNanos::from(5_000),
2812            );
2813            let mut backing = vec![hidden];
2814            backing.extend_from_slice(&deltas);
2815            backing.push(hidden);
2816            BatchView::new(Arc::new(backing), 1..4)
2817        } else {
2818            deltas.clone().into()
2819        };
2820
2821        let response = BookDeltasResponse::new(
2822            correlation_id,
2823            client_id,
2824            parent,
2825            rows,
2826            Some(UnixNanos::from(500)),
2827            Some(UnixNanos::from(4_000)),
2828            UnixNanos::from(5_000),
2829            Some(params.clone()),
2830        );
2831
2832        let responses = partition_book_deltas_response(response);
2833
2834        assert_eq!(responses.len(), 2);
2835        assert_eq!(responses[0].correlation_id, correlation_id);
2836        assert_eq!(responses[1].correlation_id, correlation_id);
2837        assert_eq!(responses[0].client_id, client_id);
2838        assert_eq!(responses[1].client_id, client_id);
2839        assert_eq!(responses[0].instrument_id, child_a);
2840        assert_eq!(responses[1].instrument_id, child_b);
2841        assert_eq!(responses[0].data.as_ref(), &vec![deltas[0], deltas[2]]);
2842        assert_eq!(responses[1].data.as_ref(), &vec![deltas[1]]);
2843        assert_eq!(responses[0].start, Some(UnixNanos::from(500)));
2844        assert_eq!(responses[1].start, Some(UnixNanos::from(500)));
2845        assert_eq!(responses[0].end, Some(UnixNanos::from(4_000)));
2846        assert_eq!(responses[1].end, Some(UnixNanos::from(4_000)));
2847        assert_eq!(responses[0].ts_init, UnixNanos::from(5_000));
2848        assert_eq!(responses[1].ts_init, UnixNanos::from(5_000));
2849        assert_eq!(responses[0].params, Some(params.clone()));
2850        assert_eq!(responses[1].params, Some(params));
2851    }
2852
2853    #[rstest]
2854    fn test_partition_book_deltas_response_preserves_homogeneous_response() {
2855        let correlation_id = UUID4::new();
2856        let client_id = ClientId::from("DATABENTO-TEST");
2857        let instrument_id = InstrumentId::from("ESM6.GLBX");
2858        let delta = OrderBookDelta::clear(
2859            instrument_id,
2860            1,
2861            UnixNanos::from(1_000),
2862            UnixNanos::from(1_000),
2863        );
2864        let response = BookDeltasResponse::new(
2865            correlation_id,
2866            client_id,
2867            instrument_id,
2868            vec![delta],
2869            Some(UnixNanos::from(500)),
2870            Some(UnixNanos::from(1_500)),
2871            UnixNanos::from(2_000),
2872            None,
2873        );
2874
2875        let responses = partition_book_deltas_response(response);
2876
2877        assert_eq!(responses.len(), 1);
2878        assert_eq!(responses[0].correlation_id, correlation_id);
2879        assert_eq!(responses[0].client_id, client_id);
2880        assert_eq!(responses[0].instrument_id, instrument_id);
2881        assert_eq!(responses[0].data.as_ref(), &vec![delta]);
2882        assert_eq!(responses[0].start, Some(UnixNanos::from(500)));
2883        assert_eq!(responses[0].end, Some(UnixNanos::from(1_500)));
2884        assert_eq!(responses[0].ts_init, UnixNanos::from(2_000));
2885        assert_eq!(responses[0].params, None);
2886    }
2887
2888    #[rstest]
2889    fn test_partition_book_deltas_response_preserves_empty_parent_response() {
2890        let correlation_id = UUID4::new();
2891        let parent = InstrumentId::from("ES.FUT.GLBX");
2892        let response = BookDeltasResponse::new(
2893            correlation_id,
2894            ClientId::from("DATABENTO-TEST"),
2895            parent,
2896            Vec::new(),
2897            None,
2898            None,
2899            UnixNanos::from(1_000),
2900            None,
2901        );
2902
2903        let responses = partition_book_deltas_response(response);
2904
2905        assert_eq!(responses.len(), 1);
2906        let response = &responses[0];
2907        assert_eq!(response.correlation_id, correlation_id);
2908        assert_eq!(response.instrument_id, parent);
2909        assert!(response.data.is_empty());
2910    }
2911
2912    #[rstest]
2913    fn test_schema_from_params_returns_default() {
2914        let schema = schema_from_params(None, dbn::Schema::Mbp1, QUOTE_SCHEMAS).unwrap();
2915
2916        assert_eq!(schema, dbn::Schema::Mbp1);
2917    }
2918
2919    #[rstest]
2920    fn test_schema_from_params_accepts_allowed_value() {
2921        let params = Params::new([(SCHEMA_PARAM.to_string(), json!("tbbo"))].into());
2922
2923        let schema = schema_from_params(Some(&params), dbn::Schema::Mbp1, QUOTE_SCHEMAS).unwrap();
2924
2925        assert_eq!(schema, dbn::Schema::Tbbo);
2926    }
2927
2928    #[rstest]
2929    fn test_schema_from_params_rejects_disallowed_value() {
2930        let params = Params::new([(SCHEMA_PARAM.to_string(), json!("mbo"))].into());
2931
2932        let result = schema_from_params(Some(&params), dbn::Schema::Mbp1, QUOTE_SCHEMAS);
2933
2934        assert!(result.is_err());
2935    }
2936
2937    #[rstest]
2938    #[case::quotes(SubscribeKind::Quotes)]
2939    #[case::trades(SubscribeKind::Trades)]
2940    fn test_invalid_subscribe_params_do_not_create_feed_handler(#[case] kind: SubscribeKind) {
2941        let mut client = test_data_client();
2942        let params = Params::new([(SCHEMA_PARAM.to_string(), json!("definition"))].into());
2943        let result = match kind {
2944            SubscribeKind::Quotes => client.subscribe_quotes(subscribe_quotes_cmd(Some(params))),
2945            SubscribeKind::Trades => client.subscribe_trades(subscribe_trades_cmd(Some(params))),
2946        };
2947
2948        assert!(result.is_err());
2949        assert!(client.cmd_channels.lock().is_empty());
2950    }
2951
2952    #[rstest]
2953    fn test_bar_list_reports_exact_full_types_and_continues_after_invalid_member() {
2954        let mut client = test_data_client();
2955        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
2956        client
2957            .cmd_channels
2958            .lock()
2959            .insert("GLBX.MDP3".to_string(), tx);
2960        let first = BarType::from("ESM4.GLBX-1-SECOND-LAST-EXTERNAL");
2961        let invalid = BarType::from("ESM4.GLBX-5-MINUTE-LAST-EXTERNAL");
2962        let last = BarType::from("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL");
2963
2964        let command = SubscribeBars::new(
2965            [first, invalid, last],
2966            Some(client.client_id),
2967            None,
2968            UUID4::new(),
2969            UnixNanos::from(17),
2970            Some(UUID4::new()),
2971            None,
2972        );
2973        let error = client.subscribe_bars(command).unwrap_err();
2974        let partial = error.downcast_ref::<SubscriptionError>().unwrap();
2975        assert_eq!(partial.bar_types_accepted, vec![first, last]);
2976        assert_eq!(partial.instrument_ids_accepted, Vec::<InstrumentId>::new());
2977
2978        for schema in [dbn::Schema::Ohlcv1S, dbn::Schema::Ohlcv1M] {
2979            let HandlerCommand::Subscribe(actual) = rx.try_recv().unwrap() else {
2980                panic!("expected bar schema subscription");
2981            };
2982
2983            let expected = Subscription::builder()
2984                .schema(schema)
2985                .symbols("ESM4")
2986                .build();
2987            assert_eq!(actual, expected);
2988        }
2989
2990        assert_eq!(
2991            rx.try_recv().unwrap_err(),
2992            tokio::sync::mpsc::error::TryRecvError::Empty
2993        );
2994    }
2995
2996    macro_rules! batch_tests {
2997        ($batch_test:ident, $failure_test:ident, $validation_test:ident, $command:ident, $subscribe:ident, $schema:ident) => {
2998            #[rstest]
2999            fn $batch_test() {
3000                let mut client = test_data_client();
3001                let (glbx_tx, mut glbx_rx) = tokio::sync::mpsc::unbounded_channel();
3002                let (opra_tx, mut opra_rx) = tokio::sync::mpsc::unbounded_channel();
3003                client
3004                    .cmd_channels
3005                    .lock()
3006                    .insert("GLBX.MDP3".to_string(), glbx_tx);
3007                client
3008                    .cmd_channels
3009                    .lock()
3010                    .insert("OPRA.PILLAR".to_string(), opra_tx);
3011                let first = InstrumentId::from("ESM4.GLBX");
3012                let second = InstrumentId::from("NQM4.GLBX");
3013                let third = InstrumentId::from("SPY_OPTION.XCBO");
3014                let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
3015                let command = $command(Some(params))
3016                    .with_instrument_ids([second, third, first, second]);
3017                let original = serde_json::to_value(&command).unwrap();
3018                client.$subscribe(command.clone()).unwrap();
3019                assert!(
3020                    matches!(glbx_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == second.symbol)
3021                );
3022                assert!(
3023                    matches!(glbx_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == first.symbol)
3024                );
3025
3026                let HandlerCommand::Subscribe(glbx_subscription) = glbx_rx.try_recv().unwrap()
3027                else {
3028                    panic!("expected GLBX subscription")
3029                };
3030
3031                assert!(
3032                    matches!(opra_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == third.symbol)
3033                );
3034
3035                let HandlerCommand::Subscribe(opra_subscription) = opra_rx.try_recv().unwrap()
3036                else {
3037                    panic!("expected OPRA subscription")
3038                };
3039
3040                let expected_glbx = Subscription::builder()
3041                    .schema(dbn::Schema::$schema)
3042                    .symbols(vec![second.symbol.to_string(), first.symbol.to_string()])
3043                    .build();
3044                let expected_opra = Subscription::builder()
3045                    .schema(dbn::Schema::$schema)
3046                    .symbols(vec![third.symbol.to_string()])
3047                    .build();
3048                assert_eq!(glbx_subscription, expected_glbx);
3049                assert_eq!(opra_subscription, expected_opra);
3050                assert_eq!(
3051                    glbx_rx.try_recv().unwrap_err(),
3052                    tokio::sync::mpsc::error::TryRecvError::Empty
3053                );
3054                assert_eq!(
3055                    opra_rx.try_recv().unwrap_err(),
3056                    tokio::sync::mpsc::error::TryRecvError::Empty
3057                );
3058                assert_eq!(client.symbol_venue_map.len(), 3);
3059
3060                for id in [first, second, third] {
3061                    assert_eq!(
3062                        client.symbol_venue_map.get_cloned(&id.symbol),
3063                        Some(id.venue)
3064                    );
3065                }
3066
3067                assert_eq!(serde_json::to_value(command).unwrap(), original);
3068            }
3069
3070            #[rstest]
3071            #[case::failure_first(true)]
3072            #[case::failure_middle(false)]
3073            fn $failure_test(
3074                #[case] failure_first: bool,
3075                #[values(false, true)] previous_mapping: bool,
3076            ) {
3077                let mut client = test_data_client();
3078                let (glbx_tx, mut glbx_rx) = tokio::sync::mpsc::unbounded_channel();
3079                let (opra_tx, opra_rx) = tokio::sync::mpsc::unbounded_channel();
3080                let (equs_tx, mut equs_rx) = tokio::sync::mpsc::unbounded_channel();
3081                drop(opra_rx);
3082                client
3083                    .cmd_channels
3084                    .lock()
3085                    .insert("GLBX.MDP3".to_string(), glbx_tx);
3086                client
3087                    .cmd_channels
3088                    .lock()
3089                    .insert("OPRA.PILLAR".to_string(), opra_tx);
3090                client
3091                    .cmd_channels
3092                    .lock()
3093                    .insert("EQUS.MINI".to_string(), equs_tx);
3094                let first = InstrumentId::from("ESM4.GLBX");
3095                let second = InstrumentId::from("SPY_OPTION.XCBO");
3096                let third = InstrumentId::from("AAPL.EQUS");
3097                let previous = previous_mapping.then_some(Venue::from("PREVIOUS"));
3098                if let Some(venue) = previous {
3099                    client.symbol_venue_map.insert(second.symbol, venue);
3100                }
3101                let ids = if failure_first {
3102                    [second, first, third]
3103                } else {
3104                    [first, second, third]
3105                };
3106                let command = $command(None).with_instrument_ids(ids);
3107                let result = client.$subscribe(command);
3108                let error = result.unwrap_err();
3109                let partial = error
3110                    .downcast_ref::<SubscriptionError>()
3111                    .expect("partial subscription error");
3112
3113                assert_eq!(partial.instrument_ids_accepted, vec![first, third]);
3114                assert_eq!(partial.bar_types_accepted, Vec::<BarType>::new());
3115                assert_eq!(
3116                    partial.source.to_string(),
3117                    "Failed to send command to dataset OPRA.PILLAR: channel closed"
3118                );
3119                assert_eq!(client.symbol_venue_map.get_cloned(&second.symbol), previous);
3120
3121                for (id, receiver) in [(first, &mut glbx_rx), (third, &mut equs_rx)] {
3122                    let HandlerCommand::Subscribe(accepted) = receiver.try_recv().unwrap() else {
3123                        panic!("expected accepted subscription");
3124                    };
3125                    let expected = Subscription::builder()
3126                        .schema(dbn::Schema::$schema)
3127                        .symbols(vec![id.symbol.to_string()])
3128                        .build();
3129                    assert_eq!(accepted, expected);
3130                    assert_eq!(client.symbol_venue_map.get_cloned(&id.symbol), Some(id.venue));
3131                    assert_eq!(
3132                        receiver.try_recv().unwrap_err(),
3133                        tokio::sync::mpsc::error::TryRecvError::Empty
3134                    );
3135                }
3136            }
3137
3138            #[rstest]
3139            #[case::unknown_dataset("UNKNOWN.UNKNOWN", "No dataset found for venue: UNKNOWN")]
3140            #[case::ambiguous_symbol(
3141                "ESM4.XCBO",
3142                "Identifiers share raw symbol ESM4 across different venues"
3143            )]
3144            fn $validation_test(#[case] invalid_id: &str, #[case] error: &str) {
3145                let mut client = test_data_client();
3146                let command = $command(None).with_instrument_ids([
3147                    InstrumentId::from("ESM4.GLBX"),
3148                    InstrumentId::from(invalid_id),
3149                ]);
3150                let result = client.$subscribe(command);
3151                assert_eq!(result.unwrap_err().to_string(), error);
3152                assert!(client.cmd_channels.lock().is_empty());
3153                assert!(client.symbol_venue_map.is_empty());
3154            }
3155        };
3156    }
3157
3158    batch_tests!(
3159        test_quote_batches_send_one_subscription_per_dataset,
3160        test_quote_batch_enqueue_failure_can_follow_accepted_dataset,
3161        test_quote_batches_validate_all_members_before_dispatch,
3162        subscribe_quotes_cmd,
3163        subscribe_quotes,
3164        Mbp1
3165    );
3166
3167    batch_tests!(
3168        test_trade_batches_send_one_subscription_per_dataset,
3169        test_trade_batch_enqueue_failure_can_follow_accepted_dataset,
3170        test_trade_batches_validate_all_members_before_dispatch,
3171        subscribe_trades_cmd,
3172        subscribe_trades,
3173        Trades
3174    );
3175
3176    #[rstest]
3177    fn test_trade_batch_unsubscribe_is_noop() {
3178        let mut client = test_data_client();
3179
3180        let command = UnsubscribeTrades::new(
3181            [
3182                InstrumentId::from("ESM4.GLBX"),
3183                InstrumentId::from("SPY_OPTION.XCBO"),
3184            ],
3185            Some(ClientId::from("DATABENTO")),
3186            None,
3187            UUID4::new(),
3188            UnixNanos::from(37),
3189            None,
3190            None,
3191        );
3192        client.unsubscribe_trades(&command).unwrap();
3193        assert!(client.cmd_channels.lock().is_empty());
3194        assert!(client.symbol_venue_map.is_empty());
3195    }
3196
3197    #[rstest]
3198    fn test_send_subscription_commands_starts_after_subscribe() {
3199        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3200        let subscription = Subscription::builder()
3201            .schema(dbn::Schema::Mbp1)
3202            .symbols(vec!["ESM4"])
3203            .build();
3204
3205        send_subscription_commands(
3206            &tx,
3207            "GLBX.MDP3",
3208            Some((Symbol::from("ESM4"), 2)),
3209            subscription,
3210            true,
3211        )
3212        .unwrap();
3213
3214        assert!(matches!(
3215            rx.try_recv().unwrap(),
3216            HandlerCommand::SetPricePrecision(symbol, 2) if symbol == Symbol::from("ESM4")
3217        ));
3218        assert!(matches!(
3219            rx.try_recv().unwrap(),
3220            HandlerCommand::Subscribe(sub) if sub.schema == dbn::Schema::Mbp1
3221        ));
3222        assert!(matches!(rx.try_recv().unwrap(), HandlerCommand::Start));
3223    }
3224
3225    #[rstest]
3226    #[case::second("ESM4.GLBX-1-SECOND-LAST-EXTERNAL", dbn::Schema::Ohlcv1S)]
3227    #[case::minute("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL", dbn::Schema::Ohlcv1M)]
3228    #[case::hour("ESM4.GLBX-1-HOUR-LAST-EXTERNAL", dbn::Schema::Ohlcv1H)]
3229    #[case::day("ESM4.GLBX-1-DAY-LAST-EXTERNAL", dbn::Schema::Ohlcv1D)]
3230    fn test_ohlcv_schema_maps_aggregation(#[case] bar_type: &str, #[case] expected: dbn::Schema) {
3231        let schema = ohlcv_schema(BarType::from(bar_type)).unwrap();
3232
3233        assert_eq!(schema, expected);
3234    }
3235
3236    #[rstest]
3237    #[case::step("ESM4.GLBX-5-MINUTE-LAST-EXTERNAL")]
3238    #[case::aggregation("ESM4.GLBX-100-TICK-LAST-EXTERNAL")]
3239    #[case::bid("ESM4.GLBX-1-MINUTE-BID-EXTERNAL")]
3240    #[case::mid("ESM4.GLBX-1-MINUTE-MID-EXTERNAL")]
3241    fn test_ohlcv_schema_rejects_unsupported_bar_type(#[case] bar_type: &str) {
3242        assert!(ohlcv_schema(BarType::from(bar_type)).is_err());
3243    }
3244
3245    #[rstest]
3246    #[case::statistics("DatabentoStatistics", dbn::Schema::Statistics)]
3247    #[case::imbalance("DatabentoImbalance", dbn::Schema::Imbalance)]
3248    fn test_subscribe_custom_data_sends_schema_subscription(
3249        #[case] type_name: &str,
3250        #[case] expected: dbn::Schema,
3251    ) {
3252        let mut client = test_data_client();
3253        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3254        client
3255            .cmd_channels
3256            .lock()
3257            .insert("GLBX.MDP3".to_string(), tx);
3258        let command = SubscribeCustomData::new(
3259            Some(ClientId::from("DATABENTO-TEST")),
3260            None,
3261            DataType::new(type_name, None, Some("ESM4.GLBX".to_string())),
3262            UUID4::new(),
3263            UnixNanos::default(),
3264            None,
3265            None,
3266        );
3267
3268        client.subscribe(command).unwrap();
3269
3270        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3271            panic!("expected subscription");
3272        };
3273        assert_eq!(subscription.schema, expected);
3274        assert_eq!(
3275            subscription.symbols,
3276            databento::Symbols::Symbols(vec!["ESM4".to_string()])
3277        );
3278        assert!(rx.try_recv().is_err());
3279    }
3280
3281    #[rstest]
3282    #[case::unsupported_type("DatabentoOther", Some("ESM4.GLBX"))]
3283    #[case::missing_identifier("DatabentoStatistics", None)]
3284    #[case::invalid_identifier("DatabentoStatistics", Some("ESM4"))]
3285    fn test_subscribe_custom_data_rejects_invalid_data_type(
3286        #[case] type_name: &str,
3287        #[case] identifier: Option<&str>,
3288    ) {
3289        let mut client = test_data_client();
3290        let command = SubscribeCustomData::new(
3291            Some(ClientId::from("DATABENTO-TEST")),
3292            None,
3293            DataType::new(type_name, None, identifier.map(str::to_string)),
3294            UUID4::new(),
3295            UnixNanos::default(),
3296            None,
3297            None,
3298        );
3299
3300        assert!(client.subscribe(command).is_err());
3301        assert!(client.cmd_channels.lock().is_empty());
3302    }
3303
3304    #[rstest]
3305    fn test_custom_data_for_instrument_matches_subscribed_topic() {
3306        let instrument_id = InstrumentId::from("ESM4.GLBX");
3307        let statistics = DatabentoStatistics::new(
3308            instrument_id,
3309            DatabentoStatisticType::SettlementPrice,
3310            DatabentoStatisticUpdateAction::Added,
3311            Some(Price::from("5000.25")),
3312            None,
3313            1,
3314            2,
3315            3,
3316            UnixNanos::from(4),
3317            5,
3318            UnixNanos::from(6),
3319            UnixNanos::from(7),
3320            UnixNanos::from(8),
3321        );
3322        let subscribed = DataType::new("DatabentoStatistics", None, Some("ESM4.GLBX".to_string()));
3323
3324        let custom = custom_data_for_instrument(Arc::new(statistics), instrument_id);
3325
3326        assert_eq!(custom.data_type, subscribed);
3327        assert_eq!(
3328            get_custom_topic(&custom.data_type),
3329            get_custom_topic(&subscribed)
3330        );
3331        assert_eq!(custom.data_type.identifier(), Some("ESM4.GLBX"));
3332    }
3333
3334    #[rstest]
3335    fn test_subscribe_schema_sends_raw_symbol() {
3336        let client = test_data_client();
3337        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3338        client
3339            .cmd_channels
3340            .lock()
3341            .insert("GLBX.MDP3".to_string(), tx);
3342
3343        client
3344            .subscribe_schema(InstrumentId::from("ESM4.GLBX"), dbn::Schema::Ohlcv1M)
3345            .unwrap();
3346
3347        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3348            panic!("expected subscription");
3349        };
3350        assert_eq!(subscription.stype_in, dbn::SType::RawSymbol);
3351        assert_eq!(
3352            subscription.symbols,
3353            databento::Symbols::Symbols(vec!["ESM4".to_string()])
3354        );
3355    }
3356
3357    #[rstest]
3358    #[case::continuous("ES.c.0.GLBX")]
3359    #[case::parent("ES.FUT.GLBX")]
3360    #[case::instrument_id("12345.GLBX")]
3361    fn test_subscribe_schema_rejects_non_raw_symbols(#[case] instrument_id: &str) {
3362        let client = test_data_client();
3363
3364        let result = client.subscribe_schema(InstrumentId::from(instrument_id), dbn::Schema::Mbp10);
3365
3366        assert!(result.is_err());
3367        assert!(client.cmd_channels.lock().is_empty());
3368    }
3369
3370    #[rstest]
3371    fn test_subscribe_custom_data_rejects_metadata() {
3372        let mut client = test_data_client();
3373        let metadata = Params::new([("venue".to_string(), json!("GLBX"))].into());
3374
3375        let command = SubscribeCustomData::new(
3376            Some(ClientId::from("DATABENTO-TEST")),
3377            None,
3378            DataType::new(
3379                "DatabentoStatistics",
3380                Some(metadata),
3381                Some("ESM4.GLBX".to_string()),
3382            ),
3383            UUID4::new(),
3384            UnixNanos::default(),
3385            None,
3386            None,
3387        );
3388
3389        assert!(client.subscribe(command).is_err());
3390        assert!(client.cmd_channels.lock().is_empty());
3391    }
3392
3393    #[rstest]
3394    fn test_subscribed_custom_data_reaches_subscriber() {
3395        let instrument_id = InstrumentId::from("ESM4.GLBX");
3396        let subscribed = DataType::new("DatabentoStatistics", None, Some("ESM4.GLBX".to_string()));
3397        let other = DataType::new("DatabentoStatistics", None, Some("ESU4.GLBX".to_string()));
3398        let handler = msgbus::stubs::get_message_saving_handler::<CustomData>(None);
3399        msgbus::subscribe_any(get_custom_topic(&subscribed).into(), handler.clone(), None);
3400        let statistics = DatabentoStatistics::new(
3401            instrument_id,
3402            DatabentoStatisticType::SettlementPrice,
3403            DatabentoStatisticUpdateAction::Added,
3404            Some(Price::from("5000.25")),
3405            None,
3406            1,
3407            2,
3408            3,
3409            UnixNanos::from(4),
3410            5,
3411            UnixNanos::from(6),
3412            UnixNanos::from(7),
3413            UnixNanos::from(8),
3414        );
3415        let custom = custom_data_for_instrument(Arc::new(statistics), instrument_id);
3416
3417        msgbus::publish_any(get_custom_topic(&custom.data_type), &custom);
3418        msgbus::publish_any(get_custom_topic(&other), &custom);
3419
3420        let received = msgbus::stubs::get_saved_messages::<CustomData>(&handler);
3421        assert_eq!(received, vec![custom]);
3422    }
3423
3424    #[rstest]
3425    fn test_subscribe_book_depth_accepts_depth_ten() {
3426        let mut client = test_data_client();
3427        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3428        client
3429            .cmd_channels
3430            .lock()
3431            .insert("GLBX.MDP3".to_string(), tx);
3432
3433        client
3434            .subscribe_book_depth(subscribe_book_depth_cmd(NonZeroUsize::new(10)))
3435            .unwrap();
3436
3437        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3438            panic!("expected subscription");
3439        };
3440        assert_eq!(subscription.schema, dbn::Schema::Mbp10);
3441    }
3442
3443    #[rstest]
3444    #[case::five(5)]
3445    #[case::twenty(20)]
3446    fn test_subscribe_book_depth_rejects_other_depths(#[case] depth: usize) {
3447        let mut client = test_data_client();
3448
3449        let result =
3450            client.subscribe_book_depth(subscribe_book_depth_cmd(NonZeroUsize::new(depth)));
3451
3452        assert!(result.is_err());
3453        assert!(client.cmd_channels.lock().is_empty());
3454    }
3455
3456    #[rstest]
3457    fn test_subscribe_bars_sends_ohlcv_subscription() {
3458        let mut client = test_data_client();
3459        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3460        client
3461            .cmd_channels
3462            .lock()
3463            .insert("GLBX.MDP3".to_string(), tx);
3464        let command = SubscribeBars::new(
3465            BarType::from("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"),
3466            Some(ClientId::from("DATABENTO-TEST")),
3467            None,
3468            UUID4::new(),
3469            UnixNanos::default(),
3470            None,
3471            None,
3472        );
3473
3474        client.subscribe_bars(command).unwrap();
3475
3476        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3477            panic!("expected subscription");
3478        };
3479        assert_eq!(subscription.schema, dbn::Schema::Ohlcv1M);
3480        assert_eq!(
3481            subscription.symbols,
3482            databento::Symbols::Symbols(vec!["ESM4".to_string()])
3483        );
3484        assert!(rx.try_recv().is_err());
3485    }
3486
3487    #[rstest]
3488    fn test_subscribe_book_depth_sends_mbp10_subscription() {
3489        let mut client = test_data_client();
3490        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3491        client
3492            .cmd_channels
3493            .lock()
3494            .insert("GLBX.MDP3".to_string(), tx);
3495        client
3496            .subscribe_book_depth(subscribe_book_depth_cmd(None))
3497            .unwrap();
3498
3499        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3500            panic!("expected subscription");
3501        };
3502        assert_eq!(subscription.schema, dbn::Schema::Mbp10);
3503        assert_eq!(
3504            subscription.symbols,
3505            databento::Symbols::Symbols(vec!["ESM4".to_string()])
3506        );
3507        assert!(rx.try_recv().is_err());
3508    }
3509
3510    #[rstest]
3511    fn test_send_subscription_commands_without_precision_or_start() {
3512        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3513        let subscription = Subscription::builder()
3514            .schema(dbn::Schema::Mbp1)
3515            .symbols(vec!["ESM4"])
3516            .build();
3517
3518        send_subscription_commands(&tx, "GLBX.MDP3", None, subscription, false).unwrap();
3519
3520        assert!(matches!(
3521            rx.try_recv().unwrap(),
3522            HandlerCommand::Subscribe(sub) if sub.schema == dbn::Schema::Mbp1
3523        ));
3524        assert!(matches!(
3525            rx.try_recv(),
3526            Err(tokio::sync::mpsc::error::TryRecvError::Empty)
3527        ));
3528    }
3529}