Skip to main content

nautilus_databento/
data.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Provides a unified data client that combines Databento's live streaming and historical data capabilities.
17//!
18//! This module implements a data client that manages connections to multiple Databento datasets,
19//! handles live market data subscriptions, and provides access to historical data on demand.
20
21use std::{
22    path::PathBuf,
23    str::FromStr,
24    sync::{
25        Arc,
26        atomic::{AtomicBool, Ordering},
27    },
28    time::Duration,
29};
30
31use ahash::AHashMap;
32use databento::{dbn, live::Subscription};
33use indexmap::IndexMap;
34use nautilus_common::{
35    clients::{DataClient, execute_bar_subscriptions, execute_instrument_subscriptions},
36    live::{runner::get_data_event_sender, sender::EventSender},
37    messages::{
38        DataEvent, DataResponse,
39        data::{
40            BarsResponse, BookDeltasResponse, BookDepthResponse, DataRequestFailed,
41            DataResponseEnvelope, InstrumentResponse, InstrumentsResponse, IntoInstrumentIds,
42            QuotesResponse, RequestBars, RequestBookDeltas, RequestBookDepth, RequestInstrument,
43            RequestInstruments, RequestQuotes, RequestTrades, SubscribeBars, SubscribeBookDeltas,
44            SubscribeBookDepth, SubscribeCustomData, SubscribeInstrument,
45            SubscribeInstrumentStatus, SubscribeQuotes, SubscribeTrades, SubscriptionError,
46            TradesResponse, UnsubscribeBars, UnsubscribeBookDeltas, UnsubscribeBookDepth,
47            UnsubscribeCustomData, UnsubscribeInstrumentStatus, UnsubscribeQuotes,
48            UnsubscribeTrades,
49        },
50    },
51};
52use nautilus_core::{
53    AtomicMap, DurationNanos, Params, UnixNanos,
54    datetime::datetime_to_unix_nanos,
55    time::{AtomicTime, get_atomic_clock_realtime},
56};
57use nautilus_live::task::TaskGroup;
58use nautilus_model::{
59    data::{BarSpecification, BarType, CustomData, Data, DataType, custom::CustomDataTrait},
60    enums::{BarAggregation, PriceType},
61    identifiers::{ClientId, InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
62    instruments::{Instrument, InstrumentAny},
63};
64use parking_lot::Mutex;
65use tokio_util::sync::CancellationToken;
66
67use crate::{
68    common::{Credential, DATABENTO_VENUE, ohlcv_schema_from_aggregation},
69    historical::{DatabentoHistoricalClient, RangeQueryParams},
70    live::{DatabentoFeedHandler, DatabentoMessage, HandlerCommand},
71    loader::DatabentoDataLoader,
72    symbology::{
73        check_consistent_symbology, infer_symbology_type, instrument_id_to_symbol_string,
74        raw_symbol,
75    },
76    types::{DatabentoImbalance, DatabentoStatistics, Dataset, PublisherId},
77};
78
79const PRICE_PRECISION_PARAM: &str = "price_precision";
80const SCHEMA_PARAM: &str = "schema";
81const MBP10_DEPTH: usize = 10;
82const QUOTE_SCHEMAS: &[dbn::Schema] = &[
83    dbn::Schema::Mbp1,
84    dbn::Schema::Bbo1S,
85    dbn::Schema::Bbo1M,
86    dbn::Schema::Cmbp1,
87    dbn::Schema::Cbbo1S,
88    dbn::Schema::Cbbo1M,
89    dbn::Schema::Tbbo,
90    dbn::Schema::Tcbbo,
91];
92const TRADE_SCHEMAS: &[dbn::Schema] = &[
93    dbn::Schema::Trades,
94    dbn::Schema::Tbbo,
95    dbn::Schema::Tcbbo,
96    dbn::Schema::Mbp1,
97    dbn::Schema::Cmbp1,
98];
99
100/// Configuration for the Databento data client.
101#[derive(Debug, Clone)]
102#[cfg_attr(
103    feature = "python",
104    pyo3::pyclass(module = "nautilus_trader.adapters.databento", from_py_object)
105)]
106#[cfg_attr(
107    feature = "python",
108    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
109)]
110pub struct DatabentoDataClientConfig {
111    /// Databento API credential.
112    pub(crate) credential: Credential,
113    /// Path to publishers.json file.
114    pub publishers_filepath: PathBuf,
115    /// Override for the Databento Historical API base URL.
116    pub historical_base_url: Option<String>,
117    /// Override for the Databento live gateway address.
118    pub live_gateway_addr: Option<String>,
119    /// Venue-to-dataset overrides applied on top of the publishers.json mappings.
120    pub venue_dataset_map: IndexMap<String, String>,
121    /// Whether to use exchange as venue for GLBX instruments.
122    pub use_exchange_as_venue: bool,
123    /// Whether to timestamp bars on close.
124    pub bars_timestamp_on_close: bool,
125    /// Reconnection timeout in minutes (None for infinite retries).
126    pub reconnect_timeout_mins: Option<u64>,
127    /// Year digits in futures IDs: 2 by default, or 1 for existing catalogs and caches.
128    pub futures_year_digits: FuturesYearDigits,
129}
130
131#[cfg(feature = "python")]
132nautilus_core::impl_pyo3_config_getters!(DatabentoDataClientConfig {
133    publishers_filepath: PathBuf,
134    historical_base_url: Option<String>,
135    live_gateway_addr: Option<String>,
136    use_exchange_as_venue: bool,
137    bars_timestamp_on_close: bool,
138    venue_dataset_map: IndexMap<String, String>,
139});
140
141impl DatabentoDataClientConfig {
142    /// Creates a new [`DatabentoDataClientConfig`] instance.
143    #[must_use]
144    pub fn new(
145        api_key: impl Into<String>,
146        publishers_filepath: PathBuf,
147        use_exchange_as_venue: bool,
148        bars_timestamp_on_close: bool,
149    ) -> Self {
150        Self {
151            futures_year_digits: FuturesYearDigits::default(),
152            credential: Credential::new(api_key),
153            publishers_filepath,
154            venue_dataset_map: IndexMap::new(),
155            use_exchange_as_venue,
156            bars_timestamp_on_close,
157            reconnect_timeout_mins: Some(10), // Default: 10 minutes
158            historical_base_url: None,
159            live_gateway_addr: None,
160        }
161    }
162
163    /// Returns the API key associated with this config.
164    #[must_use]
165    pub fn api_key(&self) -> &str {
166        self.credential.api_key()
167    }
168
169    /// Returns a masked version of the API key for logging purposes.
170    #[must_use]
171    pub fn api_key_masked(&self) -> String {
172        self.credential.api_key_masked()
173    }
174}
175
176/// A Databento data client that combines live streaming and historical data functionality.
177///
178/// This client uses the existing `DatabentoFeedHandler` for live data subscriptions
179/// and `DatabentoHistoricalClient` for historical data requests. It supports multiple
180/// datasets simultaneously, with separate feed handlers per dataset.
181#[cfg_attr(feature = "python", pyo3::pyclass)]
182#[cfg_attr(
183    feature = "python",
184    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
185)]
186#[derive(Debug)]
187pub struct DatabentoDataClient {
188    client_id: ClientId,
189    config: DatabentoDataClientConfig,
190    is_connected: AtomicBool,
191    historical: DatabentoHistoricalClient,
192    loader: DatabentoDataLoader,
193    cmd_channels: Arc<Mutex<AHashMap<String, tokio::sync::mpsc::UnboundedSender<HandlerCommand>>>>,
194    task_handles: TaskGroup,
195    cancellation_token: CancellationToken,
196    publisher_venue_map: Arc<IndexMap<PublisherId, Venue>>,
197    symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
198    data_sender: EventSender<DataEvent>,
199}
200
201impl DatabentoDataClient {
202    /// Creates a new [`DatabentoDataClient`] instance.
203    ///
204    /// # Errors
205    ///
206    /// Returns an error if client creation or publisher configuration loading fails.
207    pub fn new(
208        client_id: ClientId,
209        config: DatabentoDataClientConfig,
210        clock: &'static AtomicTime,
211    ) -> anyhow::Result<Self> {
212        let historical = match &config.historical_base_url {
213            Some(base_url) => DatabentoHistoricalClient::new_with_base_url(
214                config.credential.clone(),
215                config.publishers_filepath.clone(),
216                clock,
217                config.use_exchange_as_venue,
218                base_url,
219            )?,
220            None => DatabentoHistoricalClient::new(
221                config.credential.clone(),
222                config.publishers_filepath.clone(),
223                clock,
224                config.use_exchange_as_venue,
225            )?,
226        }
227        .with_futures_year_digits(config.futures_year_digits);
228
229        // Create data loader for venue-to-dataset mapping
230        let mut loader = DatabentoDataLoader::new(Some(config.publishers_filepath.clone()))?
231            .with_futures_year_digits(config.futures_year_digits);
232        for (venue, dataset) in &config.venue_dataset_map {
233            loader.set_dataset_for_venue(
234                Dataset::from(dataset.as_str()),
235                Venue::from(venue.as_str()),
236            );
237        }
238
239        // Load publisher configuration
240        let file_content = std::fs::read_to_string(&config.publishers_filepath)?;
241        let publishers_vec: Vec<crate::types::DatabentoPublisher> =
242            serde_json::from_str(&file_content)?;
243
244        let publisher_venue_map = publishers_vec
245            .into_iter()
246            .map(|p| (p.publisher_id, Venue::from(p.venue.as_str())))
247            .collect::<IndexMap<u16, Venue>>();
248
249        let data_sender = get_data_event_sender();
250
251        let task_handles = TaskGroup::new();
252
253        Ok(Self {
254            client_id,
255            config,
256            is_connected: AtomicBool::new(false),
257            historical,
258            loader,
259            cmd_channels: Arc::new(Mutex::new(AHashMap::new())),
260            cancellation_token: task_handles.cancellation_token(),
261            task_handles,
262            publisher_venue_map: Arc::new(publisher_venue_map),
263            symbol_venue_map: Arc::new(AtomicMap::new()),
264            data_sender,
265        })
266    }
267
268    /// Returns the API key associated with this client.
269    #[must_use]
270    pub fn api_key(&self) -> &str {
271        self.config.api_key()
272    }
273
274    /// Returns a masked version of the API key for logging purposes.
275    #[must_use]
276    pub fn api_key_masked(&self) -> String {
277        self.config.api_key_masked()
278    }
279
280    /// Gets the dataset for a given venue using the data loader.
281    ///
282    /// # Errors
283    ///
284    /// Returns an error if the venue-to-dataset mapping cannot be found.
285    fn get_dataset_for_venue(&self, venue: Venue) -> anyhow::Result<String> {
286        self.loader
287            .get_dataset_for_venue(&venue)
288            .map(ToString::to_string)
289            .ok_or_else(|| anyhow::anyhow!("No dataset found for venue: {venue}"))
290    }
291
292    /// Gets or creates a feed handler for the specified dataset.
293    fn get_or_create_feed_handler(&self, dataset: &str) -> bool {
294        let mut channels = self.cmd_channels.lock();
295
296        if !channels.contains_key(dataset) {
297            log::debug!("Creating new feed handler for dataset: {dataset}");
298            let cmd_tx = self.initialize_live_feed(dataset.to_string());
299            channels.insert(dataset.to_string(), cmd_tx);
300
301            log::debug!("Feed handler created for dataset: {dataset}, channel stored");
302            return true;
303        }
304
305        false
306    }
307
308    fn send_subscription_to_dataset(
309        &self,
310        dataset: &str,
311        price_precisions: impl IntoIterator<Item = (Symbol, u8)>,
312        mut subscription: Subscription,
313        start_after_subscribe: bool,
314    ) -> anyhow::Result<()> {
315        if subscription.stype_in == dbn::SType::RawSymbol
316            && let databento::Symbols::Symbols(symbols) = &mut subscription.symbols
317        {
318            for symbol in symbols {
319                let raw = raw_symbol(symbol, dataset, self.config.futures_year_digits);
320
321                *symbol = raw;
322            }
323        }
324
325        let price_precisions = price_precisions.into_iter().map(|(symbol, precision)| {
326            let raw = raw_symbol(symbol.as_str(), dataset, self.config.futures_year_digits);
327            (Symbol::from(raw.as_str()), precision)
328        });
329
330        let tx = {
331            let channels = self.cmd_channels.lock();
332            channels
333                .get(dataset)
334                .cloned()
335                .ok_or_else(|| anyhow::anyhow!("No feed handler found for dataset: {dataset}"))?
336        };
337
338        send_subscription_commands(
339            &tx,
340            dataset,
341            price_precisions,
342            subscription,
343            start_after_subscribe,
344        )
345    }
346
347    fn subscribe_schema(
348        &self,
349        instrument_id: InstrumentId,
350        schema: dbn::Schema,
351    ) -> anyhow::Result<()> {
352        let symbol = instrument_id.symbol.to_string();
353        // Records resolve to the underlying contract, so a continuous or parent symbol would
354        // publish on topics that its subscriber does not listen on
355        anyhow::ensure!(
356            infer_symbology_type(&symbol) == dbn::SType::RawSymbol,
357            "Unsupported symbol {symbol} for a Databento {schema} subscription, only raw symbols \
358             are available (not continuous or parent symbols)",
359        );
360
361        let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
362        let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
363
364        self.symbol_venue_map
365            .insert(instrument_id.symbol, instrument_id.venue);
366
367        let subscription = Subscription::builder()
368            .schema(schema)
369            .symbols(symbol)
370            .build();
371
372        self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)
373    }
374
375    fn send_close_to_active_feeds(&self) {
376        let channels = self.cmd_channels.lock();
377        for (dataset, tx) in channels.iter() {
378            if let Err(e) = tx.send(HandlerCommand::Close) {
379                log::warn!("Failed to send close command to dataset {dataset}: {e}");
380            }
381        }
382    }
383
384    fn clear_feed_channels(&self) {
385        let mut channels = self.cmd_channels.lock();
386        channels.clear();
387    }
388
389    fn abort_active_tasks(&self) {
390        self.task_handles.begin_shutdown();
391    }
392
393    fn spawn_task<F>(&self, future: F)
394    where
395        F: std::future::Future<Output = ()> + Send + 'static,
396    {
397        if let Err(e) = self.task_handles.spawn(future) {
398            log::debug!("Skipping Databento task after shutdown began: {e}");
399        }
400    }
401
402    /// Initializes the live feed handler for streaming data.
403    fn initialize_live_feed(
404        &self,
405        dataset: String,
406    ) -> tokio::sync::mpsc::UnboundedSender<HandlerCommand> {
407        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
408        let (msg_tx, msg_rx) = tokio::sync::mpsc::unbounded_channel();
409        let feed_dataset = dataset.clone();
410        let feed_channels = self.cmd_channels.clone();
411
412        let mut feed_handler = DatabentoFeedHandler::new(
413            self.config.credential.clone(),
414            dataset,
415            cmd_rx,
416            msg_tx,
417            (*self.publisher_venue_map).clone(),
418            self.symbol_venue_map.clone(),
419            self.config.use_exchange_as_venue,
420            self.config.bars_timestamp_on_close,
421            self.config.reconnect_timeout_mins,
422        )
423        .with_futures_year_digits(self.config.futures_year_digits);
424
425        if let Some(addr) = &self.config.live_gateway_addr {
426            feed_handler = feed_handler.with_gateway_addr(addr.clone());
427        }
428
429        let feed_future = async move {
430            // run() logs its terminal error and forwards it as `DatabentoMessage::Error`
431            let _ = feed_handler.run().await;
432            feed_channels.lock().remove(&feed_dataset);
433        };
434
435        let cancellation_token = self.cancellation_token.clone();
436        let data_sender = self.data_sender.clone();
437
438        // Spawn message processing task with cancellation support
439        let msg_future = async move {
440            let mut msg_rx = msg_rx;
441
442            loop {
443                tokio::select! {
444                    msg = msg_rx.recv() => {
445                        match msg {
446                            Some(DatabentoMessage::Data(data)) => {
447                                log::debug!("Received data: {data:?}");
448                                if let Err(e) = data_sender.send(DataEvent::Data(data)) {
449                                    log::error!("Failed to send data event: {e}");
450                                }
451                            }
452                            Some(DatabentoMessage::Instrument(instrument)) => {
453                                log::debug!("Received instrument definition: {}", instrument.id());
454                                if let Err(e) = data_sender.send(DataEvent::Data(Data::Instrument(instrument))) {
455                                    log::error!("Failed to send instrument: {e}");
456                                }
457                            }
458                            Some(DatabentoMessage::Status(status)) => {
459                                log::debug!("Received status: {status:?}");
460                                if let Err(e) =
461                                    data_sender.send(DataEvent::Data(Data::InstrumentStatus(status)))
462                                {
463                                    log::error!("Failed to send status data event: {e}");
464                                }
465                            }
466                            Some(DatabentoMessage::Imbalance(imbalance)) => {
467                                log::debug!("Received imbalance: {imbalance:?}");
468                                let instrument_id = imbalance.instrument_id;
469                                let data = Data::Custom(custom_data_for_instrument(Arc::new(imbalance), instrument_id));
470                                if let Err(e) = data_sender.send(DataEvent::Data(data)) {
471                                    log::error!("Failed to send imbalance data event: {e}");
472                                }
473                            }
474                            Some(DatabentoMessage::Statistics(statistics)) => {
475                                log::debug!("Received statistics: {statistics:?}");
476                                let instrument_id = statistics.instrument_id;
477                                let data = Data::Custom(custom_data_for_instrument(Arc::new(statistics), instrument_id));
478                                if let Err(e) = data_sender.send(DataEvent::Data(data)) {
479                                    log::error!("Failed to send statistics data event: {e}");
480                                }
481                            }
482                            Some(DatabentoMessage::SubscriptionAck(ack)) => {
483                                log::debug!("Received subscription ack: {}", ack.message);
484                            }
485                            Some(DatabentoMessage::Error(error)) => {
486                                log::error!("Feed handler error: {error}");
487                            }
488                            Some(DatabentoMessage::Close) => {
489                                log::debug!("Feed handler closed");
490                                break;
491                            }
492                            None => {
493                                log::debug!("Message channel closed");
494                                break;
495                            }
496                        }
497                    }
498                    () = cancellation_token.cancelled() => {
499                        log::debug!("Message processing cancelled");
500                        break;
501                    }
502                }
503            }
504        };
505
506        if let Err(e) = self.task_handles.spawn(feed_future) {
507            log::warn!("Skipping Databento feed task after shutdown began: {e}");
508        }
509
510        if let Err(e) = self.task_handles.spawn(msg_future) {
511            log::warn!("Skipping Databento message task after shutdown began: {e}");
512        }
513
514        cmd_tx
515    }
516}
517
518#[async_trait::async_trait(?Send)]
519impl DataClient for DatabentoDataClient {
520    /// Returns the client identifier.
521    fn client_id(&self) -> ClientId {
522        self.client_id
523    }
524
525    /// Returns the venue associated with this client (None for multi-venue clients).
526    fn venue(&self) -> Option<Venue> {
527        None
528    }
529
530    /// Starts the data client.
531    ///
532    /// # Errors
533    ///
534    /// Returns an error if the client fails to start.
535    fn start(&mut self) -> anyhow::Result<()> {
536        log::debug!("Starting");
537        Ok(())
538    }
539
540    /// Stops the data client and cancels all active subscriptions.
541    ///
542    /// # Errors
543    ///
544    /// Returns an error if the client fails to stop cleanly.
545    fn stop(&mut self) -> anyhow::Result<()> {
546        log::debug!("Stopping");
547
548        self.send_close_to_active_feeds();
549        self.clear_feed_channels();
550        self.cancellation_token.cancel();
551        self.abort_active_tasks();
552        self.is_connected.store(false, Ordering::Relaxed);
553
554        Ok(())
555    }
556
557    fn reset(&mut self) -> anyhow::Result<()> {
558        log::debug!("Resetting");
559        self.send_close_to_active_feeds();
560        self.clear_feed_channels();
561        self.abort_active_tasks();
562        self.is_connected.store(false, Ordering::Relaxed);
563        Ok(())
564    }
565
566    fn dispose(&mut self) -> anyhow::Result<()> {
567        log::debug!("Disposing");
568        self.stop()
569    }
570
571    async fn connect(&mut self) -> anyhow::Result<()> {
572        log::debug!("Connecting...");
573
574        if !self.task_handles.is_open() {
575            self.task_handles
576                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
577                .await
578                .map_err(|e| anyhow::anyhow!("Failed to terminate Databento tasks: {e}"))?;
579            self.task_handles
580                .start_generation()
581                .map_err(|e| anyhow::anyhow!("Failed to start Databento task generation: {e}"))?;
582            self.cancellation_token = self.task_handles.cancellation_token();
583        }
584
585        self.is_connected.store(true, Ordering::Relaxed);
586
587        log::info!("Connected");
588        Ok(())
589    }
590
591    async fn disconnect(&mut self) -> anyhow::Result<()> {
592        log::debug!("Disconnecting...");
593
594        self.send_close_to_active_feeds();
595        self.clear_feed_channels();
596        self.task_handles.begin_shutdown();
597
598        let tasks_result = self
599            .task_handles
600            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
601            .await
602            .map_err(|e| anyhow::anyhow!("Failed to terminate Databento tasks: {e}"));
603
604        self.is_connected.store(false, Ordering::Relaxed);
605
606        log::info!("Disconnected");
607        tasks_result
608    }
609
610    /// Returns whether the client is currently connected.
611    fn is_connected(&self) -> bool {
612        self.is_connected.load(Ordering::Relaxed)
613    }
614
615    fn is_disconnected(&self) -> bool {
616        !self.is_connected()
617    }
618
619    /// Subscribes to instrument definition data for the specified instrument.
620    ///
621    /// # Errors
622    ///
623    /// Returns an error if the subscription request fails.
624    fn subscribe_instrument(&mut self, cmd: SubscribeInstrument) -> anyhow::Result<()> {
625        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
626            let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
627            let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
628
629            self.symbol_venue_map
630                .insert(instrument_id.symbol, instrument_id.venue);
631            let symbol = instrument_id.symbol.to_string();
632
633            let subscription = Subscription::builder()
634                .schema(databento::dbn::Schema::Definition)
635                .symbols(symbol)
636                .build();
637
638            self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
639
640            Ok(())
641        })
642    }
643
644    /// Subscribes to quote tick data for the specified instruments.
645    ///
646    /// # Errors
647    ///
648    /// Returns an error if the subscription request fails.
649    fn subscribe_quotes(&mut self, cmd: SubscribeQuotes) -> anyhow::Result<()> {
650        self.subscribe_batch(
651            cmd.instrument_ids,
652            cmd.params.as_ref(),
653            dbn::Schema::Mbp1,
654            QUOTE_SCHEMAS,
655        )
656    }
657
658    /// Subscribes to trade tick data for the specified instruments.
659    ///
660    /// # Errors
661    ///
662    /// Returns an error if the subscription request fails.
663    fn subscribe_trades(&mut self, cmd: SubscribeTrades) -> anyhow::Result<()> {
664        self.subscribe_batch(
665            cmd.instrument_ids,
666            cmd.params.as_ref(),
667            dbn::Schema::Trades,
668            TRADE_SCHEMAS,
669        )
670    }
671
672    /// Subscribes to order book delta updates for the specified instruments.
673    ///
674    /// # Errors
675    ///
676    /// Returns an error if the subscription request fails.
677    fn subscribe_book_deltas(&mut self, cmd: SubscribeBookDeltas) -> anyhow::Result<()> {
678        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
679            let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
680            let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
681
682            self.symbol_venue_map
683                .insert(instrument_id.symbol, instrument_id.venue);
684            let symbol = instrument_id.symbol.to_string();
685
686            let subscription = Subscription::builder()
687                .schema(databento::dbn::Schema::Mbo) // Market by order for book deltas
688                .symbols(symbol)
689                .build();
690
691            self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
692
693            Ok(())
694        })
695    }
696
697    /// Subscribes to `DatabentoStatistics` or `DatabentoImbalance` for the instrument given by the
698    /// identifier of the data type.
699    ///
700    /// # Errors
701    ///
702    /// Returns an error if the data type is not supported, has metadata, has a missing or invalid
703    /// instrument ID identifier, or the subscription request fails.
704    fn subscribe(&mut self, cmd: SubscribeCustomData) -> anyhow::Result<()> {
705        let schema = custom_data_schema(&cmd.data_type)?;
706        let instrument_id = custom_data_instrument_id(&cmd.data_type)?;
707        self.subscribe_schema(instrument_id, schema)
708    }
709
710    /// Subscribes to 10-level order book depth snapshots for the specified instrument.
711    ///
712    /// # Errors
713    ///
714    /// Returns an error if the requested depth is not 10, the symbol is not a raw symbol, or the
715    /// subscription request fails.
716    fn subscribe_book_depth(&mut self, cmd: SubscribeBookDepth) -> anyhow::Result<()> {
717        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
718            let cmd = cmd.with_instrument_ids(instrument_id);
719            if let Some(depth) = cmd.depth {
720                anyhow::ensure!(
721                    depth.get() == MBP10_DEPTH,
722                    "Unsupported book depth {depth} for Databento `mbp-10`, only a depth of {MBP10_DEPTH} is available",
723                );
724            }
725
726            self.subscribe_schema(instrument_id, dbn::Schema::Mbp10)
727        })
728    }
729
730    /// Subscribes to OHLCV bars for the specified bar type.
731    ///
732    /// # Errors
733    ///
734    /// Returns an error if the bar type has no matching OHLCV schema or the subscription request
735    /// fails.
736    fn subscribe_bars(&mut self, cmd: SubscribeBars) -> anyhow::Result<()> {
737        execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
738            let schema = ohlcv_schema(bar_type)?;
739            self.subscribe_schema(bar_type.instrument_id(), schema)
740        })
741    }
742
743    /// Subscribes to instrument status updates for the specified instruments.
744    ///
745    /// # Errors
746    ///
747    /// Returns an error if the subscription request fails.
748    fn subscribe_instrument_status(
749        &mut self,
750        cmd: SubscribeInstrumentStatus,
751    ) -> anyhow::Result<()> {
752        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
753            let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
754            let start_after_subscribe = self.get_or_create_feed_handler(&dataset);
755
756            self.symbol_venue_map
757                .insert(instrument_id.symbol, instrument_id.venue);
758            let symbol = instrument_id.symbol.to_string();
759
760            let subscription = Subscription::builder()
761                .schema(databento::dbn::Schema::Status)
762                .symbols(symbol)
763                .build();
764
765            self.send_subscription_to_dataset(&dataset, None, subscription, start_after_subscribe)?;
766
767            Ok(())
768        })
769    }
770
771    // Unsubscribe methods
772    fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<()> {
773        log::debug!(
774            "Databento does not support granular unsubscribing, ignoring quote unsubscribe request for {:?}",
775            cmd.instrument_ids
776        );
777        Ok(())
778    }
779
780    fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> anyhow::Result<()> {
781        // Note: Databento live API doesn't support granular unsubscribing.
782        // The feed handler manages subscriptions and can handle reconnections
783        // with the appropriate subscription state.
784        log::debug!(
785            "Databento does not support granular unsubscribing - ignoring unsubscribe request for {:?}",
786            cmd.instrument_ids
787        );
788
789        Ok(())
790    }
791
792    fn unsubscribe_book_deltas(&mut self, cmd: &UnsubscribeBookDeltas) -> anyhow::Result<()> {
793        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
794            // Note: Databento live API doesn't support granular unsubscribing.
795            // The feed handler manages subscriptions and can handle reconnections
796            // with the appropriate subscription state.
797            log::debug!(
798                "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
799            );
800
801            Ok(())
802        })
803    }
804
805    fn unsubscribe(&mut self, cmd: &UnsubscribeCustomData) -> anyhow::Result<()> {
806        // Note: Databento live API doesn't support granular unsubscribing.
807        // The feed handler manages subscriptions and can handle reconnections
808        // with the appropriate subscription state.
809        log::debug!(
810            "Databento does not support granular unsubscribing - ignoring unsubscribe request for {}",
811            cmd.data_type
812        );
813
814        Ok(())
815    }
816
817    fn unsubscribe_book_depth(&mut self, cmd: &UnsubscribeBookDepth) -> anyhow::Result<()> {
818        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
819            // Note: Databento live API doesn't support granular unsubscribing.
820            // The feed handler manages subscriptions and can handle reconnections
821            // with the appropriate subscription state.
822            log::debug!(
823                "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
824            );
825
826            Ok(())
827        })
828    }
829
830    fn unsubscribe_bars(&mut self, cmd: &UnsubscribeBars) -> anyhow::Result<()> {
831        execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
832            // Note: Databento live API doesn't support granular unsubscribing.
833            // The feed handler manages subscriptions and can handle reconnections
834            // with the appropriate subscription state.
835            log::debug!(
836                "Databento does not support granular unsubscribing - ignoring unsubscribe request for {bar_type}"
837            );
838
839            Ok(())
840        })
841    }
842
843    fn unsubscribe_instrument_status(
844        &mut self,
845        cmd: &UnsubscribeInstrumentStatus,
846    ) -> anyhow::Result<()> {
847        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
848            // Note: Databento live API doesn't support granular unsubscribing.
849            // The feed handler manages subscriptions and can handle reconnections
850            // with the appropriate subscription state.
851            log::debug!(
852                "Databento does not support granular unsubscribing - ignoring unsubscribe request for {instrument_id}"
853            );
854
855            Ok(())
856        })
857    }
858
859    fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
860        log::debug!("Request instruments: {request:?}");
861
862        let historical_client = self.historical.clone();
863        let data_sender = self.data_sender.clone();
864        let dataset = request
865            .venue
866            .map(|venue| self.get_dataset_for_venue(venue))
867            .transpose()?
868            .unwrap_or_else(|| "GLBX.MDP3".to_string());
869        let request_id = request.request_id;
870        let client_id = request.client_id.unwrap_or(self.client_id);
871        let venue = request.venue.unwrap_or(*DATABENTO_VENUE);
872        let start_nanos = datetime_to_unix_nanos(request.start);
873        let end_nanos = datetime_to_unix_nanos(request.end);
874        let request_params = request.params;
875        let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
876
877        self.spawn_task(async move {
878            let query_params = instruments_query_params(dataset, query_start, query_end);
879
880            match historical_client.get_range_instruments(query_params).await {
881                Ok(instruments) => {
882                    log::debug!("Retrieved {} instruments", instruments.len());
883
884                    let response = DataResponse::Instruments(InstrumentsResponse::new(
885                        request_id,
886                        client_id,
887                        venue,
888                        instruments,
889                        start_nanos,
890                        end_nanos,
891                        get_atomic_clock_realtime().get_time_ns(),
892                        request_params,
893                    ));
894
895                    send_data_response(&data_sender, response, "instruments");
896                }
897                Err(e) => {
898                    send_request_result(&data_sender, request_id, None, &[e.to_string()]);
899                }
900            }
901        });
902
903        Ok(())
904    }
905
906    fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
907        let groups = self.group_request_instruments(&request.instrument_ids)?;
908        let historical_client = self.historical.clone();
909        let data_sender = self.data_sender.clone();
910        let client_id = request.client_id.unwrap_or(self.client_id);
911        let start = datetime_to_unix_nanos(request.start);
912        let end = datetime_to_unix_nanos(request.end);
913        let (query_start, query_end) = resolve_request_time_range(start, end);
914        self.spawn_task(async move {
915            let mut data = Vec::new();
916            let mut failures = Vec::new();
917
918            for (dataset, ids) in groups {
919                let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
920
921                let params = RangeQueryParams {
922                    dataset: dataset.clone(),
923                    symbols,
924                    start: query_start,
925                    end: query_end,
926                    limit: None,
927                    price_precision: None,
928                };
929
930                match historical_client.get_range_instruments(params).await {
931                    Ok(instruments) => {
932                        for id in ids {
933                            if let Some(instrument) = requested_instrument(&instruments, id) {
934                                data.push(instrument);
935                            } else {
936                                failures.push(format!("Instrument not found: {id}"));
937                            }
938                        }
939                    }
940                    Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
941                }
942            }
943
944            let ts_init = get_atomic_clock_realtime().get_time_ns();
945
946            let response = if request.instrument_ids.len() == 1 && data.len() == 1 {
947                let instrument = data.remove(0);
948                Some(DataResponse::Instrument(Box::new(InstrumentResponse::new(
949                    request.request_id,
950                    client_id,
951                    instrument.id(),
952                    instrument,
953                    start,
954                    end,
955                    ts_init,
956                    request.params,
957                ))))
958            } else if request.instrument_ids.len() > 1 {
959                Some(DataResponse::Instruments(InstrumentsResponse::new(
960                    request.request_id,
961                    client_id,
962                    *DATABENTO_VENUE,
963                    data,
964                    start,
965                    end,
966                    ts_init,
967                    request.params,
968                )))
969            } else {
970                None
971            };
972
973            send_request_result(&data_sender, request.request_id, response, &failures);
974        });
975
976        Ok(())
977    }
978
979    fn request_quotes(&self, request: RequestQuotes) -> anyhow::Result<()> {
980        anyhow::ensure!(
981            request.instrument_ids.len() == 1 || request.limit.is_none(),
982            "A record limit is unsupported for multi-instrument requests"
983        );
984        let groups = self.group_request_instruments(&request.instrument_ids)?;
985        let historical_client = self.historical.clone();
986        let data_sender = self.data_sender.clone();
987        let client_id = request.client_id.unwrap_or(self.client_id);
988        let start = datetime_to_unix_nanos(request.start);
989        let end = datetime_to_unix_nanos(request.end);
990        let limit = request.limit.map(|value| value.get() as u64);
991        let precision = price_precision_from_params(request.params.as_ref())?;
992        let schema = schema_from_params(request.params.as_ref(), dbn::Schema::Mbp1, QUOTE_SCHEMAS)?
993            .to_string();
994        let (query_start, query_end) = resolve_request_time_range(start, end);
995        self.spawn_task(async move {
996            let mut data = Vec::new();
997            let mut failures = Vec::new();
998
999            for (dataset, ids) in groups {
1000                for &id in &ids {
1001                    seed_price_precision_if_needed(
1002                        &historical_client,
1003                        &dataset,
1004                        id,
1005                        query_start,
1006                        query_end,
1007                        precision,
1008                    )
1009                    .await;
1010                }
1011
1012                let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
1013
1014                let params = RangeQueryParams {
1015                    dataset: dataset.clone(),
1016                    symbols,
1017                    start: query_start,
1018                    end: query_end,
1019                    limit,
1020                    price_precision: precision,
1021                };
1022
1023                match historical_client
1024                    .get_range_quotes(params, Some(schema.clone()))
1025                    .await
1026                {
1027                    Ok(rows) => data.extend(rows),
1028                    Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
1029                }
1030            }
1031
1032            data.sort_by_key(|row| (row.ts_init, row.instrument_id));
1033            let response = DataResponse::Quotes(QuotesResponse::new(
1034                request.request_id,
1035                client_id,
1036                request.instrument_ids,
1037                data,
1038                start,
1039                end,
1040                get_atomic_clock_realtime().get_time_ns(),
1041                request.params,
1042            ));
1043            send_request_result(&data_sender, request.request_id, Some(response), &failures);
1044        });
1045
1046        Ok(())
1047    }
1048
1049    fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
1050        anyhow::ensure!(
1051            request.instrument_ids.len() == 1 || request.limit.is_none(),
1052            "A record limit is unsupported for multi-instrument requests"
1053        );
1054        let groups = self.group_request_instruments(&request.instrument_ids)?;
1055        let historical_client = self.historical.clone();
1056        let data_sender = self.data_sender.clone();
1057        let client_id = request.client_id.unwrap_or(self.client_id);
1058        let start = datetime_to_unix_nanos(request.start);
1059        let end = datetime_to_unix_nanos(request.end);
1060        let limit = request.limit.map(|value| value.get() as u64);
1061        let precision = price_precision_from_params(request.params.as_ref())?;
1062        let schema =
1063            schema_from_params(request.params.as_ref(), dbn::Schema::Trades, TRADE_SCHEMAS)?
1064                .to_string();
1065        let (query_start, query_end) = resolve_request_time_range(start, end);
1066        self.spawn_task(async move {
1067            let mut data = Vec::new();
1068            let mut failures = Vec::new();
1069
1070            for (dataset, ids) in groups {
1071                for &id in &ids {
1072                    seed_price_precision_if_needed(
1073                        &historical_client,
1074                        &dataset,
1075                        id,
1076                        query_start,
1077                        query_end,
1078                        precision,
1079                    )
1080                    .await;
1081                }
1082
1083                let symbols = historical_client.prepare_symbols_from_instrument_ids(&ids);
1084
1085                let params = RangeQueryParams {
1086                    dataset: dataset.clone(),
1087                    symbols,
1088                    start: query_start,
1089                    end: query_end,
1090                    limit,
1091                    price_precision: precision,
1092                };
1093
1094                match historical_client
1095                    .get_range_trades(params, Some(schema.clone()))
1096                    .await
1097                {
1098                    Ok(rows) => data.extend(rows),
1099                    Err(e) => failures.push(format!("Dataset {dataset}: {e}")),
1100                }
1101            }
1102
1103            data.sort_by_key(|row| (row.ts_init, row.instrument_id));
1104            let response = DataResponse::Trades(TradesResponse::new(
1105                request.request_id,
1106                client_id,
1107                request.instrument_ids,
1108                data,
1109                start,
1110                end,
1111                get_atomic_clock_realtime().get_time_ns(),
1112                request.params,
1113            ));
1114            send_request_result(&data_sender, request.request_id, Some(response), &failures);
1115        });
1116
1117        Ok(())
1118    }
1119
1120    fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
1121        let bar_type = request.single_bar_type()?;
1122        let aggregation = bar_type.spec().aggregation;
1123        anyhow::ensure!(
1124            matches!(
1125                aggregation,
1126                BarAggregation::Second
1127                    | BarAggregation::Minute
1128                    | BarAggregation::Hour
1129                    | BarAggregation::Day
1130            ),
1131            "Unsupported bar aggregation: {aggregation:?}"
1132        );
1133
1134        log::debug!("Request bars: {request:?}");
1135
1136        let historical_client = self.historical.clone();
1137        let data_sender = self.data_sender.clone();
1138        let instrument_id = bar_type.instrument_id();
1139        let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1140        let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1141        let request_id = request.request_id;
1142        let client_id = request.client_id.unwrap_or(self.client_id);
1143        let start_nanos = datetime_to_unix_nanos(request.start);
1144        let end_nanos = datetime_to_unix_nanos(request.end);
1145        let limit = request.limit.map(|limit| limit.get() as u64);
1146        let request_params = request.params;
1147        let price_precision = price_precision_from_params(request_params.as_ref())?;
1148        let timestamp_on_close = self.config.bars_timestamp_on_close;
1149        let spec = BarSpecification::new(1, aggregation, PriceType::Last);
1150        let interval = DurationNanos::try_from(spec.timedelta()).expect("supported OHLCV interval");
1151        let (start, end) = resolve_request_bounds(start_nanos, end_nanos);
1152
1153        // Databento indexes bars by open, while ts_init always records the close
1154        let (query_start, query_end) = resolve_request_time_range(
1155            Some(start.saturating_sub(interval)),
1156            Some(end.saturating_sub(interval)),
1157        );
1158
1159        self.spawn_task(async move {
1160            seed_price_precision_if_needed(
1161                &historical_client,
1162                dataset.as_str(),
1163                instrument_id,
1164                query_start,
1165                query_end,
1166                price_precision,
1167            )
1168            .await;
1169
1170            let params = RangeQueryParams {
1171                dataset,
1172                symbols,
1173                start: query_start,
1174                end: query_end,
1175                limit,
1176                price_precision,
1177            };
1178
1179            match historical_client
1180                .get_range_bars(params, aggregation, timestamp_on_close)
1181                .await
1182            {
1183                Ok(bars) => {
1184                    log::debug!("Retrieved {} bars", bars.len());
1185                    let response = DataResponse::Bars(BarsResponse::new(
1186                        request_id,
1187                        client_id,
1188                        bar_type,
1189                        bars,
1190                        start_nanos,
1191                        end_nanos,
1192                        get_atomic_clock_realtime().get_time_ns(),
1193                        request_params,
1194                    ));
1195
1196                    send_data_response(&data_sender, response, "bars");
1197                }
1198                Err(e) => {
1199                    send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1200                }
1201            }
1202        });
1203
1204        Ok(())
1205    }
1206
1207    fn request_book_depth(&self, request: RequestBookDepth) -> anyhow::Result<()> {
1208        let instrument_id = request.single_instrument_id()?;
1209
1210        log::debug!("Request book depth: {request:?}");
1211
1212        let historical_client = self.historical.clone();
1213        let data_sender = self.data_sender.clone();
1214        let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1215        let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1216        let request_id = request.request_id;
1217        let client_id = request.client_id.unwrap_or(self.client_id);
1218        let start_nanos = datetime_to_unix_nanos(request.start);
1219        let end_nanos = datetime_to_unix_nanos(request.end);
1220        let limit = request.limit.map(|limit| limit.get() as u64);
1221        let depth = request.depth.map(|depth| depth.get());
1222        let request_params = request.params;
1223        let price_precision = price_precision_from_params(request_params.as_ref())?;
1224        let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
1225
1226        self.spawn_task(async move {
1227            seed_price_precision_if_needed(
1228                &historical_client,
1229                dataset.as_str(),
1230                instrument_id,
1231                query_start,
1232                query_end,
1233                price_precision,
1234            )
1235            .await;
1236
1237            let params = RangeQueryParams {
1238                dataset,
1239                symbols,
1240                start: query_start,
1241                end: query_end,
1242                limit,
1243                price_precision,
1244            };
1245
1246            match historical_client
1247                .get_range_order_book_depth(params, depth)
1248                .await
1249            {
1250                Ok(depths) => {
1251                    log::debug!("Retrieved {} order book depths", depths.len());
1252                    let response = DataResponse::BookDepth(BookDepthResponse::new(
1253                        request_id,
1254                        client_id,
1255                        instrument_id,
1256                        depths,
1257                        start_nanos,
1258                        end_nanos,
1259                        get_atomic_clock_realtime().get_time_ns(),
1260                        request_params,
1261                    ));
1262
1263                    send_data_response(&data_sender, response, "book depth");
1264                }
1265                Err(e) => {
1266                    send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1267                }
1268            }
1269        });
1270
1271        Ok(())
1272    }
1273
1274    fn request_book_deltas(&self, request: RequestBookDeltas) -> anyhow::Result<()> {
1275        let instrument_id = request.single_instrument_id()?;
1276
1277        log::debug!("Request book deltas: {request:?}");
1278
1279        let historical_client = self.historical.clone();
1280        let data_sender = self.data_sender.clone();
1281        let dataset = self.get_dataset_for_venue(instrument_id.venue)?;
1282        let symbols = historical_client.prepare_symbols_from_instrument_ids(&[instrument_id]);
1283        let request_id = request.request_id;
1284        let client_id = request.client_id.unwrap_or(self.client_id);
1285        let start_nanos = datetime_to_unix_nanos(request.start);
1286        let end_nanos = datetime_to_unix_nanos(request.end);
1287        let limit = request.limit.map(|limit| limit.get() as u64);
1288        let request_params = request.params;
1289        let price_precision = price_precision_from_params(request_params.as_ref())?;
1290        let (query_start, query_end) = resolve_request_time_range(start_nanos, end_nanos);
1291
1292        self.spawn_task(async move {
1293            seed_price_precision_if_needed(
1294                &historical_client,
1295                dataset.as_str(),
1296                instrument_id,
1297                query_start,
1298                query_end,
1299                price_precision,
1300            )
1301            .await;
1302
1303            let params = RangeQueryParams {
1304                dataset,
1305                symbols,
1306                start: query_start,
1307                end: query_end,
1308                limit,
1309                price_precision,
1310            };
1311
1312            match historical_client.get_range_order_book_deltas(params).await {
1313                Ok(deltas) => {
1314                    log::debug!("Retrieved {} order book deltas", deltas.len());
1315                    let response = BookDeltasResponse::new(
1316                        request_id,
1317                        client_id,
1318                        instrument_id,
1319                        deltas,
1320                        start_nanos,
1321                        end_nanos,
1322                        get_atomic_clock_realtime().get_time_ns(),
1323                        request_params,
1324                    );
1325
1326                    send_book_deltas_response(&data_sender, response);
1327                }
1328                Err(e) => {
1329                    send_request_result(&data_sender, request_id, None, &[e.to_string()]);
1330                }
1331            }
1332        });
1333
1334        Ok(())
1335    }
1336}
1337
1338impl DatabentoDataClient {
1339    fn subscribe_batch(
1340        &self,
1341        instrument_ids: Vec<InstrumentId>,
1342        params: Option<&Params>,
1343        default_schema: dbn::Schema,
1344        schemas: &[dbn::Schema],
1345    ) -> anyhow::Result<()> {
1346        let price_precision = price_precision_from_params(params)?;
1347        let schema = schema_from_params(params, default_schema, schemas)?;
1348        let ids = instrument_ids.into_instrument_ids()?;
1349        let groups = self.group_instruments(&ids)?;
1350
1351        let mut accepted = Vec::new();
1352        let mut failure = None;
1353
1354        for (dataset, ids) in &groups {
1355            match self.subscribe_dataset(dataset, ids, schema, price_precision) {
1356                Ok(()) => accepted.extend_from_slice(ids),
1357                Err(e) => {
1358                    log::error!(
1359                        "Databento subscription enqueue failed: dataset={dataset}, schema={schema}, instrument_ids={ids:?}"
1360                    );
1361
1362                    if failure.is_none() {
1363                        failure = Some(e);
1364                    }
1365                }
1366            }
1367        }
1368
1369        match failure {
1370            Some(e) => Err(SubscriptionError::instruments(accepted, e).into()),
1371            None => Ok(()),
1372        }
1373    }
1374
1375    fn group_request_instruments(
1376        &self,
1377        ids: &[InstrumentId],
1378    ) -> anyhow::Result<IndexMap<String, Vec<InstrumentId>>> {
1379        let groups = self.group_instruments(ids)?;
1380        for ids in groups.values() {
1381            let symbols = ids.iter().map(|id| id.symbol.as_str()).collect::<Vec<_>>();
1382            check_consistent_symbology(&symbols)?;
1383        }
1384
1385        Ok(groups)
1386    }
1387
1388    fn group_instruments(
1389        &self,
1390        ids: &[InstrumentId],
1391    ) -> anyhow::Result<IndexMap<String, Vec<InstrumentId>>> {
1392        let mut groups: IndexMap<String, Vec<InstrumentId>> = IndexMap::new();
1393        let mut symbol_venues = AHashMap::new();
1394
1395        for &id in ids {
1396            let dataset = self.get_dataset_for_venue(id.venue)?;
1397            if let Some(venue) = symbol_venues.insert(id.symbol, id.venue) {
1398                anyhow::ensure!(
1399                    venue == id.venue,
1400                    "Identifiers share raw symbol {} across different venues",
1401                    id.symbol
1402                );
1403            }
1404
1405            groups.entry(dataset).or_default().push(id);
1406        }
1407
1408        Ok(groups)
1409    }
1410
1411    fn subscribe_dataset(
1412        &self,
1413        dataset: &str,
1414        ids: &[InstrumentId],
1415        schema: dbn::Schema,
1416        price_precision: Option<u8>,
1417    ) -> anyhow::Result<()> {
1418        let symbols: Vec<String> = ids.iter().map(|id| id.symbol.to_string()).collect();
1419        let precisions: Vec<(Symbol, u8)> = ids
1420            .iter()
1421            .filter_map(|id| price_precision.map(|precision| (id.symbol, precision)))
1422            .collect();
1423        let subscription = Subscription::builder()
1424            .schema(schema)
1425            .symbols(symbols)
1426            .build();
1427        let start_after_subscribe = self.get_or_create_feed_handler(dataset);
1428        let previous_venues: Vec<_> = ids
1429            .iter()
1430            .map(|id| (id.symbol, self.symbol_venue_map.get_cloned(&id.symbol)))
1431            .collect();
1432
1433        for id in ids {
1434            self.symbol_venue_map.insert(id.symbol, id.venue);
1435        }
1436
1437        if let Err(e) = self.send_subscription_to_dataset(
1438            dataset,
1439            precisions,
1440            subscription,
1441            start_after_subscribe,
1442        ) {
1443            for (symbol, previous) in previous_venues {
1444                if let Some(venue) = previous {
1445                    self.symbol_venue_map.insert(symbol, venue);
1446                } else {
1447                    self.symbol_venue_map.remove(&symbol);
1448                }
1449            }
1450
1451            return Err(e);
1452        }
1453
1454        Ok(())
1455    }
1456}
1457
1458fn instruments_query_params(
1459    dataset: String,
1460    start_nanos: UnixNanos,
1461    end_nanos: Option<UnixNanos>,
1462) -> RangeQueryParams {
1463    RangeQueryParams {
1464        dataset,
1465        symbols: vec!["ALL_SYMBOLS".to_string()],
1466        start: start_nanos,
1467        end: end_nanos,
1468        limit: None,
1469        price_precision: None,
1470    }
1471}
1472
1473fn instrument_query_params(
1474    dataset: String,
1475    instrument_id: InstrumentId,
1476    start_nanos: UnixNanos,
1477    end_nanos: Option<UnixNanos>,
1478) -> RangeQueryParams {
1479    RangeQueryParams {
1480        dataset,
1481        symbols: vec![instrument_id_to_symbol_string(
1482            instrument_id,
1483            &mut AHashMap::new(),
1484        )],
1485        start: start_nanos,
1486        end: end_nanos,
1487        limit: None,
1488        price_precision: None,
1489    }
1490}
1491
1492fn resolve_request_time_range(
1493    start_nanos: Option<UnixNanos>,
1494    end_nanos: Option<UnixNanos>,
1495) -> (UnixNanos, Option<UnixNanos>) {
1496    let (start, end) = resolve_request_bounds(start_nanos, end_nanos);
1497
1498    // Engine bounds include the end; Databento's query excludes it
1499    (start, Some(end.saturating_add(DurationNanos::new(1))))
1500}
1501
1502fn resolve_request_bounds(
1503    start_nanos: Option<UnixNanos>,
1504    end_nanos: Option<UnixNanos>,
1505) -> (UnixNanos, UnixNanos) {
1506    let end = end_nanos.unwrap_or_else(|| get_atomic_clock_realtime().get_time_ns());
1507    let mut start = start_nanos.unwrap_or_else(|| start_of_utc_day(end));
1508
1509    if start > end {
1510        start = end;
1511    }
1512
1513    (start, end)
1514}
1515
1516fn start_of_utc_day(timestamp: UnixNanos) -> UnixNanos {
1517    timestamp.floor(DurationNanos::from_days(1))
1518}
1519
1520async fn seed_price_precision_if_needed(
1521    historical_client: &DatabentoHistoricalClient,
1522    dataset: &str,
1523    instrument_id: InstrumentId,
1524    start_nanos: UnixNanos,
1525    end_nanos: Option<UnixNanos>,
1526    price_precision: Option<u8>,
1527) {
1528    if price_precision.is_some()
1529        || historical_client
1530            .price_precision(instrument_id.symbol)
1531            .is_some()
1532    {
1533        return;
1534    }
1535
1536    let query_start = start_of_utc_day(start_nanos);
1537    let query_params =
1538        instrument_query_params(dataset.to_string(), instrument_id, query_start, end_nanos);
1539
1540    if let Err(e) = historical_client.get_range_instruments(query_params).await {
1541        log::warn!("Failed to seed price precision for {instrument_id}: {e}");
1542    }
1543}
1544
1545fn send_request_result(
1546    data_sender: &EventSender<DataEvent>,
1547    request_id: nautilus_core::UUID4,
1548    response: Option<DataResponse>,
1549    failures: &[String],
1550) {
1551    let event = if failures.is_empty() {
1552        let Some(response) = response else {
1553            return;
1554        };
1555
1556        let envelope = DataResponseEnvelope::new(response, true, None);
1557        DataEvent::ResponseEnvelope(envelope)
1558    } else {
1559        DataEvent::RequestFailed(DataRequestFailed {
1560            request_id,
1561            reason: failures.join("; "),
1562            response,
1563        })
1564    };
1565
1566    if let Err(e) = data_sender.send(event) {
1567        log::error!("Failed to send request result: {e}");
1568    }
1569}
1570
1571fn send_data_response(data_sender: &EventSender<DataEvent>, response: DataResponse, label: &str) {
1572    let envelope = DataResponseEnvelope::new(response, true, None);
1573    if let Err(e) = data_sender.send(DataEvent::ResponseEnvelope(envelope)) {
1574        log::error!("Failed to send {label} response: {e}");
1575    }
1576}
1577
1578fn send_book_deltas_response(sender: &EventSender<DataEvent>, response: BookDeltasResponse) {
1579    let responses = partition_book_deltas_response(response);
1580    let last = responses.len() - 1;
1581    for (index, response) in responses.into_iter().enumerate() {
1582        let envelope =
1583            DataResponseEnvelope::new(DataResponse::BookDeltas(response), index == last, None);
1584
1585        if let Err(e) = sender.send(DataEvent::ResponseEnvelope(envelope)) {
1586            log::error!("Failed to send book deltas response: {e}");
1587        }
1588    }
1589}
1590
1591fn partition_book_deltas_response(response: BookDeltasResponse) -> Vec<BookDeltasResponse> {
1592    if response.data.is_empty() {
1593        return vec![response];
1594    }
1595
1596    let mut partitions = IndexMap::new();
1597    for delta in response.data.iter().copied() {
1598        partitions
1599            .entry(delta.instrument_id)
1600            .or_insert_with(Vec::new)
1601            .push(delta);
1602    }
1603
1604    partitions
1605        .into_iter()
1606        .map(|(instrument_id, data)| {
1607            let mut child = response.clone();
1608            child.instrument_id = instrument_id;
1609            child.data = data.into();
1610            child
1611        })
1612        .collect()
1613}
1614
1615fn requested_instrument(
1616    instruments: &[InstrumentAny],
1617    instrument_id: InstrumentId,
1618) -> Option<InstrumentAny> {
1619    instruments
1620        .iter()
1621        .rev()
1622        .find(|instrument| instrument.id() == instrument_id)
1623        .cloned()
1624}
1625
1626fn price_precision_from_params(params: Option<&Params>) -> anyhow::Result<Option<u8>> {
1627    let Some(price_precision) = params.and_then(|params| params.get_u64(PRICE_PRECISION_PARAM))
1628    else {
1629        return Ok(None);
1630    };
1631
1632    Ok(Some(u8::try_from(price_precision).map_err(|_| {
1633        anyhow::anyhow!(
1634            "`{PRICE_PRECISION_PARAM}` must be less than or equal to {}",
1635            u8::MAX
1636        )
1637    })?))
1638}
1639
1640fn custom_data_for_instrument(
1641    data: Arc<dyn CustomDataTrait>,
1642    instrument_id: InstrumentId,
1643) -> CustomData {
1644    let data_type = DataType::new(data.type_name(), None, Some(instrument_id.to_string()));
1645    CustomData::new(data, data_type)
1646}
1647
1648fn custom_data_schema(data_type: &DataType) -> anyhow::Result<dbn::Schema> {
1649    match data_type.type_name() {
1650        name if name == DatabentoStatistics::type_name_static() => Ok(dbn::Schema::Statistics),
1651        name if name == DatabentoImbalance::type_name_static() => Ok(dbn::Schema::Imbalance),
1652        name => anyhow::bail!(
1653            "Unsupported custom data type for Databento live subscription: {name}, expected {} or {}",
1654            DatabentoStatistics::type_name_static(),
1655            DatabentoImbalance::type_name_static(),
1656        ),
1657    }
1658}
1659
1660fn custom_data_instrument_id(data_type: &DataType) -> anyhow::Result<InstrumentId> {
1661    // Records are published without metadata, so metadata would change the subscribed topic
1662    anyhow::ensure!(
1663        data_type
1664            .metadata()
1665            .is_none_or(|metadata| metadata.is_empty()),
1666        "{} subscriptions do not support metadata, received {}",
1667        data_type.type_name(),
1668        data_type.metadata_str(),
1669    );
1670
1671    let identifier = data_type.identifier().ok_or_else(|| {
1672        anyhow::anyhow!(
1673            "{} subscriptions require an instrument ID identifier",
1674            data_type.type_name()
1675        )
1676    })?;
1677
1678    InstrumentId::from_str(identifier)
1679        .map_err(|e| anyhow::anyhow!("Invalid instrument ID identifier `{identifier}`: {e}"))
1680}
1681
1682fn ohlcv_schema(bar_type: BarType) -> anyhow::Result<dbn::Schema> {
1683    let spec = bar_type.spec();
1684    anyhow::ensure!(
1685        spec.price_type == PriceType::Last,
1686        "Unsupported bar price type for Databento OHLCV: {spec}, only LAST is emitted",
1687    );
1688    anyhow::ensure!(
1689        spec.step.get() == 1,
1690        "Unsupported bar step for Databento OHLCV: {spec}, only a step of 1 is available",
1691    );
1692
1693    ohlcv_schema_from_aggregation(spec.aggregation)
1694}
1695
1696fn schema_from_params(
1697    params: Option<&Params>,
1698    default_schema: dbn::Schema,
1699    allowed_schemas: &[dbn::Schema],
1700) -> anyhow::Result<dbn::Schema> {
1701    let schema = if let Some(schema) = params.and_then(|params| params.get_str(SCHEMA_PARAM)) {
1702        dbn::Schema::from_str(schema)?
1703    } else {
1704        default_schema
1705    };
1706
1707    if allowed_schemas.contains(&schema) {
1708        return Ok(schema);
1709    }
1710
1711    let allowed = allowed_schemas
1712        .iter()
1713        .map(dbn::Schema::as_str)
1714        .collect::<Vec<_>>()
1715        .join(", ");
1716    anyhow::bail!(
1717        "Invalid `{SCHEMA_PARAM}` '{}'. Must be one of: {allowed}",
1718        schema.as_str()
1719    );
1720}
1721
1722fn send_subscription_commands(
1723    tx: &tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
1724    dataset: &str,
1725    price_precisions: impl IntoIterator<Item = (Symbol, u8)>,
1726    subscription: Subscription,
1727    start_after_subscribe: bool,
1728) -> anyhow::Result<()> {
1729    for (symbol, precision) in price_precisions {
1730        tx.send(HandlerCommand::SetPricePrecision(symbol, precision))
1731            .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1732    }
1733
1734    tx.send(HandlerCommand::Subscribe(subscription))
1735        .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1736
1737    if start_after_subscribe {
1738        tx.send(HandlerCommand::Start)
1739            .map_err(|e| anyhow::anyhow!("Failed to send command to dataset {dataset}: {e}"))?;
1740    }
1741
1742    Ok(())
1743}
1744
1745#[cfg(test)]
1746mod tests {
1747    use std::{num::NonZeroUsize, path::PathBuf};
1748
1749    use databento::dbn::{
1750        MappingInterval, SymbolMapping,
1751        decode::{DbnMetadata, DecodeRecord, dbn::Decoder},
1752        encode::EncodeRecord,
1753    };
1754    use nautilus_common::{
1755        live::runner::replace_data_event_sender,
1756        msgbus::{self, switchboard::get_custom_topic},
1757    };
1758    use nautilus_core::UUID4;
1759    use nautilus_model::{
1760        data::{BatchView, NautilusDataType, OrderBookDelta, QuoteTick},
1761        enums::BookType,
1762        identifiers::{ClientId, InstrumentId},
1763        instruments::{CurrencyPair, InstrumentAny},
1764        types::{Currency, Price, Quantity},
1765    };
1766    use rstest::rstest;
1767    use serde_json::json;
1768    use time::macros::date;
1769    use tokio::{
1770        io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
1771        net::TcpListener,
1772    };
1773
1774    use super::*;
1775    use crate::enums::{DatabentoStatisticType, DatabentoStatisticUpdateAction};
1776
1777    #[derive(Clone, Copy)]
1778    enum SubscribeKind {
1779        Quotes,
1780        Trades,
1781    }
1782
1783    fn currency_pair(instrument_id: &str) -> InstrumentAny {
1784        currency_pair_with_ts_init(instrument_id, UnixNanos::default())
1785    }
1786
1787    fn test_data_client() -> DatabentoDataClient {
1788        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
1789        replace_data_event_sender(sender);
1790
1791        let config = DatabentoDataClientConfig::new(
1792            "32-character-with-lots-of-filler",
1793            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1794            true,
1795            true,
1796        );
1797        DatabentoDataClient::new(
1798            ClientId::from("DATABENTO-TEST"),
1799            config,
1800            get_atomic_clock_realtime(),
1801        )
1802        .expect("test client should initialize")
1803    }
1804
1805    #[rstest]
1806    #[case::quotes(NautilusDataType::QuoteTick, true)]
1807    #[case::trades(NautilusDataType::TradeTick, true)]
1808    #[case::bars_close(NautilusDataType::Bar, true)]
1809    #[case::bars_open(NautilusDataType::Bar, false)]
1810    #[tokio::test]
1811    #[ignore = "Requires DATABENTO_API_KEY and GLBX.MDP3 historical access"]
1812    async fn test_historical_request_list_live(
1813        #[case] kind: NautilusDataType,
1814        #[case] timestamp_on_close: bool,
1815    ) {
1816        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
1817        replace_data_event_sender(sender);
1818        let api_key = std::env::var("DATABENTO_API_KEY").unwrap();
1819
1820        let config = DatabentoDataClientConfig::new(
1821            api_key,
1822            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1823            true,
1824            timestamp_on_close,
1825        );
1826        let client_id = ClientId::from("DATABENTO-HISTORY");
1827        let client =
1828            DatabentoDataClient::new(client_id, config, get_atomic_clock_realtime()).unwrap();
1829        let mut ids = vec![
1830            InstrumentId::from("ESM4.GLBX"),
1831            InstrumentId::from("NQM4.GLBX"),
1832        ];
1833        let start = UnixNanos::from(1_717_423_200_000_000_000);
1834
1835        let end = if kind == NautilusDataType::Bar {
1836            ids.truncate(1);
1837            start
1838        } else {
1839            start + DurationNanos::from_secs(1)
1840        };
1841
1842        let request_id = UUID4::new();
1843        let params = Params::new([(PRICE_PRECISION_PARAM.to_owned(), json!(2))].into());
1844        let bar_type = "ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"
1845            .parse::<BarType>()
1846            .unwrap();
1847
1848        match kind {
1849            NautilusDataType::QuoteTick => client
1850                .request_quotes(RequestQuotes::new(
1851                    ids.clone(),
1852                    Some(start.to_datetime_utc()),
1853                    Some(end.to_datetime_utc()),
1854                    None,
1855                    Some(client_id),
1856                    request_id,
1857                    UnixNanos::from(37),
1858                    Some(params.clone()),
1859                ))
1860                .unwrap(),
1861            NautilusDataType::TradeTick => client
1862                .request_trades(RequestTrades::new(
1863                    ids.clone(),
1864                    Some(start.to_datetime_utc()),
1865                    Some(end.to_datetime_utc()),
1866                    None,
1867                    Some(client_id),
1868                    request_id,
1869                    UnixNanos::from(37),
1870                    Some(params.clone()),
1871                ))
1872                .unwrap(),
1873            NautilusDataType::Bar => client
1874                .request_bars(RequestBars::new(
1875                    [bar_type],
1876                    Some(start.to_datetime_utc()),
1877                    Some(end.to_datetime_utc()),
1878                    None,
1879                    Some(client_id),
1880                    request_id,
1881                    UnixNanos::from(37),
1882                    Some(params.clone()),
1883                ))
1884                .unwrap(),
1885            _ => panic!("unexpected live request kind"),
1886        }
1887
1888        let event = tokio::time::timeout(Duration::from_secs(60), receiver.recv())
1889            .await
1890            .unwrap()
1891            .unwrap();
1892
1893        let DataEvent::ResponseEnvelope(envelope) = event else {
1894            panic!("expected successful historical envelope, received {event:?}");
1895        };
1896
1897        let response = envelope.response().unwrap();
1898
1899        let (response_ids, rows) = match (kind, response) {
1900            (NautilusDataType::QuoteTick, DataResponse::Quotes(response)) => (
1901                response.instrument_ids.clone(),
1902                response
1903                    .data
1904                    .iter()
1905                    .map(|row| (row.ts_init, row.instrument_id))
1906                    .collect::<Vec<_>>(),
1907            ),
1908            (NautilusDataType::TradeTick, DataResponse::Trades(response)) => (
1909                response.instrument_ids.clone(),
1910                response
1911                    .data
1912                    .iter()
1913                    .map(|row| (row.ts_init, row.instrument_id))
1914                    .collect::<Vec<_>>(),
1915            ),
1916            (NautilusDataType::Bar, DataResponse::Bars(response)) => {
1917                let expected_event = if timestamp_on_close {
1918                    start
1919                } else {
1920                    start - DurationNanos::from_mins(1)
1921                };
1922
1923                assert_eq!(response.bar_type, bar_type);
1924                assert_eq!(response.data.len(), 1);
1925                assert_eq!(response.data[0].bar_type, bar_type);
1926                assert_eq!(response.data[0].ts_init, start);
1927                assert_eq!(response.data[0].ts_event, expected_event);
1928                (
1929                    vec![response.bar_type.instrument_id()],
1930                    response
1931                        .data
1932                        .iter()
1933                        .map(|row| (row.ts_init, row.bar_type.instrument_id()))
1934                        .collect::<Vec<_>>(),
1935                )
1936            }
1937            _ => panic!("unexpected historical response variant"),
1938        };
1939
1940        let mut actual_ids = rows.iter().map(|(_, id)| *id).collect::<Vec<_>>();
1941        actual_ids.sort_unstable();
1942        actual_ids.dedup();
1943        let mut expected_ids = ids.clone();
1944        expected_ids.sort_unstable();
1945
1946        assert_eq!(envelope.correlation_id(), &request_id);
1947        assert!(envelope.completed());
1948        assert_eq!(envelope.sub_correlation_id(), None);
1949        assert_eq!(envelope.failure_reason(), None);
1950        assert_eq!(response.correlation_id(), &request_id);
1951        assert_eq!(response.client_id(), &client_id);
1952        assert_eq!(response.start(), Some(start));
1953        assert_eq!(response.end(), Some(end));
1954        assert_eq!(response.params(), Some(&params));
1955        assert_eq!(response_ids, ids);
1956        assert_eq!(actual_ids, expected_ids);
1957        assert!(rows.is_sorted());
1958        assert!(rows.iter().all(|(ts, _)| *ts >= start && *ts <= end));
1959        assert_eq!(
1960            receiver.try_recv().unwrap_err(),
1961            tokio::sync::mpsc::error::TryRecvError::Empty
1962        );
1963    }
1964
1965    #[rstest]
1966    #[case::raw("ESM4.GLBX", "NQM4.GLBX")]
1967    #[case::parent("ES.FUT.GLBX", "NQ.FUT.GLBX")]
1968    fn test_historical_request_groups_keep_dataset_and_input_order(
1969        #[case] first: &str,
1970        #[case] third: &str,
1971    ) {
1972        let client = test_data_client();
1973        let first = InstrumentId::from(first);
1974        let second = InstrumentId::from("AAPL.EQUS");
1975        let third = InstrumentId::from(third);
1976        let groups = client
1977            .group_request_instruments(&[first, second, third])
1978            .unwrap();
1979        assert_eq!(
1980            groups.into_iter().collect::<Vec<_>>(),
1981            vec![
1982                ("GLBX.MDP3".to_owned(), vec![first, third]),
1983                ("EQUS.MINI".to_owned(), vec![second])
1984            ]
1985        );
1986    }
1987
1988    #[rstest]
1989    #[case::instruments(NautilusDataType::Instrument)]
1990    #[case::quotes(NautilusDataType::QuoteTick)]
1991    #[case::trades(NautilusDataType::TradeTick)]
1992    #[tokio::test]
1993    async fn test_historical_requests_reject_mixed_symbology_before_dispatch(
1994        #[case] kind: NautilusDataType,
1995        #[values(false, true)] earlier_dataset: bool,
1996    ) {
1997        let client = test_data_client();
1998        let mut ids = Vec::new();
1999        if earlier_dataset {
2000            ids.push(InstrumentId::from("AAPL.EQUS"));
2001        }
2002
2003        ids.extend([
2004            InstrumentId::from("ES.FUT.GLBX"),
2005            InstrumentId::from("ESM4.GLBX"),
2006        ]);
2007        let request_id = UUID4::new();
2008
2009        let result = match kind {
2010            NautilusDataType::Instrument => client.request_instrument(RequestInstrument::new(
2011                ids,
2012                None,
2013                None,
2014                None,
2015                request_id,
2016                UnixNanos::from(37),
2017                None,
2018            )),
2019            NautilusDataType::QuoteTick => client.request_quotes(RequestQuotes::new(
2020                ids,
2021                None,
2022                None,
2023                None,
2024                None,
2025                request_id,
2026                UnixNanos::from(37),
2027                None,
2028            )),
2029            NautilusDataType::TradeTick => client.request_trades(RequestTrades::new(
2030                ids,
2031                None,
2032                None,
2033                None,
2034                None,
2035                request_id,
2036                UnixNanos::from(37),
2037                None,
2038            )),
2039            _ => panic!("unexpected historical request kind"),
2040        };
2041
2042        assert_eq!(
2043            result.unwrap_err().to_string(),
2044            "Inconsistent symbology types: 'parent' for ES.FUT vs 'raw_symbol' for ESM4"
2045        );
2046        assert!(client.task_handles.is_empty());
2047    }
2048
2049    #[rstest]
2050    #[case::quotes(true)]
2051    #[case::trades(false)]
2052    fn test_historical_request_lists_reject_limit_before_dispatch(#[case] quotes: bool) {
2053        let client = test_data_client();
2054        let ids = [
2055            InstrumentId::from("ESM4.GLBX"),
2056            InstrumentId::from("AAPL.EQUS"),
2057        ];
2058
2059        let result = if quotes {
2060            client.request_quotes(RequestQuotes::new(
2061                ids,
2062                None,
2063                None,
2064                NonZeroUsize::new(5),
2065                None,
2066                UUID4::new(),
2067                UnixNanos::from(37),
2068                None,
2069            ))
2070        } else {
2071            client.request_trades(RequestTrades::new(
2072                ids,
2073                None,
2074                None,
2075                NonZeroUsize::new(5),
2076                None,
2077                UUID4::new(),
2078                UnixNanos::from(37),
2079                None,
2080            ))
2081        };
2082
2083        assert_eq!(
2084            result.unwrap_err().to_string(),
2085            "A record limit is unsupported for multi-instrument requests"
2086        );
2087    }
2088
2089    #[rstest]
2090    fn test_historical_bars_reject_unsupported_aggregation_before_dispatch() {
2091        let client = test_data_client();
2092
2093        let request = RequestBars::new(
2094            BarType::from("ESM4.GLBX-100-TICK-LAST-EXTERNAL"),
2095            None,
2096            None,
2097            None,
2098            None,
2099            UUID4::new(),
2100            UnixNanos::from(37),
2101            None,
2102        );
2103        assert_eq!(
2104            client.request_bars(request).unwrap_err().to_string(),
2105            "Unsupported bar aggregation: Tick"
2106        );
2107        assert!(client.task_handles.is_empty());
2108    }
2109
2110    #[rstest]
2111    fn test_dataset_failure_preserves_successful_group_rows() {
2112        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2113        let sender: EventSender<DataEvent> = sender.into();
2114        let root = UUID4::new();
2115        let first = InstrumentId::from("ESM6.GLBX");
2116        let second = InstrumentId::from("AAPL.EQUS");
2117
2118        let quote = QuoteTick {
2119            instrument_id: first,
2120            ts_init: UnixNanos::from(17),
2121            ..Default::default()
2122        };
2123
2124        let response = QuotesResponse::new(
2125            root,
2126            ClientId::from("HISTORY"),
2127            [first, second],
2128            vec![quote],
2129            Some(UnixNanos::from(11)),
2130            Some(UnixNanos::from(29)),
2131            UnixNanos::from(31),
2132            None,
2133        );
2134
2135        send_request_result(
2136            &sender,
2137            root,
2138            Some(DataResponse::Quotes(response)),
2139            &["Dataset EQUS: source unavailable".to_owned()],
2140        );
2141
2142        let DataEvent::RequestFailed(failure) = receiver.try_recv().unwrap() else {
2143            panic!("Expected failed dataset result");
2144        };
2145
2146        assert_eq!(failure.request_id, root);
2147        assert_eq!(failure.reason, "Dataset EQUS: source unavailable");
2148
2149        let Some(DataResponse::Quotes(response)) = failure.response else {
2150            panic!("Expected successful sibling rows");
2151        };
2152
2153        assert_eq!(response.correlation_id, root);
2154        assert_eq!(response.client_id, ClientId::from("HISTORY"));
2155        assert_eq!(response.instrument_ids, vec![first, second]);
2156        assert_eq!(response.data.as_ref(), &[quote]);
2157        assert_eq!(response.start, Some(UnixNanos::from(11)));
2158        assert_eq!(response.end, Some(UnixNanos::from(29)));
2159        assert_eq!(response.ts_init, UnixNanos::from(31));
2160        assert_eq!(response.params, None);
2161        assert_eq!(
2162            receiver.try_recv().unwrap_err(),
2163            tokio::sync::mpsc::error::TryRecvError::Empty
2164        );
2165    }
2166
2167    #[rstest]
2168    fn test_book_reply_completes_after_all_partitioned_payloads() {
2169        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2170        let sender: EventSender<DataEvent> = sender.into();
2171        let root = UUID4::new();
2172        let first = InstrumentId::from("ESM6.GLBX");
2173        let second = InstrumentId::from("ESU6.GLBX");
2174        let a = OrderBookDelta::clear(first, 11, UnixNanos::from(13), UnixNanos::from(17));
2175        let b = OrderBookDelta::clear(second, 19, UnixNanos::from(23), UnixNanos::from(29));
2176        let c = OrderBookDelta::clear(first, 31, UnixNanos::from(37), UnixNanos::from(41));
2177
2178        let response = BookDeltasResponse::new(
2179            root,
2180            ClientId::from("HISTORY"),
2181            InstrumentId::from("ES.FUT.GLBX"),
2182            vec![a, b, c],
2183            None,
2184            None,
2185            UnixNanos::from(43),
2186            None,
2187        );
2188        send_book_deltas_response(&sender, response);
2189
2190        let DataEvent::ResponseEnvelope(left) = receiver.try_recv().unwrap() else {
2191            panic!("expected first envelope");
2192        };
2193
2194        let DataEvent::ResponseEnvelope(right) = receiver.try_recv().unwrap() else {
2195            panic!("expected final envelope");
2196        };
2197
2198        let Some(DataResponse::BookDeltas(left_payload)) = left.response() else {
2199            panic!("expected first book");
2200        };
2201
2202        let Some(DataResponse::BookDeltas(right_payload)) = right.response() else {
2203            panic!("expected second book");
2204        };
2205
2206        assert_eq!(
2207            (left.correlation_id(), right.correlation_id()),
2208            (&root, &root)
2209        );
2210        assert!(!left.completed());
2211        assert!(right.completed());
2212        assert_eq!(left_payload.instrument_id, first);
2213        assert_eq!(right_payload.instrument_id, second);
2214        assert_eq!(left_payload.data.as_ref(), &vec![a, c]);
2215        assert_eq!(right_payload.data.as_ref(), &vec![b]);
2216        assert_eq!(
2217            receiver.try_recv().unwrap_err(),
2218            tokio::sync::mpsc::error::TryRecvError::Empty
2219        );
2220    }
2221
2222    #[rstest]
2223    #[tokio::test]
2224    async fn test_stop_closes_task_admission_until_disconnect_drains() {
2225        let mut client = test_data_client();
2226
2227        client
2228            .task_handles
2229            .spawn(async { std::future::pending::<()>().await })
2230            .unwrap();
2231        client.is_connected.store(true, Ordering::Relaxed);
2232
2233        client.stop().unwrap();
2234
2235        assert!(!client.task_handles.is_open());
2236        assert!(client.is_disconnected());
2237
2238        client.disconnect().await.unwrap();
2239        assert!(client.task_handles.is_empty());
2240    }
2241
2242    #[rstest]
2243    #[case("EQUS", "EQUS.PLUS")] // overrides the apply_default EQUS -> EQUS.MINI mapping
2244    #[case("GLBX", "EQUS.MINI")] // overrides the apply_default GLBX -> GLBX.MDP3 mapping
2245    fn test_venue_dataset_map_overrides_default(#[case] venue: &str, #[case] dataset: &str) {
2246        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2247        replace_data_event_sender(sender);
2248
2249        let mut config = DatabentoDataClientConfig::new(
2250            "32-character-with-lots-of-filler",
2251            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2252            true,
2253            true,
2254        );
2255        config.venue_dataset_map = IndexMap::from([(venue.to_string(), dataset.to_string())]);
2256
2257        let client = DatabentoDataClient::new(
2258            ClientId::from("DATABENTO-TEST"),
2259            config,
2260            get_atomic_clock_realtime(),
2261        )
2262        .expect("test client should initialize");
2263
2264        assert_eq!(
2265            client.get_dataset_for_venue(Venue::from(venue)).unwrap(),
2266            dataset
2267        );
2268
2269        // The override is targeted: an unrelated venue keeps its default.
2270        assert_eq!(
2271            client.get_dataset_for_venue(Venue::from("XCBO")).unwrap(),
2272            "OPRA.PILLAR"
2273        );
2274    }
2275
2276    #[rstest]
2277    fn test_config_base_url_overrides_default_to_none() {
2278        let config = DatabentoDataClientConfig::new(
2279            "32-character-with-lots-of-filler",
2280            PathBuf::from("test_publishers.json"),
2281            true,
2282            true,
2283        );
2284
2285        assert!(config.historical_base_url.is_none());
2286        assert!(config.live_gateway_addr.is_none());
2287    }
2288
2289    #[rstest]
2290    fn test_historical_base_url_override_invalid_errors() {
2291        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel::<DataEvent>();
2292        replace_data_event_sender(sender);
2293
2294        let mut config = DatabentoDataClientConfig::new(
2295            "32-character-with-lots-of-filler",
2296            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2297            true,
2298            true,
2299        );
2300        config.historical_base_url = Some("not a url".to_string());
2301
2302        let result = DatabentoDataClient::new(
2303            ClientId::from("DATABENTO-TEST"),
2304            config,
2305            get_atomic_clock_realtime(),
2306        );
2307
2308        assert!(result.is_err());
2309    }
2310
2311    fn subscribe_quotes_cmd(params: Option<Params>) -> SubscribeQuotes {
2312        SubscribeQuotes::new(
2313            InstrumentId::from("ESM4.GLBX"),
2314            Some(ClientId::from("DATABENTO-TEST")),
2315            None,
2316            UUID4::new(),
2317            UnixNanos::default(),
2318            None,
2319            params,
2320        )
2321    }
2322
2323    fn subscribe_trades_cmd(params: Option<Params>) -> SubscribeTrades {
2324        SubscribeTrades::new(
2325            InstrumentId::from("ESM4.GLBX"),
2326            Some(ClientId::from("DATABENTO-TEST")),
2327            None,
2328            UUID4::new(),
2329            UnixNanos::default(),
2330            None,
2331            params,
2332        )
2333    }
2334
2335    fn subscribe_book_depth_cmd(depth: Option<NonZeroUsize>) -> SubscribeBookDepth {
2336        SubscribeBookDepth::new(
2337            InstrumentId::from("ESM4.GLBX"),
2338            BookType::L2_MBP,
2339            Some(ClientId::from("DATABENTO-TEST")),
2340            None,
2341            UUID4::new(),
2342            UnixNanos::default(),
2343            depth,
2344            true,
2345            None,
2346            None,
2347        )
2348    }
2349
2350    fn currency_pair_with_ts_init(instrument_id: &str, ts_init: UnixNanos) -> InstrumentAny {
2351        let instrument_id = InstrumentId::from(instrument_id);
2352        InstrumentAny::CurrencyPair(
2353            CurrencyPair::builder()
2354                .instrument_id(instrument_id)
2355                .raw_symbol(instrument_id.symbol)
2356                .base_currency(Currency::from("BTC"))
2357                .quote_currency(Currency::from("USDT"))
2358                .price_precision(2)
2359                .size_precision(6)
2360                .price_increment(Price::from("0.01"))
2361                .size_increment(Quantity::from("0.000001"))
2362                .ts_event(UnixNanos::default())
2363                .ts_init(ts_init)
2364                .build()
2365                .unwrap(),
2366        )
2367    }
2368
2369    #[rstest]
2370    fn test_instruments_query_params_requests_all_symbols() {
2371        let start = UnixNanos::from(1_000_000_000);
2372        let end = UnixNanos::from(2_000_000_000);
2373
2374        let params = instruments_query_params("GLBX.MDP3".to_string(), start, Some(end));
2375
2376        assert_eq!(params.dataset, "GLBX.MDP3");
2377        assert_eq!(params.symbols, vec!["ALL_SYMBOLS"]);
2378        assert_eq!(params.start, start);
2379        assert_eq!(params.end, Some(end));
2380        assert_eq!(params.limit, None);
2381        assert_eq!(params.price_precision, None);
2382    }
2383
2384    #[rstest]
2385    fn test_instrument_query_params_requests_single_symbol() {
2386        let instrument_id = InstrumentId::from("ESM4.GLBX");
2387
2388        let start = UnixNanos::from(1_000_000_000);
2389        let end = UnixNanos::from(2_000_000_000);
2390
2391        let params =
2392            instrument_query_params("GLBX.MDP3".to_string(), instrument_id, start, Some(end));
2393
2394        assert_eq!(params.dataset, "GLBX.MDP3");
2395        assert_eq!(params.symbols, vec!["ESM4"]);
2396        assert_eq!(params.start, start);
2397        assert_eq!(params.end, Some(end));
2398        assert_eq!(params.limit, None);
2399        assert_eq!(params.price_precision, None);
2400    }
2401
2402    #[rstest]
2403    #[case::intraday(1_782_302_400_000_000_001)]
2404    #[case::midnight(1_782_259_200_000_000_000)]
2405    #[case::day_end(1_782_345_599_999_999_999)]
2406    #[tokio::test]
2407    async fn test_seed_price_precision_includes_midnight(#[case] start: u64) {
2408        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2409            .join("test_data/test_data.definition.futures_contract.dbn.zst");
2410        let mut decoder = Decoder::from_zstd_file(path).unwrap();
2411        let mut metadata = decoder.metadata().clone();
2412        let record = decoder
2413            .decode_record::<dbn::InstrumentDefMsg>()
2414            .unwrap()
2415            .unwrap();
2416        metadata.symbols = vec!["ESU6".to_string()];
2417
2418        metadata.mappings = vec![SymbolMapping {
2419            raw_symbol: "ESU6".to_string(),
2420            intervals: vec![MappingInterval {
2421                start_date: date!(2026 - 06 - 24),
2422                end_date: date!(2026 - 06 - 25),
2423                symbol: record.hd.instrument_id.to_string(),
2424            }],
2425        }];
2426
2427        let mut body = Vec::new();
2428        {
2429            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
2430            encoder.encode_record(record).unwrap();
2431            encoder.flush().unwrap();
2432        }
2433
2434        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2435        let base_url = format!("http://{}/", listener.local_addr().unwrap());
2436
2437        let server = tokio::spawn(serve_historical_response(listener, body));
2438
2439        let client = DatabentoHistoricalClient::new_with_base_url(
2440            Credential::new("32-character-with-lots-of-filler"),
2441            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2442            get_atomic_clock_realtime(),
2443            false,
2444            &base_url,
2445        )
2446        .unwrap();
2447        let instrument_id = InstrumentId::from("ESU26.GLBX");
2448        let end = UnixNanos::from(1_782_345_600_000_000_123);
2449
2450        tokio::time::timeout(
2451            std::time::Duration::from_secs(5),
2452            seed_price_precision_if_needed(
2453                &client,
2454                "GLBX.MDP3",
2455                instrument_id,
2456                start.into(),
2457                Some(end),
2458                None,
2459            ),
2460        )
2461        .await
2462        .unwrap();
2463        let request = server.await.unwrap();
2464        let fields: AHashMap<_, _> = request
2465            .split('&')
2466            .map(|field| field.split_once('=').unwrap())
2467            .collect();
2468
2469        assert_eq!(fields["dataset"], "GLBX.MDP3");
2470        assert_eq!(fields["schema"], "definition");
2471        assert_eq!(fields["symbols"], "ESU6");
2472        assert_eq!(fields["start"], "1782259200000000000");
2473        assert_eq!(fields["end"], end.to_string());
2474        assert_eq!(client.price_precision(instrument_id.symbol), Some(2));
2475    }
2476
2477    async fn serve_historical_response(listener: TcpListener, body: Vec<u8>) -> String {
2478        let (stream, _) = listener.accept().await.unwrap();
2479        let mut stream = BufReader::new(stream);
2480        let mut line = String::new();
2481        stream.read_line(&mut line).await.unwrap();
2482        assert_eq!(line, "POST /v0/timeseries.get_range HTTP/1.1\r\n");
2483        let mut content_length = None;
2484
2485        loop {
2486            line.clear();
2487            stream.read_line(&mut line).await.unwrap();
2488
2489            if line == "\r\n" {
2490                break;
2491            }
2492
2493            if let Some((name, value)) = line.split_once(':')
2494                && name.eq_ignore_ascii_case("content-length")
2495            {
2496                content_length = Some(value.trim().parse::<usize>().unwrap());
2497            }
2498        }
2499
2500        let mut request = vec![0; content_length.unwrap()];
2501        stream.read_exact(&mut request).await.unwrap();
2502        let headers = format!(
2503            "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
2504            body.len(),
2505        );
2506        stream.write_all(headers.as_bytes()).await.unwrap();
2507        stream.write_all(&body).await.unwrap();
2508        stream.shutdown().await.unwrap();
2509        String::from_utf8(request).unwrap()
2510    }
2511
2512    #[rstest]
2513    #[case::explicit(Some(5), None)]
2514    #[case::cached(None, Some(3))]
2515    #[tokio::test]
2516    async fn test_seed_price_precision_skips_known_precision(
2517        #[case] price_precision: Option<u8>,
2518        #[case] cached_precision: Option<u8>,
2519    ) {
2520        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2521        let base_url = format!("http://{}/", listener.local_addr().unwrap());
2522
2523        let client = DatabentoHistoricalClient::new_with_base_url(
2524            Credential::new("32-character-with-lots-of-filler"),
2525            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2526            get_atomic_clock_realtime(),
2527            false,
2528            &base_url,
2529        )
2530        .unwrap();
2531        let instrument_id = InstrumentId::from("ESU6.GLBX");
2532        if let Some(precision) = cached_precision {
2533            client.set_price_precision(instrument_id.symbol, precision);
2534        }
2535
2536        tokio::select! {
2537            () = seed_price_precision_if_needed(
2538                &client,
2539                "GLBX.MDP3",
2540                instrument_id,
2541                1_706_443_200_000_000_001.into(),
2542                Some(1_706_486_400_000_000_123.into()),
2543                price_precision,
2544            ) => {}
2545            result = listener.accept() => panic!("Unexpected definition request: {result:?}"),
2546            () = tokio::time::sleep(std::time::Duration::from_secs(5)) => {
2547                panic!("Precision seeding did not finish");
2548            }
2549        }
2550
2551        assert_eq!(
2552            client.price_precision(instrument_id.symbol),
2553            cached_precision
2554        );
2555    }
2556
2557    #[rstest]
2558    fn test_resolve_request_time_range_defaults_to_end_day() {
2559        let end = UnixNanos::from(1_706_443_200_000_000_001);
2560
2561        let (start, resolved_end) = resolve_request_time_range(None, Some(end));
2562
2563        assert_eq!(start, UnixNanos::from(1_706_400_000_000_000_000));
2564        assert_eq!(resolved_end, Some(end + DurationNanos::new(1)));
2565    }
2566
2567    #[rstest]
2568    #[case::range(17, 23, 17, 24)]
2569    #[case::point(17, 17, 17, 18)]
2570    #[case::zero(0, 0, 0, 1)]
2571    #[case::inverted(23, 17, 17, 18)]
2572    fn test_resolve_request_time_range_converts_inclusive_bounds(
2573        #[case] start: u64,
2574        #[case] end: u64,
2575        #[case] expected_start: u64,
2576        #[case] expected_end: u64,
2577    ) {
2578        let (start, end) = resolve_request_time_range(Some(start.into()), Some(end.into()));
2579
2580        assert_eq!(start, UnixNanos::from(expected_start));
2581        assert_eq!(end, Some(UnixNanos::from(expected_end)));
2582    }
2583
2584    #[rstest]
2585    #[case::quotes(false, true, false)]
2586    #[case::bars_close(true, true, false)]
2587    #[case::bars_open(true, false, false)]
2588    #[case::bars_default_start(true, true, true)]
2589    #[tokio::test]
2590    async fn test_historical_request_sends_inclusive_bounds(
2591        #[case] bars: bool,
2592        #[case] timestamp_on_close: bool,
2593        #[case] missing_start: bool,
2594    ) {
2595        let filename = if bars {
2596            "test_data.ohlcv-1m.dbn.zst"
2597        } else {
2598            "test_data.mbp-1.dbn.zst"
2599        };
2600
2601        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2602            .join("test_data")
2603            .join(filename);
2604        let decoder = Decoder::from_zstd_file(path).unwrap();
2605        let mut metadata = decoder.metadata().clone();
2606        metadata.dataset = "GLBX.MDP3".to_owned();
2607        let mut body = Vec::new();
2608        {
2609            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
2610            encoder.flush().unwrap();
2611        }
2612
2613        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
2614        let base_url = format!("http://{}/", listener.local_addr().unwrap());
2615
2616        let server = tokio::spawn(serve_historical_response(listener, body));
2617        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
2618        replace_data_event_sender(sender);
2619
2620        let mut config = DatabentoDataClientConfig::new(
2621            "32-character-with-lots-of-filler",
2622            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
2623            true,
2624            timestamp_on_close,
2625        );
2626        config.historical_base_url = Some(base_url);
2627        let client_id = ClientId::from("DATABENTO-BOUNDS");
2628        let client =
2629            DatabentoDataClient::new(client_id, config, get_atomic_clock_realtime()).unwrap();
2630        let id = InstrumentId::from("ESM4.GLBX");
2631        let start = UnixNanos::from(1_717_423_200_000_000_000);
2632        let request_start = (!missing_start).then_some(start);
2633        let end = start + DurationNanos::from_secs(60) + DurationNanos::new(123);
2634        let request_id = UUID4::new();
2635        let params = Params::new([(PRICE_PRECISION_PARAM.to_owned(), json!(2))].into());
2636
2637        if bars {
2638            let bar_type = "ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"
2639                .parse::<BarType>()
2640                .unwrap();
2641            client
2642                .request_bars(RequestBars::new(
2643                    bar_type,
2644                    request_start.map(|timestamp| timestamp.to_datetime_utc()),
2645                    Some(end.to_datetime_utc()),
2646                    None,
2647                    Some(client_id),
2648                    request_id,
2649                    UnixNanos::from(37),
2650                    Some(params.clone()),
2651                ))
2652                .unwrap();
2653        } else {
2654            client
2655                .request_quotes(RequestQuotes::new(
2656                    id,
2657                    request_start.map(|timestamp| timestamp.to_datetime_utc()),
2658                    Some(end.to_datetime_utc()),
2659                    None,
2660                    Some(client_id),
2661                    request_id,
2662                    UnixNanos::from(37),
2663                    Some(params.clone()),
2664                ))
2665                .unwrap();
2666        }
2667
2668        let event = tokio::time::timeout(Duration::from_secs(5), receiver.recv())
2669            .await
2670            .unwrap()
2671            .unwrap();
2672
2673        let DataEvent::ResponseEnvelope(envelope) = event else {
2674            panic!("expected completed response, received {event:?}");
2675        };
2676
2677        let response = envelope.response().unwrap();
2678        let request = server.await.unwrap();
2679        let fields: AHashMap<_, _> = request
2680            .split('&')
2681            .map(|field| field.split_once('=').unwrap())
2682            .collect();
2683
2684        let offset = if bars {
2685            DurationNanos::from_secs(60)
2686        } else {
2687            DurationNanos::new(0)
2688        };
2689
2690        let expected_start = if missing_start {
2691            UnixNanos::from(1_717_372_800_000_000_000)
2692        } else {
2693            start
2694        };
2695
2696        assert_eq!(fields["dataset"], "GLBX.MDP3");
2697        assert_eq!(fields["schema"], if bars { "ohlcv-1m" } else { "mbp-1" });
2698        assert_eq!(fields["symbols"], "ESM4");
2699        assert_eq!(fields["start"], (expected_start - offset).to_string());
2700        assert_eq!(
2701            fields["end"],
2702            (end - offset + DurationNanos::new(1)).to_string()
2703        );
2704        assert_eq!(envelope.correlation_id(), &request_id);
2705        assert!(envelope.completed());
2706        assert_eq!(envelope.failure_reason(), None);
2707        assert_eq!(response.start(), request_start);
2708        assert_eq!(response.end(), Some(end));
2709        assert_eq!(response.client_id(), &client_id);
2710        assert_eq!(response.params(), Some(&params));
2711        assert_eq!(response.record_count(), Some(0));
2712    }
2713
2714    #[rstest]
2715    fn test_requested_instrument_filters_exact_id() {
2716        let requested_id = InstrumentId::from("BTCUSDT.BINANCE");
2717        let instruments = vec![
2718            currency_pair("ETHUSDT.BINANCE"),
2719            currency_pair("BTCUSDT.BINANCE"),
2720        ];
2721
2722        let instrument = requested_instrument(&instruments, requested_id).expect("instrument");
2723
2724        assert_eq!(instrument.id(), requested_id);
2725    }
2726
2727    #[rstest]
2728    fn test_requested_instrument_returns_latest_matching_id() {
2729        let requested_id = InstrumentId::from("BTCUSDT.BINANCE");
2730        let instruments = vec![
2731            currency_pair_with_ts_init("BTCUSDT.BINANCE", UnixNanos::from(1)),
2732            currency_pair_with_ts_init("BTCUSDT.BINANCE", UnixNanos::from(2)),
2733        ];
2734
2735        let instrument = requested_instrument(&instruments, requested_id).expect("instrument");
2736
2737        assert_eq!(instrument.ts_init(), UnixNanos::from(2));
2738    }
2739
2740    #[rstest]
2741    fn test_requested_instrument_returns_none_on_miss() {
2742        let instruments = vec![currency_pair("ETHUSDT.BINANCE")];
2743
2744        let instrument = requested_instrument(&instruments, InstrumentId::from("BTCUSDT.BINANCE"));
2745
2746        assert!(instrument.is_none());
2747    }
2748
2749    #[rstest]
2750    fn test_price_precision_from_params() {
2751        let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
2752
2753        let price_precision = price_precision_from_params(Some(&params)).unwrap();
2754
2755        assert_eq!(price_precision, Some(5));
2756    }
2757
2758    #[rstest]
2759    fn test_price_precision_from_params_rejects_out_of_range_value() {
2760        let params = Params::new(
2761            [(
2762                PRICE_PRECISION_PARAM.to_string(),
2763                json!(u64::from(u8::MAX) + 1),
2764            )]
2765            .into(),
2766        );
2767
2768        let result = price_precision_from_params(Some(&params));
2769
2770        assert!(result.is_err());
2771    }
2772
2773    #[rstest]
2774    fn test_partition_book_deltas_response_by_child_instrument(
2775        #[values(false, true)] narrowed: bool,
2776    ) {
2777        let correlation_id = UUID4::new();
2778        let client_id = ClientId::from("DATABENTO-TEST");
2779        let parent = InstrumentId::from("ES.FUT.GLBX");
2780        let child_a = InstrumentId::from("ESM6.GLBX");
2781        let child_b = InstrumentId::from("ESU6.GLBX");
2782        let deltas = vec![
2783            OrderBookDelta::clear(child_a, 1, UnixNanos::from(1_000), UnixNanos::from(1_000)),
2784            OrderBookDelta::clear(child_b, 2, UnixNanos::from(2_000), UnixNanos::from(2_000)),
2785            OrderBookDelta::clear(child_a, 3, UnixNanos::from(3_000), UnixNanos::from(3_000)),
2786        ];
2787        let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
2788
2789        let rows = if narrowed {
2790            let hidden = OrderBookDelta::clear(
2791                InstrumentId::from("HIDDEN.GLBX"),
2792                4,
2793                UnixNanos::from(5_000),
2794                UnixNanos::from(5_000),
2795            );
2796            let mut backing = vec![hidden];
2797            backing.extend_from_slice(&deltas);
2798            backing.push(hidden);
2799            BatchView::new(Arc::new(backing), 1..4)
2800        } else {
2801            deltas.clone().into()
2802        };
2803
2804        let response = BookDeltasResponse::new(
2805            correlation_id,
2806            client_id,
2807            parent,
2808            rows,
2809            Some(UnixNanos::from(500)),
2810            Some(UnixNanos::from(4_000)),
2811            UnixNanos::from(5_000),
2812            Some(params.clone()),
2813        );
2814
2815        let responses = partition_book_deltas_response(response);
2816
2817        assert_eq!(responses.len(), 2);
2818        assert_eq!(responses[0].correlation_id, correlation_id);
2819        assert_eq!(responses[1].correlation_id, correlation_id);
2820        assert_eq!(responses[0].client_id, client_id);
2821        assert_eq!(responses[1].client_id, client_id);
2822        assert_eq!(responses[0].instrument_id, child_a);
2823        assert_eq!(responses[1].instrument_id, child_b);
2824        assert_eq!(responses[0].data.as_ref(), &vec![deltas[0], deltas[2]]);
2825        assert_eq!(responses[1].data.as_ref(), &vec![deltas[1]]);
2826        assert_eq!(responses[0].start, Some(UnixNanos::from(500)));
2827        assert_eq!(responses[1].start, Some(UnixNanos::from(500)));
2828        assert_eq!(responses[0].end, Some(UnixNanos::from(4_000)));
2829        assert_eq!(responses[1].end, Some(UnixNanos::from(4_000)));
2830        assert_eq!(responses[0].ts_init, UnixNanos::from(5_000));
2831        assert_eq!(responses[1].ts_init, UnixNanos::from(5_000));
2832        assert_eq!(responses[0].params, Some(params.clone()));
2833        assert_eq!(responses[1].params, Some(params));
2834    }
2835
2836    #[rstest]
2837    fn test_partition_book_deltas_response_preserves_homogeneous_response() {
2838        let correlation_id = UUID4::new();
2839        let client_id = ClientId::from("DATABENTO-TEST");
2840        let instrument_id = InstrumentId::from("ESM6.GLBX");
2841        let delta = OrderBookDelta::clear(
2842            instrument_id,
2843            1,
2844            UnixNanos::from(1_000),
2845            UnixNanos::from(1_000),
2846        );
2847        let response = BookDeltasResponse::new(
2848            correlation_id,
2849            client_id,
2850            instrument_id,
2851            vec![delta],
2852            Some(UnixNanos::from(500)),
2853            Some(UnixNanos::from(1_500)),
2854            UnixNanos::from(2_000),
2855            None,
2856        );
2857
2858        let responses = partition_book_deltas_response(response);
2859
2860        assert_eq!(responses.len(), 1);
2861        assert_eq!(responses[0].correlation_id, correlation_id);
2862        assert_eq!(responses[0].client_id, client_id);
2863        assert_eq!(responses[0].instrument_id, instrument_id);
2864        assert_eq!(responses[0].data.as_ref(), &vec![delta]);
2865        assert_eq!(responses[0].start, Some(UnixNanos::from(500)));
2866        assert_eq!(responses[0].end, Some(UnixNanos::from(1_500)));
2867        assert_eq!(responses[0].ts_init, UnixNanos::from(2_000));
2868        assert_eq!(responses[0].params, None);
2869    }
2870
2871    #[rstest]
2872    fn test_partition_book_deltas_response_preserves_empty_parent_response() {
2873        let correlation_id = UUID4::new();
2874        let parent = InstrumentId::from("ES.FUT.GLBX");
2875        let response = BookDeltasResponse::new(
2876            correlation_id,
2877            ClientId::from("DATABENTO-TEST"),
2878            parent,
2879            Vec::new(),
2880            None,
2881            None,
2882            UnixNanos::from(1_000),
2883            None,
2884        );
2885
2886        let responses = partition_book_deltas_response(response);
2887
2888        assert_eq!(responses.len(), 1);
2889        let response = &responses[0];
2890        assert_eq!(response.correlation_id, correlation_id);
2891        assert_eq!(response.instrument_id, parent);
2892        assert!(response.data.is_empty());
2893    }
2894
2895    #[rstest]
2896    fn test_schema_from_params_returns_default() {
2897        let schema = schema_from_params(None, dbn::Schema::Mbp1, QUOTE_SCHEMAS).unwrap();
2898
2899        assert_eq!(schema, dbn::Schema::Mbp1);
2900    }
2901
2902    #[rstest]
2903    fn test_schema_from_params_accepts_allowed_value() {
2904        let params = Params::new([(SCHEMA_PARAM.to_string(), json!("tbbo"))].into());
2905
2906        let schema = schema_from_params(Some(&params), dbn::Schema::Mbp1, QUOTE_SCHEMAS).unwrap();
2907
2908        assert_eq!(schema, dbn::Schema::Tbbo);
2909    }
2910
2911    #[rstest]
2912    fn test_schema_from_params_rejects_disallowed_value() {
2913        let params = Params::new([(SCHEMA_PARAM.to_string(), json!("mbo"))].into());
2914
2915        let result = schema_from_params(Some(&params), dbn::Schema::Mbp1, QUOTE_SCHEMAS);
2916
2917        assert!(result.is_err());
2918    }
2919
2920    #[rstest]
2921    #[case::quotes(SubscribeKind::Quotes)]
2922    #[case::trades(SubscribeKind::Trades)]
2923    fn test_invalid_subscribe_params_do_not_create_feed_handler(#[case] kind: SubscribeKind) {
2924        let mut client = test_data_client();
2925        let params = Params::new([(SCHEMA_PARAM.to_string(), json!("definition"))].into());
2926        let result = match kind {
2927            SubscribeKind::Quotes => client.subscribe_quotes(subscribe_quotes_cmd(Some(params))),
2928            SubscribeKind::Trades => client.subscribe_trades(subscribe_trades_cmd(Some(params))),
2929        };
2930
2931        assert!(result.is_err());
2932        assert!(client.cmd_channels.lock().is_empty());
2933    }
2934
2935    #[rstest]
2936    fn test_bar_list_reports_exact_full_types_and_continues_after_invalid_member() {
2937        let mut client = test_data_client();
2938        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
2939        client
2940            .cmd_channels
2941            .lock()
2942            .insert("GLBX.MDP3".to_string(), tx);
2943        let first = BarType::from("ESM4.GLBX-1-SECOND-LAST-EXTERNAL");
2944        let invalid = BarType::from("ESM4.GLBX-5-MINUTE-LAST-EXTERNAL");
2945        let last = BarType::from("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL");
2946
2947        let command = SubscribeBars::new(
2948            [first, invalid, last],
2949            Some(client.client_id),
2950            None,
2951            UUID4::new(),
2952            UnixNanos::from(17),
2953            Some(UUID4::new()),
2954            None,
2955        );
2956        let error = client.subscribe_bars(command).unwrap_err();
2957        let partial = error.downcast_ref::<SubscriptionError>().unwrap();
2958        assert_eq!(partial.bar_types_accepted, vec![first, last]);
2959        assert_eq!(partial.instrument_ids_accepted, Vec::<InstrumentId>::new());
2960
2961        for schema in [dbn::Schema::Ohlcv1S, dbn::Schema::Ohlcv1M] {
2962            let HandlerCommand::Subscribe(actual) = rx.try_recv().unwrap() else {
2963                panic!("expected bar schema subscription");
2964            };
2965
2966            let expected = Subscription::builder()
2967                .schema(schema)
2968                .symbols("ESM4")
2969                .build();
2970            assert_eq!(actual, expected);
2971        }
2972
2973        assert_eq!(
2974            rx.try_recv().unwrap_err(),
2975            tokio::sync::mpsc::error::TryRecvError::Empty
2976        );
2977    }
2978
2979    macro_rules! batch_tests {
2980        ($batch_test:ident, $failure_test:ident, $validation_test:ident, $command:ident, $subscribe:ident, $schema:ident) => {
2981            #[rstest]
2982            fn $batch_test() {
2983                let mut client = test_data_client();
2984                let (glbx_tx, mut glbx_rx) = tokio::sync::mpsc::unbounded_channel();
2985                let (opra_tx, mut opra_rx) = tokio::sync::mpsc::unbounded_channel();
2986                client
2987                    .cmd_channels
2988                    .lock()
2989                    .insert("GLBX.MDP3".to_string(), glbx_tx);
2990                client
2991                    .cmd_channels
2992                    .lock()
2993                    .insert("OPRA.PILLAR".to_string(), opra_tx);
2994                let first = InstrumentId::from("ESM4.GLBX");
2995                let second = InstrumentId::from("NQM4.GLBX");
2996                let third = InstrumentId::from("SPY_OPTION.XCBO");
2997                let params = Params::new([(PRICE_PRECISION_PARAM.to_string(), json!(5))].into());
2998                let command = $command(Some(params))
2999                    .with_instrument_ids([second, third, first, second]);
3000                let original = serde_json::to_value(&command).unwrap();
3001                client.$subscribe(command.clone()).unwrap();
3002                assert!(
3003                    matches!(glbx_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == second.symbol)
3004                );
3005                assert!(
3006                    matches!(glbx_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == first.symbol)
3007                );
3008
3009                let HandlerCommand::Subscribe(glbx_subscription) = glbx_rx.try_recv().unwrap()
3010                else {
3011                    panic!("expected GLBX subscription")
3012                };
3013
3014                assert!(
3015                    matches!(opra_rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 5) if symbol == third.symbol)
3016                );
3017
3018                let HandlerCommand::Subscribe(opra_subscription) = opra_rx.try_recv().unwrap()
3019                else {
3020                    panic!("expected OPRA subscription")
3021                };
3022
3023                let expected_glbx = Subscription::builder()
3024                    .schema(dbn::Schema::$schema)
3025                    .symbols(vec![second.symbol.to_string(), first.symbol.to_string()])
3026                    .build();
3027                let expected_opra = Subscription::builder()
3028                    .schema(dbn::Schema::$schema)
3029                    .symbols(vec![third.symbol.to_string()])
3030                    .build();
3031                assert_eq!(glbx_subscription, expected_glbx);
3032                assert_eq!(opra_subscription, expected_opra);
3033                assert_eq!(
3034                    glbx_rx.try_recv().unwrap_err(),
3035                    tokio::sync::mpsc::error::TryRecvError::Empty
3036                );
3037                assert_eq!(
3038                    opra_rx.try_recv().unwrap_err(),
3039                    tokio::sync::mpsc::error::TryRecvError::Empty
3040                );
3041                assert_eq!(client.symbol_venue_map.len(), 3);
3042
3043                for id in [first, second, third] {
3044                    assert_eq!(
3045                        client.symbol_venue_map.get_cloned(&id.symbol),
3046                        Some(id.venue)
3047                    );
3048                }
3049
3050                assert_eq!(serde_json::to_value(command).unwrap(), original);
3051            }
3052
3053            #[rstest]
3054            #[case::failure_first(true)]
3055            #[case::failure_middle(false)]
3056            fn $failure_test(
3057                #[case] failure_first: bool,
3058                #[values(false, true)] previous_mapping: bool,
3059            ) {
3060                let mut client = test_data_client();
3061                let (glbx_tx, mut glbx_rx) = tokio::sync::mpsc::unbounded_channel();
3062                let (opra_tx, opra_rx) = tokio::sync::mpsc::unbounded_channel();
3063                let (equs_tx, mut equs_rx) = tokio::sync::mpsc::unbounded_channel();
3064                drop(opra_rx);
3065                client
3066                    .cmd_channels
3067                    .lock()
3068                    .insert("GLBX.MDP3".to_string(), glbx_tx);
3069                client
3070                    .cmd_channels
3071                    .lock()
3072                    .insert("OPRA.PILLAR".to_string(), opra_tx);
3073                client
3074                    .cmd_channels
3075                    .lock()
3076                    .insert("EQUS.MINI".to_string(), equs_tx);
3077                let first = InstrumentId::from("ESM4.GLBX");
3078                let second = InstrumentId::from("SPY_OPTION.XCBO");
3079                let third = InstrumentId::from("AAPL.EQUS");
3080                let previous = previous_mapping.then_some(Venue::from("PREVIOUS"));
3081                if let Some(venue) = previous {
3082                    client.symbol_venue_map.insert(second.symbol, venue);
3083                }
3084                let ids = if failure_first {
3085                    [second, first, third]
3086                } else {
3087                    [first, second, third]
3088                };
3089                let command = $command(None).with_instrument_ids(ids);
3090                let result = client.$subscribe(command);
3091                let error = result.unwrap_err();
3092                let partial = error
3093                    .downcast_ref::<SubscriptionError>()
3094                    .expect("partial subscription error");
3095
3096                assert_eq!(partial.instrument_ids_accepted, vec![first, third]);
3097                assert_eq!(partial.bar_types_accepted, Vec::<BarType>::new());
3098                assert_eq!(
3099                    partial.source.to_string(),
3100                    "Failed to send command to dataset OPRA.PILLAR: channel closed"
3101                );
3102                assert_eq!(client.symbol_venue_map.get_cloned(&second.symbol), previous);
3103
3104                for (id, receiver) in [(first, &mut glbx_rx), (third, &mut equs_rx)] {
3105                    let HandlerCommand::Subscribe(accepted) = receiver.try_recv().unwrap() else {
3106                        panic!("expected accepted subscription");
3107                    };
3108                    let expected = Subscription::builder()
3109                        .schema(dbn::Schema::$schema)
3110                        .symbols(vec![id.symbol.to_string()])
3111                        .build();
3112                    assert_eq!(accepted, expected);
3113                    assert_eq!(client.symbol_venue_map.get_cloned(&id.symbol), Some(id.venue));
3114                    assert_eq!(
3115                        receiver.try_recv().unwrap_err(),
3116                        tokio::sync::mpsc::error::TryRecvError::Empty
3117                    );
3118                }
3119            }
3120
3121            #[rstest]
3122            #[case::unknown_dataset("UNKNOWN.UNKNOWN", "No dataset found for venue: UNKNOWN")]
3123            #[case::ambiguous_symbol(
3124                "ESM4.XCBO",
3125                "Identifiers share raw symbol ESM4 across different venues"
3126            )]
3127            fn $validation_test(#[case] invalid_id: &str, #[case] error: &str) {
3128                let mut client = test_data_client();
3129                let command = $command(None).with_instrument_ids([
3130                    InstrumentId::from("ESM4.GLBX"),
3131                    InstrumentId::from(invalid_id),
3132                ]);
3133                let result = client.$subscribe(command);
3134                assert_eq!(result.unwrap_err().to_string(), error);
3135                assert!(client.cmd_channels.lock().is_empty());
3136                assert!(client.symbol_venue_map.is_empty());
3137            }
3138        };
3139    }
3140
3141    batch_tests!(
3142        test_quote_batches_send_one_subscription_per_dataset,
3143        test_quote_batch_enqueue_failure_can_follow_accepted_dataset,
3144        test_quote_batches_validate_all_members_before_dispatch,
3145        subscribe_quotes_cmd,
3146        subscribe_quotes,
3147        Mbp1
3148    );
3149
3150    batch_tests!(
3151        test_trade_batches_send_one_subscription_per_dataset,
3152        test_trade_batch_enqueue_failure_can_follow_accepted_dataset,
3153        test_trade_batches_validate_all_members_before_dispatch,
3154        subscribe_trades_cmd,
3155        subscribe_trades,
3156        Trades
3157    );
3158
3159    #[rstest]
3160    fn test_trade_batch_unsubscribe_is_noop() {
3161        let mut client = test_data_client();
3162
3163        let command = UnsubscribeTrades::new(
3164            [
3165                InstrumentId::from("ESM4.GLBX"),
3166                InstrumentId::from("SPY_OPTION.XCBO"),
3167            ],
3168            Some(ClientId::from("DATABENTO")),
3169            None,
3170            UUID4::new(),
3171            UnixNanos::from(37),
3172            None,
3173            None,
3174        );
3175        client.unsubscribe_trades(&command).unwrap();
3176        assert!(client.cmd_channels.lock().is_empty());
3177        assert!(client.symbol_venue_map.is_empty());
3178    }
3179
3180    #[rstest]
3181    fn test_send_subscription_commands_starts_after_subscribe() {
3182        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3183        let subscription = Subscription::builder()
3184            .schema(dbn::Schema::Mbp1)
3185            .symbols(vec!["ESM4"])
3186            .build();
3187
3188        send_subscription_commands(
3189            &tx,
3190            "GLBX.MDP3",
3191            Some((Symbol::from("ESM4"), 2)),
3192            subscription,
3193            true,
3194        )
3195        .unwrap();
3196
3197        assert!(matches!(
3198            rx.try_recv().unwrap(),
3199            HandlerCommand::SetPricePrecision(symbol, 2) if symbol == Symbol::from("ESM4")
3200        ));
3201        assert!(matches!(
3202            rx.try_recv().unwrap(),
3203            HandlerCommand::Subscribe(sub) if sub.schema == dbn::Schema::Mbp1
3204        ));
3205        assert!(matches!(rx.try_recv().unwrap(), HandlerCommand::Start));
3206    }
3207
3208    #[rstest]
3209    #[case::second("ESM4.GLBX-1-SECOND-LAST-EXTERNAL", dbn::Schema::Ohlcv1S)]
3210    #[case::minute("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL", dbn::Schema::Ohlcv1M)]
3211    #[case::hour("ESM4.GLBX-1-HOUR-LAST-EXTERNAL", dbn::Schema::Ohlcv1H)]
3212    #[case::day("ESM4.GLBX-1-DAY-LAST-EXTERNAL", dbn::Schema::Ohlcv1D)]
3213    fn test_ohlcv_schema_maps_aggregation(#[case] bar_type: &str, #[case] expected: dbn::Schema) {
3214        let schema = ohlcv_schema(BarType::from(bar_type)).unwrap();
3215
3216        assert_eq!(schema, expected);
3217    }
3218
3219    #[rstest]
3220    #[case::step("ESM4.GLBX-5-MINUTE-LAST-EXTERNAL")]
3221    #[case::aggregation("ESM4.GLBX-100-TICK-LAST-EXTERNAL")]
3222    #[case::bid("ESM4.GLBX-1-MINUTE-BID-EXTERNAL")]
3223    #[case::mid("ESM4.GLBX-1-MINUTE-MID-EXTERNAL")]
3224    fn test_ohlcv_schema_rejects_unsupported_bar_type(#[case] bar_type: &str) {
3225        assert!(ohlcv_schema(BarType::from(bar_type)).is_err());
3226    }
3227
3228    #[rstest]
3229    #[case::statistics("DatabentoStatistics", dbn::Schema::Statistics)]
3230    #[case::imbalance("DatabentoImbalance", dbn::Schema::Imbalance)]
3231    fn test_subscribe_custom_data_sends_schema_subscription(
3232        #[case] type_name: &str,
3233        #[case] expected: dbn::Schema,
3234    ) {
3235        let mut client = test_data_client();
3236        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3237        client
3238            .cmd_channels
3239            .lock()
3240            .insert("GLBX.MDP3".to_string(), tx);
3241        let command = SubscribeCustomData::new(
3242            Some(ClientId::from("DATABENTO-TEST")),
3243            None,
3244            DataType::new(type_name, None, Some("ESM4.GLBX".to_string())),
3245            UUID4::new(),
3246            UnixNanos::default(),
3247            None,
3248            None,
3249        );
3250
3251        client.subscribe(command).unwrap();
3252
3253        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3254            panic!("expected subscription");
3255        };
3256        assert_eq!(subscription.schema, expected);
3257        assert_eq!(
3258            subscription.symbols,
3259            databento::Symbols::Symbols(vec!["ESM4".to_string()])
3260        );
3261        assert!(rx.try_recv().is_err());
3262    }
3263
3264    #[rstest]
3265    #[case::unsupported_type("DatabentoOther", Some("ESM4.GLBX"))]
3266    #[case::missing_identifier("DatabentoStatistics", None)]
3267    #[case::invalid_identifier("DatabentoStatistics", Some("ESM4"))]
3268    fn test_subscribe_custom_data_rejects_invalid_data_type(
3269        #[case] type_name: &str,
3270        #[case] identifier: Option<&str>,
3271    ) {
3272        let mut client = test_data_client();
3273        let command = SubscribeCustomData::new(
3274            Some(ClientId::from("DATABENTO-TEST")),
3275            None,
3276            DataType::new(type_name, None, identifier.map(str::to_string)),
3277            UUID4::new(),
3278            UnixNanos::default(),
3279            None,
3280            None,
3281        );
3282
3283        assert!(client.subscribe(command).is_err());
3284        assert!(client.cmd_channels.lock().is_empty());
3285    }
3286
3287    #[rstest]
3288    fn test_custom_data_for_instrument_matches_subscribed_topic() {
3289        let instrument_id = InstrumentId::from("ESM4.GLBX");
3290        let statistics = DatabentoStatistics::new(
3291            instrument_id,
3292            DatabentoStatisticType::SettlementPrice,
3293            DatabentoStatisticUpdateAction::Added,
3294            Some(Price::from("5000.25")),
3295            None,
3296            1,
3297            2,
3298            3,
3299            UnixNanos::from(4),
3300            5,
3301            UnixNanos::from(6),
3302            UnixNanos::from(7),
3303            UnixNanos::from(8),
3304        );
3305        let subscribed = DataType::new("DatabentoStatistics", None, Some("ESM4.GLBX".to_string()));
3306
3307        let custom = custom_data_for_instrument(Arc::new(statistics), instrument_id);
3308
3309        assert_eq!(custom.data_type, subscribed);
3310        assert_eq!(
3311            get_custom_topic(&custom.data_type),
3312            get_custom_topic(&subscribed)
3313        );
3314        assert_eq!(custom.data_type.identifier(), Some("ESM4.GLBX"));
3315    }
3316
3317    #[rstest]
3318    fn test_subscribe_schema_sends_raw_symbol() {
3319        let client = test_data_client();
3320        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3321        client
3322            .cmd_channels
3323            .lock()
3324            .insert("GLBX.MDP3".to_string(), tx);
3325
3326        client
3327            .subscribe_schema(InstrumentId::from("ESM4.GLBX"), dbn::Schema::Ohlcv1M)
3328            .unwrap();
3329
3330        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3331            panic!("expected subscription");
3332        };
3333        assert_eq!(subscription.stype_in, dbn::SType::RawSymbol);
3334        assert_eq!(
3335            subscription.symbols,
3336            databento::Symbols::Symbols(vec!["ESM4".to_string()])
3337        );
3338    }
3339
3340    #[rstest]
3341    #[case::continuous("ES.c.0.GLBX")]
3342    #[case::parent("ES.FUT.GLBX")]
3343    #[case::instrument_id("12345.GLBX")]
3344    fn test_subscribe_schema_rejects_non_raw_symbols(#[case] instrument_id: &str) {
3345        let client = test_data_client();
3346
3347        let result = client.subscribe_schema(InstrumentId::from(instrument_id), dbn::Schema::Mbp10);
3348
3349        assert!(result.is_err());
3350        assert!(client.cmd_channels.lock().is_empty());
3351    }
3352
3353    #[rstest]
3354    fn test_subscribe_custom_data_rejects_metadata() {
3355        let mut client = test_data_client();
3356        let metadata = Params::new([("venue".to_string(), json!("GLBX"))].into());
3357
3358        let command = SubscribeCustomData::new(
3359            Some(ClientId::from("DATABENTO-TEST")),
3360            None,
3361            DataType::new(
3362                "DatabentoStatistics",
3363                Some(metadata),
3364                Some("ESM4.GLBX".to_string()),
3365            ),
3366            UUID4::new(),
3367            UnixNanos::default(),
3368            None,
3369            None,
3370        );
3371
3372        assert!(client.subscribe(command).is_err());
3373        assert!(client.cmd_channels.lock().is_empty());
3374    }
3375
3376    #[rstest]
3377    fn test_subscribed_custom_data_reaches_subscriber() {
3378        let instrument_id = InstrumentId::from("ESM4.GLBX");
3379        let subscribed = DataType::new("DatabentoStatistics", None, Some("ESM4.GLBX".to_string()));
3380        let other = DataType::new("DatabentoStatistics", None, Some("ESU4.GLBX".to_string()));
3381        let handler = msgbus::stubs::get_message_saving_handler::<CustomData>(None);
3382        msgbus::subscribe_any(get_custom_topic(&subscribed).into(), handler.clone(), None);
3383        let statistics = DatabentoStatistics::new(
3384            instrument_id,
3385            DatabentoStatisticType::SettlementPrice,
3386            DatabentoStatisticUpdateAction::Added,
3387            Some(Price::from("5000.25")),
3388            None,
3389            1,
3390            2,
3391            3,
3392            UnixNanos::from(4),
3393            5,
3394            UnixNanos::from(6),
3395            UnixNanos::from(7),
3396            UnixNanos::from(8),
3397        );
3398        let custom = custom_data_for_instrument(Arc::new(statistics), instrument_id);
3399
3400        msgbus::publish_any(get_custom_topic(&custom.data_type), &custom);
3401        msgbus::publish_any(get_custom_topic(&other), &custom);
3402
3403        let received = msgbus::stubs::get_saved_messages::<CustomData>(&handler);
3404        assert_eq!(received, vec![custom]);
3405    }
3406
3407    #[rstest]
3408    fn test_subscribe_book_depth_accepts_depth_ten() {
3409        let mut client = test_data_client();
3410        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3411        client
3412            .cmd_channels
3413            .lock()
3414            .insert("GLBX.MDP3".to_string(), tx);
3415
3416        client
3417            .subscribe_book_depth(subscribe_book_depth_cmd(NonZeroUsize::new(10)))
3418            .unwrap();
3419
3420        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3421            panic!("expected subscription");
3422        };
3423        assert_eq!(subscription.schema, dbn::Schema::Mbp10);
3424    }
3425
3426    #[rstest]
3427    #[case::five(5)]
3428    #[case::twenty(20)]
3429    fn test_subscribe_book_depth_rejects_other_depths(#[case] depth: usize) {
3430        let mut client = test_data_client();
3431
3432        let result =
3433            client.subscribe_book_depth(subscribe_book_depth_cmd(NonZeroUsize::new(depth)));
3434
3435        assert!(result.is_err());
3436        assert!(client.cmd_channels.lock().is_empty());
3437    }
3438
3439    #[rstest]
3440    fn test_subscribe_bars_sends_ohlcv_subscription() {
3441        let mut client = test_data_client();
3442        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3443        client
3444            .cmd_channels
3445            .lock()
3446            .insert("GLBX.MDP3".to_string(), tx);
3447        let command = SubscribeBars::new(
3448            BarType::from("ESM4.GLBX-1-MINUTE-LAST-EXTERNAL"),
3449            Some(ClientId::from("DATABENTO-TEST")),
3450            None,
3451            UUID4::new(),
3452            UnixNanos::default(),
3453            None,
3454            None,
3455        );
3456
3457        client.subscribe_bars(command).unwrap();
3458
3459        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3460            panic!("expected subscription");
3461        };
3462        assert_eq!(subscription.schema, dbn::Schema::Ohlcv1M);
3463        assert_eq!(
3464            subscription.symbols,
3465            databento::Symbols::Symbols(vec!["ESM4".to_string()])
3466        );
3467        assert!(rx.try_recv().is_err());
3468    }
3469
3470    #[rstest]
3471    fn test_subscribe_book_depth_sends_mbp10_subscription() {
3472        let mut client = test_data_client();
3473        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3474        client
3475            .cmd_channels
3476            .lock()
3477            .insert("GLBX.MDP3".to_string(), tx);
3478        client
3479            .subscribe_book_depth(subscribe_book_depth_cmd(None))
3480            .unwrap();
3481
3482        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
3483            panic!("expected subscription");
3484        };
3485        assert_eq!(subscription.schema, dbn::Schema::Mbp10);
3486        assert_eq!(
3487            subscription.symbols,
3488            databento::Symbols::Symbols(vec!["ESM4".to_string()])
3489        );
3490        assert!(rx.try_recv().is_err());
3491    }
3492
3493    #[rstest]
3494    fn test_send_subscription_commands_without_precision_or_start() {
3495        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
3496        let subscription = Subscription::builder()
3497            .schema(dbn::Schema::Mbp1)
3498            .symbols(vec!["ESM4"])
3499            .build();
3500
3501        send_subscription_commands(&tx, "GLBX.MDP3", None, subscription, false).unwrap();
3502
3503        assert!(matches!(
3504            rx.try_recv().unwrap(),
3505            HandlerCommand::Subscribe(sub) if sub.schema == dbn::Schema::Mbp1
3506        ));
3507        assert!(matches!(
3508            rx.try_recv(),
3509            Err(tokio::sync::mpsc::error::TryRecvError::Empty)
3510        ));
3511    }
3512}