Skip to main content

nautilus_databento/
historical.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Core Databento historical client for both Rust and Python usage.
17
18use std::{fmt::Debug, fs, num::NonZeroU64, path::PathBuf, str::FromStr, sync::Arc};
19
20use ahash::{AHashMap, AHashSet};
21use databento::{
22    dbn::{self, decode::DbnMetadata},
23    historical::timeseries::GetRangeParams,
24};
25use indexmap::IndexMap;
26use nautilus_core::{AtomicMap, UnixNanos, consts::NAUTILUS_USER_AGENT, time::AtomicTime};
27use nautilus_model::{
28    data::{Bar, Data, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
29    enums::BarAggregation,
30    identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
31    instruments::{Instrument, InstrumentAny},
32};
33
34use crate::{
35    common::{Credential, get_date_time_range, ohlcv_schema_from_aggregation},
36    decode::{
37        MboDeltaBuffer, decode_imbalance_msg, decode_instrument_def_msg, decode_mbo_msg,
38        decode_mbp10_msg, decode_record, decode_statistics_msg, decode_status_msg,
39        is_supported_stat_type,
40    },
41    symbology::{
42        MetadataCache, check_consistent_symbology, decode_nautilus_instrument_id,
43        infer_symbology_type, nautilus_symbol, raw_symbol,
44    },
45    types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, PublisherId},
46};
47
48/// Core Databento historical client for fetching historical market data.
49///
50/// This client provides both synchronous and asynchronous interfaces for fetching
51/// various types of historical market data from Databento.
52#[derive(Clone)]
53pub struct DatabentoHistoricalClient {
54    futures_year_digits: FuturesYearDigits,
55    share_class_space: bool,
56    credential: Credential,
57    clock: &'static AtomicTime,
58    inner: Arc<databento::HistoricalClient>,
59    publisher_venue_map: Arc<IndexMap<PublisherId, Venue>>,
60    symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
61    price_precisions: Arc<AtomicMap<Symbol, u8>>,
62    use_exchange_as_venue: bool,
63    futures_symbols: Arc<AtomicMap<(PublisherId, u32, time::Date), Symbol>>,
64}
65
66/// Parameters for range queries to Databento historical API.
67#[derive(Debug)]
68pub struct RangeQueryParams {
69    pub dataset: String,
70    pub symbols: Vec<String>,
71    pub start: UnixNanos,
72    pub end: Option<UnixNanos>,
73    pub limit: Option<u64>,
74    pub price_precision: Option<u8>,
75}
76
77/// Result containing dataset date range information.
78#[derive(Debug, Clone)]
79pub struct DatasetRange {
80    pub start: String,
81    pub end: String,
82}
83
84impl Debug for DatabentoHistoricalClient {
85    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86        f.debug_struct(stringify!(DatabentoHistoricalClient))
87            .field("credential", &self.credential)
88            .finish()
89    }
90}
91
92impl DatabentoHistoricalClient {
93    /// Returns the API key from the stored credential.
94    #[must_use]
95    pub fn api_key(&self) -> &str {
96        self.credential.api_key()
97    }
98
99    /// Creates a new [`DatabentoHistoricalClient`] instance.
100    ///
101    /// # Errors
102    ///
103    /// Returns an error if client creation or publisher loading fails.
104    pub fn new(
105        credential: Credential,
106        publishers_filepath: PathBuf,
107        clock: &'static AtomicTime,
108        use_exchange_as_venue: bool,
109    ) -> anyhow::Result<Self> {
110        let client = databento::HistoricalClient::builder()
111            .user_agent_extension(NAUTILUS_USER_AGENT.into())
112            .key(credential.api_key())
113            .map_err(|e| anyhow::anyhow!("Failed to create client builder: {e}"))?
114            .build()
115            .map_err(|e| anyhow::anyhow!("Failed to build client: {e}"))?;
116
117        Self::from_client(
118            credential,
119            publishers_filepath,
120            clock,
121            use_exchange_as_venue,
122            client,
123        )
124    }
125
126    /// Selects one- or two-digit years for futures instrument IDs.
127    #[must_use]
128    pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
129        self.futures_year_digits = futures_year_digits;
130        self
131    }
132
133    /// Spells a share class with a space in instrument IDs (`BRK.B` -> `BRK B`).
134    #[must_use]
135    pub fn with_share_class_space(mut self, share_class_space: bool) -> Self {
136        self.share_class_space = share_class_space;
137        self
138    }
139
140    /// Creates a new [`DatabentoHistoricalClient`] instance with a custom API base URL.
141    ///
142    /// This is intended for tests, benchmarks, and controlled deployments that route
143    /// Databento Historical API requests through a proxy.
144    ///
145    /// # Errors
146    ///
147    /// Returns an error if client creation, URL parsing, or publisher loading fails.
148    pub fn new_with_base_url(
149        credential: Credential,
150        publishers_filepath: PathBuf,
151        clock: &'static AtomicTime,
152        use_exchange_as_venue: bool,
153        base_url: &str,
154    ) -> anyhow::Result<Self> {
155        let client = databento::HistoricalClient::builder()
156            .user_agent_extension(NAUTILUS_USER_AGENT.into())
157            .base_url(base_url.parse().map_err(|e| {
158                anyhow::anyhow!("Failed to parse Databento Historical API base URL: {e}")
159            })?)
160            .key(credential.api_key())
161            .map_err(|e| anyhow::anyhow!("Failed to create client builder: {e}"))?
162            .build()
163            .map_err(|e| anyhow::anyhow!("Failed to build client: {e}"))?;
164
165        Self::from_client(
166            credential,
167            publishers_filepath,
168            clock,
169            use_exchange_as_venue,
170            client,
171        )
172    }
173
174    fn from_client(
175        credential: Credential,
176        publishers_filepath: PathBuf,
177        clock: &'static AtomicTime,
178        use_exchange_as_venue: bool,
179        client: databento::HistoricalClient,
180    ) -> anyhow::Result<Self> {
181        let file_content = fs::read_to_string(publishers_filepath)?;
182        let publishers_vec: Vec<DatabentoPublisher> = serde_json::from_str(&file_content)?;
183
184        let publisher_venue_map = publishers_vec
185            .into_iter()
186            .map(|p| (p.publisher_id, Venue::from(p.venue.as_str())))
187            .collect::<IndexMap<u16, Venue>>();
188
189        Ok(Self {
190            clock,
191            futures_year_digits: FuturesYearDigits::default(),
192            share_class_space: true,
193            inner: Arc::new(client),
194            publisher_venue_map: Arc::new(publisher_venue_map),
195            symbol_venue_map: Arc::new(AtomicMap::new()),
196            price_precisions: Arc::new(AtomicMap::new()),
197            credential,
198            use_exchange_as_venue,
199            futures_symbols: Arc::new(AtomicMap::new()),
200        })
201    }
202
203    /// Caches a `price_precision` for the given `symbol`.
204    ///
205    /// When market data is fetched without an explicit `price_precision`, the
206    /// client resolves precision per record from this cache. Instruments
207    /// returned by [`Self::get_range_instruments`] are inserted automatically.
208    pub fn set_price_precision(&self, symbol: Symbol, price_precision: u8) {
209        self.price_precisions.insert(symbol, price_precision);
210    }
211
212    /// Returns a cached `price_precision` for the given `symbol`.
213    #[must_use]
214    pub fn price_precision(&self, symbol: Symbol) -> Option<u8> {
215        self.price_precisions.load().get(&symbol).copied()
216    }
217
218    /// Resolves a price precision for the given `instrument_id`.
219    ///
220    /// Resolution order:
221    /// 1. The explicit `price_precision` argument (if `Some`).
222    /// 2. The cached precision for the instrument's symbol.
223    ///
224    /// # Errors
225    ///
226    /// Returns an error if no precision is available.
227    fn resolve_price_precision(
228        &self,
229        instrument_id: &InstrumentId,
230        price_precision: Option<u8>,
231    ) -> anyhow::Result<u8> {
232        if let Some(precision) = price_precision {
233            return Ok(precision);
234        }
235
236        let precisions = self.price_precisions.load();
237        precisions
238            .get(&instrument_id.symbol)
239            .copied()
240            .ok_or_else(|| {
241                anyhow::anyhow!(
242                    "Could not resolve `price_precision` for {instrument_id}: \
243                     pass `price_precision` explicitly, call `set_price_precision`, \
244                     or fetch the instrument definitions first via `get_range_instruments`"
245                )
246            })
247    }
248
249    fn resolve_cached_price_precision(
250        &self,
251        instrument_id: &InstrumentId,
252        price_precision: Option<u8>,
253        precision_cache: &mut AHashMap<InstrumentId, u8>,
254    ) -> anyhow::Result<u8> {
255        if let Some(precision) = price_precision {
256            return Ok(precision);
257        }
258
259        if let Some(precision) = precision_cache.get(instrument_id) {
260            return Ok(*precision);
261        }
262
263        let precision = self.resolve_price_precision(instrument_id, None)?;
264        precision_cache.insert(*instrument_id, precision);
265        Ok(precision)
266    }
267
268    /// Gets the date range for a specific dataset.
269    ///
270    /// # Errors
271    ///
272    /// Returns an error if the API request fails.
273    pub async fn get_dataset_range(&self, dataset: &str) -> anyhow::Result<DatasetRange> {
274        let mut client = (*self.inner).clone();
275        let response = client
276            .metadata()
277            .get_dataset_range(dataset)
278            .await
279            .map_err(|e| anyhow::anyhow!("Failed to get dataset range: {e}"))?;
280
281        Ok(DatasetRange {
282            start: response.start.to_string(),
283            end: response.end.to_string(),
284        })
285    }
286
287    /// Fetches instrument definitions for the given parameters.
288    ///
289    /// # Errors
290    ///
291    /// Returns an error if the API request or data processing fails.
292    pub async fn get_range_instruments(
293        &self,
294        params: RangeQueryParams,
295    ) -> anyhow::Result<Vec<InstrumentAny>> {
296        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
297        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
298        check_consistent_symbology(&symbols)?;
299
300        let first_symbol = params
301            .symbols
302            .first()
303            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
304        let stype_in = infer_symbology_type(first_symbol);
305        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
306        let time_range = get_date_time_range(params.start, end)?;
307
308        let range_params = GetRangeParams::builder()
309            .dataset(params.dataset)
310            .date_time_range(time_range)
311            .symbols(symbols)
312            .stype_in(stype_in)
313            .schema(dbn::Schema::Definition)
314            .maybe_limit(params.limit.and_then(NonZeroU64::new))
315            .build();
316
317        let mut client = (*self.inner).clone();
318        let mut decoder = client
319            .timeseries()
320            .get_range(&range_params)
321            .await
322            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
323
324        let metadata = decoder.metadata().clone();
325        let mut metadata_cache = MetadataCache::new(metadata);
326        let mut instruments = Vec::new();
327        let mut futures_symbols = Vec::new();
328
329        while let Some(msg) = decoder.decode_record::<dbn::InstrumentDefMsg>().await? {
330            let record = dbn::RecordRef::from(msg);
331            let sym_map = self.symbol_venue_map.load();
332            let mut instrument_id = decode_nautilus_instrument_id(
333                &record,
334                &mut metadata_cache,
335                &self.publisher_venue_map,
336                &sym_map,
337                self.futures_year_digits,
338                self.share_class_space,
339                &self.futures_symbols.load(),
340                Some(&requested_symbols),
341            )?;
342
343            if self.use_exchange_as_venue && instrument_id.venue == Venue::GLBX() {
344                let exchange = msg
345                    .exchange()
346                    .map_err(|e| anyhow::anyhow!("Missing exchange in record: {e}"))?;
347                let venue = Venue::from_code(exchange)
348                    .map_err(|e| anyhow::anyhow!("Venue not found for exchange {exchange}: {e}"))?;
349                instrument_id.venue = venue;
350            }
351
352            match decode_instrument_def_msg(msg, instrument_id, None, None) {
353                Ok(Some(instrument)) => {
354                    if nautilus_symbol(instrument.raw_symbol().as_str(), self.share_class_space)
355                        != instrument.id().symbol
356                    {
357                        let date = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(
358                            msg.ts_recv,
359                        ))?
360                        .date();
361                        futures_symbols.push((
362                            (msg.hd.publisher_id, msg.hd.instrument_id, date),
363                            instrument.id().symbol,
364                        ));
365                    }
366
367                    instruments.push(instrument);
368                }
369                Ok(None) => {} // Decoder logged a warning for the unsupported class
370                Err(e) => anyhow::bail!("Failed to decode instrument {instrument_id}: {e}"),
371            }
372        }
373
374        if !futures_symbols.is_empty() {
375            self.futures_symbols
376                .rcu(|symbols| symbols.extend(futures_symbols.iter().copied()));
377        }
378
379        self.cache_price_precisions(&instruments);
380
381        Ok(instruments)
382    }
383
384    fn cache_price_precisions(&self, instruments: &[InstrumentAny]) {
385        if instruments.is_empty() {
386            return;
387        }
388
389        self.price_precisions.rcu(|precisions| {
390            for instrument in instruments {
391                precisions.insert(instrument.id().symbol, instrument.price_precision());
392            }
393        });
394    }
395
396    /// Fetches quote ticks for the given parameters.
397    ///
398    /// # Errors
399    ///
400    /// Returns an error if the API request or data processing fails.
401    pub async fn get_range_quotes(
402        &self,
403        params: RangeQueryParams,
404        schema: Option<String>,
405    ) -> anyhow::Result<Vec<QuoteTick>> {
406        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
407        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
408        check_consistent_symbology(&symbols)?;
409
410        let first_symbol = params
411            .symbols
412            .first()
413            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
414        let stype_in = infer_symbology_type(first_symbol);
415        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
416        let time_range = get_date_time_range(params.start, end)?;
417        let schema = schema.unwrap_or_else(|| "mbp-1".to_string());
418        let dbn_schema = dbn::Schema::from_str(&schema)?;
419
420        match dbn_schema {
421            dbn::Schema::Mbp1
422            | dbn::Schema::Tbbo
423            | dbn::Schema::Bbo1S
424            | dbn::Schema::Bbo1M
425            | dbn::Schema::Cmbp1
426            | dbn::Schema::Tcbbo
427            | dbn::Schema::Cbbo1S
428            | dbn::Schema::Cbbo1M => (),
429            _ => anyhow::bail!(
430                "Invalid schema. Must be one of: mbp-1, tbbo, bbo-1s, bbo-1m, cmbp-1, tcbbo, cbbo-1s, cbbo-1m"
431            ),
432        }
433
434        let range_params = GetRangeParams::builder()
435            .dataset(params.dataset)
436            .date_time_range(time_range)
437            .symbols(symbols)
438            .stype_in(stype_in)
439            .schema(dbn_schema)
440            .maybe_limit(params.limit.and_then(NonZeroU64::new))
441            .build();
442
443        let price_precision_arg = params.price_precision;
444
445        let mut client = (*self.inner).clone();
446        let mut decoder = client
447            .timeseries()
448            .get_range(&range_params)
449            .await
450            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
451
452        let metadata = decoder.metadata().clone();
453        let mut metadata_cache = MetadataCache::new(metadata);
454        let mut precision_cache = AHashMap::new();
455        let mut result: Vec<QuoteTick> = Vec::new();
456
457        let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
458            let instrument_id =
459                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
460            let price_precision = self.resolve_cached_price_precision(
461                &instrument_id,
462                price_precision_arg,
463                &mut precision_cache,
464            )?;
465
466            let (data, _) = decode_record(
467                &record,
468                instrument_id,
469                price_precision,
470                None,
471                false, // Don't include trades
472                true,
473            )?;
474
475            match data {
476                Some(Data::Quote(quote)) => result.push(quote),
477                None => {} // Skip records with undefined bid/ask prices
478                _ => anyhow::bail!("Invalid data element not `QuoteTick`, was {data:?}"),
479            }
480            Ok(())
481        };
482
483        match dbn_schema {
484            dbn::Schema::Mbp1 => {
485                while let Some(msg) = decoder.decode_record::<dbn::Mbp1Msg>().await? {
486                    process_record(dbn::RecordRef::from(msg))?;
487                }
488            }
489            dbn::Schema::Tbbo => {
490                while let Some(msg) = decoder.decode_record::<dbn::TbboMsg>().await? {
491                    process_record(dbn::RecordRef::from(msg))?;
492                }
493            }
494            dbn::Schema::Cmbp1 => {
495                while let Some(msg) = decoder.decode_record::<dbn::Cmbp1Msg>().await? {
496                    process_record(dbn::RecordRef::from(msg))?;
497                }
498            }
499            dbn::Schema::Tcbbo => {
500                while let Some(msg) = decoder.decode_record::<dbn::TcbboMsg>().await? {
501                    process_record(dbn::RecordRef::from(msg))?;
502                }
503            }
504            dbn::Schema::Bbo1M => {
505                while let Some(msg) = decoder.decode_record::<dbn::Bbo1MMsg>().await? {
506                    process_record(dbn::RecordRef::from(msg))?;
507                }
508            }
509            dbn::Schema::Bbo1S => {
510                while let Some(msg) = decoder.decode_record::<dbn::Bbo1SMsg>().await? {
511                    process_record(dbn::RecordRef::from(msg))?;
512                }
513            }
514            dbn::Schema::Cbbo1S | dbn::Schema::Cbbo1M => {
515                while let Some(msg) = decoder.decode_record::<dbn::CbboMsg>().await? {
516                    process_record(dbn::RecordRef::from(msg))?;
517                }
518            }
519            _ => anyhow::bail!("Invalid schema {dbn_schema}"),
520        }
521
522        Ok(result)
523    }
524
525    /// Fetches order book depth snapshots for the given parameters.
526    ///
527    /// # Errors
528    ///
529    /// Returns an error if the API request or data processing fails.
530    pub async fn get_range_order_book_depth(
531        &self,
532        params: RangeQueryParams,
533        depth: Option<usize>,
534    ) -> anyhow::Result<Vec<OrderBookDepth>> {
535        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
536        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
537        check_consistent_symbology(&symbols)?;
538
539        let first_symbol = params
540            .symbols
541            .first()
542            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
543        let stype_in = infer_symbology_type(first_symbol);
544        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
545        let time_range = get_date_time_range(params.start, end)?;
546
547        // For now, only support MBP_10 schema for depth 10
548        let _depth = depth.unwrap_or(10);
549        if _depth != 10 {
550            anyhow::bail!("Only depth=10 is currently supported for order book depths");
551        }
552
553        let range_params = GetRangeParams::builder()
554            .dataset(params.dataset)
555            .date_time_range(time_range)
556            .symbols(symbols)
557            .stype_in(stype_in)
558            .schema(dbn::Schema::Mbp10)
559            .maybe_limit(params.limit.and_then(NonZeroU64::new))
560            .build();
561
562        let price_precision_arg = params.price_precision;
563
564        let mut client = (*self.inner).clone();
565        let mut decoder = client
566            .timeseries()
567            .get_range(&range_params)
568            .await
569            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
570
571        let metadata = decoder.metadata().clone();
572        let mut metadata_cache = MetadataCache::new(metadata);
573        let mut precision_cache = AHashMap::new();
574        let mut result: Vec<OrderBookDepth> = Vec::new();
575
576        let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
577            let instrument_id =
578                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
579            let price_precision = self.resolve_cached_price_precision(
580                &instrument_id,
581                price_precision_arg,
582                &mut precision_cache,
583            )?;
584
585            if let Some(msg) = record.get::<dbn::Mbp10Msg>() {
586                let depth = decode_mbp10_msg(msg, instrument_id, price_precision, None)?;
587                result.push(depth);
588            }
589
590            Ok(())
591        };
592
593        while let Some(msg) = decoder.decode_record::<dbn::Mbp10Msg>().await? {
594            process_record(dbn::RecordRef::from(msg))?;
595        }
596
597        Ok(result)
598    }
599
600    /// Fetches order book deltas for the given parameters.
601    ///
602    /// # Errors
603    ///
604    /// Returns an error if the API request or data processing fails.
605    pub async fn get_range_order_book_deltas(
606        &self,
607        params: RangeQueryParams,
608    ) -> anyhow::Result<Vec<OrderBookDelta>> {
609        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
610        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
611        check_consistent_symbology(&symbols)?;
612
613        let first_symbol = params
614            .symbols
615            .first()
616            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
617        let stype_in = infer_symbology_type(first_symbol);
618        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
619        let time_range = get_date_time_range(params.start, end)?;
620
621        let range_params = GetRangeParams::builder()
622            .dataset(params.dataset)
623            .date_time_range(time_range)
624            .symbols(symbols)
625            .stype_in(stype_in)
626            .schema(dbn::Schema::Mbo)
627            .maybe_limit(params.limit.and_then(NonZeroU64::new))
628            .build();
629
630        let price_precision_arg = params.price_precision;
631
632        let mut client = (*self.inner).clone();
633        let mut decoder = client
634            .timeseries()
635            .get_range(&range_params)
636            .await
637            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
638
639        let metadata = decoder.metadata().clone();
640        let mut metadata_cache = MetadataCache::new(metadata);
641        let mut precision_cache = AHashMap::new();
642        let mut result: Vec<OrderBookDelta> = Vec::new();
643        let mut delta_buffer = MboDeltaBuffer::default();
644
645        let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
646            let instrument_id =
647                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
648            let price_precision = self.resolve_cached_price_precision(
649                &instrument_id,
650                price_precision_arg,
651                &mut precision_cache,
652            )?;
653
654            if let Some(msg) = record.get::<dbn::MboMsg>() {
655                let (delta, _trade) =
656                    decode_mbo_msg(msg, instrument_id, price_precision, None, false)?;
657                delta_buffer.push(msg, instrument_id, delta);
658                while let Some(delta) = delta_buffer.pop_ready() {
659                    result.push(delta);
660                }
661            }
662
663            Ok(())
664        };
665
666        while let Some(msg) = decoder.decode_record::<dbn::MboMsg>().await? {
667            process_record(dbn::RecordRef::from(msg))?;
668        }
669
670        delta_buffer.finish();
671        while let Some(delta) = delta_buffer.pop_ready() {
672            result.push(delta);
673        }
674
675        Ok(result)
676    }
677
678    /// Fetches trade ticks for the given parameters.
679    ///
680    /// # Errors
681    ///
682    /// Returns an error if the API request or data processing fails.
683    pub async fn get_range_trades(
684        &self,
685        params: RangeQueryParams,
686        schema: Option<String>,
687    ) -> anyhow::Result<Vec<TradeTick>> {
688        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
689        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
690        check_consistent_symbology(&symbols)?;
691
692        let first_symbol = params
693            .symbols
694            .first()
695            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
696        let stype_in = infer_symbology_type(first_symbol);
697        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
698        let time_range = get_date_time_range(params.start, end)?;
699        let schema = schema.unwrap_or_else(|| "trades".to_string());
700        let dbn_schema = dbn::Schema::from_str(&schema)?;
701
702        match dbn_schema {
703            dbn::Schema::Trades
704            | dbn::Schema::Tbbo
705            | dbn::Schema::Tcbbo
706            | dbn::Schema::Mbp1
707            | dbn::Schema::Cmbp1 => (),
708            _ => {
709                anyhow::bail!("Invalid schema. Must be one of: trades, tbbo, tcbbo, mbp-1, cmbp-1")
710            }
711        }
712
713        let range_params = GetRangeParams::builder()
714            .dataset(params.dataset)
715            .date_time_range(time_range)
716            .symbols(symbols)
717            .stype_in(stype_in)
718            .schema(dbn_schema)
719            .maybe_limit(params.limit.and_then(NonZeroU64::new))
720            .build();
721
722        let price_precision_arg = params.price_precision;
723
724        let mut client = (*self.inner).clone();
725        let mut decoder = client
726            .timeseries()
727            .get_range(&range_params)
728            .await
729            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
730
731        let metadata = decoder.metadata().clone();
732        let mut metadata_cache = MetadataCache::new(metadata);
733        let mut precision_cache = AHashMap::new();
734        let mut result: Vec<TradeTick> = Vec::new();
735
736        let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
737            let instrument_id =
738                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
739            let price_precision = self.resolve_cached_price_precision(
740                &instrument_id,
741                price_precision_arg,
742                &mut precision_cache,
743            )?;
744
745            let (data, data2) =
746                decode_record(&record, instrument_id, price_precision, None, true, true)?;
747
748            match (data, data2) {
749                (Some(Data::Trade(trade)), _) | (_, Some(Data::Trade(trade))) => result.push(trade),
750                (Some(_) | None, None) => {}
751                (None, Some(data)) => {
752                    anyhow::bail!("Invalid data element not `TradeTick`, was {data:?}")
753                }
754                (Some(data), Some(_)) => {
755                    anyhow::bail!("Invalid data element not `TradeTick`, was {data:?}")
756                }
757            }
758            Ok(())
759        };
760
761        match dbn_schema {
762            dbn::Schema::Trades => {
763                while let Some(msg) = decoder.decode_record::<dbn::TradeMsg>().await? {
764                    process_record(dbn::RecordRef::from(msg))?;
765                }
766            }
767            dbn::Schema::Mbp1 => {
768                while let Some(msg) = decoder.decode_record::<dbn::Mbp1Msg>().await? {
769                    process_record(dbn::RecordRef::from(msg))?;
770                }
771            }
772            dbn::Schema::Tbbo => {
773                while let Some(msg) = decoder.decode_record::<dbn::TbboMsg>().await? {
774                    process_record(dbn::RecordRef::from(msg))?;
775                }
776            }
777            dbn::Schema::Cmbp1 => {
778                while let Some(msg) = decoder.decode_record::<dbn::Cmbp1Msg>().await? {
779                    process_record(dbn::RecordRef::from(msg))?;
780                }
781            }
782            dbn::Schema::Tcbbo => {
783                while let Some(msg) = decoder.decode_record::<dbn::TcbboMsg>().await? {
784                    process_record(dbn::RecordRef::from(msg))?;
785                }
786            }
787            _ => anyhow::bail!("Invalid schema {dbn_schema}"),
788        }
789
790        Ok(result)
791    }
792
793    /// Fetches bars for the given parameters.
794    ///
795    /// # Errors
796    ///
797    /// Returns an error if the API request or data processing fails.
798    pub async fn get_range_bars(
799        &self,
800        params: RangeQueryParams,
801        aggregation: BarAggregation,
802        timestamp_on_close: bool,
803    ) -> anyhow::Result<Vec<Bar>> {
804        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
805        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
806        check_consistent_symbology(&symbols)?;
807
808        let first_symbol = params
809            .symbols
810            .first()
811            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
812        let stype_in = infer_symbology_type(first_symbol);
813        let schema = ohlcv_schema_from_aggregation(aggregation)?;
814
815        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
816        let time_range = get_date_time_range(params.start, end)?;
817
818        let range_params = GetRangeParams::builder()
819            .dataset(params.dataset)
820            .date_time_range(time_range)
821            .symbols(symbols)
822            .stype_in(stype_in)
823            .schema(schema)
824            .maybe_limit(params.limit.and_then(NonZeroU64::new))
825            .build();
826
827        let price_precision_arg = params.price_precision;
828
829        let mut client = (*self.inner).clone();
830        let mut decoder = client
831            .timeseries()
832            .get_range(&range_params)
833            .await
834            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
835
836        let metadata = decoder.metadata().clone();
837        let mut metadata_cache = MetadataCache::new(metadata);
838        let mut precision_cache = AHashMap::new();
839        let mut result: Vec<Bar> = Vec::new();
840
841        while let Some(msg) = decoder.decode_record::<dbn::OhlcvMsg>().await? {
842            let record = dbn::RecordRef::from(msg);
843            let instrument_id =
844                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
845            let price_precision = self.resolve_cached_price_precision(
846                &instrument_id,
847                price_precision_arg,
848                &mut precision_cache,
849            )?;
850
851            let (data, _) = decode_record(
852                &record,
853                instrument_id,
854                price_precision,
855                None,
856                false, // Not applicable
857                timestamp_on_close,
858            )?;
859
860            match data {
861                Some(Data::Bar(bar)) => {
862                    result.push(bar);
863                }
864                _ => anyhow::bail!("Invalid data element not `Bar`, was {data:?}"),
865            }
866        }
867
868        Ok(result)
869    }
870
871    /// Fetches imbalance data for the given parameters.
872    ///
873    /// # Errors
874    ///
875    /// Returns an error if the API request or data processing fails.
876    pub async fn get_range_imbalance(
877        &self,
878        params: RangeQueryParams,
879    ) -> anyhow::Result<Vec<DatabentoImbalance>> {
880        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
881        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
882        check_consistent_symbology(&symbols)?;
883
884        let first_symbol = params
885            .symbols
886            .first()
887            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
888        let stype_in = infer_symbology_type(first_symbol);
889        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
890        let time_range = get_date_time_range(params.start, end)?;
891
892        let range_params = GetRangeParams::builder()
893            .dataset(params.dataset)
894            .date_time_range(time_range)
895            .symbols(symbols)
896            .stype_in(stype_in)
897            .schema(dbn::Schema::Imbalance)
898            .maybe_limit(params.limit.and_then(NonZeroU64::new))
899            .build();
900
901        let price_precision_arg = params.price_precision;
902
903        let mut client = (*self.inner).clone();
904        let mut decoder = client
905            .timeseries()
906            .get_range(&range_params)
907            .await
908            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
909
910        let metadata = decoder.metadata().clone();
911        let mut metadata_cache = MetadataCache::new(metadata);
912        let mut precision_cache = AHashMap::new();
913        let mut result: Vec<DatabentoImbalance> = Vec::new();
914
915        while let Some(msg) = decoder.decode_record::<dbn::ImbalanceMsg>().await? {
916            let record = dbn::RecordRef::from(msg);
917            let instrument_id =
918                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
919            let price_precision = self.resolve_cached_price_precision(
920                &instrument_id,
921                price_precision_arg,
922                &mut precision_cache,
923            )?;
924
925            let imbalance = decode_imbalance_msg(msg, instrument_id, price_precision, None)?;
926            result.push(imbalance);
927        }
928
929        Ok(result)
930    }
931
932    /// Fetches statistics data for the given parameters.
933    ///
934    /// # Errors
935    ///
936    /// Returns an error if the API request or data processing fails.
937    pub async fn get_range_statistics(
938        &self,
939        params: RangeQueryParams,
940    ) -> anyhow::Result<Vec<DatabentoStatistics>> {
941        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
942        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
943        check_consistent_symbology(&symbols)?;
944
945        let first_symbol = params
946            .symbols
947            .first()
948            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
949        let stype_in = infer_symbology_type(first_symbol);
950        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
951        let time_range = get_date_time_range(params.start, end)?;
952
953        let range_params = GetRangeParams::builder()
954            .dataset(params.dataset)
955            .date_time_range(time_range)
956            .symbols(symbols)
957            .stype_in(stype_in)
958            .schema(dbn::Schema::Statistics)
959            .maybe_limit(params.limit.and_then(NonZeroU64::new))
960            .build();
961
962        let price_precision_arg = params.price_precision;
963
964        let mut client = (*self.inner).clone();
965        let mut decoder = client
966            .timeseries()
967            .get_range(&range_params)
968            .await
969            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
970
971        let metadata = decoder.metadata().clone();
972        let mut metadata_cache = MetadataCache::new(metadata);
973        let mut precision_cache = AHashMap::new();
974        let mut result: Vec<DatabentoStatistics> = Vec::new();
975
976        while let Some(msg) = decoder.decode_record::<dbn::StatMsg>().await? {
977            // Precheck before precision resolution so unmodeled types skip cleanly
978            if !is_supported_stat_type(msg.stat_type) {
979                log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
980                continue;
981            }
982
983            let record = dbn::RecordRef::from(msg);
984            let instrument_id =
985                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
986            let price_precision = self.resolve_cached_price_precision(
987                &instrument_id,
988                price_precision_arg,
989                &mut precision_cache,
990            )?;
991
992            if let Some(statistics) =
993                decode_statistics_msg(msg, instrument_id, price_precision, None)?
994            {
995                result.push(statistics);
996            }
997        }
998
999        Ok(result)
1000    }
1001
1002    /// Fetches status data for the given parameters.
1003    ///
1004    /// # Errors
1005    ///
1006    /// Returns an error if the API request or data processing fails.
1007    pub async fn get_range_status(
1008        &self,
1009        params: RangeQueryParams,
1010    ) -> anyhow::Result<Vec<InstrumentStatus>> {
1011        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
1012        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
1013        check_consistent_symbology(&symbols)?;
1014
1015        let first_symbol = params
1016            .symbols
1017            .first()
1018            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
1019        let stype_in = infer_symbology_type(first_symbol);
1020        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
1021        let time_range = get_date_time_range(params.start, end)?;
1022
1023        let range_params = GetRangeParams::builder()
1024            .dataset(params.dataset)
1025            .date_time_range(time_range)
1026            .symbols(symbols)
1027            .stype_in(stype_in)
1028            .schema(dbn::Schema::Status)
1029            .maybe_limit(params.limit.and_then(NonZeroU64::new))
1030            .build();
1031
1032        let mut client = (*self.inner).clone();
1033        let mut decoder = client
1034            .timeseries()
1035            .get_range(&range_params)
1036            .await
1037            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
1038
1039        let metadata = decoder.metadata().clone();
1040        let mut metadata_cache = MetadataCache::new(metadata);
1041        let mut result: Vec<InstrumentStatus> = Vec::new();
1042
1043        while let Some(msg) = decoder.decode_record::<dbn::StatusMsg>().await? {
1044            let record = dbn::RecordRef::from(msg);
1045            let instrument_id =
1046                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
1047
1048            let status = decode_status_msg(msg, instrument_id, None)?;
1049            result.push(status);
1050        }
1051
1052        Ok(result)
1053    }
1054
1055    fn instrument_id(
1056        &self,
1057        record: &dbn::RecordRef,
1058        metadata: &mut MetadataCache,
1059        requested_symbols: &AHashSet<Symbol>,
1060    ) -> anyhow::Result<InstrumentId> {
1061        decode_nautilus_instrument_id(
1062            record,
1063            metadata,
1064            &self.publisher_venue_map,
1065            &self.symbol_venue_map.load(),
1066            self.futures_year_digits,
1067            self.share_class_space,
1068            &self.futures_symbols.load(),
1069            Some(requested_symbols),
1070        )
1071    }
1072
1073    fn prepare_symbols(&self, params: &RangeQueryParams) -> (Vec<String>, AHashSet<Symbol>) {
1074        let mut requested_symbols = AHashSet::new();
1075
1076        let raw_symbols = params
1077            .symbols
1078            .iter()
1079            .map(|symbol| {
1080                let raw = raw_symbol(
1081                    symbol,
1082                    &params.dataset,
1083                    self.futures_year_digits,
1084                    self.share_class_space,
1085                );
1086
1087                if raw != *symbol {
1088                    requested_symbols.insert(Symbol::from(symbol.as_str()));
1089                }
1090
1091                raw
1092            })
1093            .collect();
1094
1095        (raw_symbols, requested_symbols)
1096    }
1097
1098    /// Prepares symbols from instrument IDs.
1099    pub fn prepare_symbols_from_instrument_ids(
1100        &self,
1101        instrument_ids: &[InstrumentId],
1102    ) -> Vec<String> {
1103        self.symbol_venue_map.rcu(|m| {
1104            for id in instrument_ids {
1105                m.entry(id.symbol).or_insert(id.venue);
1106            }
1107        });
1108
1109        instrument_ids
1110            .iter()
1111            .map(|id| id.symbol.to_string())
1112            .collect()
1113    }
1114}
1115
1116#[cfg(test)]
1117mod tests {
1118    use std::ffi::c_char;
1119
1120    use databento::dbn::{
1121        MappingInterval, SymbolMapping, decode::DecodeRecord, encode::EncodeRecord,
1122    };
1123    use nautilus_core::time::get_atomic_clock_realtime;
1124    use nautilus_model::types::{Price, Quantity};
1125    use rstest::{fixture, rstest};
1126    use time::macros::date;
1127    use tokio::{
1128        io::{AsyncReadExt, AsyncWriteExt},
1129        net::TcpListener,
1130    };
1131
1132    use super::*;
1133
1134    fn test_api_key() -> String {
1135        "test-000000000000000000000000000".to_string()
1136    }
1137
1138    fn publishers_path() -> PathBuf {
1139        PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json")
1140    }
1141
1142    fn mbo_record(action: c_char, flags: u8, sequence: u32) -> dbn::MboMsg {
1143        let ts_event = 1_609_160_400_000_000_000;
1144        dbn::MboMsg {
1145            hd: dbn::RecordHeader::new::<dbn::MboMsg>(dbn::rtype::MBO, 1, 1, ts_event),
1146            order_id: u64::from(sequence),
1147            price: 4_800_250_000_000,
1148            size: 2,
1149            flags: dbn::FlagSet::new(flags),
1150            channel_id: 1,
1151            action,
1152            side: 'A' as c_char,
1153            ts_recv: ts_event,
1154            ts_in_delta: 0,
1155            sequence,
1156        }
1157    }
1158
1159    fn encode_mbo_response(records: &[dbn::MboMsg]) -> Vec<u8> {
1160        let metadata = dbn::Metadata::builder()
1161            .dataset("GLBX.MDP3")
1162            .schema(Some(dbn::Schema::Mbo))
1163            .start(1_609_160_400_000_000_000)
1164            .stype_in(Some(dbn::SType::RawSymbol))
1165            .stype_out(dbn::SType::InstrumentId)
1166            .symbols(vec!["ESM4".to_string()])
1167            .mappings(vec![SymbolMapping {
1168                raw_symbol: "ESM4".to_string(),
1169                intervals: vec![MappingInterval {
1170                    start_date: date!(2020 - 12 - 28),
1171                    end_date: date!(2020 - 12 - 29),
1172                    symbol: "1".to_string(),
1173                }],
1174            }])
1175            .build();
1176        let mut body = Vec::new();
1177        {
1178            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1179            encoder.encode_records(records).unwrap();
1180            encoder.flush().unwrap();
1181        }
1182        body
1183    }
1184
1185    async fn serve_response(listener: TcpListener, body: Vec<u8>) {
1186        let (mut stream, _) = listener.accept().await.unwrap();
1187        let mut request = Vec::new();
1188        let mut buffer = [0_u8; 1024];
1189        loop {
1190            let read = stream.read(&mut buffer).await.unwrap();
1191            if read == 0 {
1192                break;
1193            }
1194            request.extend_from_slice(&buffer[..read]);
1195            if request.windows(4).any(|window| window == b"\r\n\r\n") {
1196                break;
1197            }
1198        }
1199
1200        let headers = format!(
1201            "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
1202            body.len(),
1203        );
1204        stream.write_all(headers.as_bytes()).await.unwrap();
1205        stream.write_all(&body).await.unwrap();
1206        stream.shutdown().await.unwrap();
1207    }
1208
1209    #[fixture]
1210    fn historical_client() -> DatabentoHistoricalClient {
1211        DatabentoHistoricalClient::new(
1212            Credential::new(test_api_key()),
1213            publishers_path(),
1214            get_atomic_clock_realtime(),
1215            false,
1216        )
1217        .unwrap()
1218    }
1219
1220    #[rstest]
1221    fn test_new_with_base_url_rejects_invalid_url() {
1222        let err = DatabentoHistoricalClient::new_with_base_url(
1223            Credential::new(test_api_key()),
1224            publishers_path(),
1225            get_atomic_clock_realtime(),
1226            false,
1227            "://invalid",
1228        )
1229        .expect_err("expected invalid base URL to fail");
1230        let err_msg = format!("{err}");
1231
1232        assert!(
1233            err_msg.contains("Failed to parse Databento Historical API base URL"),
1234            "unexpected error message: {err_msg}",
1235        );
1236    }
1237
1238    #[rstest]
1239    #[tokio::test]
1240    async fn test_get_range_order_book_deltas_preserves_boundaries_and_drains_end() {
1241        let last = dbn::flags::LAST;
1242        let body = encode_mbo_response(&[
1243            mbo_record('C' as c_char, 0, 1),
1244            mbo_record('N' as c_char, last, 2),
1245            mbo_record('C' as c_char, 0, 3),
1246        ]);
1247        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1248        let base_url = format!("http://{}/", listener.local_addr().unwrap());
1249        let server = tokio::spawn(serve_response(listener, body));
1250        let client = DatabentoHistoricalClient::new_with_base_url(
1251            Credential::new(test_api_key()),
1252            publishers_path(),
1253            get_atomic_clock_realtime(),
1254            false,
1255            &base_url,
1256        )
1257        .unwrap();
1258
1259        let deltas = client
1260            .get_range_order_book_deltas(RangeQueryParams {
1261                dataset: "GLBX.MDP3".to_string(),
1262                symbols: vec!["ESM4".to_string()],
1263                start: 1_609_160_000_000_000_000.into(),
1264                end: Some(1_609_161_000_000_000_000.into()),
1265                limit: None,
1266                price_precision: Some(2),
1267            })
1268            .await
1269            .unwrap();
1270        server.await.unwrap();
1271
1272        assert_eq!(deltas.len(), 2);
1273        assert_eq!(deltas[0].instrument_id, InstrumentId::from("ESM24.GLBX"));
1274        assert_eq!(deltas[0].order.order_id, 1);
1275        assert_eq!(deltas[0].flags, last);
1276        assert_eq!(deltas[0].sequence, 1);
1277        assert_eq!(deltas[1].instrument_id, InstrumentId::from("ESM24.GLBX"));
1278        assert_eq!(deltas[1].order.order_id, 3);
1279        assert_eq!(deltas[1].flags, 0);
1280        assert_eq!(deltas[1].sequence, 3);
1281    }
1282
1283    #[rstest]
1284    fn test_definition_precision_batch_merges_and_last_symbol_wins(
1285        historical_client: DatabentoHistoricalClient,
1286    ) {
1287        let mut first = nautilus_model::instruments::stubs::crypto_perpetual_ethusdt();
1288        first.id = "FIRST.GLBX".into();
1289        first.price_precision = 3;
1290        let mut second = first.clone();
1291        second.id = "SECOND.GLBX".into();
1292        second.price_precision = 5;
1293        let mut latest = first.clone();
1294        latest.id = "FIRST.XNAS".into();
1295        latest.price_precision = 7;
1296        historical_client.set_price_precision("RETAINED".into(), 9);
1297        historical_client.set_price_precision("FIRST".into(), 1);
1298
1299        historical_client.cache_price_precisions(&[
1300            InstrumentAny::CryptoPerpetual(first),
1301            InstrumentAny::CryptoPerpetual(second),
1302            InstrumentAny::CryptoPerpetual(latest),
1303        ]);
1304
1305        assert_eq!(historical_client.price_precisions.len(), 3);
1306
1307        for (symbol, precision) in [("RETAINED", 9), ("FIRST", 7), ("SECOND", 5)] {
1308            assert_eq!(
1309                historical_client
1310                    .resolve_price_precision(&InstrumentId::new(symbol.into(), "GLBX".into()), None)
1311                    .unwrap(),
1312                precision
1313            );
1314        }
1315    }
1316
1317    #[rstest]
1318    fn test_set_price_precision_inserts_into_cache(historical_client: DatabentoHistoricalClient) {
1319        let symbol = Symbol::from("ESM4");
1320
1321        assert_eq!(historical_client.price_precision(symbol), None);
1322
1323        historical_client.set_price_precision(symbol, 2);
1324
1325        assert_eq!(historical_client.price_precision(symbol), Some(2));
1326    }
1327
1328    #[rstest]
1329    fn test_resolve_price_precision_explicit_arg(historical_client: DatabentoHistoricalClient) {
1330        let instrument_id = InstrumentId::from("ESM4.GLBX");
1331        // Seed a deliberately wrong cache value so we know the explicit arg wins
1332        historical_client.set_price_precision(Symbol::from("ESM4"), 9);
1333
1334        let precision = historical_client
1335            .resolve_price_precision(&instrument_id, Some(2))
1336            .unwrap();
1337        assert_eq!(precision, 2);
1338    }
1339
1340    #[rstest]
1341    fn test_resolve_price_precision_cache_hit(historical_client: DatabentoHistoricalClient) {
1342        let instrument_id = InstrumentId::from("ESM4.GLBX");
1343        historical_client.set_price_precision(Symbol::from("ESM4"), 2);
1344
1345        let precision = historical_client
1346            .resolve_price_precision(&instrument_id, None)
1347            .unwrap();
1348        assert_eq!(precision, 2);
1349    }
1350
1351    #[rstest]
1352    fn test_resolve_price_precision_cache_miss_errors(
1353        historical_client: DatabentoHistoricalClient,
1354    ) {
1355        let instrument_id = InstrumentId::from("ESM4.GLBX");
1356
1357        let err = historical_client
1358            .resolve_price_precision(&instrument_id, None)
1359            .expect_err("expected cache-miss error");
1360        let err_msg = format!("{err}");
1361        assert!(
1362            err_msg.contains("Could not resolve `price_precision`"),
1363            "unexpected error message: {err_msg}",
1364        );
1365        assert!(
1366            err_msg.contains("ESM4.GLBX"),
1367            "error should name the instrument: {err_msg}",
1368        );
1369    }
1370
1371    #[rstest]
1372    #[case(1_000)]
1373    #[case(10_000)]
1374    #[tokio::test]
1375    async fn test_historical_definition_batch_preserves_all_symbols(#[case] count: u32) {
1376        let path = publishers_path()
1377            .parent()
1378            .unwrap()
1379            .join("test_data/test_data.definition.futures_contract.dbn.zst");
1380        let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path).unwrap();
1381        let mut definition = decoder
1382            .decode_record::<dbn::InstrumentDefMsg>()
1383            .unwrap()
1384            .unwrap()
1385            .clone();
1386        definition.ts_recv = "2026-01-01T00:00:00Z"
1387            .parse::<jiff::Timestamp>()
1388            .unwrap()
1389            .as_nanosecond() as u64;
1390        definition.maturity_year = 2026;
1391        let mut definitions = Vec::new();
1392        let mut mappings = Vec::new();
1393
1394        for id in 1..=count {
1395            let raw = format!("T{id}U6");
1396            definition.hd.instrument_id = id;
1397            definition.raw_symbol.fill(0);
1398
1399            for (target, byte) in definition.raw_symbol.iter_mut().zip(raw.bytes()) {
1400                *target = byte as c_char;
1401            }
1402
1403            definitions.push(definition.clone());
1404            mappings.push(SymbolMapping {
1405                raw_symbol: raw,
1406                intervals: vec![MappingInterval {
1407                    start_date: date!(2026 - 01 - 01),
1408                    end_date: date!(2026 - 01 - 02),
1409                    symbol: id.to_string(),
1410                }],
1411            });
1412        }
1413
1414        let mut replacement = definitions[0].clone();
1415        replacement.maturity_year = 2036;
1416        definitions.push(replacement);
1417        let metadata = dbn::Metadata::builder()
1418            .dataset("GLBX.MDP3")
1419            .schema(Some(dbn::Schema::Definition))
1420            .start(definition.ts_recv)
1421            .stype_in(Some(dbn::SType::Parent))
1422            .stype_out(dbn::SType::InstrumentId)
1423            .mappings(mappings)
1424            .build();
1425        let mut body = Vec::new();
1426        {
1427            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1428            encoder.encode_records(&definitions).unwrap();
1429            encoder.flush().unwrap();
1430        }
1431
1432        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1433        let base_url = format!("http://{}/", listener.local_addr().unwrap());
1434
1435        let server = tokio::spawn(serve_response(listener, body));
1436        let client = DatabentoHistoricalClient::new_with_base_url(
1437            Credential::new(test_api_key()),
1438            publishers_path(),
1439            get_atomic_clock_realtime(),
1440            false,
1441            &base_url,
1442        )
1443        .unwrap();
1444        let publisher = definition.hd.publisher_id;
1445        let old_key = (publisher, 1, date!(2025 - 12 - 31));
1446        client
1447            .futures_symbols
1448            .insert(old_key, Symbol::from("T1U16"));
1449        let started = std::time::Instant::now();
1450
1451        let instruments = client
1452            .get_range_instruments(RangeQueryParams {
1453                dataset: "GLBX.MDP3".to_string(),
1454                symbols: vec!["T.FUT".to_string()],
1455                start: definition.ts_recv.into(),
1456                end: Some((definition.ts_recv + 1).into()),
1457                limit: None,
1458                price_precision: None,
1459            })
1460            .await
1461            .unwrap();
1462
1463        let elapsed = started.elapsed();
1464        server.await.unwrap();
1465        let symbols = client.futures_symbols.load();
1466        let mut expected = AHashMap::new();
1467        expected.insert(old_key, Symbol::from("T1U16"));
1468
1469        for id in 1..=count {
1470            let symbol = Symbol::from(format!("T{id}U26").as_str());
1471            expected.insert((publisher, id, date!(2026 - 01 - 01)), symbol);
1472            assert_eq!(
1473                instruments[(id - 1) as usize].id(),
1474                InstrumentId::new(symbol, Venue::GLBX())
1475            );
1476            assert_eq!(
1477                instruments[(id - 1) as usize].raw_symbol(),
1478                Symbol::from(format!("T{id}U6").as_str())
1479            );
1480        }
1481
1482        expected.insert((publisher, 1, date!(2026 - 01 - 01)), Symbol::from("T1U36"));
1483        assert_eq!(instruments.len(), count as usize + 1);
1484        assert_eq!(
1485            instruments.last().unwrap().id(),
1486            InstrumentId::from("T1U36.GLBX")
1487        );
1488        assert_eq!(symbols.as_ref(), &expected);
1489        eprintln!("Decoded {count} definitions plus replacement in {elapsed:?}");
1490    }
1491
1492    #[tokio::test]
1493    async fn test_historical_records_use_definition_year_for_the_same_date() {
1494        let path = publishers_path()
1495            .parent()
1496            .unwrap()
1497            .join("test_data/test_data.definition.futures_contract.dbn.zst");
1498        let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path).unwrap();
1499        let mut definition = decoder
1500            .decode_record::<dbn::InstrumentDefMsg>()
1501            .unwrap()
1502            .unwrap()
1503            .clone();
1504        definition.ts_recv = "2028-01-01T00:00:00Z"
1505            .parse::<jiff::Timestamp>()
1506            .unwrap()
1507            .as_nanosecond() as u64;
1508        let metadata = dbn::Metadata::builder()
1509            .dataset("GLBX.MDP3")
1510            .schema(Some(dbn::Schema::Definition))
1511            .start(definition.ts_recv)
1512            .stype_in(Some(dbn::SType::RawSymbol))
1513            .stype_out(dbn::SType::InstrumentId)
1514            .mappings(vec![SymbolMapping {
1515                raw_symbol: "ESU6".to_string(),
1516                intervals: vec![MappingInterval {
1517                    start_date: date!(2028 - 01 - 01),
1518                    end_date: date!(2028 - 01 - 02),
1519                    symbol: definition.hd.instrument_id.to_string(),
1520                }],
1521            }])
1522            .build();
1523        let mut body = Vec::new();
1524        {
1525            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1526            encoder.encode_record(&definition).unwrap();
1527            encoder.flush().unwrap();
1528        }
1529
1530        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1531        let base_url = format!("http://{}/", listener.local_addr().unwrap());
1532
1533        let server = tokio::spawn(serve_response(listener, body));
1534        let client = DatabentoHistoricalClient::new_with_base_url(
1535            Credential::new(test_api_key()),
1536            publishers_path(),
1537            get_atomic_clock_realtime(),
1538            false,
1539            &base_url,
1540        )
1541        .unwrap();
1542
1543        let instruments = client
1544            .get_range_instruments(RangeQueryParams {
1545                dataset: "GLBX.MDP3".to_string(),
1546                symbols: vec!["ESU26".to_string()],
1547                start: definition.ts_recv.into(),
1548                end: Some((definition.ts_recv + 1).into()),
1549                limit: Some(1),
1550                price_precision: None,
1551            })
1552            .await
1553            .unwrap();
1554
1555        server.await.unwrap();
1556        let mut record = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1557        record.hd.publisher_id = definition.hd.publisher_id;
1558        record.hd.instrument_id = definition.hd.instrument_id;
1559        record.ts_recv = definition.ts_recv;
1560        let id = client
1561            .instrument_id(
1562                &dbn::RecordRef::from(&record),
1563                &mut MetadataCache::new(metadata),
1564                &AHashSet::from_iter([Symbol::from("ESU36")]),
1565            )
1566            .unwrap();
1567        assert_eq!(instruments.len(), 1);
1568        assert_eq!(instruments[0].id(), InstrumentId::from("ESU26.GLBX"));
1569        assert_eq!(instruments[0].raw_symbol(), Symbol::from("ESU6"));
1570        assert_eq!(id, InstrumentId::from("ESU26.GLBX"));
1571    }
1572
1573    #[rstest]
1574    #[case(None, "SR3Z26.XCME", true)]
1575    #[case(Some("SR3Z36.XNYM"), "SR3Z26.XCME", true)]
1576    #[case(Some("SR3Z26.XNYM"), "SR3Z36.XCME", true)]
1577    #[case(Some("SR3Z36.XNYM"), "SR3Z26.GLBX", false)]
1578    #[tokio::test]
1579    async fn test_requested_post_year_quotes_use_cached_precision_without_definitions(
1580        #[case] previous: Option<&str>,
1581        #[case] requested: &str,
1582        #[case] register_venue: bool,
1583    ) {
1584        let ts = "2027-02-01T00:00:00Z"
1585            .parse::<jiff::Timestamp>()
1586            .unwrap()
1587            .as_nanosecond() as u64;
1588        let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1589        msg.hd.publisher_id = 1;
1590        msg.hd.instrument_id = 42;
1591        msg.hd.ts_event = ts - 1;
1592        msg.ts_recv = ts;
1593        msg.levels[0].bid_px = 95_125_000_000;
1594        msg.levels[0].ask_px = 95_250_000_000;
1595        msg.levels[0].bid_sz = 7;
1596        msg.levels[0].ask_sz = 11;
1597        let metadata = dbn::Metadata::builder()
1598            .dataset("GLBX.MDP3")
1599            .schema(Some(dbn::Schema::Bbo1S))
1600            .start(ts)
1601            .stype_in(Some(dbn::SType::RawSymbol))
1602            .stype_out(dbn::SType::InstrumentId)
1603            .mappings(vec![SymbolMapping {
1604                raw_symbol: "SR3Z6".to_string(),
1605                intervals: vec![MappingInterval {
1606                    start_date: date!(2027 - 02 - 01),
1607                    end_date: date!(2027 - 02 - 02),
1608                    symbol: "42".to_string(),
1609                }],
1610            }])
1611            .build();
1612        let mut body = Vec::new();
1613        {
1614            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1615            encoder.encode_record(&msg).unwrap();
1616            encoder.flush().unwrap();
1617        }
1618
1619        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1620        let base_url = format!("http://{}/", listener.local_addr().unwrap());
1621
1622        let server = tokio::spawn(serve_response(listener, body));
1623        let client = DatabentoHistoricalClient::new_with_base_url(
1624            Credential::new(test_api_key()),
1625            publishers_path(),
1626            get_atomic_clock_realtime(),
1627            false,
1628            &base_url,
1629        )
1630        .unwrap();
1631
1632        if let Some(previous) = previous {
1633            let previous = InstrumentId::from(previous);
1634            client.prepare_symbols_from_instrument_ids(&[previous]);
1635            client.set_price_precision(previous.symbol, 3);
1636        }
1637
1638        let id = InstrumentId::from(requested);
1639        let symbols = if register_venue {
1640            client.prepare_symbols_from_instrument_ids(&[id])
1641        } else {
1642            vec![id.symbol.to_string()]
1643        };
1644
1645        client.set_price_precision(id.symbol, 3);
1646
1647        let quotes = client
1648            .get_range_quotes(
1649                RangeQueryParams {
1650                    dataset: "GLBX.MDP3".to_string(),
1651                    symbols,
1652                    start: ts.into(),
1653                    end: Some((ts + 1).into()),
1654                    limit: Some(1),
1655                    price_precision: None,
1656                },
1657                Some("bbo-1s".to_string()),
1658            )
1659            .await
1660            .unwrap();
1661
1662        server.await.unwrap();
1663
1664        assert_eq!(
1665            quotes,
1666            vec![QuoteTick::new(
1667                id,
1668                Price::from("95.125"),
1669                Price::from("95.250"),
1670                Quantity::from(7),
1671                Quantity::from(11),
1672                ts.into(),
1673                ts.into(),
1674            )]
1675        );
1676    }
1677}