Skip to main content

nautilus_databento/
historical.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Core Databento historical client for both Rust and Python usage.
17
18use std::{fmt::Debug, fs, num::NonZeroU64, path::PathBuf, str::FromStr, sync::Arc};
19
20use ahash::{AHashMap, AHashSet};
21use databento::{
22    dbn::{self, decode::DbnMetadata},
23    historical::timeseries::GetRangeParams,
24};
25use indexmap::IndexMap;
26use nautilus_core::{AtomicMap, UnixNanos, consts::NAUTILUS_USER_AGENT, time::AtomicTime};
27use nautilus_model::{
28    data::{Bar, Data, InstrumentStatus, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick},
29    enums::BarAggregation,
30    identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
31    instruments::{Instrument, InstrumentAny},
32};
33
34use crate::{
35    common::{Credential, get_date_time_range, ohlcv_schema_from_aggregation},
36    decode::{
37        MboDeltaBuffer, decode_imbalance_msg, decode_instrument_def_msg, decode_mbo_msg,
38        decode_mbp10_msg, decode_record, decode_statistics_msg, decode_status_msg,
39        is_supported_stat_type,
40    },
41    symbology::{
42        MetadataCache, check_consistent_symbology, decode_nautilus_instrument_id,
43        infer_symbology_type, raw_symbol,
44    },
45    types::{DatabentoImbalance, DatabentoPublisher, DatabentoStatistics, PublisherId},
46};
47
48/// Core Databento historical client for fetching historical market data.
49///
50/// This client provides both synchronous and asynchronous interfaces for fetching
51/// various types of historical market data from Databento.
52#[derive(Clone)]
53pub struct DatabentoHistoricalClient {
54    futures_year_digits: FuturesYearDigits,
55    credential: Credential,
56    clock: &'static AtomicTime,
57    inner: Arc<databento::HistoricalClient>,
58    publisher_venue_map: Arc<IndexMap<PublisherId, Venue>>,
59    symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
60    price_precisions: Arc<AtomicMap<Symbol, u8>>,
61    use_exchange_as_venue: bool,
62    futures_symbols: Arc<AtomicMap<(PublisherId, u32, time::Date), Symbol>>,
63}
64
65/// Parameters for range queries to Databento historical API.
66#[derive(Debug)]
67pub struct RangeQueryParams {
68    pub dataset: String,
69    pub symbols: Vec<String>,
70    pub start: UnixNanos,
71    pub end: Option<UnixNanos>,
72    pub limit: Option<u64>,
73    pub price_precision: Option<u8>,
74}
75
76/// Result containing dataset date range information.
77#[derive(Debug, Clone)]
78pub struct DatasetRange {
79    pub start: String,
80    pub end: String,
81}
82
83impl Debug for DatabentoHistoricalClient {
84    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
85        f.debug_struct(stringify!(DatabentoHistoricalClient))
86            .field("credential", &self.credential)
87            .finish()
88    }
89}
90
91impl DatabentoHistoricalClient {
92    /// Returns the API key from the stored credential.
93    #[must_use]
94    pub fn api_key(&self) -> &str {
95        self.credential.api_key()
96    }
97
98    /// Creates a new [`DatabentoHistoricalClient`] instance.
99    ///
100    /// # Errors
101    ///
102    /// Returns an error if client creation or publisher loading fails.
103    pub fn new(
104        credential: Credential,
105        publishers_filepath: PathBuf,
106        clock: &'static AtomicTime,
107        use_exchange_as_venue: bool,
108    ) -> anyhow::Result<Self> {
109        let client = databento::HistoricalClient::builder()
110            .user_agent_extension(NAUTILUS_USER_AGENT.into())
111            .key(credential.api_key())
112            .map_err(|e| anyhow::anyhow!("Failed to create client builder: {e}"))?
113            .build()
114            .map_err(|e| anyhow::anyhow!("Failed to build client: {e}"))?;
115
116        Self::from_client(
117            credential,
118            publishers_filepath,
119            clock,
120            use_exchange_as_venue,
121            client,
122        )
123    }
124
125    /// Selects one- or two-digit years for futures instrument IDs.
126    #[must_use]
127    pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
128        self.futures_year_digits = futures_year_digits;
129        self
130    }
131
132    /// Creates a new [`DatabentoHistoricalClient`] instance with a custom API base URL.
133    ///
134    /// This is intended for tests, benchmarks, and controlled deployments that route
135    /// Databento Historical API requests through a proxy.
136    ///
137    /// # Errors
138    ///
139    /// Returns an error if client creation, URL parsing, or publisher loading fails.
140    pub fn new_with_base_url(
141        credential: Credential,
142        publishers_filepath: PathBuf,
143        clock: &'static AtomicTime,
144        use_exchange_as_venue: bool,
145        base_url: &str,
146    ) -> anyhow::Result<Self> {
147        let client = databento::HistoricalClient::builder()
148            .user_agent_extension(NAUTILUS_USER_AGENT.into())
149            .base_url(base_url.parse().map_err(|e| {
150                anyhow::anyhow!("Failed to parse Databento Historical API base URL: {e}")
151            })?)
152            .key(credential.api_key())
153            .map_err(|e| anyhow::anyhow!("Failed to create client builder: {e}"))?
154            .build()
155            .map_err(|e| anyhow::anyhow!("Failed to build client: {e}"))?;
156
157        Self::from_client(
158            credential,
159            publishers_filepath,
160            clock,
161            use_exchange_as_venue,
162            client,
163        )
164    }
165
166    fn from_client(
167        credential: Credential,
168        publishers_filepath: PathBuf,
169        clock: &'static AtomicTime,
170        use_exchange_as_venue: bool,
171        client: databento::HistoricalClient,
172    ) -> anyhow::Result<Self> {
173        let file_content = fs::read_to_string(publishers_filepath)?;
174        let publishers_vec: Vec<DatabentoPublisher> = serde_json::from_str(&file_content)?;
175
176        let publisher_venue_map = publishers_vec
177            .into_iter()
178            .map(|p| (p.publisher_id, Venue::from(p.venue.as_str())))
179            .collect::<IndexMap<u16, Venue>>();
180
181        Ok(Self {
182            clock,
183            futures_year_digits: FuturesYearDigits::default(),
184            inner: Arc::new(client),
185            publisher_venue_map: Arc::new(publisher_venue_map),
186            symbol_venue_map: Arc::new(AtomicMap::new()),
187            price_precisions: Arc::new(AtomicMap::new()),
188            credential,
189            use_exchange_as_venue,
190            futures_symbols: Arc::new(AtomicMap::new()),
191        })
192    }
193
194    /// Caches a `price_precision` for the given `symbol`.
195    ///
196    /// When market data is fetched without an explicit `price_precision`, the
197    /// client resolves precision per record from this cache. Instruments
198    /// returned by [`Self::get_range_instruments`] are inserted automatically.
199    pub fn set_price_precision(&self, symbol: Symbol, price_precision: u8) {
200        self.price_precisions.insert(symbol, price_precision);
201    }
202
203    /// Returns a cached `price_precision` for the given `symbol`.
204    #[must_use]
205    pub fn price_precision(&self, symbol: Symbol) -> Option<u8> {
206        self.price_precisions.load().get(&symbol).copied()
207    }
208
209    /// Resolves a price precision for the given `instrument_id`.
210    ///
211    /// Resolution order:
212    /// 1. The explicit `price_precision` argument (if `Some`).
213    /// 2. The cached precision for the instrument's symbol.
214    ///
215    /// # Errors
216    ///
217    /// Returns an error if no precision is available.
218    fn resolve_price_precision(
219        &self,
220        instrument_id: &InstrumentId,
221        price_precision: Option<u8>,
222    ) -> anyhow::Result<u8> {
223        if let Some(precision) = price_precision {
224            return Ok(precision);
225        }
226
227        let precisions = self.price_precisions.load();
228        precisions
229            .get(&instrument_id.symbol)
230            .copied()
231            .ok_or_else(|| {
232                anyhow::anyhow!(
233                    "Could not resolve `price_precision` for {instrument_id}: \
234                     pass `price_precision` explicitly, call `set_price_precision`, \
235                     or fetch the instrument definitions first via `get_range_instruments`"
236                )
237            })
238    }
239
240    fn resolve_cached_price_precision(
241        &self,
242        instrument_id: &InstrumentId,
243        price_precision: Option<u8>,
244        precision_cache: &mut AHashMap<InstrumentId, u8>,
245    ) -> anyhow::Result<u8> {
246        if let Some(precision) = price_precision {
247            return Ok(precision);
248        }
249
250        if let Some(precision) = precision_cache.get(instrument_id) {
251            return Ok(*precision);
252        }
253
254        let precision = self.resolve_price_precision(instrument_id, None)?;
255        precision_cache.insert(*instrument_id, precision);
256        Ok(precision)
257    }
258
259    /// Gets the date range for a specific dataset.
260    ///
261    /// # Errors
262    ///
263    /// Returns an error if the API request fails.
264    pub async fn get_dataset_range(&self, dataset: &str) -> anyhow::Result<DatasetRange> {
265        let mut client = (*self.inner).clone();
266        let response = client
267            .metadata()
268            .get_dataset_range(dataset)
269            .await
270            .map_err(|e| anyhow::anyhow!("Failed to get dataset range: {e}"))?;
271
272        Ok(DatasetRange {
273            start: response.start.to_string(),
274            end: response.end.to_string(),
275        })
276    }
277
278    /// Fetches instrument definitions for the given parameters.
279    ///
280    /// # Errors
281    ///
282    /// Returns an error if the API request or data processing fails.
283    pub async fn get_range_instruments(
284        &self,
285        params: RangeQueryParams,
286    ) -> anyhow::Result<Vec<InstrumentAny>> {
287        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
288        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
289        check_consistent_symbology(&symbols)?;
290
291        let first_symbol = params
292            .symbols
293            .first()
294            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
295        let stype_in = infer_symbology_type(first_symbol);
296        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
297        let time_range = get_date_time_range(params.start, end)?;
298
299        let range_params = GetRangeParams::builder()
300            .dataset(params.dataset)
301            .date_time_range(time_range)
302            .symbols(symbols)
303            .stype_in(stype_in)
304            .schema(dbn::Schema::Definition)
305            .maybe_limit(params.limit.and_then(NonZeroU64::new))
306            .build();
307
308        let mut client = (*self.inner).clone();
309        let mut decoder = client
310            .timeseries()
311            .get_range(&range_params)
312            .await
313            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
314
315        let metadata = decoder.metadata().clone();
316        let mut metadata_cache = MetadataCache::new(metadata);
317        let mut instruments = Vec::new();
318        let mut futures_symbols = Vec::new();
319
320        while let Some(msg) = decoder.decode_record::<dbn::InstrumentDefMsg>().await? {
321            let record = dbn::RecordRef::from(msg);
322            let sym_map = self.symbol_venue_map.load();
323            let mut instrument_id = decode_nautilus_instrument_id(
324                &record,
325                &mut metadata_cache,
326                &self.publisher_venue_map,
327                &sym_map,
328                self.futures_year_digits,
329                &self.futures_symbols.load(),
330                Some(&requested_symbols),
331            )?;
332
333            if self.use_exchange_as_venue && instrument_id.venue == Venue::GLBX() {
334                let exchange = msg
335                    .exchange()
336                    .map_err(|e| anyhow::anyhow!("Missing exchange in record: {e}"))?;
337                let venue = Venue::from_code(exchange)
338                    .map_err(|e| anyhow::anyhow!("Venue not found for exchange {exchange}: {e}"))?;
339                instrument_id.venue = venue;
340            }
341
342            match decode_instrument_def_msg(msg, instrument_id, None, None) {
343                Ok(Some(instrument)) => {
344                    if instrument.raw_symbol() != instrument.id().symbol {
345                        let date = time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(
346                            msg.ts_recv,
347                        ))?
348                        .date();
349                        futures_symbols.push((
350                            (msg.hd.publisher_id, msg.hd.instrument_id, date),
351                            instrument.id().symbol,
352                        ));
353                    }
354
355                    instruments.push(instrument);
356                }
357                Ok(None) => {} // Decoder logged a warning for the unsupported class
358                Err(e) => anyhow::bail!("Failed to decode instrument {instrument_id}: {e}"),
359            }
360        }
361
362        if !futures_symbols.is_empty() {
363            self.futures_symbols
364                .rcu(|symbols| symbols.extend(futures_symbols.iter().copied()));
365        }
366
367        self.cache_price_precisions(&instruments);
368
369        Ok(instruments)
370    }
371
372    fn cache_price_precisions(&self, instruments: &[InstrumentAny]) {
373        if instruments.is_empty() {
374            return;
375        }
376
377        self.price_precisions.rcu(|precisions| {
378            for instrument in instruments {
379                precisions.insert(instrument.id().symbol, instrument.price_precision());
380            }
381        });
382    }
383
384    /// Fetches quote ticks for the given parameters.
385    ///
386    /// # Errors
387    ///
388    /// Returns an error if the API request or data processing fails.
389    pub async fn get_range_quotes(
390        &self,
391        params: RangeQueryParams,
392        schema: Option<String>,
393    ) -> anyhow::Result<Vec<QuoteTick>> {
394        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
395        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
396        check_consistent_symbology(&symbols)?;
397
398        let first_symbol = params
399            .symbols
400            .first()
401            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
402        let stype_in = infer_symbology_type(first_symbol);
403        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
404        let time_range = get_date_time_range(params.start, end)?;
405        let schema = schema.unwrap_or_else(|| "mbp-1".to_string());
406        let dbn_schema = dbn::Schema::from_str(&schema)?;
407
408        match dbn_schema {
409            dbn::Schema::Mbp1
410            | dbn::Schema::Tbbo
411            | dbn::Schema::Bbo1S
412            | dbn::Schema::Bbo1M
413            | dbn::Schema::Cmbp1
414            | dbn::Schema::Tcbbo
415            | dbn::Schema::Cbbo1S
416            | dbn::Schema::Cbbo1M => (),
417            _ => anyhow::bail!(
418                "Invalid schema. Must be one of: mbp-1, tbbo, bbo-1s, bbo-1m, cmbp-1, tcbbo, cbbo-1s, cbbo-1m"
419            ),
420        }
421
422        let range_params = GetRangeParams::builder()
423            .dataset(params.dataset)
424            .date_time_range(time_range)
425            .symbols(symbols)
426            .stype_in(stype_in)
427            .schema(dbn_schema)
428            .maybe_limit(params.limit.and_then(NonZeroU64::new))
429            .build();
430
431        let price_precision_arg = params.price_precision;
432
433        let mut client = (*self.inner).clone();
434        let mut decoder = client
435            .timeseries()
436            .get_range(&range_params)
437            .await
438            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
439
440        let metadata = decoder.metadata().clone();
441        let mut metadata_cache = MetadataCache::new(metadata);
442        let mut precision_cache = AHashMap::new();
443        let mut result: Vec<QuoteTick> = Vec::new();
444
445        let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
446            let instrument_id =
447                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
448            let price_precision = self.resolve_cached_price_precision(
449                &instrument_id,
450                price_precision_arg,
451                &mut precision_cache,
452            )?;
453
454            let (data, _) = decode_record(
455                &record,
456                instrument_id,
457                price_precision,
458                None,
459                false, // Don't include trades
460                true,
461            )?;
462
463            match data {
464                Some(Data::Quote(quote)) => result.push(quote),
465                None => {} // Skip records with undefined bid/ask prices
466                _ => anyhow::bail!("Invalid data element not `QuoteTick`, was {data:?}"),
467            }
468            Ok(())
469        };
470
471        match dbn_schema {
472            dbn::Schema::Mbp1 => {
473                while let Some(msg) = decoder.decode_record::<dbn::Mbp1Msg>().await? {
474                    process_record(dbn::RecordRef::from(msg))?;
475                }
476            }
477            dbn::Schema::Tbbo => {
478                while let Some(msg) = decoder.decode_record::<dbn::TbboMsg>().await? {
479                    process_record(dbn::RecordRef::from(msg))?;
480                }
481            }
482            dbn::Schema::Cmbp1 => {
483                while let Some(msg) = decoder.decode_record::<dbn::Cmbp1Msg>().await? {
484                    process_record(dbn::RecordRef::from(msg))?;
485                }
486            }
487            dbn::Schema::Tcbbo => {
488                while let Some(msg) = decoder.decode_record::<dbn::TcbboMsg>().await? {
489                    process_record(dbn::RecordRef::from(msg))?;
490                }
491            }
492            dbn::Schema::Bbo1M => {
493                while let Some(msg) = decoder.decode_record::<dbn::Bbo1MMsg>().await? {
494                    process_record(dbn::RecordRef::from(msg))?;
495                }
496            }
497            dbn::Schema::Bbo1S => {
498                while let Some(msg) = decoder.decode_record::<dbn::Bbo1SMsg>().await? {
499                    process_record(dbn::RecordRef::from(msg))?;
500                }
501            }
502            dbn::Schema::Cbbo1S | dbn::Schema::Cbbo1M => {
503                while let Some(msg) = decoder.decode_record::<dbn::CbboMsg>().await? {
504                    process_record(dbn::RecordRef::from(msg))?;
505                }
506            }
507            _ => anyhow::bail!("Invalid schema {dbn_schema}"),
508        }
509
510        Ok(result)
511    }
512
513    /// Fetches order book depth snapshots for the given parameters.
514    ///
515    /// # Errors
516    ///
517    /// Returns an error if the API request or data processing fails.
518    pub async fn get_range_order_book_depth(
519        &self,
520        params: RangeQueryParams,
521        depth: Option<usize>,
522    ) -> anyhow::Result<Vec<OrderBookDepth>> {
523        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
524        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
525        check_consistent_symbology(&symbols)?;
526
527        let first_symbol = params
528            .symbols
529            .first()
530            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
531        let stype_in = infer_symbology_type(first_symbol);
532        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
533        let time_range = get_date_time_range(params.start, end)?;
534
535        // For now, only support MBP_10 schema for depth 10
536        let _depth = depth.unwrap_or(10);
537        if _depth != 10 {
538            anyhow::bail!("Only depth=10 is currently supported for order book depths");
539        }
540
541        let range_params = GetRangeParams::builder()
542            .dataset(params.dataset)
543            .date_time_range(time_range)
544            .symbols(symbols)
545            .stype_in(stype_in)
546            .schema(dbn::Schema::Mbp10)
547            .maybe_limit(params.limit.and_then(NonZeroU64::new))
548            .build();
549
550        let price_precision_arg = params.price_precision;
551
552        let mut client = (*self.inner).clone();
553        let mut decoder = client
554            .timeseries()
555            .get_range(&range_params)
556            .await
557            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
558
559        let metadata = decoder.metadata().clone();
560        let mut metadata_cache = MetadataCache::new(metadata);
561        let mut precision_cache = AHashMap::new();
562        let mut result: Vec<OrderBookDepth> = Vec::new();
563
564        let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
565            let instrument_id =
566                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
567            let price_precision = self.resolve_cached_price_precision(
568                &instrument_id,
569                price_precision_arg,
570                &mut precision_cache,
571            )?;
572
573            if let Some(msg) = record.get::<dbn::Mbp10Msg>() {
574                let depth = decode_mbp10_msg(msg, instrument_id, price_precision, None)?;
575                result.push(depth);
576            }
577
578            Ok(())
579        };
580
581        while let Some(msg) = decoder.decode_record::<dbn::Mbp10Msg>().await? {
582            process_record(dbn::RecordRef::from(msg))?;
583        }
584
585        Ok(result)
586    }
587
588    /// Fetches order book deltas for the given parameters.
589    ///
590    /// # Errors
591    ///
592    /// Returns an error if the API request or data processing fails.
593    pub async fn get_range_order_book_deltas(
594        &self,
595        params: RangeQueryParams,
596    ) -> anyhow::Result<Vec<OrderBookDelta>> {
597        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
598        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
599        check_consistent_symbology(&symbols)?;
600
601        let first_symbol = params
602            .symbols
603            .first()
604            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
605        let stype_in = infer_symbology_type(first_symbol);
606        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
607        let time_range = get_date_time_range(params.start, end)?;
608
609        let range_params = GetRangeParams::builder()
610            .dataset(params.dataset)
611            .date_time_range(time_range)
612            .symbols(symbols)
613            .stype_in(stype_in)
614            .schema(dbn::Schema::Mbo)
615            .maybe_limit(params.limit.and_then(NonZeroU64::new))
616            .build();
617
618        let price_precision_arg = params.price_precision;
619
620        let mut client = (*self.inner).clone();
621        let mut decoder = client
622            .timeseries()
623            .get_range(&range_params)
624            .await
625            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
626
627        let metadata = decoder.metadata().clone();
628        let mut metadata_cache = MetadataCache::new(metadata);
629        let mut precision_cache = AHashMap::new();
630        let mut result: Vec<OrderBookDelta> = Vec::new();
631        let mut delta_buffer = MboDeltaBuffer::default();
632
633        let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
634            let instrument_id =
635                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
636            let price_precision = self.resolve_cached_price_precision(
637                &instrument_id,
638                price_precision_arg,
639                &mut precision_cache,
640            )?;
641
642            if let Some(msg) = record.get::<dbn::MboMsg>() {
643                let (delta, _trade) =
644                    decode_mbo_msg(msg, instrument_id, price_precision, None, false)?;
645                delta_buffer.push(msg, instrument_id, delta);
646                while let Some(delta) = delta_buffer.pop_ready() {
647                    result.push(delta);
648                }
649            }
650
651            Ok(())
652        };
653
654        while let Some(msg) = decoder.decode_record::<dbn::MboMsg>().await? {
655            process_record(dbn::RecordRef::from(msg))?;
656        }
657
658        delta_buffer.finish();
659        while let Some(delta) = delta_buffer.pop_ready() {
660            result.push(delta);
661        }
662
663        Ok(result)
664    }
665
666    /// Fetches trade ticks for the given parameters.
667    ///
668    /// # Errors
669    ///
670    /// Returns an error if the API request or data processing fails.
671    pub async fn get_range_trades(
672        &self,
673        params: RangeQueryParams,
674        schema: Option<String>,
675    ) -> anyhow::Result<Vec<TradeTick>> {
676        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
677        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
678        check_consistent_symbology(&symbols)?;
679
680        let first_symbol = params
681            .symbols
682            .first()
683            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
684        let stype_in = infer_symbology_type(first_symbol);
685        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
686        let time_range = get_date_time_range(params.start, end)?;
687        let schema = schema.unwrap_or_else(|| "trades".to_string());
688        let dbn_schema = dbn::Schema::from_str(&schema)?;
689
690        match dbn_schema {
691            dbn::Schema::Trades
692            | dbn::Schema::Tbbo
693            | dbn::Schema::Tcbbo
694            | dbn::Schema::Mbp1
695            | dbn::Schema::Cmbp1 => (),
696            _ => {
697                anyhow::bail!("Invalid schema. Must be one of: trades, tbbo, tcbbo, mbp-1, cmbp-1")
698            }
699        }
700
701        let range_params = GetRangeParams::builder()
702            .dataset(params.dataset)
703            .date_time_range(time_range)
704            .symbols(symbols)
705            .stype_in(stype_in)
706            .schema(dbn_schema)
707            .maybe_limit(params.limit.and_then(NonZeroU64::new))
708            .build();
709
710        let price_precision_arg = params.price_precision;
711
712        let mut client = (*self.inner).clone();
713        let mut decoder = client
714            .timeseries()
715            .get_range(&range_params)
716            .await
717            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
718
719        let metadata = decoder.metadata().clone();
720        let mut metadata_cache = MetadataCache::new(metadata);
721        let mut precision_cache = AHashMap::new();
722        let mut result: Vec<TradeTick> = Vec::new();
723
724        let mut process_record = |record: dbn::RecordRef| -> anyhow::Result<()> {
725            let instrument_id =
726                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
727            let price_precision = self.resolve_cached_price_precision(
728                &instrument_id,
729                price_precision_arg,
730                &mut precision_cache,
731            )?;
732
733            let (data, data2) =
734                decode_record(&record, instrument_id, price_precision, None, true, true)?;
735
736            match (data, data2) {
737                (Some(Data::Trade(trade)), _) | (_, Some(Data::Trade(trade))) => result.push(trade),
738                (Some(_) | None, None) => {}
739                (None, Some(data)) => {
740                    anyhow::bail!("Invalid data element not `TradeTick`, was {data:?}")
741                }
742                (Some(data), Some(_)) => {
743                    anyhow::bail!("Invalid data element not `TradeTick`, was {data:?}")
744                }
745            }
746            Ok(())
747        };
748
749        match dbn_schema {
750            dbn::Schema::Trades => {
751                while let Some(msg) = decoder.decode_record::<dbn::TradeMsg>().await? {
752                    process_record(dbn::RecordRef::from(msg))?;
753                }
754            }
755            dbn::Schema::Mbp1 => {
756                while let Some(msg) = decoder.decode_record::<dbn::Mbp1Msg>().await? {
757                    process_record(dbn::RecordRef::from(msg))?;
758                }
759            }
760            dbn::Schema::Tbbo => {
761                while let Some(msg) = decoder.decode_record::<dbn::TbboMsg>().await? {
762                    process_record(dbn::RecordRef::from(msg))?;
763                }
764            }
765            dbn::Schema::Cmbp1 => {
766                while let Some(msg) = decoder.decode_record::<dbn::Cmbp1Msg>().await? {
767                    process_record(dbn::RecordRef::from(msg))?;
768                }
769            }
770            dbn::Schema::Tcbbo => {
771                while let Some(msg) = decoder.decode_record::<dbn::TcbboMsg>().await? {
772                    process_record(dbn::RecordRef::from(msg))?;
773                }
774            }
775            _ => anyhow::bail!("Invalid schema {dbn_schema}"),
776        }
777
778        Ok(result)
779    }
780
781    /// Fetches bars for the given parameters.
782    ///
783    /// # Errors
784    ///
785    /// Returns an error if the API request or data processing fails.
786    pub async fn get_range_bars(
787        &self,
788        params: RangeQueryParams,
789        aggregation: BarAggregation,
790        timestamp_on_close: bool,
791    ) -> anyhow::Result<Vec<Bar>> {
792        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
793        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
794        check_consistent_symbology(&symbols)?;
795
796        let first_symbol = params
797            .symbols
798            .first()
799            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
800        let stype_in = infer_symbology_type(first_symbol);
801        let schema = ohlcv_schema_from_aggregation(aggregation)?;
802
803        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
804        let time_range = get_date_time_range(params.start, end)?;
805
806        let range_params = GetRangeParams::builder()
807            .dataset(params.dataset)
808            .date_time_range(time_range)
809            .symbols(symbols)
810            .stype_in(stype_in)
811            .schema(schema)
812            .maybe_limit(params.limit.and_then(NonZeroU64::new))
813            .build();
814
815        let price_precision_arg = params.price_precision;
816
817        let mut client = (*self.inner).clone();
818        let mut decoder = client
819            .timeseries()
820            .get_range(&range_params)
821            .await
822            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
823
824        let metadata = decoder.metadata().clone();
825        let mut metadata_cache = MetadataCache::new(metadata);
826        let mut precision_cache = AHashMap::new();
827        let mut result: Vec<Bar> = Vec::new();
828
829        while let Some(msg) = decoder.decode_record::<dbn::OhlcvMsg>().await? {
830            let record = dbn::RecordRef::from(msg);
831            let instrument_id =
832                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
833            let price_precision = self.resolve_cached_price_precision(
834                &instrument_id,
835                price_precision_arg,
836                &mut precision_cache,
837            )?;
838
839            let (data, _) = decode_record(
840                &record,
841                instrument_id,
842                price_precision,
843                None,
844                false, // Not applicable
845                timestamp_on_close,
846            )?;
847
848            match data {
849                Some(Data::Bar(bar)) => {
850                    result.push(bar);
851                }
852                _ => anyhow::bail!("Invalid data element not `Bar`, was {data:?}"),
853            }
854        }
855
856        Ok(result)
857    }
858
859    /// Fetches imbalance data for the given parameters.
860    ///
861    /// # Errors
862    ///
863    /// Returns an error if the API request or data processing fails.
864    pub async fn get_range_imbalance(
865        &self,
866        params: RangeQueryParams,
867    ) -> anyhow::Result<Vec<DatabentoImbalance>> {
868        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
869        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
870        check_consistent_symbology(&symbols)?;
871
872        let first_symbol = params
873            .symbols
874            .first()
875            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
876        let stype_in = infer_symbology_type(first_symbol);
877        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
878        let time_range = get_date_time_range(params.start, end)?;
879
880        let range_params = GetRangeParams::builder()
881            .dataset(params.dataset)
882            .date_time_range(time_range)
883            .symbols(symbols)
884            .stype_in(stype_in)
885            .schema(dbn::Schema::Imbalance)
886            .maybe_limit(params.limit.and_then(NonZeroU64::new))
887            .build();
888
889        let price_precision_arg = params.price_precision;
890
891        let mut client = (*self.inner).clone();
892        let mut decoder = client
893            .timeseries()
894            .get_range(&range_params)
895            .await
896            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
897
898        let metadata = decoder.metadata().clone();
899        let mut metadata_cache = MetadataCache::new(metadata);
900        let mut precision_cache = AHashMap::new();
901        let mut result: Vec<DatabentoImbalance> = Vec::new();
902
903        while let Some(msg) = decoder.decode_record::<dbn::ImbalanceMsg>().await? {
904            let record = dbn::RecordRef::from(msg);
905            let instrument_id =
906                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
907            let price_precision = self.resolve_cached_price_precision(
908                &instrument_id,
909                price_precision_arg,
910                &mut precision_cache,
911            )?;
912
913            let imbalance = decode_imbalance_msg(msg, instrument_id, price_precision, None)?;
914            result.push(imbalance);
915        }
916
917        Ok(result)
918    }
919
920    /// Fetches statistics data for the given parameters.
921    ///
922    /// # Errors
923    ///
924    /// Returns an error if the API request or data processing fails.
925    pub async fn get_range_statistics(
926        &self,
927        params: RangeQueryParams,
928    ) -> anyhow::Result<Vec<DatabentoStatistics>> {
929        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
930        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
931        check_consistent_symbology(&symbols)?;
932
933        let first_symbol = params
934            .symbols
935            .first()
936            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
937        let stype_in = infer_symbology_type(first_symbol);
938        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
939        let time_range = get_date_time_range(params.start, end)?;
940
941        let range_params = GetRangeParams::builder()
942            .dataset(params.dataset)
943            .date_time_range(time_range)
944            .symbols(symbols)
945            .stype_in(stype_in)
946            .schema(dbn::Schema::Statistics)
947            .maybe_limit(params.limit.and_then(NonZeroU64::new))
948            .build();
949
950        let price_precision_arg = params.price_precision;
951
952        let mut client = (*self.inner).clone();
953        let mut decoder = client
954            .timeseries()
955            .get_range(&range_params)
956            .await
957            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
958
959        let metadata = decoder.metadata().clone();
960        let mut metadata_cache = MetadataCache::new(metadata);
961        let mut precision_cache = AHashMap::new();
962        let mut result: Vec<DatabentoStatistics> = Vec::new();
963
964        while let Some(msg) = decoder.decode_record::<dbn::StatMsg>().await? {
965            // Precheck before precision resolution so unmodeled types skip cleanly
966            if !is_supported_stat_type(msg.stat_type) {
967                log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
968                continue;
969            }
970
971            let record = dbn::RecordRef::from(msg);
972            let instrument_id =
973                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
974            let price_precision = self.resolve_cached_price_precision(
975                &instrument_id,
976                price_precision_arg,
977                &mut precision_cache,
978            )?;
979
980            if let Some(statistics) =
981                decode_statistics_msg(msg, instrument_id, price_precision, None)?
982            {
983                result.push(statistics);
984            }
985        }
986
987        Ok(result)
988    }
989
990    /// Fetches status data for the given parameters.
991    ///
992    /// # Errors
993    ///
994    /// Returns an error if the API request or data processing fails.
995    pub async fn get_range_status(
996        &self,
997        params: RangeQueryParams,
998    ) -> anyhow::Result<Vec<InstrumentStatus>> {
999        let (raw_symbols, requested_symbols) = self.prepare_symbols(&params);
1000        let symbols: Vec<&str> = raw_symbols.iter().map(String::as_str).collect();
1001        check_consistent_symbology(&symbols)?;
1002
1003        let first_symbol = params
1004            .symbols
1005            .first()
1006            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
1007        let stype_in = infer_symbology_type(first_symbol);
1008        let end = params.end.unwrap_or_else(|| self.clock.get_time_ns());
1009        let time_range = get_date_time_range(params.start, end)?;
1010
1011        let range_params = GetRangeParams::builder()
1012            .dataset(params.dataset)
1013            .date_time_range(time_range)
1014            .symbols(symbols)
1015            .stype_in(stype_in)
1016            .schema(dbn::Schema::Status)
1017            .maybe_limit(params.limit.and_then(NonZeroU64::new))
1018            .build();
1019
1020        let mut client = (*self.inner).clone();
1021        let mut decoder = client
1022            .timeseries()
1023            .get_range(&range_params)
1024            .await
1025            .map_err(|e| anyhow::anyhow!("Failed to get range: {e}"))?;
1026
1027        let metadata = decoder.metadata().clone();
1028        let mut metadata_cache = MetadataCache::new(metadata);
1029        let mut result: Vec<InstrumentStatus> = Vec::new();
1030
1031        while let Some(msg) = decoder.decode_record::<dbn::StatusMsg>().await? {
1032            let record = dbn::RecordRef::from(msg);
1033            let instrument_id =
1034                self.instrument_id(&record, &mut metadata_cache, &requested_symbols)?;
1035
1036            let status = decode_status_msg(msg, instrument_id, None)?;
1037            result.push(status);
1038        }
1039
1040        Ok(result)
1041    }
1042
1043    fn instrument_id(
1044        &self,
1045        record: &dbn::RecordRef,
1046        metadata: &mut MetadataCache,
1047        requested_symbols: &AHashSet<Symbol>,
1048    ) -> anyhow::Result<InstrumentId> {
1049        decode_nautilus_instrument_id(
1050            record,
1051            metadata,
1052            &self.publisher_venue_map,
1053            &self.symbol_venue_map.load(),
1054            self.futures_year_digits,
1055            &self.futures_symbols.load(),
1056            Some(requested_symbols),
1057        )
1058    }
1059
1060    fn prepare_symbols(&self, params: &RangeQueryParams) -> (Vec<String>, AHashSet<Symbol>) {
1061        let mut requested_symbols = AHashSet::new();
1062
1063        let raw_symbols = params
1064            .symbols
1065            .iter()
1066            .map(|symbol| {
1067                let raw = raw_symbol(symbol, &params.dataset, self.futures_year_digits);
1068                if raw != *symbol {
1069                    requested_symbols.insert(Symbol::from(symbol.as_str()));
1070                }
1071
1072                raw
1073            })
1074            .collect();
1075
1076        (raw_symbols, requested_symbols)
1077    }
1078
1079    /// Prepares symbols from instrument IDs.
1080    pub fn prepare_symbols_from_instrument_ids(
1081        &self,
1082        instrument_ids: &[InstrumentId],
1083    ) -> Vec<String> {
1084        self.symbol_venue_map.rcu(|m| {
1085            for id in instrument_ids {
1086                m.entry(id.symbol).or_insert(id.venue);
1087            }
1088        });
1089
1090        instrument_ids
1091            .iter()
1092            .map(|id| id.symbol.to_string())
1093            .collect()
1094    }
1095}
1096
1097#[cfg(test)]
1098mod tests {
1099    use std::ffi::c_char;
1100
1101    use databento::dbn::{
1102        MappingInterval, SymbolMapping, decode::DecodeRecord, encode::EncodeRecord,
1103    };
1104    use nautilus_core::time::get_atomic_clock_realtime;
1105    use nautilus_model::types::{Price, Quantity};
1106    use rstest::{fixture, rstest};
1107    use time::macros::date;
1108    use tokio::{
1109        io::{AsyncReadExt, AsyncWriteExt},
1110        net::TcpListener,
1111    };
1112
1113    use super::*;
1114
1115    fn test_api_key() -> String {
1116        "test-000000000000000000000000000".to_string()
1117    }
1118
1119    fn publishers_path() -> PathBuf {
1120        PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json")
1121    }
1122
1123    fn mbo_record(action: c_char, flags: u8, sequence: u32) -> dbn::MboMsg {
1124        let ts_event = 1_609_160_400_000_000_000;
1125        dbn::MboMsg {
1126            hd: dbn::RecordHeader::new::<dbn::MboMsg>(dbn::rtype::MBO, 1, 1, ts_event),
1127            order_id: u64::from(sequence),
1128            price: 4_800_250_000_000,
1129            size: 2,
1130            flags: dbn::FlagSet::new(flags),
1131            channel_id: 1,
1132            action,
1133            side: 'A' as c_char,
1134            ts_recv: ts_event,
1135            ts_in_delta: 0,
1136            sequence,
1137        }
1138    }
1139
1140    fn encode_mbo_response(records: &[dbn::MboMsg]) -> Vec<u8> {
1141        let metadata = dbn::Metadata::builder()
1142            .dataset("GLBX.MDP3")
1143            .schema(Some(dbn::Schema::Mbo))
1144            .start(1_609_160_400_000_000_000)
1145            .stype_in(Some(dbn::SType::RawSymbol))
1146            .stype_out(dbn::SType::InstrumentId)
1147            .symbols(vec!["ESM4".to_string()])
1148            .mappings(vec![SymbolMapping {
1149                raw_symbol: "ESM4".to_string(),
1150                intervals: vec![MappingInterval {
1151                    start_date: date!(2020 - 12 - 28),
1152                    end_date: date!(2020 - 12 - 29),
1153                    symbol: "1".to_string(),
1154                }],
1155            }])
1156            .build();
1157        let mut body = Vec::new();
1158        {
1159            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1160            encoder.encode_records(records).unwrap();
1161            encoder.flush().unwrap();
1162        }
1163        body
1164    }
1165
1166    async fn serve_response(listener: TcpListener, body: Vec<u8>) {
1167        let (mut stream, _) = listener.accept().await.unwrap();
1168        let mut request = Vec::new();
1169        let mut buffer = [0_u8; 1024];
1170        loop {
1171            let read = stream.read(&mut buffer).await.unwrap();
1172            if read == 0 {
1173                break;
1174            }
1175            request.extend_from_slice(&buffer[..read]);
1176            if request.windows(4).any(|window| window == b"\r\n\r\n") {
1177                break;
1178            }
1179        }
1180
1181        let headers = format!(
1182            "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
1183            body.len(),
1184        );
1185        stream.write_all(headers.as_bytes()).await.unwrap();
1186        stream.write_all(&body).await.unwrap();
1187        stream.shutdown().await.unwrap();
1188    }
1189
1190    #[fixture]
1191    fn historical_client() -> DatabentoHistoricalClient {
1192        DatabentoHistoricalClient::new(
1193            Credential::new(test_api_key()),
1194            publishers_path(),
1195            get_atomic_clock_realtime(),
1196            false,
1197        )
1198        .unwrap()
1199    }
1200
1201    #[rstest]
1202    fn test_new_with_base_url_rejects_invalid_url() {
1203        let err = DatabentoHistoricalClient::new_with_base_url(
1204            Credential::new(test_api_key()),
1205            publishers_path(),
1206            get_atomic_clock_realtime(),
1207            false,
1208            "://invalid",
1209        )
1210        .expect_err("expected invalid base URL to fail");
1211        let err_msg = format!("{err}");
1212
1213        assert!(
1214            err_msg.contains("Failed to parse Databento Historical API base URL"),
1215            "unexpected error message: {err_msg}",
1216        );
1217    }
1218
1219    #[rstest]
1220    #[tokio::test]
1221    async fn test_get_range_order_book_deltas_preserves_boundaries_and_drains_end() {
1222        let last = dbn::flags::LAST;
1223        let body = encode_mbo_response(&[
1224            mbo_record('C' as c_char, 0, 1),
1225            mbo_record('N' as c_char, last, 2),
1226            mbo_record('C' as c_char, 0, 3),
1227        ]);
1228        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1229        let base_url = format!("http://{}/", listener.local_addr().unwrap());
1230        let server = tokio::spawn(serve_response(listener, body));
1231        let client = DatabentoHistoricalClient::new_with_base_url(
1232            Credential::new(test_api_key()),
1233            publishers_path(),
1234            get_atomic_clock_realtime(),
1235            false,
1236            &base_url,
1237        )
1238        .unwrap();
1239
1240        let deltas = client
1241            .get_range_order_book_deltas(RangeQueryParams {
1242                dataset: "GLBX.MDP3".to_string(),
1243                symbols: vec!["ESM4".to_string()],
1244                start: 1_609_160_000_000_000_000.into(),
1245                end: Some(1_609_161_000_000_000_000.into()),
1246                limit: None,
1247                price_precision: Some(2),
1248            })
1249            .await
1250            .unwrap();
1251        server.await.unwrap();
1252
1253        assert_eq!(deltas.len(), 2);
1254        assert_eq!(deltas[0].instrument_id, InstrumentId::from("ESM24.GLBX"));
1255        assert_eq!(deltas[0].order.order_id, 1);
1256        assert_eq!(deltas[0].flags, last);
1257        assert_eq!(deltas[0].sequence, 1);
1258        assert_eq!(deltas[1].instrument_id, InstrumentId::from("ESM24.GLBX"));
1259        assert_eq!(deltas[1].order.order_id, 3);
1260        assert_eq!(deltas[1].flags, 0);
1261        assert_eq!(deltas[1].sequence, 3);
1262    }
1263
1264    #[rstest]
1265    fn test_definition_precision_batch_merges_and_last_symbol_wins(
1266        historical_client: DatabentoHistoricalClient,
1267    ) {
1268        let mut first = nautilus_model::instruments::stubs::crypto_perpetual_ethusdt();
1269        first.id = "FIRST.GLBX".into();
1270        first.price_precision = 3;
1271        let mut second = first.clone();
1272        second.id = "SECOND.GLBX".into();
1273        second.price_precision = 5;
1274        let mut latest = first.clone();
1275        latest.id = "FIRST.XNAS".into();
1276        latest.price_precision = 7;
1277        historical_client.set_price_precision("RETAINED".into(), 9);
1278        historical_client.set_price_precision("FIRST".into(), 1);
1279
1280        historical_client.cache_price_precisions(&[
1281            InstrumentAny::CryptoPerpetual(first),
1282            InstrumentAny::CryptoPerpetual(second),
1283            InstrumentAny::CryptoPerpetual(latest),
1284        ]);
1285
1286        assert_eq!(historical_client.price_precisions.len(), 3);
1287
1288        for (symbol, precision) in [("RETAINED", 9), ("FIRST", 7), ("SECOND", 5)] {
1289            assert_eq!(
1290                historical_client
1291                    .resolve_price_precision(&InstrumentId::new(symbol.into(), "GLBX".into()), None)
1292                    .unwrap(),
1293                precision
1294            );
1295        }
1296    }
1297
1298    #[rstest]
1299    fn test_set_price_precision_inserts_into_cache(historical_client: DatabentoHistoricalClient) {
1300        let symbol = Symbol::from("ESM4");
1301
1302        assert_eq!(historical_client.price_precision(symbol), None);
1303
1304        historical_client.set_price_precision(symbol, 2);
1305
1306        assert_eq!(historical_client.price_precision(symbol), Some(2));
1307    }
1308
1309    #[rstest]
1310    fn test_resolve_price_precision_explicit_arg(historical_client: DatabentoHistoricalClient) {
1311        let instrument_id = InstrumentId::from("ESM4.GLBX");
1312        // Seed a deliberately wrong cache value so we know the explicit arg wins
1313        historical_client.set_price_precision(Symbol::from("ESM4"), 9);
1314
1315        let precision = historical_client
1316            .resolve_price_precision(&instrument_id, Some(2))
1317            .unwrap();
1318        assert_eq!(precision, 2);
1319    }
1320
1321    #[rstest]
1322    fn test_resolve_price_precision_cache_hit(historical_client: DatabentoHistoricalClient) {
1323        let instrument_id = InstrumentId::from("ESM4.GLBX");
1324        historical_client.set_price_precision(Symbol::from("ESM4"), 2);
1325
1326        let precision = historical_client
1327            .resolve_price_precision(&instrument_id, None)
1328            .unwrap();
1329        assert_eq!(precision, 2);
1330    }
1331
1332    #[rstest]
1333    fn test_resolve_price_precision_cache_miss_errors(
1334        historical_client: DatabentoHistoricalClient,
1335    ) {
1336        let instrument_id = InstrumentId::from("ESM4.GLBX");
1337
1338        let err = historical_client
1339            .resolve_price_precision(&instrument_id, None)
1340            .expect_err("expected cache-miss error");
1341        let err_msg = format!("{err}");
1342        assert!(
1343            err_msg.contains("Could not resolve `price_precision`"),
1344            "unexpected error message: {err_msg}",
1345        );
1346        assert!(
1347            err_msg.contains("ESM4.GLBX"),
1348            "error should name the instrument: {err_msg}",
1349        );
1350    }
1351
1352    #[rstest]
1353    #[case(1_000)]
1354    #[case(10_000)]
1355    #[tokio::test]
1356    async fn test_historical_definition_batch_preserves_all_symbols(#[case] count: u32) {
1357        let path = publishers_path()
1358            .parent()
1359            .unwrap()
1360            .join("test_data/test_data.definition.futures_contract.dbn.zst");
1361        let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path).unwrap();
1362        let mut definition = decoder
1363            .decode_record::<dbn::InstrumentDefMsg>()
1364            .unwrap()
1365            .unwrap()
1366            .clone();
1367        definition.ts_recv = "2026-01-01T00:00:00Z"
1368            .parse::<jiff::Timestamp>()
1369            .unwrap()
1370            .as_nanosecond() as u64;
1371        definition.maturity_year = 2026;
1372        let mut definitions = Vec::new();
1373        let mut mappings = Vec::new();
1374
1375        for id in 1..=count {
1376            let raw = format!("T{id}U6");
1377            definition.hd.instrument_id = id;
1378            definition.raw_symbol.fill(0);
1379
1380            for (target, byte) in definition.raw_symbol.iter_mut().zip(raw.bytes()) {
1381                *target = byte as c_char;
1382            }
1383
1384            definitions.push(definition.clone());
1385            mappings.push(SymbolMapping {
1386                raw_symbol: raw,
1387                intervals: vec![MappingInterval {
1388                    start_date: date!(2026 - 01 - 01),
1389                    end_date: date!(2026 - 01 - 02),
1390                    symbol: id.to_string(),
1391                }],
1392            });
1393        }
1394
1395        let mut replacement = definitions[0].clone();
1396        replacement.maturity_year = 2036;
1397        definitions.push(replacement);
1398        let metadata = dbn::Metadata::builder()
1399            .dataset("GLBX.MDP3")
1400            .schema(Some(dbn::Schema::Definition))
1401            .start(definition.ts_recv)
1402            .stype_in(Some(dbn::SType::Parent))
1403            .stype_out(dbn::SType::InstrumentId)
1404            .mappings(mappings)
1405            .build();
1406        let mut body = Vec::new();
1407        {
1408            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1409            encoder.encode_records(&definitions).unwrap();
1410            encoder.flush().unwrap();
1411        }
1412
1413        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1414        let base_url = format!("http://{}/", listener.local_addr().unwrap());
1415
1416        let server = tokio::spawn(serve_response(listener, body));
1417        let client = DatabentoHistoricalClient::new_with_base_url(
1418            Credential::new(test_api_key()),
1419            publishers_path(),
1420            get_atomic_clock_realtime(),
1421            false,
1422            &base_url,
1423        )
1424        .unwrap();
1425        let publisher = definition.hd.publisher_id;
1426        let old_key = (publisher, 1, date!(2025 - 12 - 31));
1427        client
1428            .futures_symbols
1429            .insert(old_key, Symbol::from("T1U16"));
1430        let started = std::time::Instant::now();
1431
1432        let instruments = client
1433            .get_range_instruments(RangeQueryParams {
1434                dataset: "GLBX.MDP3".to_string(),
1435                symbols: vec!["T.FUT".to_string()],
1436                start: definition.ts_recv.into(),
1437                end: Some((definition.ts_recv + 1).into()),
1438                limit: None,
1439                price_precision: None,
1440            })
1441            .await
1442            .unwrap();
1443
1444        let elapsed = started.elapsed();
1445        server.await.unwrap();
1446        let symbols = client.futures_symbols.load();
1447        let mut expected = AHashMap::new();
1448        expected.insert(old_key, Symbol::from("T1U16"));
1449
1450        for id in 1..=count {
1451            let symbol = Symbol::from(format!("T{id}U26").as_str());
1452            expected.insert((publisher, id, date!(2026 - 01 - 01)), symbol);
1453            assert_eq!(
1454                instruments[(id - 1) as usize].id(),
1455                InstrumentId::new(symbol, Venue::GLBX())
1456            );
1457            assert_eq!(
1458                instruments[(id - 1) as usize].raw_symbol(),
1459                Symbol::from(format!("T{id}U6").as_str())
1460            );
1461        }
1462
1463        expected.insert((publisher, 1, date!(2026 - 01 - 01)), Symbol::from("T1U36"));
1464        assert_eq!(instruments.len(), count as usize + 1);
1465        assert_eq!(
1466            instruments.last().unwrap().id(),
1467            InstrumentId::from("T1U36.GLBX")
1468        );
1469        assert_eq!(symbols.as_ref(), &expected);
1470        eprintln!("Decoded {count} definitions plus replacement in {elapsed:?}");
1471    }
1472
1473    #[tokio::test]
1474    async fn test_historical_records_use_definition_year_for_the_same_date() {
1475        let path = publishers_path()
1476            .parent()
1477            .unwrap()
1478            .join("test_data/test_data.definition.futures_contract.dbn.zst");
1479        let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path).unwrap();
1480        let mut definition = decoder
1481            .decode_record::<dbn::InstrumentDefMsg>()
1482            .unwrap()
1483            .unwrap()
1484            .clone();
1485        definition.ts_recv = "2028-01-01T00:00:00Z"
1486            .parse::<jiff::Timestamp>()
1487            .unwrap()
1488            .as_nanosecond() as u64;
1489        let metadata = dbn::Metadata::builder()
1490            .dataset("GLBX.MDP3")
1491            .schema(Some(dbn::Schema::Definition))
1492            .start(definition.ts_recv)
1493            .stype_in(Some(dbn::SType::RawSymbol))
1494            .stype_out(dbn::SType::InstrumentId)
1495            .mappings(vec![SymbolMapping {
1496                raw_symbol: "ESU6".to_string(),
1497                intervals: vec![MappingInterval {
1498                    start_date: date!(2028 - 01 - 01),
1499                    end_date: date!(2028 - 01 - 02),
1500                    symbol: definition.hd.instrument_id.to_string(),
1501                }],
1502            }])
1503            .build();
1504        let mut body = Vec::new();
1505        {
1506            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1507            encoder.encode_record(&definition).unwrap();
1508            encoder.flush().unwrap();
1509        }
1510
1511        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1512        let base_url = format!("http://{}/", listener.local_addr().unwrap());
1513
1514        let server = tokio::spawn(serve_response(listener, body));
1515        let client = DatabentoHistoricalClient::new_with_base_url(
1516            Credential::new(test_api_key()),
1517            publishers_path(),
1518            get_atomic_clock_realtime(),
1519            false,
1520            &base_url,
1521        )
1522        .unwrap();
1523
1524        let instruments = client
1525            .get_range_instruments(RangeQueryParams {
1526                dataset: "GLBX.MDP3".to_string(),
1527                symbols: vec!["ESU26".to_string()],
1528                start: definition.ts_recv.into(),
1529                end: Some((definition.ts_recv + 1).into()),
1530                limit: Some(1),
1531                price_precision: None,
1532            })
1533            .await
1534            .unwrap();
1535
1536        server.await.unwrap();
1537        let mut record = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1538        record.hd.publisher_id = definition.hd.publisher_id;
1539        record.hd.instrument_id = definition.hd.instrument_id;
1540        record.ts_recv = definition.ts_recv;
1541        let id = client
1542            .instrument_id(
1543                &dbn::RecordRef::from(&record),
1544                &mut MetadataCache::new(metadata),
1545                &AHashSet::from_iter([Symbol::from("ESU36")]),
1546            )
1547            .unwrap();
1548        assert_eq!(instruments.len(), 1);
1549        assert_eq!(instruments[0].id(), InstrumentId::from("ESU26.GLBX"));
1550        assert_eq!(instruments[0].raw_symbol(), Symbol::from("ESU6"));
1551        assert_eq!(id, InstrumentId::from("ESU26.GLBX"));
1552    }
1553
1554    #[rstest]
1555    #[case(None, "SR3Z26.XCME", true)]
1556    #[case(Some("SR3Z36.XNYM"), "SR3Z26.XCME", true)]
1557    #[case(Some("SR3Z26.XNYM"), "SR3Z36.XCME", true)]
1558    #[case(Some("SR3Z36.XNYM"), "SR3Z26.GLBX", false)]
1559    #[tokio::test]
1560    async fn test_requested_post_year_quotes_use_cached_precision_without_definitions(
1561        #[case] previous: Option<&str>,
1562        #[case] requested: &str,
1563        #[case] register_venue: bool,
1564    ) {
1565        let ts = "2027-02-01T00:00:00Z"
1566            .parse::<jiff::Timestamp>()
1567            .unwrap()
1568            .as_nanosecond() as u64;
1569        let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
1570        msg.hd.publisher_id = 1;
1571        msg.hd.instrument_id = 42;
1572        msg.hd.ts_event = ts - 1;
1573        msg.ts_recv = ts;
1574        msg.levels[0].bid_px = 95_125_000_000;
1575        msg.levels[0].ask_px = 95_250_000_000;
1576        msg.levels[0].bid_sz = 7;
1577        msg.levels[0].ask_sz = 11;
1578        let metadata = dbn::Metadata::builder()
1579            .dataset("GLBX.MDP3")
1580            .schema(Some(dbn::Schema::Bbo1S))
1581            .start(ts)
1582            .stype_in(Some(dbn::SType::RawSymbol))
1583            .stype_out(dbn::SType::InstrumentId)
1584            .mappings(vec![SymbolMapping {
1585                raw_symbol: "SR3Z6".to_string(),
1586                intervals: vec![MappingInterval {
1587                    start_date: date!(2027 - 02 - 01),
1588                    end_date: date!(2027 - 02 - 02),
1589                    symbol: "42".to_string(),
1590                }],
1591            }])
1592            .build();
1593        let mut body = Vec::new();
1594        {
1595            let mut encoder = dbn::encode::dbn::Encoder::with_zstd(&mut body, &metadata).unwrap();
1596            encoder.encode_record(&msg).unwrap();
1597            encoder.flush().unwrap();
1598        }
1599
1600        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1601        let base_url = format!("http://{}/", listener.local_addr().unwrap());
1602
1603        let server = tokio::spawn(serve_response(listener, body));
1604        let client = DatabentoHistoricalClient::new_with_base_url(
1605            Credential::new(test_api_key()),
1606            publishers_path(),
1607            get_atomic_clock_realtime(),
1608            false,
1609            &base_url,
1610        )
1611        .unwrap();
1612
1613        if let Some(previous) = previous {
1614            let previous = InstrumentId::from(previous);
1615            client.prepare_symbols_from_instrument_ids(&[previous]);
1616            client.set_price_precision(previous.symbol, 3);
1617        }
1618
1619        let id = InstrumentId::from(requested);
1620        let symbols = if register_venue {
1621            client.prepare_symbols_from_instrument_ids(&[id])
1622        } else {
1623            vec![id.symbol.to_string()]
1624        };
1625
1626        client.set_price_precision(id.symbol, 3);
1627
1628        let quotes = client
1629            .get_range_quotes(
1630                RangeQueryParams {
1631                    dataset: "GLBX.MDP3".to_string(),
1632                    symbols,
1633                    start: ts.into(),
1634                    end: Some((ts + 1).into()),
1635                    limit: Some(1),
1636                    price_precision: None,
1637                },
1638                Some("bbo-1s".to_string()),
1639            )
1640            .await
1641            .unwrap();
1642
1643        server.await.unwrap();
1644
1645        assert_eq!(
1646            quotes,
1647            vec![QuoteTick::new(
1648                id,
1649                Price::from("95.125"),
1650                Price::from("95.250"),
1651                Quantity::from(7),
1652                Quantity::from(11),
1653                ts.into(),
1654                ts.into(),
1655            )]
1656        );
1657    }
1658}