1use std::{
32 fmt::{Debug, Display},
33 path::PathBuf,
34 str::FromStr,
35 sync::Arc,
36 time::Duration,
37};
38
39use ahash::{AHashMap, HashSet, HashSetExt};
40use databento::{
41 dbn::{self, PitSymbolMap, SymbolIndex},
42 live::Subscription,
43};
44use indexmap::IndexMap;
45use nautilus_core::{
46 AtomicMap, UnixNanos, consts::NAUTILUS_USER_AGENT, string::secret::SecretString,
47 time::get_atomic_clock_realtime,
48};
49use nautilus_model::{
50 data::{Data, InstrumentStatus, OrderBookDelta, OrderBookDeltas},
51 enums::RecordFlag,
52 identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
53 instruments::{Instrument, InstrumentAny},
54 types::Currency,
55};
56use nautilus_network::backoff::ExponentialBackoff;
57use time::OffsetDateTime;
58
59use super::{
60 decode::{
61 decode_imbalance_msg, decode_statistics_msg, decode_status_msg, is_supported_stat_type,
62 },
63 types::{DatabentoImbalance, DatabentoStatistics, SubscriptionAckEvent},
64};
65use crate::{
66 common::{Credential, build_publisher_venue_map, load_publishers},
67 decode::{decode_instrument_def_msg, decode_record},
68 symbology::{
69 check_consistent_symbology, dataset_uses_futures_symbology, infer_symbology_type,
70 raw_symbol, resolve_futures_instrument_id,
71 },
72 types::PublisherId,
73};
74
75#[derive(Debug)]
76pub enum HandlerCommand {
77 Subscribe(Subscription),
78 SetPricePrecision(Symbol, u8),
79 Start,
80 Close,
81}
82
83#[derive(Debug)]
84pub enum DatabentoMessage {
85 Data(Data),
86 Instrument(Box<InstrumentAny>),
87 Status(InstrumentStatus),
88 Imbalance(DatabentoImbalance),
89 Statistics(DatabentoStatistics),
90 SubscriptionAck(SubscriptionAckEvent),
91 Error(anyhow::Error),
92 Close,
93}
94
95#[cfg_attr(
96 feature = "python",
97 pyo3::pyclass(module = "nautilus_trader.adapters.databento")
98)]
99#[cfg_attr(
100 feature = "python",
101 pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
102)]
103pub struct DatabentoLiveClient {
104 futures_year_digits: FuturesYearDigits,
105 credential: Credential,
106 pub dataset: String,
107 is_running: bool,
108 is_closed: bool,
109 cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
110 cmd_rx: Option<tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>>,
111 publisher_venue_map: IndexMap<PublisherId, Venue>,
112 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
113 use_exchange_as_venue: bool,
114 bars_timestamp_on_close: bool,
115 reconnect_timeout_mins: Option<u64>,
116}
117
118impl Debug for DatabentoLiveClient {
119 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
120 f.debug_struct(stringify!(DatabentoLiveClient))
121 .field("credential", &self.credential)
122 .field("dataset", &self.dataset)
123 .field("is_running", &self.is_running)
124 .field("is_closed", &self.is_closed)
125 .finish()
126 }
127}
128
129impl DatabentoLiveClient {
130 pub fn new(
136 key: String,
137 dataset: String,
138 publishers_filepath: PathBuf,
139 use_exchange_as_venue: bool,
140 bars_timestamp_on_close: Option<bool>,
141 reconnect_timeout_mins: Option<i64>,
142 ) -> anyhow::Result<Self> {
143 let publishers = load_publishers(publishers_filepath)?;
144 let publisher_venue_map = build_publisher_venue_map(&publishers);
145
146 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
147
148 let reconnect_timeout_mins = reconnect_timeout_mins
149 .and_then(|mins| if mins >= 0 { Some(mins as u64) } else { None });
150
151 Ok(Self {
152 futures_year_digits: FuturesYearDigits::default(),
153 credential: Credential::new(key),
154 dataset,
155 cmd_tx,
156 cmd_rx: Some(cmd_rx),
157 is_running: false,
158 is_closed: false,
159 publisher_venue_map,
160 symbol_venue_map: Arc::new(AtomicMap::new()),
161 use_exchange_as_venue,
162 bars_timestamp_on_close: bars_timestamp_on_close.unwrap_or(true),
163 reconnect_timeout_mins,
164 })
165 }
166
167 #[must_use]
169 pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
170 self.futures_year_digits = futures_year_digits;
171 self
172 }
173
174 #[must_use]
175 pub const fn is_running(&self) -> bool {
176 self.is_running
177 }
178
179 #[must_use]
180 pub const fn is_closed(&self) -> bool {
181 self.is_closed
182 }
183
184 #[expect(clippy::needless_pass_by_value)]
191 pub fn subscribe(
192 &mut self,
193 schema: String,
194 instrument_ids: Vec<InstrumentId>,
195 start: Option<u64>,
196 snapshot: Option<bool>,
197 price_precisions: Option<Vec<Option<u8>>>,
198 stype_in: Option<String>,
199 ) -> anyhow::Result<()> {
200 if let Some(precisions) = &price_precisions
201 && precisions.len() != instrument_ids.len()
202 {
203 anyhow::bail!(
204 "`price_precisions` length ({}) must match `instrument_ids` length ({})",
205 precisions.len(),
206 instrument_ids.len()
207 );
208 }
209
210 let symbols: Vec<String> = instrument_ids
211 .iter()
212 .map(|id| raw_symbol(id.symbol.as_str(), &self.dataset, self.futures_year_digits))
213 .collect();
214 let first_symbol = symbols
215 .first()
216 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
217 let stype_in = match stype_in {
218 Some(stype_in) => dbn::SType::from_str(&stype_in)?,
219 None => infer_symbology_type(first_symbol),
220 };
221 let symbols: Vec<&str> = symbols.iter().map(String::as_str).collect();
222 check_consistent_symbology(symbols.as_slice())?;
223 let mut sub = Subscription::builder()
224 .symbols(symbols)
225 .schema(dbn::Schema::from_str(&schema)?)
226 .stype_in(stype_in)
227 .build();
228
229 if let Some(start) = start {
230 sub.start = Some(OffsetDateTime::from_unix_timestamp_nanos(i128::from(
231 start,
232 ))?);
233 }
234 sub.use_snapshot = snapshot.unwrap_or(false);
235
236 self.symbol_venue_map.rcu(|m| {
237 for id in &instrument_ids {
238 m.entry(id.symbol).or_insert(id.venue);
239 }
240 });
241
242 if let Some(precisions) = price_precisions {
243 for (instrument_id, precision) in instrument_ids.iter().zip(precisions) {
244 if let Some(precision) = precision {
245 let raw = raw_symbol(
246 instrument_id.symbol.as_str(),
247 &self.dataset,
248 self.futures_year_digits,
249 );
250 self.send_command(HandlerCommand::SetPricePrecision(
251 Symbol::from(raw.as_str()),
252 precision,
253 ))?;
254 }
255 }
256 }
257
258 self.send_command(HandlerCommand::Subscribe(sub))
259 }
260
261 pub fn start(
267 &mut self,
268 ) -> anyhow::Result<(
269 DatabentoFeedHandler,
270 tokio::sync::mpsc::UnboundedReceiver<DatabentoMessage>,
271 )> {
272 if self.is_closed {
273 anyhow::bail!("Client already closed");
274 }
275
276 if self.is_running {
277 anyhow::bail!("Client already running");
278 }
279
280 log::debug!("Starting client");
281
282 let (msg_tx, msg_rx) = tokio::sync::mpsc::unbounded_channel::<DatabentoMessage>();
283 let cmd_rx = self
284 .cmd_rx
285 .take()
286 .ok_or_else(|| anyhow::anyhow!("Command receiver already taken"))?;
287
288 let feed_handler = DatabentoFeedHandler::new(
289 self.credential.clone(),
290 self.dataset.clone(),
291 cmd_rx,
292 msg_tx,
293 self.publisher_venue_map.clone(),
294 self.symbol_venue_map.clone(),
295 self.use_exchange_as_venue,
296 self.bars_timestamp_on_close,
297 self.reconnect_timeout_mins,
298 )
299 .with_futures_year_digits(self.futures_year_digits);
300
301 self.send_command(HandlerCommand::Start)?;
302 self.is_running = true;
303
304 Ok((feed_handler, msg_rx))
305 }
306
307 pub fn close(&mut self) -> anyhow::Result<()> {
314 if !self.is_running {
315 anyhow::bail!("Client never started");
316 }
317
318 if self.is_closed {
319 anyhow::bail!("Client already closed");
320 }
321
322 log::debug!("Closing client");
323
324 if !self.cmd_tx.is_closed() {
325 self.send_command(HandlerCommand::Close)?;
326 }
327
328 self.is_running = false;
329 self.is_closed = true;
330
331 Ok(())
332 }
333
334 fn send_command(&self, cmd: HandlerCommand) -> anyhow::Result<()> {
335 self.cmd_tx.send(cmd).map_err(|e| {
336 anyhow::Error::new(CommandSendError {
337 details: e.to_string(),
338 })
339 })
340 }
341}
342
343#[cfg(any(test, feature = "python"))]
344pub(crate) fn is_command_send_error(error: &anyhow::Error) -> bool {
345 error.is::<CommandSendError>()
346}
347
348#[derive(Debug)]
349struct CommandSendError {
350 details: String,
351}
352
353impl Display for CommandSendError {
354 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
355 write!(
356 f,
357 "Failed to send command to Databento feed handler: {}",
358 self.details
359 )
360 }
361}
362
363impl std::error::Error for CommandSendError {}
364
365pub struct DatabentoFeedHandler {
377 futures_year_digits: FuturesYearDigits,
378 credential: Credential,
379 dataset: String,
380 cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
381 msg_tx: tokio::sync::mpsc::UnboundedSender<DatabentoMessage>,
382 publisher_venue_map: IndexMap<PublisherId, Venue>,
383 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
384 replay: bool,
385 use_exchange_as_venue: bool,
386 bars_timestamp_on_close: bool,
387 reconnect_timeout_mins: Option<u64>,
388 backoff: ExponentialBackoff,
389 subscriptions: Vec<Subscription>,
390 buffered_commands: Vec<HandlerCommand>,
391 price_precision_overrides: AHashMap<Symbol, u8>,
392 gateway_addr: Option<String>,
393 success_threshold: Duration,
394}
395
396impl Debug for DatabentoFeedHandler {
397 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
398 f.debug_struct(stringify!(DatabentoFeedHandler))
399 .field("credential", &self.credential)
400 .field("dataset", &self.dataset)
401 .field("replay", &self.replay)
402 .field("reconnect_timeout_mins", &self.reconnect_timeout_mins)
403 .field("subscriptions", &self.subscriptions.len())
404 .finish()
405 }
406}
407
408impl DatabentoFeedHandler {
409 #[must_use]
411 pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
412 self.futures_year_digits = futures_year_digits;
413 self
414 }
415
416 #[must_use]
422 #[expect(clippy::too_many_arguments)]
423 pub fn new(
424 credential: Credential,
425 dataset: String,
426 rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
427 tx: tokio::sync::mpsc::UnboundedSender<DatabentoMessage>,
428 publisher_venue_map: IndexMap<PublisherId, Venue>,
429 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
430 use_exchange_as_venue: bool,
431 bars_timestamp_on_close: bool,
432 reconnect_timeout_mins: Option<u64>,
433 ) -> Self {
434 let delay_max = if reconnect_timeout_mins.is_some() {
438 Duration::from_secs(60)
439 } else {
440 Duration::from_secs(600)
441 };
442
443 let backoff = ExponentialBackoff::new(Duration::from_secs(1), delay_max, 2.0, 1000, false)
444 .expect("hardcoded backoff parameters are valid");
445
446 Self {
447 futures_year_digits: FuturesYearDigits::default(),
448 credential,
449 dataset,
450 cmd_rx: rx,
451 msg_tx: tx,
452 publisher_venue_map,
453 symbol_venue_map,
454 replay: false,
455 use_exchange_as_venue,
456 bars_timestamp_on_close,
457 reconnect_timeout_mins,
458 backoff,
459 subscriptions: Vec::new(),
460 buffered_commands: Vec::new(),
461 price_precision_overrides: AHashMap::new(),
462 gateway_addr: None,
463 success_threshold: Duration::from_secs(60),
464 }
465 }
466
467 #[must_use]
469 pub fn with_gateway_addr(mut self, addr: String) -> Self {
470 self.gateway_addr = Some(addr);
471 self
472 }
473
474 #[must_use]
479 pub fn with_success_threshold(mut self, threshold: Duration) -> Self {
480 self.success_threshold = threshold;
481 self
482 }
483
484 pub async fn run(&mut self) -> anyhow::Result<()> {
497 log::debug!("Running feed handler");
498
499 let mut reconnect_start: Option<tokio::time::Instant> = None;
500 let mut attempt = 0;
501
502 loop {
503 attempt += 1;
504
505 match self.run_session(attempt).await {
506 Ok(ran_successfully) => {
507 if ran_successfully {
508 log::info!("Resetting reconnection cycle after successful session");
509 reconnect_start = None;
510 attempt = 0;
511 self.backoff.reset();
512 } else {
513 log::debug!("Session ended normally");
514 break Ok(());
515 }
516 }
517 Err(e) => {
518 let cycle_start = reconnect_start.get_or_insert_with(tokio::time::Instant::now);
519
520 if let Some(timeout_mins) = self.reconnect_timeout_mins {
521 let elapsed = cycle_start.elapsed();
522 let timeout = Duration::from_mins(timeout_mins);
523
524 if elapsed >= timeout {
525 log::error!("Giving up reconnection after {timeout_mins} minutes");
526 self.send_msg(DatabentoMessage::Error(anyhow::anyhow!(
527 "Reconnection timeout after {timeout_mins} minutes: {e}"
528 )));
529 break Err(e);
530 }
531 }
532
533 let delay = self.backoff.next_duration();
534
535 log::warn!(
536 "Connection lost (attempt {}): {}. Reconnecting in {}s...",
537 attempt,
538 e,
539 delay.as_secs()
540 );
541
542 let sleep = tokio::time::sleep(delay);
543 tokio::pin!(sleep);
544
545 loop {
546 tokio::select! {
547 () = &mut sleep => break,
548 cmd = self.cmd_rx.recv() => {
549 match cmd {
550 Some(HandlerCommand::Close) => {
551 log::debug!("Close received during backoff");
552 return Ok(());
553 }
554 None => {
555 log::debug!("Command channel closed during backoff");
556 return Ok(());
557 }
558 Some(cmd) => {
559 log::debug!("Buffering command received during backoff: {cmd:?}");
560 self.buffered_commands.push(cmd);
561 }
562 }
563 }
564 }
565 }
566 }
567 }
568 }
569 }
570
571 async fn run_session(&mut self, attempt: usize) -> anyhow::Result<bool> {
580 if attempt > 1 {
581 log::info!("Reconnecting (attempt {attempt})...");
582 }
583
584 let session_start = tokio::time::Instant::now();
585 let clock = get_atomic_clock_realtime();
586 let mut symbol_map = PitSymbolMap::new();
587 let mut instrument_id_map: AHashMap<u32, CachedInstrumentId> = AHashMap::new();
588 let mut instrument_def_price_precision_map: AHashMap<u32, u8> = AHashMap::new();
589 let mut subscription_price_precision_map: AHashMap<u32, u8> = AHashMap::new();
590
591 let mut buffering_start = None;
592 let mut buffered_deltas: AHashMap<InstrumentId, Vec<OrderBookDelta>> = AHashMap::new();
593 let mut initialized_books = HashSet::new();
594 let mut trade_sources: AHashMap<InstrumentId, u8> = AHashMap::new();
595 let timeout = Duration::from_secs(5); let gateway_addr = self.gateway_addr.clone();
598 let api_key = SecretString::from(self.credential.api_key().to_owned());
599 let dataset = self.dataset.clone();
600
601 let result = tokio::time::timeout(timeout, async move {
602 let base = databento::LiveClient::builder();
603 let base = if let Some(addr) = gateway_addr {
604 base.addr(addr).await?
605 } else {
606 base
607 };
608 base.user_agent_extension(NAUTILUS_USER_AGENT.into())
609 .key(api_key.expose_secret().to_owned())?
610 .dataset(dataset)
611 .build()
612 .await
613 })
614 .await?;
615
616 let mut client = match result {
617 Ok(client) => {
618 if attempt > 1 {
619 log::info!("Reconnected successfully");
620 } else {
621 log::info!("Connected");
622 }
623 client
624 }
625 Err(e) => {
626 anyhow::bail!("Failed to connect to Databento LSG: {e}");
627 }
628 };
629
630 let mut start_buffered = false;
632
633 if !self.buffered_commands.is_empty() {
634 log::debug!(
635 "Processing {} buffered commands",
636 self.buffered_commands.len()
637 );
638
639 for cmd in self.buffered_commands.drain(..) {
640 match cmd {
641 HandlerCommand::Subscribe(sub) => {
642 if !self.replay && sub.start.is_some() {
643 self.replay = true;
644 }
645 self.subscriptions.push(sub);
646 }
647 HandlerCommand::SetPricePrecision(symbol, precision) => {
648 self.price_precision_overrides.insert(symbol, precision);
649 }
650 HandlerCommand::Start => {
651 start_buffered = true;
652 }
653 HandlerCommand::Close => {
654 log::warn!("Close command was buffered, shutting down");
655 return Ok(false);
656 }
657 }
658 }
659 }
660
661 let mut running = false;
662
663 if !self.subscriptions.is_empty() {
664 log::info!(
665 "Resubscribing to {} subscriptions",
666 self.subscriptions.len()
667 );
668
669 let replay = self.subscriptions.iter().any(|sub| sub.start.is_some());
671
672 for sub in self.subscriptions.clone() {
673 client.subscribe(sub).await?;
674 }
675 for sub in &mut self.subscriptions {
677 sub.start = None;
678 }
679
680 buffering_start = replay.then(|| clock.get_time_ns());
681 client.start().await?;
682 running = true;
683 log::info!("Resubscription complete");
684 } else if start_buffered {
685 log::debug!("Starting session from buffered Start command");
686 buffering_start = if self.replay {
687 Some(clock.get_time_ns())
688 } else {
689 None
690 };
691 client.start().await?;
692 running = true;
693 }
694
695 loop {
696 if self.msg_tx.is_closed() {
697 log::debug!("Message channel was closed: stopping");
698 return Ok(false);
699 }
700
701 if !running {
706 match self.cmd_rx.recv().await {
707 Some(HandlerCommand::Subscribe(sub)) => {
708 log::debug!("Received command: Subscribe");
709
710 if !self.replay && sub.start.is_some() {
711 self.replay = true;
712 }
713 client.subscribe(sub.clone()).await?;
714 let mut sub_for_reconnect = sub;
715 sub_for_reconnect.start = None;
716 self.subscriptions.push(sub_for_reconnect);
717 continue;
718 }
719 Some(HandlerCommand::SetPricePrecision(symbol, precision)) => {
720 log::debug!(
721 "Received command: SetPricePrecision for {symbol} to {precision}"
722 );
723 self.price_precision_overrides.insert(symbol, precision);
724 continue;
725 }
726 Some(HandlerCommand::Start) => {
727 log::debug!("Received command: Start");
728 buffering_start = if self.replay {
729 Some(clock.get_time_ns())
730 } else {
731 None
732 };
733 client.start().await?;
734 running = true;
735 continue;
736 }
737 Some(HandlerCommand::Close) => {
738 self.send_close_msg();
739 return Ok(false);
740 }
741 None => {
742 log::debug!("Command channel disconnected");
743 return Ok(false);
744 }
745 }
746 }
747
748 let record_opt = tokio::select! {
749 cmd = self.cmd_rx.recv() =>
750 match cmd {
751 Some(HandlerCommand::Subscribe(sub)) => {
752 log::debug!("Received command: Subscribe");
753
754 if sub.start.is_some() {
755 self.replay = true;
756 log::error!(
757 "Ignoring `start` on {} subscribe, session already running, Databento drops replay anchors sent after session start",
758 self.dataset,
759 );
760 }
761 client.subscribe(sub.clone()).await?;
762 let mut sub_for_reconnect = sub;
763 sub_for_reconnect.start = None;
764 self.subscriptions.push(sub_for_reconnect);
765 continue;
766 }
767 Some(HandlerCommand::SetPricePrecision(symbol, precision)) => {
768 log::debug!(
769 "Received command: SetPricePrecision for {symbol} to {precision}"
770 );
771 self.price_precision_overrides.insert(symbol, precision);
772 continue;
773 }
774 Some(HandlerCommand::Start) => {
775 log::warn!("Received Start command but session already running");
776 continue;
777 }
778 Some(HandlerCommand::Close) => {
779 self.send_close_msg();
780 client.close().await?;
781 log::debug!("Closed inner client");
782 return Ok(false);
783 }
784 None => {
785 log::debug!("Command channel disconnected");
786 return Ok(false);
787 }
788 },
789 result = client.next_record() => result,
790 };
791
792 let record = match record_opt {
793 Ok(Some(record)) => record,
794 Ok(None) => {
795 if session_start.elapsed() >= self.success_threshold {
796 log::debug!("Session ended after successful run");
797 return Ok(true);
798 }
799 anyhow::bail!("Session ended by gateway");
800 }
801 Err(e) => {
802 if session_start.elapsed() >= self.success_threshold {
803 log::debug!("Connection error after successful run: {e}");
804 return Ok(true);
805 }
806 anyhow::bail!("Connection error: {e}");
807 }
808 };
809
810 let ts_init = clock.get_time_ns();
811
812 if let Err(e) =
813 self.cache_futures_instrument_id(&record, &symbol_map, &mut instrument_id_map)
814 {
815 log::error!("Error resolving futures instrument ID: {e}");
816 continue;
817 }
818
819 if let Some(msg) = record.get::<dbn::ErrorMsg>() {
821 handle_error_msg(msg);
822 } else if let Some(msg) = record.get::<dbn::SystemMsg>() {
823 if let Some(ack) = handle_system_msg(msg, ts_init) {
824 self.send_msg(DatabentoMessage::SubscriptionAck(ack));
825 }
826 } else if let Some(msg) = record.get::<dbn::SymbolMappingMsg>() {
827 instrument_def_price_precision_map.remove(&msg.hd.instrument_id);
828 update_price_precision_map_with_symbol_mapping_msg(
829 msg,
830 &self.price_precision_overrides,
831 &mut subscription_price_precision_map,
832 )?;
833 handle_symbol_mapping_msg(
834 msg,
835 &mut symbol_map,
836 &self.symbol_venue_map,
837 &mut instrument_id_map,
838 )?;
839 } else if let Some(msg) = record.get::<dbn::InstrumentDefMsg>() {
840 let maybe_data = self.handle_instrument_def_msg(
841 msg,
842 &record,
843 &symbol_map,
844 &mut instrument_id_map,
845 ts_init,
846 )?;
847
848 if let Some(data) = maybe_data {
849 instrument_def_price_precision_map
850 .insert(msg.hd.instrument_id, data.price_precision());
851 self.send_msg(DatabentoMessage::Instrument(Box::new(data)));
852 }
853 } else if let Some(msg) = record.get::<dbn::StatusMsg>() {
854 let data = handle_status_msg(
855 msg,
856 &record,
857 &symbol_map,
858 &self.publisher_venue_map,
859 &self.symbol_venue_map,
860 &mut instrument_id_map,
861 ts_init,
862 )?;
863 self.send_msg(DatabentoMessage::Status(data));
864 } else if let Some(msg) = record.get::<dbn::ImbalanceMsg>() {
865 handle_imbalance_msg(
866 msg,
867 &record,
868 &symbol_map,
869 &self.publisher_venue_map,
870 &self.symbol_venue_map,
871 &mut instrument_id_map,
872 &instrument_def_price_precision_map,
873 &subscription_price_precision_map,
874 &self.price_precision_overrides,
875 ts_init,
876 )
877 .map_or_else(
878 |e| log::error!("Error decoding imbalance: {e}"),
879 |data| self.send_msg(DatabentoMessage::Imbalance(data)),
880 );
881 } else if let Some(msg) = record.get::<dbn::StatMsg>() {
882 let maybe_data = handle_statistics_msg(
883 msg,
884 &record,
885 &symbol_map,
886 &self.publisher_venue_map,
887 &self.symbol_venue_map,
888 &mut instrument_id_map,
889 &instrument_def_price_precision_map,
890 &subscription_price_precision_map,
891 &self.price_precision_overrides,
892 ts_init,
893 )?;
894
895 if let Some(data) = maybe_data {
896 self.send_msg(DatabentoMessage::Statistics(data));
897 }
898 } else {
899 let res = handle_record(
901 record,
902 &symbol_map,
903 &self.publisher_venue_map,
904 &self.symbol_venue_map,
905 &mut instrument_id_map,
906 &instrument_def_price_precision_map,
907 &subscription_price_precision_map,
908 &self.price_precision_overrides,
909 ts_init,
910 &initialized_books,
911 self.bars_timestamp_on_close,
912 );
913 let (mut data1, data2) = match res {
914 Ok(decoded) => decoded,
915 Err(e) => {
916 log::error!("Error decoding record: {e}");
917
918 (None, None)
920 }
921 };
922
923 if let Some(msg) = record.get::<dbn::MboMsg>() {
924 if let Some(Data::BookDelta(delta)) = &data1 {
925 initialized_books.insert(delta.instrument_id);
926
927 log::trace!(
928 "Buffering delta: {} {buffering_start:?} flags={}",
929 delta.ts_event,
930 msg.flags.raw(),
931 );
932
933 match process_mbo_delta(
934 *delta,
935 msg.flags.raw(),
936 &mut buffering_start,
937 &mut buffered_deltas,
938 ) {
939 Some(deltas) => data1 = Some(Data::BookDeltas(Box::new(deltas))),
940 None => continue,
941 }
942 } else {
943 let deltas = instrument_id_map.get(&msg.hd.instrument_id).and_then(|id| {
950 flush_mbo_event_boundary(
951 id.id(),
952 msg.ts_recv.into(),
953 msg.flags.raw(),
954 &mut buffering_start,
955 &mut buffered_deltas,
956 )
957 });
958
959 if let Some(deltas) = deltas {
960 self.send_msg(DatabentoMessage::Data(Data::BookDeltas(Box::new(
961 deltas,
962 ))));
963 }
964
965 if buffering_start.is_some_and(|start| msg.ts_recv <= start) {
967 continue;
968 }
969 }
970 }
971
972 let rtype = record.header().rtype;
973
974 for data in [data1, data2].into_iter().flatten() {
975 if claim_trade_source(&data, rtype, &mut trade_sources) {
976 self.send_msg(DatabentoMessage::Data(data));
977 }
978 }
979 }
980 }
981 }
982
983 fn handle_instrument_def_msg(
989 &self,
990 msg: &dbn::InstrumentDefMsg,
991 record: &dbn::RecordRef,
992 symbol_map: &PitSymbolMap,
993 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
994 ts_init: UnixNanos,
995 ) -> anyhow::Result<Option<InstrumentAny>> {
996 instrument_id_map.remove(&msg.hd.instrument_id);
997 let exchange = if self.use_exchange_as_venue {
998 msg.exchange()?
999 } else {
1000 ""
1001 };
1002
1003 if !exchange.is_empty() {
1004 update_instrument_id_map_with_exchange(
1005 symbol_map,
1006 &self.symbol_venue_map,
1007 instrument_id_map,
1008 msg.hd.instrument_id,
1009 exchange,
1010 )?;
1011 }
1012
1013 let instrument_id = update_instrument_id_map(
1014 record,
1015 symbol_map,
1016 &self.publisher_venue_map,
1017 &self.symbol_venue_map,
1018 instrument_id_map,
1019 )?;
1020 let mut instrument_id = resolve_futures_instrument_id(
1021 record,
1022 instrument_id,
1023 self.futures_year_digits,
1024 &self.symbol_venue_map.load(),
1025 None,
1026 )?;
1027
1028 if exchange.is_empty()
1029 && let Some(venue) = self.symbol_venue_map.get_cloned(&instrument_id.symbol)
1030 {
1031 instrument_id.venue = venue;
1032 }
1033
1034 let instrument = decode_instrument_def_msg(msg, instrument_id, Some(ts_init), None)?;
1035
1036 let cached = if instrument.is_some()
1037 && self.futures_year_digits == FuturesYearDigits::Two
1038 && dataset_uses_futures_symbology(&self.dataset)
1039 {
1040 if exchange.is_empty() {
1041 CachedInstrumentId::Definition(instrument_id)
1042 } else {
1043 CachedInstrumentId::Exchange(instrument_id)
1044 }
1045 } else {
1046 CachedInstrumentId::Inferred(instrument_id)
1047 };
1048
1049 instrument_id_map.insert(record.header().instrument_id, cached);
1050 Ok(instrument)
1051 }
1052
1053 fn cache_futures_instrument_id(
1054 &self,
1055 record: &dbn::RecordRef,
1056 symbol_map: &PitSymbolMap,
1057 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1058 ) -> anyhow::Result<()> {
1059 if self.futures_year_digits != FuturesYearDigits::Two
1060 || !dataset_uses_futures_symbology(&self.dataset)
1061 || record.get::<dbn::InstrumentDefMsg>().is_some()
1062 || record.get::<dbn::SymbolMappingMsg>().is_some()
1063 || instrument_id_map.contains_key(&record.header().instrument_id)
1064 || symbol_map.get_for_rec(record).is_none()
1065 {
1066 return Ok(());
1067 }
1068
1069 let instrument_id = update_instrument_id_map(
1070 record,
1071 symbol_map,
1072 &self.publisher_venue_map,
1073 &self.symbol_venue_map,
1074 instrument_id_map,
1075 )?;
1076 let mut instrument_id = resolve_futures_instrument_id(
1077 record,
1078 instrument_id,
1079 self.futures_year_digits,
1080 &self.symbol_venue_map.load(),
1081 None,
1082 )?;
1083
1084 if let Some(venue) = self.symbol_venue_map.get_cloned(&instrument_id.symbol) {
1085 instrument_id.venue = venue;
1086 }
1087
1088 instrument_id_map.insert(
1089 record.header().instrument_id,
1090 CachedInstrumentId::Inferred(instrument_id),
1091 );
1092 Ok(())
1093 }
1094
1095 fn send_msg(&self, msg: DatabentoMessage) {
1097 log::trace!("Sending {msg:?}");
1098 match self.msg_tx.send(msg) {
1099 Ok(()) => {}
1100 Err(e) => log::debug!("Error sending message: {e}"),
1101 }
1102 }
1103
1104 fn send_close_msg(&self) {
1105 if let Err(e) = self.msg_tx.send(DatabentoMessage::Close) {
1106 log::debug!("Could not send close message: {e}");
1107 }
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy)]
1112enum CachedInstrumentId {
1113 Inferred(InstrumentId),
1114 Definition(InstrumentId),
1115 Exchange(InstrumentId),
1116}
1117
1118fn handle_error_msg(msg: &dbn::ErrorMsg) {
1120 log::error!("{msg:?}");
1121}
1122
1123fn handle_system_msg(msg: &dbn::SystemMsg, ts_received: UnixNanos) -> Option<SubscriptionAckEvent> {
1125 match msg.code() {
1126 Ok(dbn::SystemCode::SubscriptionAck) => {
1127 let message = msg.msg().unwrap_or("<invalid utf-8>");
1128 log::debug!("Subscription acknowledged: {message}");
1129
1130 let schema = parse_ack_message(message);
1131
1132 Some(SubscriptionAckEvent {
1133 schema,
1134 message: message.to_string(),
1135 ts_received,
1136 })
1137 }
1138 Ok(dbn::SystemCode::Heartbeat) => {
1139 log::trace!("Heartbeat received");
1140 None
1141 }
1142 Ok(dbn::SystemCode::SlowReaderWarning) => {
1143 let message = msg.msg().unwrap_or("<invalid utf-8>");
1144 log::warn!("Slow reader warning: {message}");
1145 None
1146 }
1147 Ok(dbn::SystemCode::ReplayCompleted) => {
1148 let message = msg.msg().unwrap_or("<invalid utf-8>");
1149 log::debug!("Replay completed: {message}");
1150 None
1151 }
1152 _ => {
1153 log::debug!("{msg:?}");
1154 None
1155 }
1156 }
1157}
1158
1159fn parse_ack_message(message: &str) -> String {
1161 message
1163 .strip_circumfix("Subscription request ", " data succeeded")
1164 .and_then(|rest| rest.split_once(" for "))
1165 .map(|(_, schema)| schema.trim().to_string())
1166 .unwrap_or_default()
1167}
1168
1169fn handle_symbol_mapping_msg(
1175 msg: &dbn::SymbolMappingMsg,
1176 symbol_map: &mut PitSymbolMap,
1177 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1178 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1179) -> anyhow::Result<()> {
1180 let changed =
1181 symbol_map.get(msg.hd.instrument_id).map(String::as_str) != Some(msg.stype_out_symbol()?);
1182 symbol_map
1183 .on_symbol_mapping(msg)
1184 .map_err(|e| anyhow::anyhow!("on_symbol_mapping failed for {msg:?}: {e}"))?;
1185
1186 match instrument_id_map.get(&msg.hd.instrument_id).copied() {
1187 Some(CachedInstrumentId::Definition(mut id)) if !changed => {
1188 if let Some(venue) = symbol_venue_map.get_cloned(&id.symbol) {
1189 id.venue = venue;
1190 }
1191
1192 instrument_id_map.insert(msg.hd.instrument_id, CachedInstrumentId::Definition(id));
1193 }
1194 Some(CachedInstrumentId::Exchange(_)) if !changed => {}
1195 _ => {
1196 instrument_id_map.remove(&msg.hd.instrument_id);
1197 }
1198 }
1199
1200 Ok(())
1201}
1202
1203fn update_price_precision_map_with_symbol_mapping_msg(
1204 msg: &dbn::SymbolMappingMsg,
1205 price_precision_overrides: &AHashMap<Symbol, u8>,
1206 subscription_price_precision_map: &mut AHashMap<u32, u8>,
1207) -> anyhow::Result<()> {
1208 subscription_price_precision_map.remove(&msg.hd.instrument_id);
1209
1210 if price_precision_overrides.is_empty() {
1211 return Ok(());
1212 }
1213
1214 let stype_in_symbol = msg
1215 .stype_in_symbol()
1216 .map_err(|e| anyhow::anyhow!("Error decoding `stype_in_symbol`: {e}"))?;
1217 let stype_out_symbol = msg
1218 .stype_out_symbol()
1219 .map_err(|e| anyhow::anyhow!("Error decoding `stype_out_symbol`: {e}"))?;
1220
1221 let price_precision = [stype_in_symbol, stype_out_symbol]
1222 .into_iter()
1223 .find_map(|symbol| {
1224 price_precision_overrides
1225 .get(&Symbol::from_str_unchecked(symbol))
1226 .copied()
1227 });
1228
1229 if let Some(price_precision) = price_precision {
1230 subscription_price_precision_map.insert(msg.hd.instrument_id, price_precision);
1231 }
1232
1233 Ok(())
1234}
1235
1236fn update_instrument_id_map_with_exchange(
1238 symbol_map: &PitSymbolMap,
1239 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1240 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1241 raw_instrument_id: u32,
1242 exchange: &str,
1243) -> anyhow::Result<InstrumentId> {
1244 let raw_symbol = symbol_map.get(raw_instrument_id).ok_or_else(|| {
1245 anyhow::anyhow!("Cannot resolve raw_symbol for instrument_id {raw_instrument_id}")
1246 })?;
1247 let symbol = Symbol::from(raw_symbol.as_str());
1248 let venue = Venue::from_code(exchange)
1249 .map_err(|e| anyhow::anyhow!("Invalid venue code '{exchange}': {e}"))?;
1250 let instrument_id = InstrumentId::new(symbol, venue);
1251 symbol_venue_map.rcu(|m| {
1252 m.entry(symbol).or_insert(venue);
1253 });
1254
1255 instrument_id_map.insert(
1256 raw_instrument_id,
1257 CachedInstrumentId::Exchange(instrument_id),
1258 );
1259 Ok(instrument_id)
1260}
1261
1262fn update_instrument_id_map(
1263 record: &dbn::RecordRef,
1264 symbol_map: &PitSymbolMap,
1265 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1266 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1267 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1268) -> anyhow::Result<InstrumentId> {
1269 let header = record.header();
1270
1271 if let Some(&instrument_id) = instrument_id_map.get(&header.instrument_id) {
1273 return Ok(instrument_id.id());
1274 }
1275
1276 let raw_symbol = symbol_map.get_for_rec(record).ok_or_else(|| {
1277 anyhow::anyhow!(
1278 "Cannot resolve `raw_symbol` from `symbol_map` for instrument_id {}",
1279 header.instrument_id
1280 )
1281 })?;
1282
1283 let symbol = Symbol::from_str_unchecked(raw_symbol);
1284
1285 let publisher_id = header.publisher_id;
1286 let venue = if let Some(venue) = symbol_venue_map.get_cloned(&symbol) {
1287 venue
1288 } else {
1289 let venue = publisher_venue_map
1290 .get(&publisher_id)
1291 .ok_or_else(|| anyhow::anyhow!("No venue found for `publisher_id` {publisher_id}"))?;
1292 *venue
1293 };
1294 let instrument_id = InstrumentId::new(symbol, venue);
1295
1296 instrument_id_map.insert(
1297 header.instrument_id,
1298 CachedInstrumentId::Inferred(instrument_id),
1299 );
1300 Ok(instrument_id)
1301}
1302
1303impl CachedInstrumentId {
1304 fn id(self) -> InstrumentId {
1305 match self {
1306 Self::Inferred(id) | Self::Definition(id) | Self::Exchange(id) => id,
1307 }
1308 }
1309}
1310
1311fn handle_status_msg(
1312 msg: &dbn::StatusMsg,
1313 record: &dbn::RecordRef,
1314 symbol_map: &PitSymbolMap,
1315 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1316 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1317 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1318 ts_init: UnixNanos,
1319) -> anyhow::Result<InstrumentStatus> {
1320 let instrument_id = update_instrument_id_map(
1321 record,
1322 symbol_map,
1323 publisher_venue_map,
1324 symbol_venue_map,
1325 instrument_id_map,
1326 )?;
1327
1328 decode_status_msg(msg, instrument_id, Some(ts_init))
1329}
1330
1331#[expect(clippy::too_many_arguments)]
1332fn handle_imbalance_msg(
1333 msg: &dbn::ImbalanceMsg,
1334 record: &dbn::RecordRef,
1335 symbol_map: &PitSymbolMap,
1336 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1337 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1338 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1339 instrument_def_price_precision_map: &AHashMap<u32, u8>,
1340 subscription_price_precision_map: &AHashMap<u32, u8>,
1341 price_precision_overrides: &AHashMap<Symbol, u8>,
1342 ts_init: UnixNanos,
1343) -> anyhow::Result<DatabentoImbalance> {
1344 let instrument_id = update_instrument_id_map(
1345 record,
1346 symbol_map,
1347 publisher_venue_map,
1348 symbol_venue_map,
1349 instrument_id_map,
1350 )?;
1351
1352 let price_precision = resolve_price_precision(
1353 msg.hd.instrument_id,
1354 symbol_map.get_for_rec(record).map(String::as_str),
1355 instrument_def_price_precision_map,
1356 subscription_price_precision_map,
1357 price_precision_overrides,
1358 );
1359
1360 decode_imbalance_msg(msg, instrument_id, price_precision, Some(ts_init))
1361}
1362
1363#[expect(clippy::too_many_arguments)]
1364fn handle_statistics_msg(
1365 msg: &dbn::StatMsg,
1366 record: &dbn::RecordRef,
1367 symbol_map: &PitSymbolMap,
1368 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1369 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1370 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1371 instrument_def_price_precision_map: &AHashMap<u32, u8>,
1372 subscription_price_precision_map: &AHashMap<u32, u8>,
1373 price_precision_overrides: &AHashMap<Symbol, u8>,
1374 ts_init: UnixNanos,
1375) -> anyhow::Result<Option<DatabentoStatistics>> {
1376 if !is_supported_stat_type(msg.stat_type) {
1378 log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
1379 return Ok(None);
1380 }
1381
1382 let instrument_id = update_instrument_id_map(
1383 record,
1384 symbol_map,
1385 publisher_venue_map,
1386 symbol_venue_map,
1387 instrument_id_map,
1388 )?;
1389
1390 let price_precision = resolve_price_precision(
1391 msg.hd.instrument_id,
1392 symbol_map.get_for_rec(record).map(String::as_str),
1393 instrument_def_price_precision_map,
1394 subscription_price_precision_map,
1395 price_precision_overrides,
1396 );
1397
1398 decode_statistics_msg(msg, instrument_id, price_precision, Some(ts_init))
1399}
1400
1401#[expect(clippy::too_many_arguments)]
1402fn handle_record(
1403 record: dbn::RecordRef,
1404 symbol_map: &PitSymbolMap,
1405 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1406 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1407 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1408 instrument_def_price_precision_map: &AHashMap<u32, u8>,
1409 subscription_price_precision_map: &AHashMap<u32, u8>,
1410 price_precision_overrides: &AHashMap<Symbol, u8>,
1411 ts_init: UnixNanos,
1412 initialized_books: &HashSet<InstrumentId>,
1413 bars_timestamp_on_close: bool,
1414) -> anyhow::Result<(Option<Data>, Option<Data>)> {
1415 let instrument_id = update_instrument_id_map(
1416 &record,
1417 symbol_map,
1418 publisher_venue_map,
1419 symbol_venue_map,
1420 instrument_id_map,
1421 )?;
1422
1423 let price_precision = resolve_price_precision(
1424 record.header().instrument_id,
1425 symbol_map.get_for_rec(&record).map(String::as_str),
1426 instrument_def_price_precision_map,
1427 subscription_price_precision_map,
1428 price_precision_overrides,
1429 );
1430
1431 let include_trades = if record.get::<dbn::Mbp1Msg>().is_some()
1434 || record.get::<dbn::TbboMsg>().is_some()
1435 || record.get::<dbn::Cmbp1Msg>().is_some()
1436 {
1437 true } else {
1439 initialized_books.contains(&instrument_id) };
1441
1442 decode_record(
1443 &record,
1444 instrument_id,
1445 price_precision,
1446 Some(ts_init),
1447 include_trades,
1448 bars_timestamp_on_close,
1449 )
1450}
1451
1452fn claim_trade_source(
1462 data: &Data,
1463 rtype: u8,
1464 trade_sources: &mut AHashMap<InstrumentId, u8>,
1465) -> bool {
1466 match data {
1467 Data::Trade(trade) => *trade_sources.entry(trade.instrument_id).or_insert(rtype) == rtype,
1468 _ => true,
1469 }
1470}
1471
1472fn resolve_price_precision(
1473 record_instrument_id: u32,
1474 raw_symbol: Option<&str>,
1475 instrument_def_price_precision_map: &AHashMap<u32, u8>,
1476 subscription_price_precision_map: &AHashMap<u32, u8>,
1477 price_precision_overrides: &AHashMap<Symbol, u8>,
1478) -> u8 {
1479 subscription_price_precision_map
1480 .get(&record_instrument_id)
1481 .copied()
1482 .or_else(|| {
1483 raw_symbol.and_then(|symbol| {
1484 price_precision_overrides
1485 .get(&Symbol::from_str_unchecked(symbol))
1486 .copied()
1487 })
1488 })
1489 .or_else(|| {
1490 instrument_def_price_precision_map
1491 .get(&record_instrument_id)
1492 .copied()
1493 })
1494 .unwrap_or(Currency::USD().precision)
1495}
1496
1497fn process_mbo_delta(
1502 delta: OrderBookDelta,
1503 flags: u8,
1504 buffering_start: &mut Option<UnixNanos>,
1505 buffered_deltas: &mut AHashMap<InstrumentId, Vec<OrderBookDelta>>,
1506) -> Option<OrderBookDeltas> {
1507 let is_last = RecordFlag::F_LAST.matches(flags);
1508 let is_snapshot = RecordFlag::F_SNAPSHOT.matches(flags);
1509
1510 if is_last
1512 && !is_snapshot
1513 && buffering_start.is_none()
1514 && !buffered_deltas.contains_key(&delta.instrument_id)
1515 {
1516 let deltas = OrderBookDeltas::new(delta.instrument_id, vec![delta]);
1517 return Some(deltas);
1518 }
1519
1520 let buffer = buffered_deltas.entry(delta.instrument_id).or_default();
1521 buffer.push(delta);
1522
1523 if is_snapshot {
1524 return None;
1525 }
1526
1527 flush_mbo_event_boundary(
1528 delta.instrument_id,
1529 delta.ts_event,
1530 flags,
1531 buffering_start,
1532 buffered_deltas,
1533 )
1534}
1535
1536fn flush_mbo_event_boundary(
1544 instrument_id: InstrumentId,
1545 ts_event: UnixNanos,
1546 flags: u8,
1547 buffering_start: &mut Option<UnixNanos>,
1548 buffered_deltas: &mut AHashMap<InstrumentId, Vec<OrderBookDelta>>,
1549) -> Option<OrderBookDeltas> {
1550 if !RecordFlag::F_LAST.matches(flags) || RecordFlag::F_SNAPSHOT.matches(flags) {
1551 return None;
1552 }
1553
1554 let buffer = buffered_deltas.get(&instrument_id)?;
1555
1556 if buffer.is_empty() {
1557 return None;
1558 }
1559
1560 if let Some(start_ns) = *buffering_start {
1561 if ts_event <= start_ns {
1562 return None;
1563 }
1564 *buffering_start = None;
1565 }
1566
1567 let buffer = buffered_deltas.remove(&instrument_id)?;
1568 let deltas = OrderBookDeltas::new(instrument_id, buffer);
1569 Some(deltas)
1570}
1571
1572#[cfg(test)]
1573mod tests {
1574 use std::path::PathBuf;
1575
1576 use databento::{dbn::enums::rtype, live::Subscription};
1577 use indexmap::IndexMap;
1578 use nautilus_model::types::{Price, Quantity};
1579 use rstest::*;
1580 use time::macros::datetime;
1581
1582 use super::*;
1583
1584 fn stub_delta(instrument_id: InstrumentId, ts: u64) -> OrderBookDelta {
1585 use nautilus_model::{
1586 data::BookOrder,
1587 enums::{BookAction, OrderSide},
1588 types::{Price, Quantity},
1589 };
1590
1591 OrderBookDelta::new(
1592 instrument_id,
1593 BookAction::Delete,
1594 BookOrder::new(
1595 OrderSide::Sell,
1596 Price::from("100.00"),
1597 Quantity::from("5"),
1598 42,
1599 ),
1600 0, 1,
1602 ts.into(),
1603 ts.into(),
1604 )
1605 }
1606
1607 #[rstest]
1608 fn test_boundary_flag_on_recordless_message_flushes_buffered_event() {
1609 let instrument_id = InstrumentId::from("TEST.GLBX");
1613 let mut buffering_start = None;
1614 let mut buffered = AHashMap::new();
1615
1616 let buffered_result = process_mbo_delta(
1617 stub_delta(instrument_id, 1),
1618 0,
1619 &mut buffering_start,
1620 &mut buffered,
1621 );
1622 assert!(buffered_result.is_none());
1623
1624 let flushed = flush_mbo_event_boundary(
1625 instrument_id,
1626 2.into(),
1627 RecordFlag::F_LAST as u8,
1628 &mut buffering_start,
1629 &mut buffered,
1630 );
1631 assert!(flushed.is_some());
1632 assert!(buffered.is_empty());
1633 }
1634
1635 #[rstest]
1636 fn test_boundary_flag_with_empty_buffer_is_noop() {
1637 let instrument_id = InstrumentId::from("TEST.GLBX");
1638 let start = UnixNanos::from(1);
1639 let mut buffering_start = Some(start);
1640 let mut buffered: AHashMap<InstrumentId, Vec<OrderBookDelta>> = AHashMap::new();
1641 buffered.insert(instrument_id, Vec::new());
1642
1643 let flushed = flush_mbo_event_boundary(
1644 instrument_id,
1645 2.into(),
1646 RecordFlag::F_LAST as u8,
1647 &mut buffering_start,
1648 &mut buffered,
1649 );
1650
1651 assert!(flushed.is_none());
1652 assert_eq!(buffering_start, Some(start));
1653 }
1654
1655 #[rstest]
1656 fn test_empty_boundary_preserves_replay_gate_for_buffered_instrument() {
1657 let empty_id = InstrumentId::from("EMPTY.GLBX");
1658 let buffered_id = InstrumentId::from("BUFFERED.GLBX");
1659 let start = UnixNanos::from(10);
1660 let mut buffering_start = Some(start);
1661 let mut buffered = AHashMap::new();
1662 let _ = process_mbo_delta(
1663 stub_delta(buffered_id, 1),
1664 0,
1665 &mut buffering_start,
1666 &mut buffered,
1667 );
1668
1669 let empty_boundary = flush_mbo_event_boundary(
1670 empty_id,
1671 11.into(),
1672 RecordFlag::F_LAST as u8,
1673 &mut buffering_start,
1674 &mut buffered,
1675 );
1676 let buffered_boundary = flush_mbo_event_boundary(
1677 buffered_id,
1678 5.into(),
1679 RecordFlag::F_LAST as u8,
1680 &mut buffering_start,
1681 &mut buffered,
1682 );
1683
1684 assert!(empty_boundary.is_none());
1685 assert!(buffered_boundary.is_none());
1686 assert_eq!(buffering_start, Some(start));
1687 assert!(buffered.contains_key(&buffered_id));
1688 }
1689
1690 #[rstest]
1691 fn test_boundary_flag_respects_replay_buffering_gate() {
1692 let instrument_id = InstrumentId::from("TEST.GLBX");
1693 let mut buffering_start = Some(UnixNanos::from(10));
1694 let mut buffered = AHashMap::new();
1695 let _ = process_mbo_delta(
1696 stub_delta(instrument_id, 1),
1697 0,
1698 &mut buffering_start,
1699 &mut buffered,
1700 );
1701
1702 let flushed = flush_mbo_event_boundary(
1704 instrument_id,
1705 5.into(),
1706 RecordFlag::F_LAST as u8,
1707 &mut buffering_start,
1708 &mut buffered,
1709 );
1710 assert!(flushed.is_none());
1711 assert!(!buffered.is_empty());
1712
1713 let flushed = flush_mbo_event_boundary(
1715 instrument_id,
1716 11.into(),
1717 RecordFlag::F_LAST as u8,
1718 &mut buffering_start,
1719 &mut buffered,
1720 );
1721 assert!(flushed.is_some());
1722 assert!(buffering_start.is_none());
1723 }
1724
1725 #[rstest]
1726 fn test_boundary_snapshot_flag_never_flushes() {
1727 let instrument_id = InstrumentId::from("TEST.GLBX");
1728 let mut buffering_start = None;
1729 let mut buffered = AHashMap::new();
1730 let _ = process_mbo_delta(
1731 stub_delta(instrument_id, 1),
1732 0,
1733 &mut buffering_start,
1734 &mut buffered,
1735 );
1736
1737 let flags = RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8;
1738 let flushed = flush_mbo_event_boundary(
1739 instrument_id,
1740 2.into(),
1741 flags,
1742 &mut buffering_start,
1743 &mut buffered,
1744 );
1745 assert!(flushed.is_none());
1746 }
1747
1748 fn stub_trade(instrument_id: InstrumentId) -> Data {
1749 use nautilus_model::{
1750 data::TradeTick,
1751 enums::AggressorSide,
1752 identifiers::TradeId,
1753 types::{Price, Quantity},
1754 };
1755
1756 Data::Trade(TradeTick::new(
1757 instrument_id,
1758 Price::from("100.00"),
1759 Quantity::from("5"),
1760 AggressorSide::Buy,
1761 TradeId::new("1"),
1762 1.into(),
1763 1.into(),
1764 ))
1765 }
1766
1767 #[rstest]
1768 fn test_trade_source_is_first_record_type_to_deliver_a_trade() {
1769 let trade = stub_trade(InstrumentId::from("TEST.GLBX"));
1770 let mut trade_sources = AHashMap::new();
1771
1772 let sent: Vec<bool> = [rtype::MBP_1, rtype::MBP_0, rtype::MBO, rtype::MBP_1]
1773 .into_iter()
1774 .map(|rtype| claim_trade_source(&trade, rtype, &mut trade_sources))
1775 .collect();
1776
1777 assert_eq!(sent, [true, false, false, true]);
1778 }
1779
1780 #[rstest]
1781 fn test_trade_source_is_kept_per_instrument() {
1782 let es_trade = stub_trade(InstrumentId::from("ESM4.GLBX"));
1783 let nq_trade = stub_trade(InstrumentId::from("NQM4.GLBX"));
1784 let mut trade_sources = AHashMap::new();
1785
1786 let sent: Vec<bool> = [
1787 (&es_trade, rtype::MBP_0),
1788 (&nq_trade, rtype::MBO),
1789 (&es_trade, rtype::MBO),
1790 (&nq_trade, rtype::MBP_0),
1791 ]
1792 .into_iter()
1793 .map(|(trade, rtype)| claim_trade_source(trade, rtype, &mut trade_sources))
1794 .collect();
1795
1796 assert_eq!(sent, [true, true, false, false]);
1797 }
1798
1799 #[rstest]
1800 fn test_trade_source_passes_data_other_than_trades() {
1801 let instrument_id = InstrumentId::from("TEST.GLBX");
1802 let mut trade_sources = AHashMap::new();
1803 let _ = claim_trade_source(&stub_trade(instrument_id), rtype::MBP_0, &mut trade_sources);
1804 let delta = Data::BookDelta(stub_delta(instrument_id, 1));
1805
1806 let sent = claim_trade_source(&delta, rtype::MBO, &mut trade_sources);
1807
1808 assert!(sent);
1809 }
1810
1811 fn create_test_handler(reconnect_timeout_mins: Option<u64>) -> DatabentoFeedHandler {
1812 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1813 let (msg_tx, _msg_rx) = tokio::sync::mpsc::unbounded_channel();
1814
1815 DatabentoFeedHandler::new(
1816 Credential::new("test_key"),
1817 "GLBX.MDP3".to_string(),
1818 cmd_rx,
1819 msg_tx,
1820 IndexMap::new(),
1821 Arc::new(AtomicMap::new()),
1822 false,
1823 false,
1824 reconnect_timeout_mins,
1825 )
1826 }
1827
1828 fn create_test_client() -> DatabentoLiveClient {
1829 DatabentoLiveClient::new(
1830 "test-api-key".to_string(),
1831 "GLBX.MDP3".to_string(),
1832 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1833 true,
1834 None,
1835 None,
1836 )
1837 .unwrap()
1838 }
1839
1840 #[rstest]
1841 #[case(Some(10))]
1842 #[case(None)]
1843 fn test_backoff_initialization(#[case] reconnect_timeout_mins: Option<u64>) {
1844 let handler = create_test_handler(reconnect_timeout_mins);
1845
1846 assert_eq!(handler.reconnect_timeout_mins, reconnect_timeout_mins);
1847 assert!(handler.subscriptions.is_empty());
1848 assert!(handler.buffered_commands.is_empty());
1849 }
1850
1851 #[rstest]
1852 fn test_subscription_with_and_without_start() {
1853 let start_time = datetime!(2024-01-01 00:00:00 UTC);
1854 let sub_with_start = Subscription::builder()
1855 .symbols("ES.FUT")
1856 .schema(databento::dbn::Schema::Mbp1)
1857 .start(start_time)
1858 .build();
1859
1860 let mut sub_without_start = sub_with_start.clone();
1861 sub_without_start.start = None;
1862
1863 assert!(sub_with_start.start.is_some());
1864 assert!(sub_without_start.start.is_none());
1865 assert_eq!(sub_with_start.schema, sub_without_start.schema);
1866 assert_eq!(sub_with_start.symbols, sub_without_start.symbols);
1867 }
1868
1869 #[rstest]
1870 fn test_handler_initialization_state() {
1871 let handler = create_test_handler(Some(10));
1872
1873 assert!(!handler.replay);
1874 assert_eq!(handler.dataset, "GLBX.MDP3");
1875 assert_eq!(handler.credential.api_key(), "test_key");
1876 assert!(handler.subscriptions.is_empty());
1877 assert!(handler.buffered_commands.is_empty());
1878 }
1879
1880 #[rstest]
1881 fn test_handler_with_no_timeout() {
1882 let handler = create_test_handler(None);
1883
1884 assert_eq!(handler.reconnect_timeout_mins, None);
1885 assert!(!handler.replay);
1886 }
1887
1888 #[rstest]
1889 fn test_handler_with_zero_timeout() {
1890 let handler = create_test_handler(Some(0));
1891
1892 assert_eq!(handler.reconnect_timeout_mins, Some(0));
1893 assert!(!handler.replay);
1894 }
1895
1896 #[rstest]
1897 fn test_subscribe_uses_explicit_parent_stype() {
1898 let mut client = create_test_client();
1899
1900 client
1901 .subscribe(
1902 "definition".to_string(),
1903 vec![InstrumentId::from("ES.FUT.GLBX")],
1904 None,
1905 None,
1906 None,
1907 Some("parent".to_string()),
1908 )
1909 .unwrap();
1910
1911 let command = client.cmd_rx.as_mut().unwrap().try_recv().unwrap();
1912 match command {
1913 HandlerCommand::Subscribe(sub) => {
1914 assert_eq!(sub.schema, dbn::Schema::Definition);
1915 assert_eq!(sub.stype_in, dbn::SType::Parent);
1916 assert_eq!(sub.symbols.to_api_string(), "ES.FUT");
1917 }
1918 other => panic!("expected HandlerCommand::Subscribe, was {other:?}"),
1919 }
1920 }
1921
1922 #[rstest]
1923 fn test_subscribe_rejects_invalid_stype() {
1924 let mut client = create_test_client();
1925
1926 let err = client
1927 .subscribe(
1928 "definition".to_string(),
1929 vec![InstrumentId::from("ES.FUT.GLBX")],
1930 None,
1931 None,
1932 None,
1933 Some("not-a-stype".to_string()),
1934 )
1935 .unwrap_err();
1936
1937 assert!(err.to_string().contains("not-a-stype"));
1938 assert!(!is_command_send_error(&err));
1939 assert!(matches!(
1940 client.cmd_rx.as_mut().unwrap().try_recv(),
1941 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
1942 ));
1943 }
1944
1945 #[rstest]
1946 fn test_subscribe_classifies_command_send_errors() {
1947 let mut client = create_test_client();
1948 client.cmd_rx = None;
1949
1950 let err = client
1951 .subscribe(
1952 "definition".to_string(),
1953 vec![InstrumentId::from("ES.FUT.GLBX")],
1954 None,
1955 None,
1956 None,
1957 Some("parent".to_string()),
1958 )
1959 .unwrap_err();
1960
1961 assert!(is_command_send_error(&err));
1962 }
1963
1964 #[rstest]
1965 fn test_close_after_handler_exit_marks_closed() {
1966 let mut client = create_test_client();
1967 let (handler, _msg_rx) = client.start().unwrap();
1968 drop(handler);
1969
1970 client.close().unwrap();
1971
1972 assert!(!client.is_running());
1973 assert!(client.is_closed());
1974 }
1975
1976 fn test_delta(instrument_id: InstrumentId, ts_event: u64) -> OrderBookDelta {
1977 OrderBookDelta::clear(instrument_id, 0, ts_event.into(), 0.into())
1978 }
1979
1980 #[rstest]
1981 fn test_mbo_delta_without_f_last_buffers() {
1982 let instrument_id = InstrumentId::from("ESM4.GLBX");
1983 let delta = test_delta(instrument_id, 1_000_000_000);
1984 let mut buffering_start = None;
1985 let mut buffered = AHashMap::new();
1986
1987 let result = process_mbo_delta(delta, 0, &mut buffering_start, &mut buffered);
1988
1989 assert!(result.is_none());
1990 assert_eq!(buffered[&instrument_id].len(), 1);
1991 }
1992
1993 #[rstest]
1994 fn test_mbo_single_f_last_emits_without_buffering() {
1995 let instrument_id = InstrumentId::from("ESM4.GLBX");
1996 let ts_event = 1_000_000_000;
1997 let mut delta = test_delta(instrument_id, ts_event);
1998 delta.flags = 128;
1999 delta.sequence = 42;
2000 let mut buffering_start = None;
2001 let mut buffered = AHashMap::new();
2002
2003 let result = process_mbo_delta(delta, delta.flags, &mut buffering_start, &mut buffered);
2004
2005 let emitted = result.expect("single F_LAST delta should emit");
2006 assert_eq!(emitted.instrument_id, instrument_id);
2007 assert_eq!(emitted.deltas.len(), 1);
2008 assert_eq!(emitted.flags, 128);
2009 assert_eq!(emitted.sequence, 42);
2010 assert_eq!(emitted.ts_event, UnixNanos::from(ts_event));
2011 assert_eq!(emitted.deltas[0].instrument_id, instrument_id);
2012 assert_eq!(emitted.deltas[0].flags, 128);
2013 assert_eq!(emitted.deltas[0].sequence, 42);
2014 assert_eq!(emitted.deltas[0].ts_event, UnixNanos::from(ts_event));
2015 assert!(buffering_start.is_none());
2016 assert!(buffered.is_empty());
2017 }
2018
2019 #[rstest]
2020 fn test_mbo_delta_with_f_last_emits() {
2021 let instrument_id = InstrumentId::from("ESM4.GLBX");
2022 let mut buffering_start = None;
2023 let mut buffered = AHashMap::new();
2024
2025 let _ = process_mbo_delta(
2026 test_delta(instrument_id, 1_000_000_000),
2027 0,
2028 &mut buffering_start,
2029 &mut buffered,
2030 );
2031
2032 let result = process_mbo_delta(
2033 test_delta(instrument_id, 2_000_000_000),
2034 128, &mut buffering_start,
2036 &mut buffered,
2037 );
2038
2039 assert!(result.is_some());
2040 assert_eq!(result.unwrap().deltas.len(), 2);
2041 assert!(buffered.is_empty());
2042 }
2043
2044 #[rstest]
2045 fn test_mbo_snapshot_with_f_last_buffers() {
2046 let instrument_id = InstrumentId::from("ESM4.GLBX");
2047 let mut buffering_start = None;
2048 let mut buffered = AHashMap::new();
2049
2050 let result = process_mbo_delta(
2051 test_delta(instrument_id, 1_000_000_000),
2052 128 | 32, &mut buffering_start,
2054 &mut buffered,
2055 );
2056
2057 assert!(result.is_none());
2058 assert_eq!(buffered[&instrument_id].len(), 1);
2059 }
2060
2061 #[rstest]
2062 fn test_mbo_replay_buffers_until_past_start() {
2063 let instrument_id = InstrumentId::from("ESM4.GLBX");
2064 let start_ns = 5_000_000_000u64;
2065 let mut buffering_start = Some(start_ns.into());
2066 let mut buffered = AHashMap::new();
2067
2068 let result = process_mbo_delta(
2069 test_delta(instrument_id, 4_000_000_000),
2070 128, &mut buffering_start,
2072 &mut buffered,
2073 );
2074 assert!(result.is_none());
2075
2076 let result = process_mbo_delta(
2077 test_delta(instrument_id, 5_000_000_000),
2078 128,
2079 &mut buffering_start,
2080 &mut buffered,
2081 );
2082 assert!(result.is_none());
2083
2084 let result = process_mbo_delta(
2086 test_delta(instrument_id, 6_000_000_000),
2087 128,
2088 &mut buffering_start,
2089 &mut buffered,
2090 );
2091 assert!(result.is_some());
2092 assert!(buffering_start.is_none());
2093 }
2094
2095 #[rstest]
2096 fn test_mbo_multiple_deltas_accumulated() {
2097 let instrument_id = InstrumentId::from("ESM4.GLBX");
2098 let mut buffering_start = None;
2099 let mut buffered = AHashMap::new();
2100
2101 for i in 0..5 {
2102 process_mbo_delta(
2103 test_delta(instrument_id, 1_000_000_000 + i),
2104 0,
2105 &mut buffering_start,
2106 &mut buffered,
2107 );
2108 }
2109
2110 let result = process_mbo_delta(
2111 test_delta(instrument_id, 2_000_000_000),
2112 128,
2113 &mut buffering_start,
2114 &mut buffered,
2115 );
2116
2117 assert!(result.is_some());
2118 assert_eq!(result.unwrap().deltas.len(), 6);
2119 }
2120
2121 #[rstest]
2122 fn test_mbo_multi_instrument_isolation() {
2123 let id_a = InstrumentId::from("ESM4.GLBX");
2124 let id_b = InstrumentId::from("NQM4.GLBX");
2125 let mut buffering_start = None;
2126 let mut buffered = AHashMap::new();
2127
2128 process_mbo_delta(
2129 test_delta(id_a, 1_000_000_000),
2130 0,
2131 &mut buffering_start,
2132 &mut buffered,
2133 );
2134 process_mbo_delta(
2135 test_delta(id_b, 1_000_000_000),
2136 0,
2137 &mut buffering_start,
2138 &mut buffered,
2139 );
2140
2141 let result = process_mbo_delta(
2143 test_delta(id_a, 2_000_000_000),
2144 128,
2145 &mut buffering_start,
2146 &mut buffered,
2147 );
2148
2149 assert!(result.is_some());
2150 assert_eq!(result.unwrap().instrument_id, id_a);
2151 assert!(buffered.contains_key(&id_b));
2152 assert!(!buffered.contains_key(&id_a));
2153 }
2154
2155 mod property_tests {
2156 use proptest::prelude::*;
2157 use rstest::rstest;
2158
2159 use super::*;
2160
2161 proptest! {
2162 #[rstest]
2163 fn mbo_buffering_conserves_deltas(
2164 num_non_last in 0usize..=20,
2165 ) {
2166 let instrument_id = InstrumentId::from("ESM4.GLBX");
2167 let mut buffering_start = None;
2168 let mut buffered = AHashMap::new();
2169 let total = num_non_last + 1;
2170
2171 for i in 0..num_non_last {
2172 let result = process_mbo_delta(
2173 test_delta(instrument_id, 1_000_000_000 + i as u64),
2174 0, &mut buffering_start,
2176 &mut buffered,
2177 );
2178 prop_assert!(result.is_none());
2179 }
2180
2181 let result = process_mbo_delta(
2182 test_delta(instrument_id, 2_000_000_000),
2183 128, &mut buffering_start,
2185 &mut buffered,
2186 );
2187
2188 prop_assert!(result.is_some());
2189 let emitted = result.unwrap();
2190 prop_assert_eq!(emitted.deltas.len(), total);
2191 prop_assert!(buffered.is_empty());
2192 }
2193
2194 #[rstest]
2195 fn mbo_snapshots_never_emit(
2196 num_snapshots in 1usize..=20,
2197 ) {
2198 let instrument_id = InstrumentId::from("ESM4.GLBX");
2199 let mut buffering_start = None;
2200 let mut buffered = AHashMap::new();
2201
2202 for i in 0..num_snapshots {
2203 let result = process_mbo_delta(
2204 test_delta(instrument_id, 1_000_000_000 + i as u64),
2205 128 | 32, &mut buffering_start,
2207 &mut buffered,
2208 );
2209 prop_assert!(result.is_none());
2210 }
2211
2212 prop_assert_eq!(buffered[&instrument_id].len(), num_snapshots);
2213 }
2214
2215 #[rstest]
2216 fn mbo_replay_delays_emission(
2217 start_offset in 1u64..=100,
2218 num_before in 1usize..=10,
2219 ) {
2220 let instrument_id = InstrumentId::from("ESM4.GLBX");
2221 let start_ns = 1_000_000_000u64 * start_offset;
2222 let mut buffering_start = Some(start_ns.into());
2223 let mut buffered = AHashMap::new();
2224
2225 for i in 0..num_before {
2226 let ts = start_ns - (num_before as u64 - i as u64);
2227 let result = process_mbo_delta(
2228 test_delta(instrument_id, ts),
2229 128, &mut buffering_start,
2231 &mut buffered,
2232 );
2233 prop_assert!(result.is_none());
2234 }
2235
2236 let result = process_mbo_delta(
2237 test_delta(instrument_id, start_ns + 1),
2238 128,
2239 &mut buffering_start,
2240 &mut buffered,
2241 );
2242
2243 prop_assert!(result.is_some());
2244 prop_assert!(buffering_start.is_none());
2245 let total = num_before + 1;
2246 prop_assert_eq!(result.unwrap().deltas.len(), total);
2247 }
2248 }
2249 }
2250
2251 #[rstest]
2252 #[case(2, "CLZ29-CLF30.XNYM")]
2253 #[case(1, "CLZ9-CLF0.XNYM")]
2254 fn test_live_subscription_preserves_native_symbols(
2255 #[case] digits: u8,
2256 #[case] id: InstrumentId,
2257 ) {
2258 let mut client = create_test_client().with_futures_year_digits(digits.try_into().unwrap());
2259 client
2260 .subscribe(
2261 "mbp-1".to_string(),
2262 vec![id],
2263 None,
2264 None,
2265 Some(vec![Some(3)]),
2266 None,
2267 )
2268 .unwrap();
2269 let rx = client.cmd_rx.as_mut().unwrap();
2270 assert!(
2271 matches!(rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 3) if symbol == Symbol::from("CLZ9-CLF0"))
2272 );
2273
2274 let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
2275 panic!("expected subscription");
2276 };
2277
2278 let expected = Subscription::builder()
2279 .schema(dbn::Schema::Mbp1)
2280 .symbols(vec!["CLZ9-CLF0".to_string()])
2281 .build();
2282 assert_eq!(subscription, expected);
2283 assert_eq!(
2284 client.symbol_venue_map.get_cloned(&id.symbol),
2285 Some(Venue::from("XNYM"))
2286 );
2287 assert_eq!(
2288 rx.try_recv().unwrap_err(),
2289 tokio::sync::mpsc::error::TryRecvError::Empty
2290 );
2291 }
2292
2293 #[rstest]
2294 fn test_futures_cache_uses_requested_venue_after_year_conversion() {
2295 let mut handler = create_test_handler(None);
2296 handler.publisher_venue_map.insert(1, Venue::from("GLBX"));
2297 handler
2298 .symbol_venue_map
2299 .insert(Symbol::from("CLF26"), Venue::from("XNYM"));
2300 let mut mapping = dbn::PitSymbolMap::new();
2301 mapping.inner_mut().insert(42, "CLF6".to_string());
2302 let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
2303 msg.hd.publisher_id = 1;
2304 msg.hd.instrument_id = 42;
2305 msg.ts_recv = 1_766_102_400_000_000_000;
2306 let mut ids = AHashMap::new();
2307 handler
2308 .cache_futures_instrument_id(&dbn::RecordRef::from(&msg), &mapping, &mut ids)
2309 .unwrap();
2310 assert_eq!(
2311 ids.get(&42).copied().map(CachedInstrumentId::id),
2312 Some(InstrumentId::from("CLF26.XNYM"))
2313 );
2314 }
2315
2316 #[rstest]
2317 #[case(Some(2), Some(5), Some(4), 5)]
2318 #[case(Some(2), None, Some(5), 5)]
2319 #[case(Some(5), Some(2), None, 2)]
2320 #[case(Some(5), None, Some(2), 2)]
2321 #[case(Some(3), None, None, 3)]
2322 #[case(None, None, None, 2)]
2323 fn test_explicit_precision_precedes_definitions(
2324 #[case] definition: Option<u8>,
2325 #[case] mapped: Option<u8>,
2326 #[case] native: Option<u8>,
2327 #[case] expected: u8,
2328 ) {
2329 let definitions = AHashMap::from_iter(definition.map(|precision| (42, precision)));
2330 let subscriptions = AHashMap::from_iter(mapped.map(|precision| (42, precision)));
2331 let overrides =
2332 AHashMap::from_iter(native.map(|precision| (Symbol::from("CLF6"), precision)));
2333
2334 let actual =
2335 resolve_price_precision(42, Some("CLF6"), &definitions, &subscriptions, &overrides);
2336
2337 assert_eq!(actual, expected);
2338 }
2339
2340 #[rstest]
2341 fn test_futures_precision_override_uses_native_symbol() {
2342 let mut handler = create_test_handler(None);
2343 handler.publisher_venue_map.insert(1, Venue::from("GLBX"));
2344 let mut mapping = dbn::PitSymbolMap::new();
2345 mapping.inner_mut().insert(42, "CLF6".to_string());
2346 let mut msg = dbn::TradeMsg::default();
2347 msg.hd.publisher_id = 1;
2348 msg.hd.instrument_id = 42;
2349 msg.hd.ts_event = 1_766_102_400_000_000_000;
2350 msg.ts_recv = msg.hd.ts_event;
2351 msg.price = 75_125_000_000;
2352 msg.size = 7;
2353 msg.side = b'A' as std::ffi::c_char;
2354 let record = dbn::RecordRef::from(&msg);
2355 let mut ids = AHashMap::new();
2356 handler
2357 .cache_futures_instrument_id(&record, &mapping, &mut ids)
2358 .unwrap();
2359 let overrides = AHashMap::from_iter([(Symbol::from("CLF6"), 3)]);
2360 let (data, extra) = handle_record(
2361 record,
2362 &mapping,
2363 &handler.publisher_venue_map,
2364 &handler.symbol_venue_map,
2365 &mut ids,
2366 &AHashMap::new(),
2367 &AHashMap::new(),
2368 &overrides,
2369 1_766_102_400_000_000_001.into(),
2370 &HashSet::new(),
2371 true,
2372 )
2373 .unwrap();
2374
2375 let Some(Data::Trade(trade)) = data else {
2376 panic!("expected trade");
2377 };
2378
2379 assert_eq!(trade.instrument_id, InstrumentId::from("CLF26.GLBX"));
2380 assert_eq!(trade.price.precision, 3);
2381 assert_eq!(trade.price, Price::from("75.125"));
2382 assert_eq!(trade.size, Quantity::from(7));
2383 assert_eq!(trade.ts_event, msg.hd.ts_event);
2384 assert_eq!(trade.ts_init, 1_766_102_400_000_000_001);
2385 assert!(extra.is_none());
2386 }
2387}