1use std::{
32 fmt::{Debug, Display},
33 path::PathBuf,
34 str::FromStr,
35 sync::Arc,
36 time::Duration,
37};
38
39use ahash::{AHashMap, HashSet, HashSetExt};
40use databento::{
41 dbn::{self, PitSymbolMap, SymbolIndex},
42 live::Subscription,
43};
44use indexmap::IndexMap;
45use nautilus_core::{
46 AtomicMap, UnixNanos, consts::NAUTILUS_USER_AGENT, string::secret::SecretString,
47 time::get_atomic_clock_realtime,
48};
49use nautilus_model::{
50 data::{Data, InstrumentStatus, OrderBookDelta, OrderBookDeltas},
51 enums::RecordFlag,
52 identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
53 instruments::{Instrument, InstrumentAny},
54 types::Currency,
55};
56use nautilus_network::backoff::ExponentialBackoff;
57use time::OffsetDateTime;
58
59use super::{
60 decode::{
61 decode_imbalance_msg, decode_statistics_msg, decode_status_msg, is_supported_stat_type,
62 },
63 types::{DatabentoImbalance, DatabentoStatistics, SubscriptionAckEvent},
64};
65use crate::{
66 common::{Credential, build_publisher_venue_map, load_publishers},
67 decode::{decode_instrument_def_msg, decode_record},
68 symbology::{
69 check_consistent_symbology, dataset_uses_futures_symbology, infer_symbology_type,
70 nautilus_symbol, raw_symbol, resolve_futures_instrument_id,
71 },
72 types::PublisherId,
73};
74
75#[derive(Debug)]
76pub enum HandlerCommand {
77 Subscribe(Subscription),
78 SetPricePrecision(Symbol, u8),
79 Start,
80 Close,
81}
82
83#[derive(Debug)]
84pub enum DatabentoMessage {
85 Data(Data),
86 Instrument(Box<InstrumentAny>),
87 Status(InstrumentStatus),
88 Imbalance(DatabentoImbalance),
89 Statistics(DatabentoStatistics),
90 SubscriptionAck(SubscriptionAckEvent),
91 Error(anyhow::Error),
92 Close,
93}
94
95#[cfg_attr(
96 feature = "python",
97 pyo3::pyclass(module = "nautilus_trader.adapters.databento")
98)]
99#[cfg_attr(
100 feature = "python",
101 pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
102)]
103pub struct DatabentoLiveClient {
104 futures_year_digits: FuturesYearDigits,
105 share_class_space: bool,
106 credential: Credential,
107 pub dataset: String,
108 is_running: bool,
109 is_closed: bool,
110 cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
111 cmd_rx: Option<tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>>,
112 publisher_venue_map: IndexMap<PublisherId, Venue>,
113 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
114 use_exchange_as_venue: bool,
115 bars_timestamp_on_close: bool,
116 reconnect_timeout_mins: Option<u64>,
117}
118
119impl Debug for DatabentoLiveClient {
120 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
121 f.debug_struct(stringify!(DatabentoLiveClient))
122 .field("credential", &self.credential)
123 .field("dataset", &self.dataset)
124 .field("is_running", &self.is_running)
125 .field("is_closed", &self.is_closed)
126 .finish()
127 }
128}
129
130impl DatabentoLiveClient {
131 pub fn new(
137 key: String,
138 dataset: String,
139 publishers_filepath: PathBuf,
140 use_exchange_as_venue: bool,
141 bars_timestamp_on_close: Option<bool>,
142 reconnect_timeout_mins: Option<i64>,
143 ) -> anyhow::Result<Self> {
144 let publishers = load_publishers(publishers_filepath)?;
145 let publisher_venue_map = build_publisher_venue_map(&publishers);
146
147 let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
148
149 let reconnect_timeout_mins = reconnect_timeout_mins
150 .and_then(|mins| if mins >= 0 { Some(mins as u64) } else { None });
151
152 Ok(Self {
153 futures_year_digits: FuturesYearDigits::default(),
154 share_class_space: true,
155 credential: Credential::new(key),
156 dataset,
157 cmd_tx,
158 cmd_rx: Some(cmd_rx),
159 is_running: false,
160 is_closed: false,
161 publisher_venue_map,
162 symbol_venue_map: Arc::new(AtomicMap::new()),
163 use_exchange_as_venue,
164 bars_timestamp_on_close: bars_timestamp_on_close.unwrap_or(true),
165 reconnect_timeout_mins,
166 })
167 }
168
169 #[must_use]
171 pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
172 self.futures_year_digits = futures_year_digits;
173 self
174 }
175
176 #[must_use]
178 pub fn with_share_class_space(mut self, share_class_space: bool) -> Self {
179 self.share_class_space = share_class_space;
180 self
181 }
182
183 #[must_use]
184 pub const fn is_running(&self) -> bool {
185 self.is_running
186 }
187
188 #[must_use]
189 pub const fn is_closed(&self) -> bool {
190 self.is_closed
191 }
192
193 #[expect(clippy::needless_pass_by_value)]
200 pub fn subscribe(
201 &mut self,
202 schema: String,
203 instrument_ids: Vec<InstrumentId>,
204 start: Option<u64>,
205 snapshot: Option<bool>,
206 price_precisions: Option<Vec<Option<u8>>>,
207 stype_in: Option<String>,
208 ) -> anyhow::Result<()> {
209 if let Some(precisions) = &price_precisions
210 && precisions.len() != instrument_ids.len()
211 {
212 anyhow::bail!(
213 "`price_precisions` length ({}) must match `instrument_ids` length ({})",
214 precisions.len(),
215 instrument_ids.len()
216 );
217 }
218
219 let symbols: Vec<String> = instrument_ids
220 .iter()
221 .map(|id| {
222 raw_symbol(
223 id.symbol.as_str(),
224 &self.dataset,
225 self.futures_year_digits,
226 self.share_class_space,
227 )
228 })
229 .collect();
230 let first_symbol = symbols
231 .first()
232 .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
233 let stype_in = match stype_in {
234 Some(stype_in) => dbn::SType::from_str(&stype_in)?,
235 None => infer_symbology_type(first_symbol),
236 };
237 let symbols: Vec<&str> = symbols.iter().map(String::as_str).collect();
238 check_consistent_symbology(symbols.as_slice())?;
239 let mut sub = Subscription::builder()
240 .symbols(symbols)
241 .schema(dbn::Schema::from_str(&schema)?)
242 .stype_in(stype_in)
243 .build();
244
245 if let Some(start) = start {
246 sub.start = Some(OffsetDateTime::from_unix_timestamp_nanos(i128::from(
247 start,
248 ))?);
249 }
250 sub.use_snapshot = snapshot.unwrap_or(false);
251
252 self.symbol_venue_map.rcu(|m| {
253 for id in &instrument_ids {
254 m.entry(id.symbol).or_insert(id.venue);
255 }
256 });
257
258 if let Some(precisions) = price_precisions {
259 for (instrument_id, precision) in instrument_ids.iter().zip(precisions) {
260 if let Some(precision) = precision {
261 let raw = raw_symbol(
262 instrument_id.symbol.as_str(),
263 &self.dataset,
264 self.futures_year_digits,
265 self.share_class_space,
266 );
267 self.send_command(HandlerCommand::SetPricePrecision(
268 Symbol::from(raw.as_str()),
269 precision,
270 ))?;
271 }
272 }
273 }
274
275 self.send_command(HandlerCommand::Subscribe(sub))
276 }
277
278 pub fn start(
284 &mut self,
285 ) -> anyhow::Result<(
286 DatabentoFeedHandler,
287 tokio::sync::mpsc::UnboundedReceiver<DatabentoMessage>,
288 )> {
289 if self.is_closed {
290 anyhow::bail!("Client already closed");
291 }
292
293 if self.is_running {
294 anyhow::bail!("Client already running");
295 }
296
297 log::debug!("Starting client");
298
299 let (msg_tx, msg_rx) = tokio::sync::mpsc::unbounded_channel::<DatabentoMessage>();
300 let cmd_rx = self
301 .cmd_rx
302 .take()
303 .ok_or_else(|| anyhow::anyhow!("Command receiver already taken"))?;
304
305 let feed_handler = DatabentoFeedHandler::new(
306 self.credential.clone(),
307 self.dataset.clone(),
308 cmd_rx,
309 msg_tx,
310 self.publisher_venue_map.clone(),
311 self.symbol_venue_map.clone(),
312 self.use_exchange_as_venue,
313 self.bars_timestamp_on_close,
314 self.reconnect_timeout_mins,
315 )
316 .with_futures_year_digits(self.futures_year_digits)
317 .with_share_class_space(self.share_class_space);
318
319 self.send_command(HandlerCommand::Start)?;
320 self.is_running = true;
321
322 Ok((feed_handler, msg_rx))
323 }
324
325 pub fn close(&mut self) -> anyhow::Result<()> {
332 if !self.is_running {
333 anyhow::bail!("Client never started");
334 }
335
336 if self.is_closed {
337 anyhow::bail!("Client already closed");
338 }
339
340 log::debug!("Closing client");
341
342 if !self.cmd_tx.is_closed() {
343 self.send_command(HandlerCommand::Close)?;
344 }
345
346 self.is_running = false;
347 self.is_closed = true;
348
349 Ok(())
350 }
351
352 fn send_command(&self, cmd: HandlerCommand) -> anyhow::Result<()> {
353 self.cmd_tx.send(cmd).map_err(|e| {
354 anyhow::Error::new(CommandSendError {
355 details: e.to_string(),
356 })
357 })
358 }
359}
360
361#[cfg(any(test, feature = "python"))]
362pub(crate) fn is_command_send_error(error: &anyhow::Error) -> bool {
363 error.is::<CommandSendError>()
364}
365
366#[derive(Debug)]
367struct CommandSendError {
368 details: String,
369}
370
371impl Display for CommandSendError {
372 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
373 write!(
374 f,
375 "Failed to send command to Databento feed handler: {}",
376 self.details
377 )
378 }
379}
380
381impl std::error::Error for CommandSendError {}
382
383pub struct DatabentoFeedHandler {
395 futures_year_digits: FuturesYearDigits,
396 share_class_space: bool,
397 credential: Credential,
398 dataset: String,
399 cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
400 msg_tx: tokio::sync::mpsc::UnboundedSender<DatabentoMessage>,
401 publisher_venue_map: IndexMap<PublisherId, Venue>,
402 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
403 replay: bool,
404 use_exchange_as_venue: bool,
405 bars_timestamp_on_close: bool,
406 reconnect_timeout_mins: Option<u64>,
407 backoff: ExponentialBackoff,
408 subscriptions: Vec<Subscription>,
409 buffered_commands: Vec<HandlerCommand>,
410 price_precision_overrides: AHashMap<Symbol, u8>,
411 gateway_addr: Option<String>,
412 success_threshold: Duration,
413}
414
415impl Debug for DatabentoFeedHandler {
416 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
417 f.debug_struct(stringify!(DatabentoFeedHandler))
418 .field("credential", &self.credential)
419 .field("dataset", &self.dataset)
420 .field("replay", &self.replay)
421 .field("reconnect_timeout_mins", &self.reconnect_timeout_mins)
422 .field("subscriptions", &self.subscriptions.len())
423 .finish()
424 }
425}
426
427impl DatabentoFeedHandler {
428 #[must_use]
430 pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
431 self.futures_year_digits = futures_year_digits;
432 self
433 }
434
435 #[must_use]
437 pub fn with_share_class_space(mut self, share_class_space: bool) -> Self {
438 self.share_class_space = share_class_space;
439 self
440 }
441
442 #[must_use]
448 #[expect(clippy::too_many_arguments)]
449 pub fn new(
450 credential: Credential,
451 dataset: String,
452 rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
453 tx: tokio::sync::mpsc::UnboundedSender<DatabentoMessage>,
454 publisher_venue_map: IndexMap<PublisherId, Venue>,
455 symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
456 use_exchange_as_venue: bool,
457 bars_timestamp_on_close: bool,
458 reconnect_timeout_mins: Option<u64>,
459 ) -> Self {
460 let delay_max = if reconnect_timeout_mins.is_some() {
464 Duration::from_secs(60)
465 } else {
466 Duration::from_secs(600)
467 };
468
469 let backoff = ExponentialBackoff::new(Duration::from_secs(1), delay_max, 2.0, 1000, false)
470 .expect("hardcoded backoff parameters are valid");
471
472 Self {
473 futures_year_digits: FuturesYearDigits::default(),
474 share_class_space: true,
475 credential,
476 dataset,
477 cmd_rx: rx,
478 msg_tx: tx,
479 publisher_venue_map,
480 symbol_venue_map,
481 replay: false,
482 use_exchange_as_venue,
483 bars_timestamp_on_close,
484 reconnect_timeout_mins,
485 backoff,
486 subscriptions: Vec::new(),
487 buffered_commands: Vec::new(),
488 price_precision_overrides: AHashMap::new(),
489 gateway_addr: None,
490 success_threshold: Duration::from_secs(60),
491 }
492 }
493
494 #[must_use]
496 pub fn with_gateway_addr(mut self, addr: String) -> Self {
497 self.gateway_addr = Some(addr);
498 self
499 }
500
501 #[must_use]
506 pub fn with_success_threshold(mut self, threshold: Duration) -> Self {
507 self.success_threshold = threshold;
508 self
509 }
510
511 pub async fn run(&mut self) -> anyhow::Result<()> {
524 log::debug!("Running feed handler");
525
526 let mut reconnect_start: Option<tokio::time::Instant> = None;
527 let mut attempt = 0;
528
529 loop {
530 attempt += 1;
531
532 match self.run_session(attempt).await {
533 Ok(ran_successfully) => {
534 if ran_successfully {
535 log::info!("Resetting reconnection cycle after successful session");
536 reconnect_start = None;
537 attempt = 0;
538 self.backoff.reset();
539 } else {
540 log::debug!("Session ended normally");
541 break Ok(());
542 }
543 }
544 Err(e) => {
545 let cycle_start = reconnect_start.get_or_insert_with(tokio::time::Instant::now);
546
547 if let Some(timeout_mins) = self.reconnect_timeout_mins {
548 let elapsed = cycle_start.elapsed();
549 let timeout = Duration::from_mins(timeout_mins);
550
551 if elapsed >= timeout {
552 log::error!("Giving up reconnection after {timeout_mins} minutes");
553 self.send_msg(DatabentoMessage::Error(anyhow::anyhow!(
554 "Reconnection timeout after {timeout_mins} minutes: {e}"
555 )));
556 break Err(e);
557 }
558 }
559
560 let delay = self.backoff.next_duration();
561
562 log::warn!(
563 "Connection lost (attempt {}): {}. Reconnecting in {}s...",
564 attempt,
565 e,
566 delay.as_secs()
567 );
568
569 let sleep = tokio::time::sleep(delay);
570 tokio::pin!(sleep);
571
572 loop {
573 tokio::select! {
574 () = &mut sleep => break,
575 cmd = self.cmd_rx.recv() => {
576 match cmd {
577 Some(HandlerCommand::Close) => {
578 log::debug!("Close received during backoff");
579 return Ok(());
580 }
581 None => {
582 log::debug!("Command channel closed during backoff");
583 return Ok(());
584 }
585 Some(cmd) => {
586 log::debug!("Buffering command received during backoff: {cmd:?}");
587 self.buffered_commands.push(cmd);
588 }
589 }
590 }
591 }
592 }
593 }
594 }
595 }
596 }
597
598 async fn run_session(&mut self, attempt: usize) -> anyhow::Result<bool> {
607 if attempt > 1 {
608 log::info!("Reconnecting (attempt {attempt})...");
609 }
610
611 let session_start = tokio::time::Instant::now();
612 let clock = get_atomic_clock_realtime();
613 let mut symbol_map = PitSymbolMap::new();
614 let mut instrument_id_map: AHashMap<u32, CachedInstrumentId> = AHashMap::new();
615 let mut instrument_def_price_precision_map: AHashMap<u32, u8> = AHashMap::new();
616 let mut subscription_price_precision_map: AHashMap<u32, u8> = AHashMap::new();
617
618 let mut buffering_start = None;
619 let mut buffered_deltas: AHashMap<InstrumentId, Vec<OrderBookDelta>> = AHashMap::new();
620 let mut initialized_books = HashSet::new();
621 let mut trade_sources: AHashMap<InstrumentId, u8> = AHashMap::new();
622 let timeout = Duration::from_secs(5); let gateway_addr = self.gateway_addr.clone();
625 let api_key = SecretString::from(self.credential.api_key().to_owned());
626 let dataset = self.dataset.clone();
627
628 let result = tokio::time::timeout(timeout, async move {
629 let base = databento::LiveClient::builder();
630 let base = if let Some(addr) = gateway_addr {
631 base.addr(addr).await?
632 } else {
633 base
634 };
635 base.user_agent_extension(NAUTILUS_USER_AGENT.into())
636 .key(api_key.expose_secret().to_owned())?
637 .dataset(dataset)
638 .build()
639 .await
640 })
641 .await?;
642
643 let mut client = match result {
644 Ok(client) => {
645 if attempt > 1 {
646 log::info!("Reconnected successfully");
647 } else {
648 log::info!("Connected");
649 }
650 client
651 }
652 Err(e) => {
653 anyhow::bail!("Failed to connect to Databento LSG: {e}");
654 }
655 };
656
657 let mut start_buffered = false;
659
660 if !self.buffered_commands.is_empty() {
661 log::debug!(
662 "Processing {} buffered commands",
663 self.buffered_commands.len()
664 );
665
666 for cmd in self.buffered_commands.drain(..) {
667 match cmd {
668 HandlerCommand::Subscribe(sub) => {
669 if !self.replay && sub.start.is_some() {
670 self.replay = true;
671 }
672 self.subscriptions.push(sub);
673 }
674 HandlerCommand::SetPricePrecision(symbol, precision) => {
675 self.price_precision_overrides.insert(symbol, precision);
676 }
677 HandlerCommand::Start => {
678 start_buffered = true;
679 }
680 HandlerCommand::Close => {
681 log::warn!("Close command was buffered, shutting down");
682 return Ok(false);
683 }
684 }
685 }
686 }
687
688 let mut running = false;
689
690 if !self.subscriptions.is_empty() {
691 log::info!(
692 "Resubscribing to {} subscriptions",
693 self.subscriptions.len()
694 );
695
696 let replay = self.subscriptions.iter().any(|sub| sub.start.is_some());
698
699 for sub in self.subscriptions.clone() {
700 client.subscribe(sub).await?;
701 }
702 for sub in &mut self.subscriptions {
704 sub.start = None;
705 }
706
707 buffering_start = replay.then(|| clock.get_time_ns());
708 client.start().await?;
709 running = true;
710 log::info!("Resubscription complete");
711 } else if start_buffered {
712 log::debug!("Starting session from buffered Start command");
713 buffering_start = if self.replay {
714 Some(clock.get_time_ns())
715 } else {
716 None
717 };
718 client.start().await?;
719 running = true;
720 }
721
722 loop {
723 if self.msg_tx.is_closed() {
724 log::debug!("Message channel was closed: stopping");
725 return Ok(false);
726 }
727
728 if !running {
733 match self.cmd_rx.recv().await {
734 Some(HandlerCommand::Subscribe(sub)) => {
735 log::debug!("Received command: Subscribe");
736
737 if !self.replay && sub.start.is_some() {
738 self.replay = true;
739 }
740 client.subscribe(sub.clone()).await?;
741 let mut sub_for_reconnect = sub;
742 sub_for_reconnect.start = None;
743 self.subscriptions.push(sub_for_reconnect);
744 continue;
745 }
746 Some(HandlerCommand::SetPricePrecision(symbol, precision)) => {
747 log::debug!(
748 "Received command: SetPricePrecision for {symbol} to {precision}"
749 );
750 self.price_precision_overrides.insert(symbol, precision);
751 continue;
752 }
753 Some(HandlerCommand::Start) => {
754 log::debug!("Received command: Start");
755 buffering_start = if self.replay {
756 Some(clock.get_time_ns())
757 } else {
758 None
759 };
760 client.start().await?;
761 running = true;
762 continue;
763 }
764 Some(HandlerCommand::Close) => {
765 self.send_close_msg();
766 return Ok(false);
767 }
768 None => {
769 log::debug!("Command channel disconnected");
770 return Ok(false);
771 }
772 }
773 }
774
775 let record_opt = tokio::select! {
776 cmd = self.cmd_rx.recv() =>
777 match cmd {
778 Some(HandlerCommand::Subscribe(sub)) => {
779 log::debug!("Received command: Subscribe");
780
781 if sub.start.is_some() {
782 self.replay = true;
783 log::error!(
784 "Ignoring `start` on {} subscribe, session already running, Databento drops replay anchors sent after session start",
785 self.dataset,
786 );
787 }
788 client.subscribe(sub.clone()).await?;
789 let mut sub_for_reconnect = sub;
790 sub_for_reconnect.start = None;
791 self.subscriptions.push(sub_for_reconnect);
792 continue;
793 }
794 Some(HandlerCommand::SetPricePrecision(symbol, precision)) => {
795 log::debug!(
796 "Received command: SetPricePrecision for {symbol} to {precision}"
797 );
798 self.price_precision_overrides.insert(symbol, precision);
799 continue;
800 }
801 Some(HandlerCommand::Start) => {
802 log::warn!("Received Start command but session already running");
803 continue;
804 }
805 Some(HandlerCommand::Close) => {
806 self.send_close_msg();
807 client.close().await?;
808 log::debug!("Closed inner client");
809 return Ok(false);
810 }
811 None => {
812 log::debug!("Command channel disconnected");
813 return Ok(false);
814 }
815 },
816 result = client.next_record() => result,
817 };
818
819 let record = match record_opt {
820 Ok(Some(record)) => record,
821 Ok(None) => {
822 if session_start.elapsed() >= self.success_threshold {
823 log::debug!("Session ended after successful run");
824 return Ok(true);
825 }
826 anyhow::bail!("Session ended by gateway");
827 }
828 Err(e) => {
829 if session_start.elapsed() >= self.success_threshold {
830 log::debug!("Connection error after successful run: {e}");
831 return Ok(true);
832 }
833 anyhow::bail!("Connection error: {e}");
834 }
835 };
836
837 let ts_init = clock.get_time_ns();
838
839 if let Err(e) =
840 self.cache_futures_instrument_id(&record, &symbol_map, &mut instrument_id_map)
841 {
842 log::error!("Error resolving futures instrument ID: {e}");
843 continue;
844 }
845
846 if let Some(msg) = record.get::<dbn::ErrorMsg>() {
848 handle_error_msg(msg);
849 } else if let Some(msg) = record.get::<dbn::SystemMsg>() {
850 if let Some(ack) = handle_system_msg(msg, ts_init) {
851 self.send_msg(DatabentoMessage::SubscriptionAck(ack));
852 }
853 } else if let Some(msg) = record.get::<dbn::SymbolMappingMsg>() {
854 instrument_def_price_precision_map.remove(&msg.hd.instrument_id);
855 update_price_precision_map_with_symbol_mapping_msg(
856 msg,
857 &self.price_precision_overrides,
858 &mut subscription_price_precision_map,
859 )?;
860 handle_symbol_mapping_msg(
861 msg,
862 &mut symbol_map,
863 &self.symbol_venue_map,
864 &mut instrument_id_map,
865 )?;
866 } else if let Some(msg) = record.get::<dbn::InstrumentDefMsg>() {
867 let maybe_data = self.handle_instrument_def_msg(
868 msg,
869 &record,
870 &symbol_map,
871 &mut instrument_id_map,
872 ts_init,
873 )?;
874
875 if let Some(data) = maybe_data {
876 instrument_def_price_precision_map
877 .insert(msg.hd.instrument_id, data.price_precision());
878 self.send_msg(DatabentoMessage::Instrument(Box::new(data)));
879 }
880 } else if let Some(msg) = record.get::<dbn::StatusMsg>() {
881 let data = handle_status_msg(
882 msg,
883 &record,
884 &symbol_map,
885 &self.publisher_venue_map,
886 &self.symbol_venue_map,
887 &mut instrument_id_map,
888 ts_init,
889 self.share_class_space,
890 )?;
891 self.send_msg(DatabentoMessage::Status(data));
892 } else if let Some(msg) = record.get::<dbn::ImbalanceMsg>() {
893 handle_imbalance_msg(
894 msg,
895 &record,
896 &symbol_map,
897 &self.publisher_venue_map,
898 &self.symbol_venue_map,
899 &mut instrument_id_map,
900 &instrument_def_price_precision_map,
901 &subscription_price_precision_map,
902 &self.price_precision_overrides,
903 ts_init,
904 self.share_class_space,
905 )
906 .map_or_else(
907 |e| log::error!("Error decoding imbalance: {e}"),
908 |data| self.send_msg(DatabentoMessage::Imbalance(data)),
909 );
910 } else if let Some(msg) = record.get::<dbn::StatMsg>() {
911 let maybe_data = handle_statistics_msg(
912 msg,
913 &record,
914 &symbol_map,
915 &self.publisher_venue_map,
916 &self.symbol_venue_map,
917 &mut instrument_id_map,
918 &instrument_def_price_precision_map,
919 &subscription_price_precision_map,
920 &self.price_precision_overrides,
921 ts_init,
922 self.share_class_space,
923 )?;
924
925 if let Some(data) = maybe_data {
926 self.send_msg(DatabentoMessage::Statistics(data));
927 }
928 } else {
929 let res = handle_record(
931 record,
932 &symbol_map,
933 &self.publisher_venue_map,
934 &self.symbol_venue_map,
935 &mut instrument_id_map,
936 &instrument_def_price_precision_map,
937 &subscription_price_precision_map,
938 &self.price_precision_overrides,
939 ts_init,
940 &initialized_books,
941 self.bars_timestamp_on_close,
942 self.share_class_space,
943 );
944 let (mut data1, data2) = match res {
945 Ok(decoded) => decoded,
946 Err(e) => {
947 log::error!("Error decoding record: {e}");
948
949 (None, None)
951 }
952 };
953
954 if let Some(msg) = record.get::<dbn::MboMsg>() {
955 if let Some(Data::BookDelta(delta)) = &data1 {
956 initialized_books.insert(delta.instrument_id);
957
958 log::trace!(
959 "Buffering delta: {} {buffering_start:?} flags={}",
960 delta.ts_event,
961 msg.flags.raw(),
962 );
963
964 match process_mbo_delta(
965 *delta,
966 msg.flags.raw(),
967 &mut buffering_start,
968 &mut buffered_deltas,
969 ) {
970 Some(deltas) => data1 = Some(Data::BookDeltas(Box::new(deltas))),
971 None => continue,
972 }
973 } else {
974 let deltas = instrument_id_map.get(&msg.hd.instrument_id).and_then(|id| {
981 flush_mbo_event_boundary(
982 id.id(),
983 msg.ts_recv.into(),
984 msg.flags.raw(),
985 &mut buffering_start,
986 &mut buffered_deltas,
987 )
988 });
989
990 if let Some(deltas) = deltas {
991 self.send_msg(DatabentoMessage::Data(Data::BookDeltas(Box::new(
992 deltas,
993 ))));
994 }
995
996 if buffering_start.is_some_and(|start| msg.ts_recv <= start) {
998 continue;
999 }
1000 }
1001 }
1002
1003 let rtype = record.header().rtype;
1004
1005 for data in [data1, data2].into_iter().flatten() {
1006 if claim_trade_source(&data, rtype, &mut trade_sources) {
1007 self.send_msg(DatabentoMessage::Data(data));
1008 }
1009 }
1010 }
1011 }
1012 }
1013
1014 fn handle_instrument_def_msg(
1020 &self,
1021 msg: &dbn::InstrumentDefMsg,
1022 record: &dbn::RecordRef,
1023 symbol_map: &PitSymbolMap,
1024 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1025 ts_init: UnixNanos,
1026 ) -> anyhow::Result<Option<InstrumentAny>> {
1027 instrument_id_map.remove(&msg.hd.instrument_id);
1028 let exchange = if self.use_exchange_as_venue {
1029 msg.exchange()?
1030 } else {
1031 ""
1032 };
1033
1034 if !exchange.is_empty() {
1035 update_instrument_id_map_with_exchange(
1036 symbol_map,
1037 &self.symbol_venue_map,
1038 instrument_id_map,
1039 msg.hd.instrument_id,
1040 exchange,
1041 self.share_class_space,
1042 )?;
1043 }
1044
1045 let instrument_id = update_instrument_id_map(
1046 record,
1047 symbol_map,
1048 &self.publisher_venue_map,
1049 &self.symbol_venue_map,
1050 instrument_id_map,
1051 self.share_class_space,
1052 )?;
1053 let mut instrument_id = resolve_futures_instrument_id(
1054 record,
1055 instrument_id,
1056 self.futures_year_digits,
1057 &self.symbol_venue_map.load(),
1058 None,
1059 )?;
1060
1061 if exchange.is_empty()
1062 && let Some(venue) = self.symbol_venue_map.get_cloned(&instrument_id.symbol)
1063 {
1064 instrument_id.venue = venue;
1065 }
1066
1067 let instrument = decode_instrument_def_msg(msg, instrument_id, Some(ts_init), None)?;
1068
1069 let cached = if instrument.is_some()
1070 && self.futures_year_digits == FuturesYearDigits::Two
1071 && dataset_uses_futures_symbology(&self.dataset)
1072 {
1073 if exchange.is_empty() {
1074 CachedInstrumentId::Definition(instrument_id)
1075 } else {
1076 CachedInstrumentId::Exchange(instrument_id)
1077 }
1078 } else {
1079 CachedInstrumentId::Inferred(instrument_id)
1080 };
1081
1082 instrument_id_map.insert(record.header().instrument_id, cached);
1083 Ok(instrument)
1084 }
1085
1086 fn cache_futures_instrument_id(
1087 &self,
1088 record: &dbn::RecordRef,
1089 symbol_map: &PitSymbolMap,
1090 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1091 ) -> anyhow::Result<()> {
1092 if self.futures_year_digits != FuturesYearDigits::Two
1093 || !dataset_uses_futures_symbology(&self.dataset)
1094 || record.get::<dbn::InstrumentDefMsg>().is_some()
1095 || record.get::<dbn::SymbolMappingMsg>().is_some()
1096 || instrument_id_map.contains_key(&record.header().instrument_id)
1097 || symbol_map.get_for_rec(record).is_none()
1098 {
1099 return Ok(());
1100 }
1101
1102 let instrument_id = update_instrument_id_map(
1103 record,
1104 symbol_map,
1105 &self.publisher_venue_map,
1106 &self.symbol_venue_map,
1107 instrument_id_map,
1108 self.share_class_space,
1109 )?;
1110 let mut instrument_id = resolve_futures_instrument_id(
1111 record,
1112 instrument_id,
1113 self.futures_year_digits,
1114 &self.symbol_venue_map.load(),
1115 None,
1116 )?;
1117
1118 if let Some(venue) = self.symbol_venue_map.get_cloned(&instrument_id.symbol) {
1119 instrument_id.venue = venue;
1120 }
1121
1122 instrument_id_map.insert(
1123 record.header().instrument_id,
1124 CachedInstrumentId::Inferred(instrument_id),
1125 );
1126 Ok(())
1127 }
1128
1129 fn send_msg(&self, msg: DatabentoMessage) {
1131 log::trace!("Sending {msg:?}");
1132 match self.msg_tx.send(msg) {
1133 Ok(()) => {}
1134 Err(e) => log::debug!("Error sending message: {e}"),
1135 }
1136 }
1137
1138 fn send_close_msg(&self) {
1139 if let Err(e) = self.msg_tx.send(DatabentoMessage::Close) {
1140 log::debug!("Could not send close message: {e}");
1141 }
1142 }
1143}
1144
1145#[derive(Debug, Clone, Copy)]
1146enum CachedInstrumentId {
1147 Inferred(InstrumentId),
1148 Definition(InstrumentId),
1149 Exchange(InstrumentId),
1150}
1151
1152fn handle_error_msg(msg: &dbn::ErrorMsg) {
1154 log::error!("{msg:?}");
1155}
1156
1157fn handle_system_msg(msg: &dbn::SystemMsg, ts_received: UnixNanos) -> Option<SubscriptionAckEvent> {
1159 match msg.code() {
1160 Ok(dbn::SystemCode::SubscriptionAck) => {
1161 let message = msg.msg().unwrap_or("<invalid utf-8>");
1162 log::debug!("Subscription acknowledged: {message}");
1163
1164 let schema = parse_ack_message(message);
1165
1166 Some(SubscriptionAckEvent {
1167 schema,
1168 message: message.to_string(),
1169 ts_received,
1170 })
1171 }
1172 Ok(dbn::SystemCode::Heartbeat) => {
1173 log::trace!("Heartbeat received");
1174 None
1175 }
1176 Ok(dbn::SystemCode::SlowReaderWarning) => {
1177 let message = msg.msg().unwrap_or("<invalid utf-8>");
1178 log::warn!("Slow reader warning: {message}");
1179 None
1180 }
1181 Ok(dbn::SystemCode::ReplayCompleted) => {
1182 let message = msg.msg().unwrap_or("<invalid utf-8>");
1183 log::debug!("Replay completed: {message}");
1184 None
1185 }
1186 _ => {
1187 log::debug!("{msg:?}");
1188 None
1189 }
1190 }
1191}
1192
1193fn parse_ack_message(message: &str) -> String {
1195 message
1197 .strip_circumfix("Subscription request ", " data succeeded")
1198 .and_then(|rest| rest.split_once(" for "))
1199 .map_or_default(|(_, schema)| schema.trim().to_string())
1200}
1201
1202fn handle_symbol_mapping_msg(
1208 msg: &dbn::SymbolMappingMsg,
1209 symbol_map: &mut PitSymbolMap,
1210 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1211 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1212) -> anyhow::Result<()> {
1213 let changed =
1214 symbol_map.get(msg.hd.instrument_id).map(String::as_str) != Some(msg.stype_out_symbol()?);
1215 symbol_map
1216 .on_symbol_mapping(msg)
1217 .map_err(|e| anyhow::anyhow!("on_symbol_mapping failed for {msg:?}: {e}"))?;
1218
1219 match instrument_id_map.get(&msg.hd.instrument_id).copied() {
1220 Some(CachedInstrumentId::Definition(mut id)) if !changed => {
1221 if let Some(venue) = symbol_venue_map.get_cloned(&id.symbol) {
1222 id.venue = venue;
1223 }
1224
1225 instrument_id_map.insert(msg.hd.instrument_id, CachedInstrumentId::Definition(id));
1226 }
1227 Some(CachedInstrumentId::Exchange(_)) if !changed => {}
1228 _ => {
1229 instrument_id_map.remove(&msg.hd.instrument_id);
1230 }
1231 }
1232
1233 Ok(())
1234}
1235
1236fn update_price_precision_map_with_symbol_mapping_msg(
1237 msg: &dbn::SymbolMappingMsg,
1238 price_precision_overrides: &AHashMap<Symbol, u8>,
1239 subscription_price_precision_map: &mut AHashMap<u32, u8>,
1240) -> anyhow::Result<()> {
1241 subscription_price_precision_map.remove(&msg.hd.instrument_id);
1242
1243 if price_precision_overrides.is_empty() {
1244 return Ok(());
1245 }
1246
1247 let stype_in_symbol = msg
1248 .stype_in_symbol()
1249 .map_err(|e| anyhow::anyhow!("Error decoding `stype_in_symbol`: {e}"))?;
1250 let stype_out_symbol = msg
1251 .stype_out_symbol()
1252 .map_err(|e| anyhow::anyhow!("Error decoding `stype_out_symbol`: {e}"))?;
1253
1254 let price_precision = [stype_in_symbol, stype_out_symbol]
1255 .into_iter()
1256 .find_map(|symbol| {
1257 price_precision_overrides
1258 .get(&Symbol::from_str_unchecked(symbol))
1259 .copied()
1260 });
1261
1262 if let Some(price_precision) = price_precision {
1263 subscription_price_precision_map.insert(msg.hd.instrument_id, price_precision);
1264 }
1265
1266 Ok(())
1267}
1268
1269fn update_instrument_id_map_with_exchange(
1271 symbol_map: &PitSymbolMap,
1272 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1273 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1274 raw_instrument_id: u32,
1275 exchange: &str,
1276 share_class_space: bool,
1277) -> anyhow::Result<InstrumentId> {
1278 let raw_symbol = symbol_map.get(raw_instrument_id).ok_or_else(|| {
1279 anyhow::anyhow!("Cannot resolve raw_symbol for instrument_id {raw_instrument_id}")
1280 })?;
1281 let symbol = nautilus_symbol(raw_symbol, share_class_space);
1282 let venue = Venue::from_code(exchange)
1283 .map_err(|e| anyhow::anyhow!("Invalid venue code '{exchange}': {e}"))?;
1284 let instrument_id = InstrumentId::new(symbol, venue);
1285 symbol_venue_map.rcu(|m| {
1286 m.entry(symbol).or_insert(venue);
1287 });
1288
1289 instrument_id_map.insert(
1290 raw_instrument_id,
1291 CachedInstrumentId::Exchange(instrument_id),
1292 );
1293 Ok(instrument_id)
1294}
1295
1296fn update_instrument_id_map(
1297 record: &dbn::RecordRef,
1298 symbol_map: &PitSymbolMap,
1299 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1300 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1301 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1302 share_class_space: bool,
1303) -> anyhow::Result<InstrumentId> {
1304 let header = record.header();
1305
1306 if let Some(&instrument_id) = instrument_id_map.get(&header.instrument_id) {
1308 return Ok(instrument_id.id());
1309 }
1310
1311 let raw_symbol = symbol_map.get_for_rec(record).ok_or_else(|| {
1312 anyhow::anyhow!(
1313 "Cannot resolve `raw_symbol` from `symbol_map` for instrument_id {}",
1314 header.instrument_id
1315 )
1316 })?;
1317
1318 let symbol = nautilus_symbol(raw_symbol, share_class_space);
1319
1320 let publisher_id = header.publisher_id;
1321 let venue = if let Some(venue) = symbol_venue_map.get_cloned(&symbol) {
1322 venue
1323 } else {
1324 let venue = publisher_venue_map
1325 .get(&publisher_id)
1326 .ok_or_else(|| anyhow::anyhow!("No venue found for `publisher_id` {publisher_id}"))?;
1327 *venue
1328 };
1329 let instrument_id = InstrumentId::new(symbol, venue);
1330
1331 instrument_id_map.insert(
1332 header.instrument_id,
1333 CachedInstrumentId::Inferred(instrument_id),
1334 );
1335 Ok(instrument_id)
1336}
1337
1338impl CachedInstrumentId {
1339 fn id(self) -> InstrumentId {
1340 match self {
1341 Self::Inferred(id) | Self::Definition(id) | Self::Exchange(id) => id,
1342 }
1343 }
1344}
1345
1346#[expect(clippy::too_many_arguments)]
1347fn handle_status_msg(
1348 msg: &dbn::StatusMsg,
1349 record: &dbn::RecordRef,
1350 symbol_map: &PitSymbolMap,
1351 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1352 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1353 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1354 ts_init: UnixNanos,
1355 share_class_space: bool,
1356) -> anyhow::Result<InstrumentStatus> {
1357 let instrument_id = update_instrument_id_map(
1358 record,
1359 symbol_map,
1360 publisher_venue_map,
1361 symbol_venue_map,
1362 instrument_id_map,
1363 share_class_space,
1364 )?;
1365
1366 decode_status_msg(msg, instrument_id, Some(ts_init))
1367}
1368
1369#[expect(clippy::too_many_arguments)]
1370fn handle_imbalance_msg(
1371 msg: &dbn::ImbalanceMsg,
1372 record: &dbn::RecordRef,
1373 symbol_map: &PitSymbolMap,
1374 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1375 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1376 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1377 instrument_def_price_precision_map: &AHashMap<u32, u8>,
1378 subscription_price_precision_map: &AHashMap<u32, u8>,
1379 price_precision_overrides: &AHashMap<Symbol, u8>,
1380 ts_init: UnixNanos,
1381 share_class_space: bool,
1382) -> anyhow::Result<DatabentoImbalance> {
1383 let instrument_id = update_instrument_id_map(
1384 record,
1385 symbol_map,
1386 publisher_venue_map,
1387 symbol_venue_map,
1388 instrument_id_map,
1389 share_class_space,
1390 )?;
1391
1392 let price_precision = resolve_price_precision(
1393 msg.hd.instrument_id,
1394 symbol_map.get_for_rec(record).map(String::as_str),
1395 instrument_def_price_precision_map,
1396 subscription_price_precision_map,
1397 price_precision_overrides,
1398 );
1399
1400 decode_imbalance_msg(msg, instrument_id, price_precision, Some(ts_init))
1401}
1402
1403#[expect(clippy::too_many_arguments)]
1404fn handle_statistics_msg(
1405 msg: &dbn::StatMsg,
1406 record: &dbn::RecordRef,
1407 symbol_map: &PitSymbolMap,
1408 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1409 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1410 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1411 instrument_def_price_precision_map: &AHashMap<u32, u8>,
1412 subscription_price_precision_map: &AHashMap<u32, u8>,
1413 price_precision_overrides: &AHashMap<Symbol, u8>,
1414 ts_init: UnixNanos,
1415 share_class_space: bool,
1416) -> anyhow::Result<Option<DatabentoStatistics>> {
1417 if !is_supported_stat_type(msg.stat_type) {
1419 log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
1420 return Ok(None);
1421 }
1422
1423 let instrument_id = update_instrument_id_map(
1424 record,
1425 symbol_map,
1426 publisher_venue_map,
1427 symbol_venue_map,
1428 instrument_id_map,
1429 share_class_space,
1430 )?;
1431
1432 let price_precision = resolve_price_precision(
1433 msg.hd.instrument_id,
1434 symbol_map.get_for_rec(record).map(String::as_str),
1435 instrument_def_price_precision_map,
1436 subscription_price_precision_map,
1437 price_precision_overrides,
1438 );
1439
1440 decode_statistics_msg(msg, instrument_id, price_precision, Some(ts_init))
1441}
1442
1443#[expect(clippy::too_many_arguments)]
1444fn handle_record(
1445 record: dbn::RecordRef,
1446 symbol_map: &PitSymbolMap,
1447 publisher_venue_map: &IndexMap<PublisherId, Venue>,
1448 symbol_venue_map: &AtomicMap<Symbol, Venue>,
1449 instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1450 instrument_def_price_precision_map: &AHashMap<u32, u8>,
1451 subscription_price_precision_map: &AHashMap<u32, u8>,
1452 price_precision_overrides: &AHashMap<Symbol, u8>,
1453 ts_init: UnixNanos,
1454 initialized_books: &HashSet<InstrumentId>,
1455 bars_timestamp_on_close: bool,
1456 share_class_space: bool,
1457) -> anyhow::Result<(Option<Data>, Option<Data>)> {
1458 let instrument_id = update_instrument_id_map(
1459 &record,
1460 symbol_map,
1461 publisher_venue_map,
1462 symbol_venue_map,
1463 instrument_id_map,
1464 share_class_space,
1465 )?;
1466
1467 let price_precision = resolve_price_precision(
1468 record.header().instrument_id,
1469 symbol_map.get_for_rec(&record).map(String::as_str),
1470 instrument_def_price_precision_map,
1471 subscription_price_precision_map,
1472 price_precision_overrides,
1473 );
1474
1475 let include_trades = if record.get::<dbn::Mbp1Msg>().is_some()
1478 || record.get::<dbn::TbboMsg>().is_some()
1479 || record.get::<dbn::Cmbp1Msg>().is_some()
1480 {
1481 true } else {
1483 initialized_books.contains(&instrument_id) };
1485
1486 decode_record(
1487 &record,
1488 instrument_id,
1489 price_precision,
1490 Some(ts_init),
1491 include_trades,
1492 bars_timestamp_on_close,
1493 )
1494}
1495
1496fn claim_trade_source(
1506 data: &Data,
1507 rtype: u8,
1508 trade_sources: &mut AHashMap<InstrumentId, u8>,
1509) -> bool {
1510 match data {
1511 Data::Trade(trade) => *trade_sources.entry(trade.instrument_id).or_insert(rtype) == rtype,
1512 _ => true,
1513 }
1514}
1515
1516fn resolve_price_precision(
1517 record_instrument_id: u32,
1518 raw_symbol: Option<&str>,
1519 instrument_def_price_precision_map: &AHashMap<u32, u8>,
1520 subscription_price_precision_map: &AHashMap<u32, u8>,
1521 price_precision_overrides: &AHashMap<Symbol, u8>,
1522) -> u8 {
1523 subscription_price_precision_map
1524 .get(&record_instrument_id)
1525 .copied()
1526 .or_else(|| {
1527 raw_symbol.and_then(|symbol| {
1528 price_precision_overrides
1529 .get(&Symbol::from_str_unchecked(symbol))
1530 .copied()
1531 })
1532 })
1533 .or_else(|| {
1534 instrument_def_price_precision_map
1535 .get(&record_instrument_id)
1536 .copied()
1537 })
1538 .unwrap_or(Currency::USD().precision)
1539}
1540
1541fn process_mbo_delta(
1546 delta: OrderBookDelta,
1547 flags: u8,
1548 buffering_start: &mut Option<UnixNanos>,
1549 buffered_deltas: &mut AHashMap<InstrumentId, Vec<OrderBookDelta>>,
1550) -> Option<OrderBookDeltas> {
1551 let is_last = RecordFlag::F_LAST.matches(flags);
1552 let is_snapshot = RecordFlag::F_SNAPSHOT.matches(flags);
1553
1554 if is_last
1556 && !is_snapshot
1557 && buffering_start.is_none()
1558 && !buffered_deltas.contains_key(&delta.instrument_id)
1559 {
1560 let deltas = OrderBookDeltas::new(delta.instrument_id, vec![delta]);
1561 return Some(deltas);
1562 }
1563
1564 let buffer = buffered_deltas.entry(delta.instrument_id).or_default();
1565 buffer.push(delta);
1566
1567 if is_snapshot {
1568 return None;
1569 }
1570
1571 flush_mbo_event_boundary(
1572 delta.instrument_id,
1573 delta.ts_event,
1574 flags,
1575 buffering_start,
1576 buffered_deltas,
1577 )
1578}
1579
1580fn flush_mbo_event_boundary(
1588 instrument_id: InstrumentId,
1589 ts_event: UnixNanos,
1590 flags: u8,
1591 buffering_start: &mut Option<UnixNanos>,
1592 buffered_deltas: &mut AHashMap<InstrumentId, Vec<OrderBookDelta>>,
1593) -> Option<OrderBookDeltas> {
1594 if !RecordFlag::F_LAST.matches(flags) || RecordFlag::F_SNAPSHOT.matches(flags) {
1595 return None;
1596 }
1597
1598 let buffer = buffered_deltas.get(&instrument_id)?;
1599
1600 if buffer.is_empty() {
1601 return None;
1602 }
1603
1604 if let Some(start_ns) = *buffering_start {
1605 if ts_event <= start_ns {
1606 return None;
1607 }
1608 *buffering_start = None;
1609 }
1610
1611 let buffer = buffered_deltas.remove(&instrument_id)?;
1612 let deltas = OrderBookDeltas::new(instrument_id, buffer);
1613 Some(deltas)
1614}
1615
1616#[cfg(test)]
1617mod tests {
1618 use std::path::PathBuf;
1619
1620 use databento::{dbn::enums::rtype, live::Subscription};
1621 use indexmap::IndexMap;
1622 use nautilus_model::types::{Price, Quantity};
1623 use rstest::*;
1624 use time::macros::datetime;
1625
1626 use super::*;
1627
1628 fn stub_delta(instrument_id: InstrumentId, ts: u64) -> OrderBookDelta {
1629 use nautilus_model::{
1630 data::BookOrder,
1631 enums::{BookAction, OrderSide},
1632 types::{Price, Quantity},
1633 };
1634
1635 OrderBookDelta::new(
1636 instrument_id,
1637 BookAction::Delete,
1638 BookOrder::new(
1639 OrderSide::Sell,
1640 Price::from("100.00"),
1641 Quantity::from("5"),
1642 42,
1643 ),
1644 0, 1,
1646 ts.into(),
1647 ts.into(),
1648 )
1649 }
1650
1651 #[rstest]
1652 fn test_boundary_flag_on_recordless_message_flushes_buffered_event() {
1653 let instrument_id = InstrumentId::from("TEST.GLBX");
1657 let mut buffering_start = None;
1658 let mut buffered = AHashMap::new();
1659
1660 let buffered_result = process_mbo_delta(
1661 stub_delta(instrument_id, 1),
1662 0,
1663 &mut buffering_start,
1664 &mut buffered,
1665 );
1666 assert!(buffered_result.is_none());
1667
1668 let flushed = flush_mbo_event_boundary(
1669 instrument_id,
1670 2.into(),
1671 RecordFlag::F_LAST as u8,
1672 &mut buffering_start,
1673 &mut buffered,
1674 );
1675 assert!(flushed.is_some());
1676 assert!(buffered.is_empty());
1677 }
1678
1679 #[rstest]
1680 fn test_boundary_flag_with_empty_buffer_is_noop() {
1681 let instrument_id = InstrumentId::from("TEST.GLBX");
1682 let start = UnixNanos::from(1);
1683 let mut buffering_start = Some(start);
1684 let mut buffered: AHashMap<InstrumentId, Vec<OrderBookDelta>> = AHashMap::new();
1685 buffered.insert(instrument_id, Vec::new());
1686
1687 let flushed = flush_mbo_event_boundary(
1688 instrument_id,
1689 2.into(),
1690 RecordFlag::F_LAST as u8,
1691 &mut buffering_start,
1692 &mut buffered,
1693 );
1694
1695 assert!(flushed.is_none());
1696 assert_eq!(buffering_start, Some(start));
1697 }
1698
1699 #[rstest]
1700 fn test_empty_boundary_preserves_replay_gate_for_buffered_instrument() {
1701 let empty_id = InstrumentId::from("EMPTY.GLBX");
1702 let buffered_id = InstrumentId::from("BUFFERED.GLBX");
1703 let start = UnixNanos::from(10);
1704 let mut buffering_start = Some(start);
1705 let mut buffered = AHashMap::new();
1706 let _ = process_mbo_delta(
1707 stub_delta(buffered_id, 1),
1708 0,
1709 &mut buffering_start,
1710 &mut buffered,
1711 );
1712
1713 let empty_boundary = flush_mbo_event_boundary(
1714 empty_id,
1715 11.into(),
1716 RecordFlag::F_LAST as u8,
1717 &mut buffering_start,
1718 &mut buffered,
1719 );
1720 let buffered_boundary = flush_mbo_event_boundary(
1721 buffered_id,
1722 5.into(),
1723 RecordFlag::F_LAST as u8,
1724 &mut buffering_start,
1725 &mut buffered,
1726 );
1727
1728 assert!(empty_boundary.is_none());
1729 assert!(buffered_boundary.is_none());
1730 assert_eq!(buffering_start, Some(start));
1731 assert!(buffered.contains_key(&buffered_id));
1732 }
1733
1734 #[rstest]
1735 fn test_boundary_flag_respects_replay_buffering_gate() {
1736 let instrument_id = InstrumentId::from("TEST.GLBX");
1737 let mut buffering_start = Some(UnixNanos::from(10));
1738 let mut buffered = AHashMap::new();
1739 let _ = process_mbo_delta(
1740 stub_delta(instrument_id, 1),
1741 0,
1742 &mut buffering_start,
1743 &mut buffered,
1744 );
1745
1746 let flushed = flush_mbo_event_boundary(
1748 instrument_id,
1749 5.into(),
1750 RecordFlag::F_LAST as u8,
1751 &mut buffering_start,
1752 &mut buffered,
1753 );
1754 assert!(flushed.is_none());
1755 assert!(!buffered.is_empty());
1756
1757 let flushed = flush_mbo_event_boundary(
1759 instrument_id,
1760 11.into(),
1761 RecordFlag::F_LAST as u8,
1762 &mut buffering_start,
1763 &mut buffered,
1764 );
1765 assert!(flushed.is_some());
1766 assert!(buffering_start.is_none());
1767 }
1768
1769 #[rstest]
1770 fn test_boundary_snapshot_flag_never_flushes() {
1771 let instrument_id = InstrumentId::from("TEST.GLBX");
1772 let mut buffering_start = None;
1773 let mut buffered = AHashMap::new();
1774 let _ = process_mbo_delta(
1775 stub_delta(instrument_id, 1),
1776 0,
1777 &mut buffering_start,
1778 &mut buffered,
1779 );
1780
1781 let flags = RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8;
1782 let flushed = flush_mbo_event_boundary(
1783 instrument_id,
1784 2.into(),
1785 flags,
1786 &mut buffering_start,
1787 &mut buffered,
1788 );
1789 assert!(flushed.is_none());
1790 }
1791
1792 fn stub_trade(instrument_id: InstrumentId) -> Data {
1793 use nautilus_model::{
1794 data::TradeTick,
1795 enums::AggressorSide,
1796 identifiers::TradeId,
1797 types::{Price, Quantity},
1798 };
1799
1800 Data::Trade(TradeTick::new(
1801 instrument_id,
1802 Price::from("100.00"),
1803 Quantity::from("5"),
1804 AggressorSide::Buy,
1805 TradeId::new("1"),
1806 1.into(),
1807 1.into(),
1808 ))
1809 }
1810
1811 #[rstest]
1812 fn test_trade_source_is_first_record_type_to_deliver_a_trade() {
1813 let trade = stub_trade(InstrumentId::from("TEST.GLBX"));
1814 let mut trade_sources = AHashMap::new();
1815
1816 let sent: Vec<bool> = [rtype::MBP_1, rtype::MBP_0, rtype::MBO, rtype::MBP_1]
1817 .into_iter()
1818 .map(|rtype| claim_trade_source(&trade, rtype, &mut trade_sources))
1819 .collect();
1820
1821 assert_eq!(sent, [true, false, false, true]);
1822 }
1823
1824 #[rstest]
1825 fn test_trade_source_is_kept_per_instrument() {
1826 let es_trade = stub_trade(InstrumentId::from("ESM4.GLBX"));
1827 let nq_trade = stub_trade(InstrumentId::from("NQM4.GLBX"));
1828 let mut trade_sources = AHashMap::new();
1829
1830 let sent: Vec<bool> = [
1831 (&es_trade, rtype::MBP_0),
1832 (&nq_trade, rtype::MBO),
1833 (&es_trade, rtype::MBO),
1834 (&nq_trade, rtype::MBP_0),
1835 ]
1836 .into_iter()
1837 .map(|(trade, rtype)| claim_trade_source(trade, rtype, &mut trade_sources))
1838 .collect();
1839
1840 assert_eq!(sent, [true, true, false, false]);
1841 }
1842
1843 #[rstest]
1844 fn test_trade_source_passes_data_other_than_trades() {
1845 let instrument_id = InstrumentId::from("TEST.GLBX");
1846 let mut trade_sources = AHashMap::new();
1847 let _ = claim_trade_source(&stub_trade(instrument_id), rtype::MBP_0, &mut trade_sources);
1848 let delta = Data::BookDelta(stub_delta(instrument_id, 1));
1849
1850 let sent = claim_trade_source(&delta, rtype::MBO, &mut trade_sources);
1851
1852 assert!(sent);
1853 }
1854
1855 fn create_test_handler(reconnect_timeout_mins: Option<u64>) -> DatabentoFeedHandler {
1856 let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1857 let (msg_tx, _msg_rx) = tokio::sync::mpsc::unbounded_channel();
1858
1859 DatabentoFeedHandler::new(
1860 Credential::new("test_key"),
1861 "GLBX.MDP3".to_string(),
1862 cmd_rx,
1863 msg_tx,
1864 IndexMap::new(),
1865 Arc::new(AtomicMap::new()),
1866 false,
1867 false,
1868 reconnect_timeout_mins,
1869 )
1870 }
1871
1872 fn create_test_client() -> DatabentoLiveClient {
1873 DatabentoLiveClient::new(
1874 "test-api-key".to_string(),
1875 "GLBX.MDP3".to_string(),
1876 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1877 true,
1878 None,
1879 None,
1880 )
1881 .unwrap()
1882 }
1883
1884 #[rstest]
1885 #[case(Some(10))]
1886 #[case(None)]
1887 fn test_backoff_initialization(#[case] reconnect_timeout_mins: Option<u64>) {
1888 let handler = create_test_handler(reconnect_timeout_mins);
1889
1890 assert_eq!(handler.reconnect_timeout_mins, reconnect_timeout_mins);
1891 assert!(handler.subscriptions.is_empty());
1892 assert!(handler.buffered_commands.is_empty());
1893 }
1894
1895 #[rstest]
1896 fn test_subscription_with_and_without_start() {
1897 let start_time = datetime!(2024-01-01 00:00:00 UTC);
1898 let sub_with_start = Subscription::builder()
1899 .symbols("ES.FUT")
1900 .schema(databento::dbn::Schema::Mbp1)
1901 .start(start_time)
1902 .build();
1903
1904 let mut sub_without_start = sub_with_start.clone();
1905 sub_without_start.start = None;
1906
1907 assert!(sub_with_start.start.is_some());
1908 assert!(sub_without_start.start.is_none());
1909 assert_eq!(sub_with_start.schema, sub_without_start.schema);
1910 assert_eq!(sub_with_start.symbols, sub_without_start.symbols);
1911 }
1912
1913 #[rstest]
1914 fn test_handler_initialization_state() {
1915 let handler = create_test_handler(Some(10));
1916
1917 assert!(!handler.replay);
1918 assert_eq!(handler.dataset, "GLBX.MDP3");
1919 assert_eq!(handler.credential.api_key(), "test_key");
1920 assert!(handler.subscriptions.is_empty());
1921 assert!(handler.buffered_commands.is_empty());
1922 }
1923
1924 #[rstest]
1925 fn test_handler_with_no_timeout() {
1926 let handler = create_test_handler(None);
1927
1928 assert_eq!(handler.reconnect_timeout_mins, None);
1929 assert!(!handler.replay);
1930 }
1931
1932 #[rstest]
1933 fn test_handler_with_zero_timeout() {
1934 let handler = create_test_handler(Some(0));
1935
1936 assert_eq!(handler.reconnect_timeout_mins, Some(0));
1937 assert!(!handler.replay);
1938 }
1939
1940 #[rstest]
1941 fn test_subscribe_uses_explicit_parent_stype() {
1942 let mut client = create_test_client();
1943
1944 client
1945 .subscribe(
1946 "definition".to_string(),
1947 vec![InstrumentId::from("ES.FUT.GLBX")],
1948 None,
1949 None,
1950 None,
1951 Some("parent".to_string()),
1952 )
1953 .unwrap();
1954
1955 let command = client.cmd_rx.as_mut().unwrap().try_recv().unwrap();
1956 match command {
1957 HandlerCommand::Subscribe(sub) => {
1958 assert_eq!(sub.schema, dbn::Schema::Definition);
1959 assert_eq!(sub.stype_in, dbn::SType::Parent);
1960 assert_eq!(sub.symbols.to_api_string(), "ES.FUT");
1961 }
1962 other => panic!("expected HandlerCommand::Subscribe, was {other:?}"),
1963 }
1964 }
1965
1966 #[rstest]
1967 fn test_subscribe_rejects_invalid_stype() {
1968 let mut client = create_test_client();
1969
1970 let err = client
1971 .subscribe(
1972 "definition".to_string(),
1973 vec![InstrumentId::from("ES.FUT.GLBX")],
1974 None,
1975 None,
1976 None,
1977 Some("not-a-stype".to_string()),
1978 )
1979 .unwrap_err();
1980
1981 assert!(err.to_string().contains("not-a-stype"));
1982 assert!(!is_command_send_error(&err));
1983 assert!(matches!(
1984 client.cmd_rx.as_mut().unwrap().try_recv(),
1985 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
1986 ));
1987 }
1988
1989 #[rstest]
1990 fn test_subscribe_classifies_command_send_errors() {
1991 let mut client = create_test_client();
1992 client.cmd_rx = None;
1993
1994 let err = client
1995 .subscribe(
1996 "definition".to_string(),
1997 vec![InstrumentId::from("ES.FUT.GLBX")],
1998 None,
1999 None,
2000 None,
2001 Some("parent".to_string()),
2002 )
2003 .unwrap_err();
2004
2005 assert!(is_command_send_error(&err));
2006 }
2007
2008 #[rstest]
2009 fn test_close_after_handler_exit_marks_closed() {
2010 let mut client = create_test_client();
2011 let (handler, _msg_rx) = client.start().unwrap();
2012 drop(handler);
2013
2014 client.close().unwrap();
2015
2016 assert!(!client.is_running());
2017 assert!(client.is_closed());
2018 }
2019
2020 fn test_delta(instrument_id: InstrumentId, ts_event: u64) -> OrderBookDelta {
2021 OrderBookDelta::clear(instrument_id, 0, ts_event.into(), 0.into())
2022 }
2023
2024 #[rstest]
2025 fn test_mbo_delta_without_f_last_buffers() {
2026 let instrument_id = InstrumentId::from("ESM4.GLBX");
2027 let delta = test_delta(instrument_id, 1_000_000_000);
2028 let mut buffering_start = None;
2029 let mut buffered = AHashMap::new();
2030
2031 let result = process_mbo_delta(delta, 0, &mut buffering_start, &mut buffered);
2032
2033 assert!(result.is_none());
2034 assert_eq!(buffered[&instrument_id].len(), 1);
2035 }
2036
2037 #[rstest]
2038 fn test_mbo_single_f_last_emits_without_buffering() {
2039 let instrument_id = InstrumentId::from("ESM4.GLBX");
2040 let ts_event = 1_000_000_000;
2041 let mut delta = test_delta(instrument_id, ts_event);
2042 delta.flags = 128;
2043 delta.sequence = 42;
2044 let mut buffering_start = None;
2045 let mut buffered = AHashMap::new();
2046
2047 let result = process_mbo_delta(delta, delta.flags, &mut buffering_start, &mut buffered);
2048
2049 let emitted = result.expect("single F_LAST delta should emit");
2050 assert_eq!(emitted.instrument_id, instrument_id);
2051 assert_eq!(emitted.deltas.len(), 1);
2052 assert_eq!(emitted.flags, 128);
2053 assert_eq!(emitted.sequence, 42);
2054 assert_eq!(emitted.ts_event, UnixNanos::from(ts_event));
2055 assert_eq!(emitted.deltas[0].instrument_id, instrument_id);
2056 assert_eq!(emitted.deltas[0].flags, 128);
2057 assert_eq!(emitted.deltas[0].sequence, 42);
2058 assert_eq!(emitted.deltas[0].ts_event, UnixNanos::from(ts_event));
2059 assert!(buffering_start.is_none());
2060 assert!(buffered.is_empty());
2061 }
2062
2063 #[rstest]
2064 fn test_mbo_delta_with_f_last_emits() {
2065 let instrument_id = InstrumentId::from("ESM4.GLBX");
2066 let mut buffering_start = None;
2067 let mut buffered = AHashMap::new();
2068
2069 let _ = process_mbo_delta(
2070 test_delta(instrument_id, 1_000_000_000),
2071 0,
2072 &mut buffering_start,
2073 &mut buffered,
2074 );
2075
2076 let result = process_mbo_delta(
2077 test_delta(instrument_id, 2_000_000_000),
2078 128, &mut buffering_start,
2080 &mut buffered,
2081 );
2082
2083 assert!(result.is_some());
2084 assert_eq!(result.unwrap().deltas.len(), 2);
2085 assert!(buffered.is_empty());
2086 }
2087
2088 #[rstest]
2089 fn test_mbo_snapshot_with_f_last_buffers() {
2090 let instrument_id = InstrumentId::from("ESM4.GLBX");
2091 let mut buffering_start = None;
2092 let mut buffered = AHashMap::new();
2093
2094 let result = process_mbo_delta(
2095 test_delta(instrument_id, 1_000_000_000),
2096 128 | 32, &mut buffering_start,
2098 &mut buffered,
2099 );
2100
2101 assert!(result.is_none());
2102 assert_eq!(buffered[&instrument_id].len(), 1);
2103 }
2104
2105 #[rstest]
2106 fn test_mbo_replay_buffers_until_past_start() {
2107 let instrument_id = InstrumentId::from("ESM4.GLBX");
2108 let start_ns = 5_000_000_000u64;
2109 let mut buffering_start = Some(start_ns.into());
2110 let mut buffered = AHashMap::new();
2111
2112 let result = process_mbo_delta(
2113 test_delta(instrument_id, 4_000_000_000),
2114 128, &mut buffering_start,
2116 &mut buffered,
2117 );
2118 assert!(result.is_none());
2119
2120 let result = process_mbo_delta(
2121 test_delta(instrument_id, 5_000_000_000),
2122 128,
2123 &mut buffering_start,
2124 &mut buffered,
2125 );
2126 assert!(result.is_none());
2127
2128 let result = process_mbo_delta(
2130 test_delta(instrument_id, 6_000_000_000),
2131 128,
2132 &mut buffering_start,
2133 &mut buffered,
2134 );
2135 assert!(result.is_some());
2136 assert!(buffering_start.is_none());
2137 }
2138
2139 #[rstest]
2140 fn test_mbo_multiple_deltas_accumulated() {
2141 let instrument_id = InstrumentId::from("ESM4.GLBX");
2142 let mut buffering_start = None;
2143 let mut buffered = AHashMap::new();
2144
2145 for i in 0..5 {
2146 process_mbo_delta(
2147 test_delta(instrument_id, 1_000_000_000 + i),
2148 0,
2149 &mut buffering_start,
2150 &mut buffered,
2151 );
2152 }
2153
2154 let result = process_mbo_delta(
2155 test_delta(instrument_id, 2_000_000_000),
2156 128,
2157 &mut buffering_start,
2158 &mut buffered,
2159 );
2160
2161 assert!(result.is_some());
2162 assert_eq!(result.unwrap().deltas.len(), 6);
2163 }
2164
2165 #[rstest]
2166 fn test_mbo_multi_instrument_isolation() {
2167 let id_a = InstrumentId::from("ESM4.GLBX");
2168 let id_b = InstrumentId::from("NQM4.GLBX");
2169 let mut buffering_start = None;
2170 let mut buffered = AHashMap::new();
2171
2172 process_mbo_delta(
2173 test_delta(id_a, 1_000_000_000),
2174 0,
2175 &mut buffering_start,
2176 &mut buffered,
2177 );
2178 process_mbo_delta(
2179 test_delta(id_b, 1_000_000_000),
2180 0,
2181 &mut buffering_start,
2182 &mut buffered,
2183 );
2184
2185 let result = process_mbo_delta(
2187 test_delta(id_a, 2_000_000_000),
2188 128,
2189 &mut buffering_start,
2190 &mut buffered,
2191 );
2192
2193 assert!(result.is_some());
2194 assert_eq!(result.unwrap().instrument_id, id_a);
2195 assert!(buffered.contains_key(&id_b));
2196 assert!(!buffered.contains_key(&id_a));
2197 }
2198
2199 mod property_tests {
2200 use proptest::prelude::*;
2201 use rstest::rstest;
2202
2203 use super::*;
2204
2205 proptest! {
2206 #[rstest]
2207 fn mbo_buffering_conserves_deltas(
2208 num_non_last in 0usize..=20,
2209 ) {
2210 let instrument_id = InstrumentId::from("ESM4.GLBX");
2211 let mut buffering_start = None;
2212 let mut buffered = AHashMap::new();
2213 let total = num_non_last + 1;
2214
2215 for i in 0..num_non_last {
2216 let result = process_mbo_delta(
2217 test_delta(instrument_id, 1_000_000_000 + i as u64),
2218 0, &mut buffering_start,
2220 &mut buffered,
2221 );
2222 prop_assert!(result.is_none());
2223 }
2224
2225 let result = process_mbo_delta(
2226 test_delta(instrument_id, 2_000_000_000),
2227 128, &mut buffering_start,
2229 &mut buffered,
2230 );
2231
2232 prop_assert!(result.is_some());
2233 let emitted = result.unwrap();
2234 prop_assert_eq!(emitted.deltas.len(), total);
2235 prop_assert!(buffered.is_empty());
2236 }
2237
2238 #[rstest]
2239 fn mbo_snapshots_never_emit(
2240 num_snapshots in 1usize..=20,
2241 ) {
2242 let instrument_id = InstrumentId::from("ESM4.GLBX");
2243 let mut buffering_start = None;
2244 let mut buffered = AHashMap::new();
2245
2246 for i in 0..num_snapshots {
2247 let result = process_mbo_delta(
2248 test_delta(instrument_id, 1_000_000_000 + i as u64),
2249 128 | 32, &mut buffering_start,
2251 &mut buffered,
2252 );
2253 prop_assert!(result.is_none());
2254 }
2255
2256 prop_assert_eq!(buffered[&instrument_id].len(), num_snapshots);
2257 }
2258
2259 #[rstest]
2260 fn mbo_replay_delays_emission(
2261 start_offset in 1u64..=100,
2262 num_before in 1usize..=10,
2263 ) {
2264 let instrument_id = InstrumentId::from("ESM4.GLBX");
2265 let start_ns = 1_000_000_000u64 * start_offset;
2266 let mut buffering_start = Some(start_ns.into());
2267 let mut buffered = AHashMap::new();
2268
2269 for i in 0..num_before {
2270 let ts = start_ns - (num_before as u64 - i as u64);
2271 let result = process_mbo_delta(
2272 test_delta(instrument_id, ts),
2273 128, &mut buffering_start,
2275 &mut buffered,
2276 );
2277 prop_assert!(result.is_none());
2278 }
2279
2280 let result = process_mbo_delta(
2281 test_delta(instrument_id, start_ns + 1),
2282 128,
2283 &mut buffering_start,
2284 &mut buffered,
2285 );
2286
2287 prop_assert!(result.is_some());
2288 prop_assert!(buffering_start.is_none());
2289 let total = num_before + 1;
2290 prop_assert_eq!(result.unwrap().deltas.len(), total);
2291 }
2292 }
2293 }
2294
2295 #[rstest]
2296 #[case(2, "CLZ29-CLF30.XNYM")]
2297 #[case(1, "CLZ9-CLF0.XNYM")]
2298 fn test_live_subscription_preserves_native_symbols(
2299 #[case] digits: u8,
2300 #[case] id: InstrumentId,
2301 ) {
2302 let mut client = create_test_client().with_futures_year_digits(digits.try_into().unwrap());
2303 client
2304 .subscribe(
2305 "mbp-1".to_string(),
2306 vec![id],
2307 None,
2308 None,
2309 Some(vec![Some(3)]),
2310 None,
2311 )
2312 .unwrap();
2313 let rx = client.cmd_rx.as_mut().unwrap();
2314 assert!(
2315 matches!(rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 3) if symbol == Symbol::from("CLZ9-CLF0"))
2316 );
2317
2318 let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
2319 panic!("expected subscription");
2320 };
2321
2322 let expected = Subscription::builder()
2323 .schema(dbn::Schema::Mbp1)
2324 .symbols(vec!["CLZ9-CLF0".to_string()])
2325 .build();
2326 assert_eq!(subscription, expected);
2327 assert_eq!(
2328 client.symbol_venue_map.get_cloned(&id.symbol),
2329 Some(Venue::from("XNYM"))
2330 );
2331 assert_eq!(
2332 rx.try_recv().unwrap_err(),
2333 tokio::sync::mpsc::error::TryRecvError::Empty
2334 );
2335 }
2336
2337 #[rstest]
2338 fn test_futures_cache_uses_requested_venue_after_year_conversion() {
2339 let mut handler = create_test_handler(None);
2340 handler.publisher_venue_map.insert(1, Venue::from("GLBX"));
2341 handler
2342 .symbol_venue_map
2343 .insert(Symbol::from("CLF26"), Venue::from("XNYM"));
2344 let mut mapping = dbn::PitSymbolMap::new();
2345 mapping.inner_mut().insert(42, "CLF6".to_string());
2346 let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
2347 msg.hd.publisher_id = 1;
2348 msg.hd.instrument_id = 42;
2349 msg.ts_recv = 1_766_102_400_000_000_000;
2350 let mut ids = AHashMap::new();
2351 handler
2352 .cache_futures_instrument_id(&dbn::RecordRef::from(&msg), &mapping, &mut ids)
2353 .unwrap();
2354 assert_eq!(
2355 ids.get(&42).copied().map(CachedInstrumentId::id),
2356 Some(InstrumentId::from("CLF26.XNYM"))
2357 );
2358 }
2359
2360 #[rstest]
2361 #[case(Some(2), Some(5), Some(4), 5)]
2362 #[case(Some(2), None, Some(5), 5)]
2363 #[case(Some(5), Some(2), None, 2)]
2364 #[case(Some(5), None, Some(2), 2)]
2365 #[case(Some(3), None, None, 3)]
2366 #[case(None, None, None, 2)]
2367 fn test_explicit_precision_precedes_definitions(
2368 #[case] definition: Option<u8>,
2369 #[case] mapped: Option<u8>,
2370 #[case] native: Option<u8>,
2371 #[case] expected: u8,
2372 ) {
2373 let definitions = AHashMap::from_iter(definition.map(|precision| (42, precision)));
2374 let subscriptions = AHashMap::from_iter(mapped.map(|precision| (42, precision)));
2375 let overrides =
2376 AHashMap::from_iter(native.map(|precision| (Symbol::from("CLF6"), precision)));
2377
2378 let actual =
2379 resolve_price_precision(42, Some("CLF6"), &definitions, &subscriptions, &overrides);
2380
2381 assert_eq!(actual, expected);
2382 }
2383
2384 #[rstest]
2385 #[case(true, "BRK B.EQUS")]
2386 #[case(false, "BRK.B.EQUS")]
2387 fn test_handle_record_spells_share_class_with_space(
2388 #[case] share_class_space: bool,
2389 #[case] expected: &str,
2390 ) {
2391 let handler = create_test_handler(None);
2392 let mut publisher_venue_map = IndexMap::new();
2393 publisher_venue_map.insert(1, Venue::from("EQUS"));
2394 let mut mapping = dbn::PitSymbolMap::new();
2395 mapping.inner_mut().insert(42, "BRK.B".to_string());
2396 let mut msg = dbn::TradeMsg::default();
2397 msg.hd.publisher_id = 1;
2398 msg.hd.instrument_id = 42;
2399 msg.hd.ts_event = 1_766_102_400_000_000_000;
2400 msg.ts_recv = msg.hd.ts_event;
2401 msg.price = 480_250_000_000;
2402 msg.size = 10;
2403 msg.side = b'A' as std::ffi::c_char;
2404 let record = dbn::RecordRef::from(&msg);
2405 let mut ids = AHashMap::new();
2406
2407 let (data, _) = handle_record(
2408 record,
2409 &mapping,
2410 &publisher_venue_map,
2411 &handler.symbol_venue_map,
2412 &mut ids,
2413 &AHashMap::new(),
2414 &AHashMap::new(),
2415 &AHashMap::new(),
2416 1_766_102_400_000_000_001.into(),
2417 &HashSet::new(),
2418 true,
2419 share_class_space,
2420 )
2421 .unwrap();
2422
2423 let Some(Data::Trade(trade)) = data else {
2424 panic!("expected trade");
2425 };
2426 assert_eq!(trade.instrument_id, InstrumentId::from(expected));
2427 }
2428
2429 #[rstest]
2430 fn test_futures_precision_override_uses_native_symbol() {
2431 let mut handler = create_test_handler(None);
2432 handler.publisher_venue_map.insert(1, Venue::from("GLBX"));
2433 let mut mapping = dbn::PitSymbolMap::new();
2434 mapping.inner_mut().insert(42, "CLF6".to_string());
2435 let mut msg = dbn::TradeMsg::default();
2436 msg.hd.publisher_id = 1;
2437 msg.hd.instrument_id = 42;
2438 msg.hd.ts_event = 1_766_102_400_000_000_000;
2439 msg.ts_recv = msg.hd.ts_event;
2440 msg.price = 75_125_000_000;
2441 msg.size = 7;
2442 msg.side = b'A' as std::ffi::c_char;
2443 let record = dbn::RecordRef::from(&msg);
2444 let mut ids = AHashMap::new();
2445 handler
2446 .cache_futures_instrument_id(&record, &mapping, &mut ids)
2447 .unwrap();
2448 let overrides = AHashMap::from_iter([(Symbol::from("CLF6"), 3)]);
2449 let (data, extra) = handle_record(
2450 record,
2451 &mapping,
2452 &handler.publisher_venue_map,
2453 &handler.symbol_venue_map,
2454 &mut ids,
2455 &AHashMap::new(),
2456 &AHashMap::new(),
2457 &overrides,
2458 1_766_102_400_000_000_001.into(),
2459 &HashSet::new(),
2460 true,
2461 false,
2462 )
2463 .unwrap();
2464
2465 let Some(Data::Trade(trade)) = data else {
2466 panic!("expected trade");
2467 };
2468
2469 assert_eq!(trade.instrument_id, InstrumentId::from("CLF26.GLBX"));
2470 assert_eq!(trade.price.precision, 3);
2471 assert_eq!(trade.price, Price::from("75.125"));
2472 assert_eq!(trade.size, Quantity::from(7));
2473 assert_eq!(trade.ts_event, msg.hd.ts_event);
2474 assert_eq!(trade.ts_init, 1_766_102_400_000_000_001);
2475 assert!(extra.is_none());
2476 }
2477}