Skip to main content

nautilus_databento/
live.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Databento live feed handler.
17//!
18//! The feed handler runs a single async task per dataset. It receives
19//! [`HandlerCommand`] messages over an unbounded channel and streams decoded
20//! market data back as [`DatabentoMessage`]s on an unbounded tokio channel.
21//!
22//! The inner loop uses `tokio::select!` to concurrently await the next record
23//! from the Databento gateway and the next command from the engine, giving
24//! near-zero idle CPU and immediate command responsiveness.
25//!
26//! Heartbeat detection is delegated to the upstream `databento` client, which
27//! returns `Error::HeartbeatTimeout` when no data arrives within
28//! `heartbeat_interval + 5 s` (default 35 s). The handler treats this as a
29//! connection error and enters the reconnection backoff loop.
30
31use std::{
32    fmt::{Debug, Display},
33    path::PathBuf,
34    str::FromStr,
35    sync::Arc,
36    time::Duration,
37};
38
39use ahash::{AHashMap, HashSet, HashSetExt};
40use databento::{
41    dbn::{self, PitSymbolMap, SymbolIndex},
42    live::Subscription,
43};
44use indexmap::IndexMap;
45use nautilus_core::{
46    AtomicMap, UnixNanos, consts::NAUTILUS_USER_AGENT, string::secret::SecretString,
47    time::get_atomic_clock_realtime,
48};
49use nautilus_model::{
50    data::{Data, InstrumentStatus, OrderBookDelta, OrderBookDeltas},
51    enums::RecordFlag,
52    identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
53    instruments::{Instrument, InstrumentAny},
54    types::Currency,
55};
56use nautilus_network::backoff::ExponentialBackoff;
57use time::OffsetDateTime;
58
59use super::{
60    decode::{
61        decode_imbalance_msg, decode_statistics_msg, decode_status_msg, is_supported_stat_type,
62    },
63    types::{DatabentoImbalance, DatabentoStatistics, SubscriptionAckEvent},
64};
65use crate::{
66    common::{Credential, build_publisher_venue_map, load_publishers},
67    decode::{decode_instrument_def_msg, decode_record},
68    symbology::{
69        check_consistent_symbology, dataset_uses_futures_symbology, infer_symbology_type,
70        raw_symbol, resolve_futures_instrument_id,
71    },
72    types::PublisherId,
73};
74
75#[derive(Debug)]
76pub enum HandlerCommand {
77    Subscribe(Subscription),
78    SetPricePrecision(Symbol, u8),
79    Start,
80    Close,
81}
82
83#[derive(Debug)]
84pub enum DatabentoMessage {
85    Data(Data),
86    Instrument(Box<InstrumentAny>),
87    Status(InstrumentStatus),
88    Imbalance(DatabentoImbalance),
89    Statistics(DatabentoStatistics),
90    SubscriptionAck(SubscriptionAckEvent),
91    Error(anyhow::Error),
92    Close,
93}
94
95#[cfg_attr(
96    feature = "python",
97    pyo3::pyclass(module = "nautilus_trader.adapters.databento")
98)]
99#[cfg_attr(
100    feature = "python",
101    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
102)]
103pub struct DatabentoLiveClient {
104    futures_year_digits: FuturesYearDigits,
105    credential: Credential,
106    pub dataset: String,
107    is_running: bool,
108    is_closed: bool,
109    cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
110    cmd_rx: Option<tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>>,
111    publisher_venue_map: IndexMap<PublisherId, Venue>,
112    symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
113    use_exchange_as_venue: bool,
114    bars_timestamp_on_close: bool,
115    reconnect_timeout_mins: Option<u64>,
116}
117
118impl Debug for DatabentoLiveClient {
119    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
120        f.debug_struct(stringify!(DatabentoLiveClient))
121            .field("credential", &self.credential)
122            .field("dataset", &self.dataset)
123            .field("is_running", &self.is_running)
124            .field("is_closed", &self.is_closed)
125            .finish()
126    }
127}
128
129impl DatabentoLiveClient {
130    /// Creates a new [`DatabentoLiveClient`] instance.
131    ///
132    /// # Errors
133    ///
134    /// Returns an error if reading or parsing the publishers file fails.
135    pub fn new(
136        key: String,
137        dataset: String,
138        publishers_filepath: PathBuf,
139        use_exchange_as_venue: bool,
140        bars_timestamp_on_close: Option<bool>,
141        reconnect_timeout_mins: Option<i64>,
142    ) -> anyhow::Result<Self> {
143        let publishers = load_publishers(publishers_filepath)?;
144        let publisher_venue_map = build_publisher_venue_map(&publishers);
145
146        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
147
148        let reconnect_timeout_mins = reconnect_timeout_mins
149            .and_then(|mins| if mins >= 0 { Some(mins as u64) } else { None });
150
151        Ok(Self {
152            futures_year_digits: FuturesYearDigits::default(),
153            credential: Credential::new(key),
154            dataset,
155            cmd_tx,
156            cmd_rx: Some(cmd_rx),
157            is_running: false,
158            is_closed: false,
159            publisher_venue_map,
160            symbol_venue_map: Arc::new(AtomicMap::new()),
161            use_exchange_as_venue,
162            bars_timestamp_on_close: bars_timestamp_on_close.unwrap_or(true),
163            reconnect_timeout_mins,
164        })
165    }
166
167    /// Selects one- or two-digit years for futures instrument IDs.
168    #[must_use]
169    pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
170        self.futures_year_digits = futures_year_digits;
171        self
172    }
173
174    #[must_use]
175    pub const fn is_running(&self) -> bool {
176        self.is_running
177    }
178
179    #[must_use]
180    pub const fn is_closed(&self) -> bool {
181        self.is_closed
182    }
183
184    /// Subscribes to Databento live data for the requested instruments.
185    ///
186    /// # Errors
187    ///
188    /// Returns an error if symbology, schema, timestamp, or precision inputs are invalid,
189    /// or if the command cannot be sent to the feed handler.
190    #[expect(clippy::needless_pass_by_value)]
191    pub fn subscribe(
192        &mut self,
193        schema: String,
194        instrument_ids: Vec<InstrumentId>,
195        start: Option<u64>,
196        snapshot: Option<bool>,
197        price_precisions: Option<Vec<Option<u8>>>,
198        stype_in: Option<String>,
199    ) -> anyhow::Result<()> {
200        if let Some(precisions) = &price_precisions
201            && precisions.len() != instrument_ids.len()
202        {
203            anyhow::bail!(
204                "`price_precisions` length ({}) must match `instrument_ids` length ({})",
205                precisions.len(),
206                instrument_ids.len()
207            );
208        }
209
210        let symbols: Vec<String> = instrument_ids
211            .iter()
212            .map(|id| raw_symbol(id.symbol.as_str(), &self.dataset, self.futures_year_digits))
213            .collect();
214        let first_symbol = symbols
215            .first()
216            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
217        let stype_in = match stype_in {
218            Some(stype_in) => dbn::SType::from_str(&stype_in)?,
219            None => infer_symbology_type(first_symbol),
220        };
221        let symbols: Vec<&str> = symbols.iter().map(String::as_str).collect();
222        check_consistent_symbology(symbols.as_slice())?;
223        let mut sub = Subscription::builder()
224            .symbols(symbols)
225            .schema(dbn::Schema::from_str(&schema)?)
226            .stype_in(stype_in)
227            .build();
228
229        if let Some(start) = start {
230            sub.start = Some(OffsetDateTime::from_unix_timestamp_nanos(i128::from(
231                start,
232            ))?);
233        }
234        sub.use_snapshot = snapshot.unwrap_or(false);
235
236        self.symbol_venue_map.rcu(|m| {
237            for id in &instrument_ids {
238                m.entry(id.symbol).or_insert(id.venue);
239            }
240        });
241
242        if let Some(precisions) = price_precisions {
243            for (instrument_id, precision) in instrument_ids.iter().zip(precisions) {
244                if let Some(precision) = precision {
245                    let raw = raw_symbol(
246                        instrument_id.symbol.as_str(),
247                        &self.dataset,
248                        self.futures_year_digits,
249                    );
250                    self.send_command(HandlerCommand::SetPricePrecision(
251                        Symbol::from(raw.as_str()),
252                        precision,
253                    ))?;
254                }
255            }
256        }
257
258        self.send_command(HandlerCommand::Subscribe(sub))
259    }
260
261    /// Starts the live feed handler and returns its message receiver.
262    ///
263    /// # Errors
264    ///
265    /// Returns an error if the client is already closed, already running, or cannot start.
266    pub fn start(
267        &mut self,
268    ) -> anyhow::Result<(
269        DatabentoFeedHandler,
270        tokio::sync::mpsc::UnboundedReceiver<DatabentoMessage>,
271    )> {
272        if self.is_closed {
273            anyhow::bail!("Client already closed");
274        }
275
276        if self.is_running {
277            anyhow::bail!("Client already running");
278        }
279
280        log::debug!("Starting client");
281
282        let (msg_tx, msg_rx) = tokio::sync::mpsc::unbounded_channel::<DatabentoMessage>();
283        let cmd_rx = self
284            .cmd_rx
285            .take()
286            .ok_or_else(|| anyhow::anyhow!("Command receiver already taken"))?;
287
288        let feed_handler = DatabentoFeedHandler::new(
289            self.credential.clone(),
290            self.dataset.clone(),
291            cmd_rx,
292            msg_tx,
293            self.publisher_venue_map.clone(),
294            self.symbol_venue_map.clone(),
295            self.use_exchange_as_venue,
296            self.bars_timestamp_on_close,
297            self.reconnect_timeout_mins,
298        )
299        .with_futures_year_digits(self.futures_year_digits);
300
301        self.send_command(HandlerCommand::Start)?;
302        self.is_running = true;
303
304        Ok((feed_handler, msg_rx))
305    }
306
307    /// Closes the live client.
308    ///
309    /// # Errors
310    ///
311    /// Returns an error if the client was never started, is already closed, or cannot send
312    /// the close command to the feed handler.
313    pub fn close(&mut self) -> anyhow::Result<()> {
314        if !self.is_running {
315            anyhow::bail!("Client never started");
316        }
317
318        if self.is_closed {
319            anyhow::bail!("Client already closed");
320        }
321
322        log::debug!("Closing client");
323
324        if !self.cmd_tx.is_closed() {
325            self.send_command(HandlerCommand::Close)?;
326        }
327
328        self.is_running = false;
329        self.is_closed = true;
330
331        Ok(())
332    }
333
334    fn send_command(&self, cmd: HandlerCommand) -> anyhow::Result<()> {
335        self.cmd_tx.send(cmd).map_err(|e| {
336            anyhow::Error::new(CommandSendError {
337                details: e.to_string(),
338            })
339        })
340    }
341}
342
343#[cfg(any(test, feature = "python"))]
344pub(crate) fn is_command_send_error(error: &anyhow::Error) -> bool {
345    error.is::<CommandSendError>()
346}
347
348#[derive(Debug)]
349struct CommandSendError {
350    details: String,
351}
352
353impl Display for CommandSendError {
354    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
355        write!(
356            f,
357            "Failed to send command to Databento feed handler: {}",
358            self.details
359        )
360    }
361}
362
363impl std::error::Error for CommandSendError {}
364
365/// Handles a raw TCP data feed from the Databento LSG for a single dataset.
366///
367/// [`HandlerCommand`] messages are received synchronously across a channel,
368/// decoded records are sent asynchronously on a tokio channel as [`DatabentoMessage`]s
369/// back to a message processing task.
370///
371/// # Crash Policy
372///
373/// This handler intentionally avoids applying downstream backpressure to the
374/// live feed. If decoded output cannot be drained, memory pressure is the hard
375/// failure mode instead of arbitrary queue limits or delayed market data.
376pub struct DatabentoFeedHandler {
377    futures_year_digits: FuturesYearDigits,
378    credential: Credential,
379    dataset: String,
380    cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
381    msg_tx: tokio::sync::mpsc::UnboundedSender<DatabentoMessage>,
382    publisher_venue_map: IndexMap<PublisherId, Venue>,
383    symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
384    replay: bool,
385    use_exchange_as_venue: bool,
386    bars_timestamp_on_close: bool,
387    reconnect_timeout_mins: Option<u64>,
388    backoff: ExponentialBackoff,
389    subscriptions: Vec<Subscription>,
390    buffered_commands: Vec<HandlerCommand>,
391    price_precision_overrides: AHashMap<Symbol, u8>,
392    gateway_addr: Option<String>,
393    success_threshold: Duration,
394}
395
396impl Debug for DatabentoFeedHandler {
397    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
398        f.debug_struct(stringify!(DatabentoFeedHandler))
399            .field("credential", &self.credential)
400            .field("dataset", &self.dataset)
401            .field("replay", &self.replay)
402            .field("reconnect_timeout_mins", &self.reconnect_timeout_mins)
403            .field("subscriptions", &self.subscriptions.len())
404            .finish()
405    }
406}
407
408impl DatabentoFeedHandler {
409    /// Selects one- or two-digit years for futures instrument IDs.
410    #[must_use]
411    pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
412        self.futures_year_digits = futures_year_digits;
413        self
414    }
415
416    /// Creates a new [`DatabentoFeedHandler`] instance.
417    ///
418    /// # Panics
419    ///
420    /// Panics if exponential backoff creation fails (should never happen with valid hardcoded parameters).
421    #[must_use]
422    #[expect(clippy::too_many_arguments)]
423    pub fn new(
424        credential: Credential,
425        dataset: String,
426        rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
427        tx: tokio::sync::mpsc::UnboundedSender<DatabentoMessage>,
428        publisher_venue_map: IndexMap<PublisherId, Venue>,
429        symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
430        use_exchange_as_venue: bool,
431        bars_timestamp_on_close: bool,
432        reconnect_timeout_mins: Option<u64>,
433    ) -> Self {
434        // Choose max delay based on timeout configuration:
435        // - With timeout: 60s max (quick recovery to reconnect within window)
436        // - Without timeout (None): 600s max (patient recovery, respectful of infrastructure)
437        let delay_max = if reconnect_timeout_mins.is_some() {
438            Duration::from_secs(60)
439        } else {
440            Duration::from_secs(600)
441        };
442
443        let backoff = ExponentialBackoff::new(Duration::from_secs(1), delay_max, 2.0, 1000, false)
444            .expect("hardcoded backoff parameters are valid");
445
446        Self {
447            futures_year_digits: FuturesYearDigits::default(),
448            credential,
449            dataset,
450            cmd_rx: rx,
451            msg_tx: tx,
452            publisher_venue_map,
453            symbol_venue_map,
454            replay: false,
455            use_exchange_as_venue,
456            bars_timestamp_on_close,
457            reconnect_timeout_mins,
458            backoff,
459            subscriptions: Vec::new(),
460            buffered_commands: Vec::new(),
461            price_precision_overrides: AHashMap::new(),
462            gateway_addr: None,
463            success_threshold: Duration::from_secs(60),
464        }
465    }
466
467    /// Sets a custom gateway address, overriding the default Databento LSG endpoint.
468    #[must_use]
469    pub fn with_gateway_addr(mut self, addr: String) -> Self {
470        self.gateway_addr = Some(addr);
471        self
472    }
473
474    /// Sets the duration a session must run before it counts as successful.
475    ///
476    /// A successful session resets the reconnection backoff cycle.
477    /// Defaults to 60 seconds.
478    #[must_use]
479    pub fn with_success_threshold(mut self, threshold: Duration) -> Self {
480        self.success_threshold = threshold;
481        self
482    }
483
484    /// Runs the feed handler main loop, processing commands and streaming market data.
485    ///
486    /// Establishes a connection to the Databento LSG, subscribes to requested data feeds,
487    /// and continuously processes incoming market data messages until shutdown.
488    ///
489    /// Implements automatic reconnection with exponential backoff (1s to 60s with jitter).
490    /// Each successful session resets the reconnection cycle, giving the next disconnect
491    /// a fresh timeout window. Gives up after `reconnect_timeout_mins` if configured.
492    ///
493    /// # Errors
494    ///
495    /// Returns an error if any client operation or message handling fails.
496    pub async fn run(&mut self) -> anyhow::Result<()> {
497        log::debug!("Running feed handler");
498
499        let mut reconnect_start: Option<tokio::time::Instant> = None;
500        let mut attempt = 0;
501
502        loop {
503            attempt += 1;
504
505            match self.run_session(attempt).await {
506                Ok(ran_successfully) => {
507                    if ran_successfully {
508                        log::info!("Resetting reconnection cycle after successful session");
509                        reconnect_start = None;
510                        attempt = 0;
511                        self.backoff.reset();
512                    } else {
513                        log::debug!("Session ended normally");
514                        break Ok(());
515                    }
516                }
517                Err(e) => {
518                    let cycle_start = reconnect_start.get_or_insert_with(tokio::time::Instant::now);
519
520                    if let Some(timeout_mins) = self.reconnect_timeout_mins {
521                        let elapsed = cycle_start.elapsed();
522                        let timeout = Duration::from_mins(timeout_mins);
523
524                        if elapsed >= timeout {
525                            log::error!("Giving up reconnection after {timeout_mins} minutes");
526                            self.send_msg(DatabentoMessage::Error(anyhow::anyhow!(
527                                "Reconnection timeout after {timeout_mins} minutes: {e}"
528                            )));
529                            break Err(e);
530                        }
531                    }
532
533                    let delay = self.backoff.next_duration();
534
535                    log::warn!(
536                        "Connection lost (attempt {}): {}. Reconnecting in {}s...",
537                        attempt,
538                        e,
539                        delay.as_secs()
540                    );
541
542                    let sleep = tokio::time::sleep(delay);
543                    tokio::pin!(sleep);
544
545                    loop {
546                        tokio::select! {
547                            () = &mut sleep => break,
548                            cmd = self.cmd_rx.recv() => {
549                                match cmd {
550                                    Some(HandlerCommand::Close) => {
551                                        log::debug!("Close received during backoff");
552                                        return Ok(());
553                                    }
554                                    None => {
555                                        log::debug!("Command channel closed during backoff");
556                                        return Ok(());
557                                    }
558                                    Some(cmd) => {
559                                        log::debug!("Buffering command received during backoff: {cmd:?}");
560                                        self.buffered_commands.push(cmd);
561                                    }
562                                }
563                            }
564                        }
565                    }
566                }
567            }
568        }
569    }
570
571    /// Runs a single session, handling connection, subscriptions, and data streaming.
572    ///
573    /// Returns `Ok(bool)` where the bool indicates if the session ran successfully
574    /// for a meaningful duration (true) or was intentionally closed (false).
575    ///
576    /// # Errors
577    ///
578    /// Returns an error if connection fails, subscription fails, or data streaming encounters an error.
579    async fn run_session(&mut self, attempt: usize) -> anyhow::Result<bool> {
580        if attempt > 1 {
581            log::info!("Reconnecting (attempt {attempt})...");
582        }
583
584        let session_start = tokio::time::Instant::now();
585        let clock = get_atomic_clock_realtime();
586        let mut symbol_map = PitSymbolMap::new();
587        let mut instrument_id_map: AHashMap<u32, CachedInstrumentId> = AHashMap::new();
588        let mut instrument_def_price_precision_map: AHashMap<u32, u8> = AHashMap::new();
589        let mut subscription_price_precision_map: AHashMap<u32, u8> = AHashMap::new();
590
591        let mut buffering_start = None;
592        let mut buffered_deltas: AHashMap<InstrumentId, Vec<OrderBookDelta>> = AHashMap::new();
593        let mut initialized_books = HashSet::new();
594        let mut trade_sources: AHashMap<InstrumentId, u8> = AHashMap::new();
595        let timeout = Duration::from_secs(5); // Hardcoded timeout for now
596
597        let gateway_addr = self.gateway_addr.clone();
598        let api_key = SecretString::from(self.credential.api_key().to_owned());
599        let dataset = self.dataset.clone();
600
601        let result = tokio::time::timeout(timeout, async move {
602            let base = databento::LiveClient::builder();
603            let base = if let Some(addr) = gateway_addr {
604                base.addr(addr).await?
605            } else {
606                base
607            };
608            base.user_agent_extension(NAUTILUS_USER_AGENT.into())
609                .key(api_key.expose_secret().to_owned())?
610                .dataset(dataset)
611                .build()
612                .await
613        })
614        .await?;
615
616        let mut client = match result {
617            Ok(client) => {
618                if attempt > 1 {
619                    log::info!("Reconnected successfully");
620                } else {
621                    log::info!("Connected");
622                }
623                client
624            }
625            Err(e) => {
626                anyhow::bail!("Failed to connect to Databento LSG: {e}");
627            }
628        };
629
630        // Process any commands buffered during reconnection backoff
631        let mut start_buffered = false;
632
633        if !self.buffered_commands.is_empty() {
634            log::debug!(
635                "Processing {} buffered commands",
636                self.buffered_commands.len()
637            );
638
639            for cmd in self.buffered_commands.drain(..) {
640                match cmd {
641                    HandlerCommand::Subscribe(sub) => {
642                        if !self.replay && sub.start.is_some() {
643                            self.replay = true;
644                        }
645                        self.subscriptions.push(sub);
646                    }
647                    HandlerCommand::SetPricePrecision(symbol, precision) => {
648                        self.price_precision_overrides.insert(symbol, precision);
649                    }
650                    HandlerCommand::Start => {
651                        start_buffered = true;
652                    }
653                    HandlerCommand::Close => {
654                        log::warn!("Close command was buffered, shutting down");
655                        return Ok(false);
656                    }
657                }
658            }
659        }
660
661        let mut running = false;
662
663        if !self.subscriptions.is_empty() {
664            log::info!(
665                "Resubscribing to {} subscriptions",
666                self.subscriptions.len()
667            );
668
669            // Anchors remain only when the first connection failed, so this session replays
670            let replay = self.subscriptions.iter().any(|sub| sub.start.is_some());
671
672            for sub in self.subscriptions.clone() {
673                client.subscribe(sub).await?;
674            }
675            // Strip start timestamps after successful subscription to avoid replaying history on future reconnects
676            for sub in &mut self.subscriptions {
677                sub.start = None;
678            }
679
680            buffering_start = replay.then(|| clock.get_time_ns());
681            client.start().await?;
682            running = true;
683            log::info!("Resubscription complete");
684        } else if start_buffered {
685            log::debug!("Starting session from buffered Start command");
686            buffering_start = if self.replay {
687                Some(clock.get_time_ns())
688            } else {
689                None
690            };
691            client.start().await?;
692            running = true;
693        }
694
695        loop {
696            if self.msg_tx.is_closed() {
697                log::debug!("Message channel was closed: stopping");
698                return Ok(false);
699            }
700
701            // Wait for either a command or a record. When the session has not
702            // started yet (`!running`), only commands are awaited. Once running,
703            // `next_record` is cancel-safe so `tokio::select!` can safely
704            // race both futures.
705            if !running {
706                match self.cmd_rx.recv().await {
707                    Some(HandlerCommand::Subscribe(sub)) => {
708                        log::debug!("Received command: Subscribe");
709
710                        if !self.replay && sub.start.is_some() {
711                            self.replay = true;
712                        }
713                        client.subscribe(sub.clone()).await?;
714                        let mut sub_for_reconnect = sub;
715                        sub_for_reconnect.start = None;
716                        self.subscriptions.push(sub_for_reconnect);
717                        continue;
718                    }
719                    Some(HandlerCommand::SetPricePrecision(symbol, precision)) => {
720                        log::debug!(
721                            "Received command: SetPricePrecision for {symbol} to {precision}"
722                        );
723                        self.price_precision_overrides.insert(symbol, precision);
724                        continue;
725                    }
726                    Some(HandlerCommand::Start) => {
727                        log::debug!("Received command: Start");
728                        buffering_start = if self.replay {
729                            Some(clock.get_time_ns())
730                        } else {
731                            None
732                        };
733                        client.start().await?;
734                        running = true;
735                        continue;
736                    }
737                    Some(HandlerCommand::Close) => {
738                        self.send_close_msg();
739                        return Ok(false);
740                    }
741                    None => {
742                        log::debug!("Command channel disconnected");
743                        return Ok(false);
744                    }
745                }
746            }
747
748            let record_opt = tokio::select! {
749                cmd = self.cmd_rx.recv() =>
750                match cmd {
751                    Some(HandlerCommand::Subscribe(sub)) => {
752                        log::debug!("Received command: Subscribe");
753
754                        if sub.start.is_some() {
755                            self.replay = true;
756                            log::error!(
757                                "Ignoring `start` on {} subscribe, session already running, Databento drops replay anchors sent after session start",
758                                self.dataset,
759                            );
760                        }
761                        client.subscribe(sub.clone()).await?;
762                        let mut sub_for_reconnect = sub;
763                        sub_for_reconnect.start = None;
764                        self.subscriptions.push(sub_for_reconnect);
765                        continue;
766                    }
767                    Some(HandlerCommand::SetPricePrecision(symbol, precision)) => {
768                        log::debug!(
769                            "Received command: SetPricePrecision for {symbol} to {precision}"
770                        );
771                        self.price_precision_overrides.insert(symbol, precision);
772                        continue;
773                    }
774                    Some(HandlerCommand::Start) => {
775                        log::warn!("Received Start command but session already running");
776                        continue;
777                    }
778                    Some(HandlerCommand::Close) => {
779                        self.send_close_msg();
780                        client.close().await?;
781                        log::debug!("Closed inner client");
782                        return Ok(false);
783                    }
784                    None => {
785                        log::debug!("Command channel disconnected");
786                        return Ok(false);
787                    }
788                },
789                result = client.next_record() => result,
790            };
791
792            let record = match record_opt {
793                Ok(Some(record)) => record,
794                Ok(None) => {
795                    if session_start.elapsed() >= self.success_threshold {
796                        log::debug!("Session ended after successful run");
797                        return Ok(true);
798                    }
799                    anyhow::bail!("Session ended by gateway");
800                }
801                Err(e) => {
802                    if session_start.elapsed() >= self.success_threshold {
803                        log::debug!("Connection error after successful run: {e}");
804                        return Ok(true);
805                    }
806                    anyhow::bail!("Connection error: {e}");
807                }
808            };
809
810            let ts_init = clock.get_time_ns();
811
812            if let Err(e) =
813                self.cache_futures_instrument_id(&record, &symbol_map, &mut instrument_id_map)
814            {
815                log::error!("Error resolving futures instrument ID: {e}");
816                continue;
817            }
818
819            // Decode record
820            if let Some(msg) = record.get::<dbn::ErrorMsg>() {
821                handle_error_msg(msg);
822            } else if let Some(msg) = record.get::<dbn::SystemMsg>() {
823                if let Some(ack) = handle_system_msg(msg, ts_init) {
824                    self.send_msg(DatabentoMessage::SubscriptionAck(ack));
825                }
826            } else if let Some(msg) = record.get::<dbn::SymbolMappingMsg>() {
827                instrument_def_price_precision_map.remove(&msg.hd.instrument_id);
828                update_price_precision_map_with_symbol_mapping_msg(
829                    msg,
830                    &self.price_precision_overrides,
831                    &mut subscription_price_precision_map,
832                )?;
833                handle_symbol_mapping_msg(
834                    msg,
835                    &mut symbol_map,
836                    &self.symbol_venue_map,
837                    &mut instrument_id_map,
838                )?;
839            } else if let Some(msg) = record.get::<dbn::InstrumentDefMsg>() {
840                let maybe_data = self.handle_instrument_def_msg(
841                    msg,
842                    &record,
843                    &symbol_map,
844                    &mut instrument_id_map,
845                    ts_init,
846                )?;
847
848                if let Some(data) = maybe_data {
849                    instrument_def_price_precision_map
850                        .insert(msg.hd.instrument_id, data.price_precision());
851                    self.send_msg(DatabentoMessage::Instrument(Box::new(data)));
852                }
853            } else if let Some(msg) = record.get::<dbn::StatusMsg>() {
854                let data = handle_status_msg(
855                    msg,
856                    &record,
857                    &symbol_map,
858                    &self.publisher_venue_map,
859                    &self.symbol_venue_map,
860                    &mut instrument_id_map,
861                    ts_init,
862                )?;
863                self.send_msg(DatabentoMessage::Status(data));
864            } else if let Some(msg) = record.get::<dbn::ImbalanceMsg>() {
865                handle_imbalance_msg(
866                    msg,
867                    &record,
868                    &symbol_map,
869                    &self.publisher_venue_map,
870                    &self.symbol_venue_map,
871                    &mut instrument_id_map,
872                    &instrument_def_price_precision_map,
873                    &subscription_price_precision_map,
874                    &self.price_precision_overrides,
875                    ts_init,
876                )
877                .map_or_else(
878                    |e| log::error!("Error decoding imbalance: {e}"),
879                    |data| self.send_msg(DatabentoMessage::Imbalance(data)),
880                );
881            } else if let Some(msg) = record.get::<dbn::StatMsg>() {
882                let maybe_data = handle_statistics_msg(
883                    msg,
884                    &record,
885                    &symbol_map,
886                    &self.publisher_venue_map,
887                    &self.symbol_venue_map,
888                    &mut instrument_id_map,
889                    &instrument_def_price_precision_map,
890                    &subscription_price_precision_map,
891                    &self.price_precision_overrides,
892                    ts_init,
893                )?;
894
895                if let Some(data) = maybe_data {
896                    self.send_msg(DatabentoMessage::Statistics(data));
897                }
898            } else {
899                // Decode a generic record with possible errors
900                let res = handle_record(
901                    record,
902                    &symbol_map,
903                    &self.publisher_venue_map,
904                    &self.symbol_venue_map,
905                    &mut instrument_id_map,
906                    &instrument_def_price_precision_map,
907                    &subscription_price_precision_map,
908                    &self.price_precision_overrides,
909                    ts_init,
910                    &initialized_books,
911                    self.bars_timestamp_on_close,
912                );
913                let (mut data1, data2) = match res {
914                    Ok(decoded) => decoded,
915                    Err(e) => {
916                        log::error!("Error decoding record: {e}");
917
918                        // Preserve raw MBO event boundaries when the payload is rejected
919                        (None, None)
920                    }
921                };
922
923                if let Some(msg) = record.get::<dbn::MboMsg>() {
924                    if let Some(Data::BookDelta(delta)) = &data1 {
925                        initialized_books.insert(delta.instrument_id);
926
927                        log::trace!(
928                            "Buffering delta: {} {buffering_start:?} flags={}",
929                            delta.ts_event,
930                            msg.flags.raw(),
931                        );
932
933                        match process_mbo_delta(
934                            *delta,
935                            msg.flags.raw(),
936                            &mut buffering_start,
937                            &mut buffered_deltas,
938                        ) {
939                            Some(deltas) => data1 = Some(Data::BookDeltas(Box::new(deltas))),
940                            None => continue,
941                        }
942                    } else {
943                        // Records that decode to no delta (`Action::Fill`
944                        // attribution, `Action::None` status, or trades) may
945                        // still carry the match-event boundary: honor the raw
946                        // `F_LAST` flag or a buffered partial event is
947                        // stranded and merged into the next event (e.g. a
948                        // non-terminal 'C' followed by 'N' | F_LAST).
949                        let deltas = instrument_id_map.get(&msg.hd.instrument_id).and_then(|id| {
950                            flush_mbo_event_boundary(
951                                id.id(),
952                                msg.ts_recv.into(),
953                                msg.flags.raw(),
954                                &mut buffering_start,
955                                &mut buffered_deltas,
956                            )
957                        });
958
959                        if let Some(deltas) = deltas {
960                            self.send_msg(DatabentoMessage::Data(Data::BookDeltas(Box::new(
961                                deltas,
962                            ))));
963                        }
964
965                        // Replayed trades stay out of the live stream
966                        if buffering_start.is_some_and(|start| msg.ts_recv <= start) {
967                            continue;
968                        }
969                    }
970                }
971
972                let rtype = record.header().rtype;
973
974                for data in [data1, data2].into_iter().flatten() {
975                    if claim_trade_source(&data, rtype, &mut trade_sources) {
976                        self.send_msg(DatabentoMessage::Data(data));
977                    }
978                }
979            }
980        }
981    }
982
983    /// Handles instrument definition messages and decodes them into Nautilus instruments.
984    ///
985    /// # Errors
986    ///
987    /// Returns an error if instrument decoding fails.
988    fn handle_instrument_def_msg(
989        &self,
990        msg: &dbn::InstrumentDefMsg,
991        record: &dbn::RecordRef,
992        symbol_map: &PitSymbolMap,
993        instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
994        ts_init: UnixNanos,
995    ) -> anyhow::Result<Option<InstrumentAny>> {
996        instrument_id_map.remove(&msg.hd.instrument_id);
997        let exchange = if self.use_exchange_as_venue {
998            msg.exchange()?
999        } else {
1000            ""
1001        };
1002
1003        if !exchange.is_empty() {
1004            update_instrument_id_map_with_exchange(
1005                symbol_map,
1006                &self.symbol_venue_map,
1007                instrument_id_map,
1008                msg.hd.instrument_id,
1009                exchange,
1010            )?;
1011        }
1012
1013        let instrument_id = update_instrument_id_map(
1014            record,
1015            symbol_map,
1016            &self.publisher_venue_map,
1017            &self.symbol_venue_map,
1018            instrument_id_map,
1019        )?;
1020        let mut instrument_id = resolve_futures_instrument_id(
1021            record,
1022            instrument_id,
1023            self.futures_year_digits,
1024            &self.symbol_venue_map.load(),
1025            None,
1026        )?;
1027
1028        if exchange.is_empty()
1029            && let Some(venue) = self.symbol_venue_map.get_cloned(&instrument_id.symbol)
1030        {
1031            instrument_id.venue = venue;
1032        }
1033
1034        let instrument = decode_instrument_def_msg(msg, instrument_id, Some(ts_init), None)?;
1035
1036        let cached = if instrument.is_some()
1037            && self.futures_year_digits == FuturesYearDigits::Two
1038            && dataset_uses_futures_symbology(&self.dataset)
1039        {
1040            if exchange.is_empty() {
1041                CachedInstrumentId::Definition(instrument_id)
1042            } else {
1043                CachedInstrumentId::Exchange(instrument_id)
1044            }
1045        } else {
1046            CachedInstrumentId::Inferred(instrument_id)
1047        };
1048
1049        instrument_id_map.insert(record.header().instrument_id, cached);
1050        Ok(instrument)
1051    }
1052
1053    fn cache_futures_instrument_id(
1054        &self,
1055        record: &dbn::RecordRef,
1056        symbol_map: &PitSymbolMap,
1057        instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1058    ) -> anyhow::Result<()> {
1059        if self.futures_year_digits != FuturesYearDigits::Two
1060            || !dataset_uses_futures_symbology(&self.dataset)
1061            || record.get::<dbn::InstrumentDefMsg>().is_some()
1062            || record.get::<dbn::SymbolMappingMsg>().is_some()
1063            || instrument_id_map.contains_key(&record.header().instrument_id)
1064            || symbol_map.get_for_rec(record).is_none()
1065        {
1066            return Ok(());
1067        }
1068
1069        let instrument_id = update_instrument_id_map(
1070            record,
1071            symbol_map,
1072            &self.publisher_venue_map,
1073            &self.symbol_venue_map,
1074            instrument_id_map,
1075        )?;
1076        let mut instrument_id = resolve_futures_instrument_id(
1077            record,
1078            instrument_id,
1079            self.futures_year_digits,
1080            &self.symbol_venue_map.load(),
1081            None,
1082        )?;
1083
1084        if let Some(venue) = self.symbol_venue_map.get_cloned(&instrument_id.symbol) {
1085            instrument_id.venue = venue;
1086        }
1087
1088        instrument_id_map.insert(
1089            record.header().instrument_id,
1090            CachedInstrumentId::Inferred(instrument_id),
1091        );
1092        Ok(())
1093    }
1094
1095    /// Sends a message to the message processing task.
1096    fn send_msg(&self, msg: DatabentoMessage) {
1097        log::trace!("Sending {msg:?}");
1098        match self.msg_tx.send(msg) {
1099            Ok(()) => {}
1100            Err(e) => log::debug!("Error sending message: {e}"),
1101        }
1102    }
1103
1104    fn send_close_msg(&self) {
1105        if let Err(e) = self.msg_tx.send(DatabentoMessage::Close) {
1106            log::debug!("Could not send close message: {e}");
1107        }
1108    }
1109}
1110
1111#[derive(Debug, Clone, Copy)]
1112enum CachedInstrumentId {
1113    Inferred(InstrumentId),
1114    Definition(InstrumentId),
1115    Exchange(InstrumentId),
1116}
1117
1118/// Handles Databento error messages by logging them.
1119fn handle_error_msg(msg: &dbn::ErrorMsg) {
1120    log::error!("{msg:?}");
1121}
1122
1123/// Handles Databento system messages, returning a subscription ack event if applicable.
1124fn handle_system_msg(msg: &dbn::SystemMsg, ts_received: UnixNanos) -> Option<SubscriptionAckEvent> {
1125    match msg.code() {
1126        Ok(dbn::SystemCode::SubscriptionAck) => {
1127            let message = msg.msg().unwrap_or("<invalid utf-8>");
1128            log::debug!("Subscription acknowledged: {message}");
1129
1130            let schema = parse_ack_message(message);
1131
1132            Some(SubscriptionAckEvent {
1133                schema,
1134                message: message.to_string(),
1135                ts_received,
1136            })
1137        }
1138        Ok(dbn::SystemCode::Heartbeat) => {
1139            log::trace!("Heartbeat received");
1140            None
1141        }
1142        Ok(dbn::SystemCode::SlowReaderWarning) => {
1143            let message = msg.msg().unwrap_or("<invalid utf-8>");
1144            log::warn!("Slow reader warning: {message}");
1145            None
1146        }
1147        Ok(dbn::SystemCode::ReplayCompleted) => {
1148            let message = msg.msg().unwrap_or("<invalid utf-8>");
1149            log::debug!("Replay completed: {message}");
1150            None
1151        }
1152        _ => {
1153            log::debug!("{msg:?}");
1154            None
1155        }
1156    }
1157}
1158
1159/// Parses a subscription ack message to extract the schema.
1160fn parse_ack_message(message: &str) -> String {
1161    // Format: "Subscription request N for <schema> data succeeded"
1162    message
1163        .strip_circumfix("Subscription request ", " data succeeded")
1164        .and_then(|rest| rest.split_once(" for "))
1165        .map(|(_, schema)| schema.trim().to_string())
1166        .unwrap_or_default()
1167}
1168
1169/// Handles symbol mapping messages and updates the instrument ID map.
1170///
1171/// # Errors
1172///
1173/// Returns an error if symbol mapping fails.
1174fn handle_symbol_mapping_msg(
1175    msg: &dbn::SymbolMappingMsg,
1176    symbol_map: &mut PitSymbolMap,
1177    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1178    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1179) -> anyhow::Result<()> {
1180    let changed =
1181        symbol_map.get(msg.hd.instrument_id).map(String::as_str) != Some(msg.stype_out_symbol()?);
1182    symbol_map
1183        .on_symbol_mapping(msg)
1184        .map_err(|e| anyhow::anyhow!("on_symbol_mapping failed for {msg:?}: {e}"))?;
1185
1186    match instrument_id_map.get(&msg.hd.instrument_id).copied() {
1187        Some(CachedInstrumentId::Definition(mut id)) if !changed => {
1188            if let Some(venue) = symbol_venue_map.get_cloned(&id.symbol) {
1189                id.venue = venue;
1190            }
1191
1192            instrument_id_map.insert(msg.hd.instrument_id, CachedInstrumentId::Definition(id));
1193        }
1194        Some(CachedInstrumentId::Exchange(_)) if !changed => {}
1195        _ => {
1196            instrument_id_map.remove(&msg.hd.instrument_id);
1197        }
1198    }
1199
1200    Ok(())
1201}
1202
1203fn update_price_precision_map_with_symbol_mapping_msg(
1204    msg: &dbn::SymbolMappingMsg,
1205    price_precision_overrides: &AHashMap<Symbol, u8>,
1206    subscription_price_precision_map: &mut AHashMap<u32, u8>,
1207) -> anyhow::Result<()> {
1208    subscription_price_precision_map.remove(&msg.hd.instrument_id);
1209
1210    if price_precision_overrides.is_empty() {
1211        return Ok(());
1212    }
1213
1214    let stype_in_symbol = msg
1215        .stype_in_symbol()
1216        .map_err(|e| anyhow::anyhow!("Error decoding `stype_in_symbol`: {e}"))?;
1217    let stype_out_symbol = msg
1218        .stype_out_symbol()
1219        .map_err(|e| anyhow::anyhow!("Error decoding `stype_out_symbol`: {e}"))?;
1220
1221    let price_precision = [stype_in_symbol, stype_out_symbol]
1222        .into_iter()
1223        .find_map(|symbol| {
1224            price_precision_overrides
1225                .get(&Symbol::from_str_unchecked(symbol))
1226                .copied()
1227        });
1228
1229    if let Some(price_precision) = price_precision {
1230        subscription_price_precision_map.insert(msg.hd.instrument_id, price_precision);
1231    }
1232
1233    Ok(())
1234}
1235
1236/// Updates the instrument ID map using exchange information from the symbol map.
1237fn update_instrument_id_map_with_exchange(
1238    symbol_map: &PitSymbolMap,
1239    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1240    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1241    raw_instrument_id: u32,
1242    exchange: &str,
1243) -> anyhow::Result<InstrumentId> {
1244    let raw_symbol = symbol_map.get(raw_instrument_id).ok_or_else(|| {
1245        anyhow::anyhow!("Cannot resolve raw_symbol for instrument_id {raw_instrument_id}")
1246    })?;
1247    let symbol = Symbol::from(raw_symbol.as_str());
1248    let venue = Venue::from_code(exchange)
1249        .map_err(|e| anyhow::anyhow!("Invalid venue code '{exchange}': {e}"))?;
1250    let instrument_id = InstrumentId::new(symbol, venue);
1251    symbol_venue_map.rcu(|m| {
1252        m.entry(symbol).or_insert(venue);
1253    });
1254
1255    instrument_id_map.insert(
1256        raw_instrument_id,
1257        CachedInstrumentId::Exchange(instrument_id),
1258    );
1259    Ok(instrument_id)
1260}
1261
1262fn update_instrument_id_map(
1263    record: &dbn::RecordRef,
1264    symbol_map: &PitSymbolMap,
1265    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1266    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1267    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1268) -> anyhow::Result<InstrumentId> {
1269    let header = record.header();
1270
1271    // Check if instrument ID is already in the map
1272    if let Some(&instrument_id) = instrument_id_map.get(&header.instrument_id) {
1273        return Ok(instrument_id.id());
1274    }
1275
1276    let raw_symbol = symbol_map.get_for_rec(record).ok_or_else(|| {
1277        anyhow::anyhow!(
1278            "Cannot resolve `raw_symbol` from `symbol_map` for instrument_id {}",
1279            header.instrument_id
1280        )
1281    })?;
1282
1283    let symbol = Symbol::from_str_unchecked(raw_symbol);
1284
1285    let publisher_id = header.publisher_id;
1286    let venue = if let Some(venue) = symbol_venue_map.get_cloned(&symbol) {
1287        venue
1288    } else {
1289        let venue = publisher_venue_map
1290            .get(&publisher_id)
1291            .ok_or_else(|| anyhow::anyhow!("No venue found for `publisher_id` {publisher_id}"))?;
1292        *venue
1293    };
1294    let instrument_id = InstrumentId::new(symbol, venue);
1295
1296    instrument_id_map.insert(
1297        header.instrument_id,
1298        CachedInstrumentId::Inferred(instrument_id),
1299    );
1300    Ok(instrument_id)
1301}
1302
1303impl CachedInstrumentId {
1304    fn id(self) -> InstrumentId {
1305        match self {
1306            Self::Inferred(id) | Self::Definition(id) | Self::Exchange(id) => id,
1307        }
1308    }
1309}
1310
1311fn handle_status_msg(
1312    msg: &dbn::StatusMsg,
1313    record: &dbn::RecordRef,
1314    symbol_map: &PitSymbolMap,
1315    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1316    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1317    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1318    ts_init: UnixNanos,
1319) -> anyhow::Result<InstrumentStatus> {
1320    let instrument_id = update_instrument_id_map(
1321        record,
1322        symbol_map,
1323        publisher_venue_map,
1324        symbol_venue_map,
1325        instrument_id_map,
1326    )?;
1327
1328    decode_status_msg(msg, instrument_id, Some(ts_init))
1329}
1330
1331#[expect(clippy::too_many_arguments)]
1332fn handle_imbalance_msg(
1333    msg: &dbn::ImbalanceMsg,
1334    record: &dbn::RecordRef,
1335    symbol_map: &PitSymbolMap,
1336    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1337    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1338    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1339    instrument_def_price_precision_map: &AHashMap<u32, u8>,
1340    subscription_price_precision_map: &AHashMap<u32, u8>,
1341    price_precision_overrides: &AHashMap<Symbol, u8>,
1342    ts_init: UnixNanos,
1343) -> anyhow::Result<DatabentoImbalance> {
1344    let instrument_id = update_instrument_id_map(
1345        record,
1346        symbol_map,
1347        publisher_venue_map,
1348        symbol_venue_map,
1349        instrument_id_map,
1350    )?;
1351
1352    let price_precision = resolve_price_precision(
1353        msg.hd.instrument_id,
1354        symbol_map.get_for_rec(record).map(String::as_str),
1355        instrument_def_price_precision_map,
1356        subscription_price_precision_map,
1357        price_precision_overrides,
1358    );
1359
1360    decode_imbalance_msg(msg, instrument_id, price_precision, Some(ts_init))
1361}
1362
1363#[expect(clippy::too_many_arguments)]
1364fn handle_statistics_msg(
1365    msg: &dbn::StatMsg,
1366    record: &dbn::RecordRef,
1367    symbol_map: &PitSymbolMap,
1368    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1369    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1370    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1371    instrument_def_price_precision_map: &AHashMap<u32, u8>,
1372    subscription_price_precision_map: &AHashMap<u32, u8>,
1373    price_precision_overrides: &AHashMap<Symbol, u8>,
1374    ts_init: UnixNanos,
1375) -> anyhow::Result<Option<DatabentoStatistics>> {
1376    // Precheck before symbol resolution so unmodeled types skip cleanly
1377    if !is_supported_stat_type(msg.stat_type) {
1378        log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
1379        return Ok(None);
1380    }
1381
1382    let instrument_id = update_instrument_id_map(
1383        record,
1384        symbol_map,
1385        publisher_venue_map,
1386        symbol_venue_map,
1387        instrument_id_map,
1388    )?;
1389
1390    let price_precision = resolve_price_precision(
1391        msg.hd.instrument_id,
1392        symbol_map.get_for_rec(record).map(String::as_str),
1393        instrument_def_price_precision_map,
1394        subscription_price_precision_map,
1395        price_precision_overrides,
1396    );
1397
1398    decode_statistics_msg(msg, instrument_id, price_precision, Some(ts_init))
1399}
1400
1401#[expect(clippy::too_many_arguments)]
1402fn handle_record(
1403    record: dbn::RecordRef,
1404    symbol_map: &PitSymbolMap,
1405    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1406    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1407    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1408    instrument_def_price_precision_map: &AHashMap<u32, u8>,
1409    subscription_price_precision_map: &AHashMap<u32, u8>,
1410    price_precision_overrides: &AHashMap<Symbol, u8>,
1411    ts_init: UnixNanos,
1412    initialized_books: &HashSet<InstrumentId>,
1413    bars_timestamp_on_close: bool,
1414) -> anyhow::Result<(Option<Data>, Option<Data>)> {
1415    let instrument_id = update_instrument_id_map(
1416        &record,
1417        symbol_map,
1418        publisher_venue_map,
1419        symbol_venue_map,
1420        instrument_id_map,
1421    )?;
1422
1423    let price_precision = resolve_price_precision(
1424        record.header().instrument_id,
1425        symbol_map.get_for_rec(&record).map(String::as_str),
1426        instrument_def_price_precision_map,
1427        subscription_price_precision_map,
1428        price_precision_overrides,
1429    );
1430
1431    // For MBP-1 and quote-based schemas, always include trades since they're integral to the data
1432    // For MBO, only include trades after the book is initialized to maintain consistency
1433    let include_trades = if record.get::<dbn::Mbp1Msg>().is_some()
1434        || record.get::<dbn::TbboMsg>().is_some()
1435        || record.get::<dbn::Cmbp1Msg>().is_some()
1436    {
1437        true // These schemas include trade information directly
1438    } else {
1439        initialized_books.contains(&instrument_id) // MBO requires initialized book
1440    };
1441
1442    decode_record(
1443        &record,
1444        instrument_id,
1445        price_precision,
1446        Some(ts_init),
1447        include_trades,
1448        bars_timestamp_on_close,
1449    )
1450}
1451
1452/// Claims the instrument's trade source for a trade decoded from a record of `rtype`, returning
1453/// whether `data` should be sent.
1454///
1455/// The `trades`, `mbp-1`, `tbbo`, `cmbp-1`, `tcbbo`, and `mbo` schemas all carry the same venue
1456/// trades, so an instrument subscribed on several of them would receive each trade once per schema.
1457/// The first record type to deliver a trade for an instrument carries its trades for the session,
1458/// and trades decoded from any other record type are dropped. A later subscription never takes
1459/// over, since its stream starts at an unknown point and a handover could drop or repeat trades.
1460/// Data other than trades always passes.
1461fn claim_trade_source(
1462    data: &Data,
1463    rtype: u8,
1464    trade_sources: &mut AHashMap<InstrumentId, u8>,
1465) -> bool {
1466    match data {
1467        Data::Trade(trade) => *trade_sources.entry(trade.instrument_id).or_insert(rtype) == rtype,
1468        _ => true,
1469    }
1470}
1471
1472fn resolve_price_precision(
1473    record_instrument_id: u32,
1474    raw_symbol: Option<&str>,
1475    instrument_def_price_precision_map: &AHashMap<u32, u8>,
1476    subscription_price_precision_map: &AHashMap<u32, u8>,
1477    price_precision_overrides: &AHashMap<Symbol, u8>,
1478) -> u8 {
1479    subscription_price_precision_map
1480        .get(&record_instrument_id)
1481        .copied()
1482        .or_else(|| {
1483            raw_symbol.and_then(|symbol| {
1484                price_precision_overrides
1485                    .get(&Symbol::from_str_unchecked(symbol))
1486                    .copied()
1487            })
1488        })
1489        .or_else(|| {
1490            instrument_def_price_precision_map
1491                .get(&record_instrument_id)
1492                .copied()
1493        })
1494        .unwrap_or(Currency::USD().precision)
1495}
1496
1497/// Processes an MBO delta through the buffering state machine.
1498///
1499/// Returns `Some(deltas)` when a complete batch is ready to emit (non-snapshot
1500/// F_LAST with replay buffering complete), or `None` when still accumulating.
1501fn process_mbo_delta(
1502    delta: OrderBookDelta,
1503    flags: u8,
1504    buffering_start: &mut Option<UnixNanos>,
1505    buffered_deltas: &mut AHashMap<InstrumentId, Vec<OrderBookDelta>>,
1506) -> Option<OrderBookDeltas> {
1507    let is_last = RecordFlag::F_LAST.matches(flags);
1508    let is_snapshot = RecordFlag::F_SNAPSHOT.matches(flags);
1509
1510    // Most live MBO events are single non-snapshot deltas, avoid map churn on that path
1511    if is_last
1512        && !is_snapshot
1513        && buffering_start.is_none()
1514        && !buffered_deltas.contains_key(&delta.instrument_id)
1515    {
1516        let deltas = OrderBookDeltas::new(delta.instrument_id, vec![delta]);
1517        return Some(deltas);
1518    }
1519
1520    let buffer = buffered_deltas.entry(delta.instrument_id).or_default();
1521    buffer.push(delta);
1522
1523    if is_snapshot {
1524        return None;
1525    }
1526
1527    flush_mbo_event_boundary(
1528        delta.instrument_id,
1529        delta.ts_event,
1530        flags,
1531        buffering_start,
1532        buffered_deltas,
1533    )
1534}
1535
1536/// Flushes the buffered deltas for `instrument_id` when `flags` marks a
1537/// non-snapshot match-event boundary (`F_LAST`).
1538///
1539/// Databento documents that records which decode to no delta (`Action::Fill`,
1540/// `Action::None`) may carry `F_LAST`, so the boundary must be processed
1541/// independently of the decoded payload or a buffered partial event is
1542/// stranded (follow-up to #4445).
1543fn flush_mbo_event_boundary(
1544    instrument_id: InstrumentId,
1545    ts_event: UnixNanos,
1546    flags: u8,
1547    buffering_start: &mut Option<UnixNanos>,
1548    buffered_deltas: &mut AHashMap<InstrumentId, Vec<OrderBookDelta>>,
1549) -> Option<OrderBookDeltas> {
1550    if !RecordFlag::F_LAST.matches(flags) || RecordFlag::F_SNAPSHOT.matches(flags) {
1551        return None;
1552    }
1553
1554    let buffer = buffered_deltas.get(&instrument_id)?;
1555
1556    if buffer.is_empty() {
1557        return None;
1558    }
1559
1560    if let Some(start_ns) = *buffering_start {
1561        if ts_event <= start_ns {
1562            return None;
1563        }
1564        *buffering_start = None;
1565    }
1566
1567    let buffer = buffered_deltas.remove(&instrument_id)?;
1568    let deltas = OrderBookDeltas::new(instrument_id, buffer);
1569    Some(deltas)
1570}
1571
1572#[cfg(test)]
1573mod tests {
1574    use std::path::PathBuf;
1575
1576    use databento::{dbn::enums::rtype, live::Subscription};
1577    use indexmap::IndexMap;
1578    use nautilus_model::types::{Price, Quantity};
1579    use rstest::*;
1580    use time::macros::datetime;
1581
1582    use super::*;
1583
1584    fn stub_delta(instrument_id: InstrumentId, ts: u64) -> OrderBookDelta {
1585        use nautilus_model::{
1586            data::BookOrder,
1587            enums::{BookAction, OrderSide},
1588            types::{Price, Quantity},
1589        };
1590
1591        OrderBookDelta::new(
1592            instrument_id,
1593            BookAction::Delete,
1594            BookOrder::new(
1595                OrderSide::Sell,
1596                Price::from("100.00"),
1597                Quantity::from("5"),
1598                42,
1599            ),
1600            0, // non-terminal: no F_LAST
1601            1,
1602            ts.into(),
1603            ts.into(),
1604        )
1605    }
1606
1607    #[rstest]
1608    fn test_boundary_flag_on_recordless_message_flushes_buffered_event() {
1609        // A non-terminal 'C' delta buffers; the event terminates on an 'N'
1610        // record carrying F_LAST which decodes to no delta - the raw flag
1611        // must still flush the buffer (follow-up to #4445).
1612        let instrument_id = InstrumentId::from("TEST.GLBX");
1613        let mut buffering_start = None;
1614        let mut buffered = AHashMap::new();
1615
1616        let buffered_result = process_mbo_delta(
1617            stub_delta(instrument_id, 1),
1618            0,
1619            &mut buffering_start,
1620            &mut buffered,
1621        );
1622        assert!(buffered_result.is_none());
1623
1624        let flushed = flush_mbo_event_boundary(
1625            instrument_id,
1626            2.into(),
1627            RecordFlag::F_LAST as u8,
1628            &mut buffering_start,
1629            &mut buffered,
1630        );
1631        assert!(flushed.is_some());
1632        assert!(buffered.is_empty());
1633    }
1634
1635    #[rstest]
1636    fn test_boundary_flag_with_empty_buffer_is_noop() {
1637        let instrument_id = InstrumentId::from("TEST.GLBX");
1638        let start = UnixNanos::from(1);
1639        let mut buffering_start = Some(start);
1640        let mut buffered: AHashMap<InstrumentId, Vec<OrderBookDelta>> = AHashMap::new();
1641        buffered.insert(instrument_id, Vec::new());
1642
1643        let flushed = flush_mbo_event_boundary(
1644            instrument_id,
1645            2.into(),
1646            RecordFlag::F_LAST as u8,
1647            &mut buffering_start,
1648            &mut buffered,
1649        );
1650
1651        assert!(flushed.is_none());
1652        assert_eq!(buffering_start, Some(start));
1653    }
1654
1655    #[rstest]
1656    fn test_empty_boundary_preserves_replay_gate_for_buffered_instrument() {
1657        let empty_id = InstrumentId::from("EMPTY.GLBX");
1658        let buffered_id = InstrumentId::from("BUFFERED.GLBX");
1659        let start = UnixNanos::from(10);
1660        let mut buffering_start = Some(start);
1661        let mut buffered = AHashMap::new();
1662        let _ = process_mbo_delta(
1663            stub_delta(buffered_id, 1),
1664            0,
1665            &mut buffering_start,
1666            &mut buffered,
1667        );
1668
1669        let empty_boundary = flush_mbo_event_boundary(
1670            empty_id,
1671            11.into(),
1672            RecordFlag::F_LAST as u8,
1673            &mut buffering_start,
1674            &mut buffered,
1675        );
1676        let buffered_boundary = flush_mbo_event_boundary(
1677            buffered_id,
1678            5.into(),
1679            RecordFlag::F_LAST as u8,
1680            &mut buffering_start,
1681            &mut buffered,
1682        );
1683
1684        assert!(empty_boundary.is_none());
1685        assert!(buffered_boundary.is_none());
1686        assert_eq!(buffering_start, Some(start));
1687        assert!(buffered.contains_key(&buffered_id));
1688    }
1689
1690    #[rstest]
1691    fn test_boundary_flag_respects_replay_buffering_gate() {
1692        let instrument_id = InstrumentId::from("TEST.GLBX");
1693        let mut buffering_start = Some(UnixNanos::from(10));
1694        let mut buffered = AHashMap::new();
1695        let _ = process_mbo_delta(
1696            stub_delta(instrument_id, 1),
1697            0,
1698            &mut buffering_start,
1699            &mut buffered,
1700        );
1701
1702        // Boundary inside the replay gate: keep buffering
1703        let flushed = flush_mbo_event_boundary(
1704            instrument_id,
1705            5.into(),
1706            RecordFlag::F_LAST as u8,
1707            &mut buffering_start,
1708            &mut buffered,
1709        );
1710        assert!(flushed.is_none());
1711        assert!(!buffered.is_empty());
1712
1713        // Boundary past the gate: flush and clear the gate
1714        let flushed = flush_mbo_event_boundary(
1715            instrument_id,
1716            11.into(),
1717            RecordFlag::F_LAST as u8,
1718            &mut buffering_start,
1719            &mut buffered,
1720        );
1721        assert!(flushed.is_some());
1722        assert!(buffering_start.is_none());
1723    }
1724
1725    #[rstest]
1726    fn test_boundary_snapshot_flag_never_flushes() {
1727        let instrument_id = InstrumentId::from("TEST.GLBX");
1728        let mut buffering_start = None;
1729        let mut buffered = AHashMap::new();
1730        let _ = process_mbo_delta(
1731            stub_delta(instrument_id, 1),
1732            0,
1733            &mut buffering_start,
1734            &mut buffered,
1735        );
1736
1737        let flags = RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8;
1738        let flushed = flush_mbo_event_boundary(
1739            instrument_id,
1740            2.into(),
1741            flags,
1742            &mut buffering_start,
1743            &mut buffered,
1744        );
1745        assert!(flushed.is_none());
1746    }
1747
1748    fn stub_trade(instrument_id: InstrumentId) -> Data {
1749        use nautilus_model::{
1750            data::TradeTick,
1751            enums::AggressorSide,
1752            identifiers::TradeId,
1753            types::{Price, Quantity},
1754        };
1755
1756        Data::Trade(TradeTick::new(
1757            instrument_id,
1758            Price::from("100.00"),
1759            Quantity::from("5"),
1760            AggressorSide::Buy,
1761            TradeId::new("1"),
1762            1.into(),
1763            1.into(),
1764        ))
1765    }
1766
1767    #[rstest]
1768    fn test_trade_source_is_first_record_type_to_deliver_a_trade() {
1769        let trade = stub_trade(InstrumentId::from("TEST.GLBX"));
1770        let mut trade_sources = AHashMap::new();
1771
1772        let sent: Vec<bool> = [rtype::MBP_1, rtype::MBP_0, rtype::MBO, rtype::MBP_1]
1773            .into_iter()
1774            .map(|rtype| claim_trade_source(&trade, rtype, &mut trade_sources))
1775            .collect();
1776
1777        assert_eq!(sent, [true, false, false, true]);
1778    }
1779
1780    #[rstest]
1781    fn test_trade_source_is_kept_per_instrument() {
1782        let es_trade = stub_trade(InstrumentId::from("ESM4.GLBX"));
1783        let nq_trade = stub_trade(InstrumentId::from("NQM4.GLBX"));
1784        let mut trade_sources = AHashMap::new();
1785
1786        let sent: Vec<bool> = [
1787            (&es_trade, rtype::MBP_0),
1788            (&nq_trade, rtype::MBO),
1789            (&es_trade, rtype::MBO),
1790            (&nq_trade, rtype::MBP_0),
1791        ]
1792        .into_iter()
1793        .map(|(trade, rtype)| claim_trade_source(trade, rtype, &mut trade_sources))
1794        .collect();
1795
1796        assert_eq!(sent, [true, true, false, false]);
1797    }
1798
1799    #[rstest]
1800    fn test_trade_source_passes_data_other_than_trades() {
1801        let instrument_id = InstrumentId::from("TEST.GLBX");
1802        let mut trade_sources = AHashMap::new();
1803        let _ = claim_trade_source(&stub_trade(instrument_id), rtype::MBP_0, &mut trade_sources);
1804        let delta = Data::BookDelta(stub_delta(instrument_id, 1));
1805
1806        let sent = claim_trade_source(&delta, rtype::MBO, &mut trade_sources);
1807
1808        assert!(sent);
1809    }
1810
1811    fn create_test_handler(reconnect_timeout_mins: Option<u64>) -> DatabentoFeedHandler {
1812        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1813        let (msg_tx, _msg_rx) = tokio::sync::mpsc::unbounded_channel();
1814
1815        DatabentoFeedHandler::new(
1816            Credential::new("test_key"),
1817            "GLBX.MDP3".to_string(),
1818            cmd_rx,
1819            msg_tx,
1820            IndexMap::new(),
1821            Arc::new(AtomicMap::new()),
1822            false,
1823            false,
1824            reconnect_timeout_mins,
1825        )
1826    }
1827
1828    fn create_test_client() -> DatabentoLiveClient {
1829        DatabentoLiveClient::new(
1830            "test-api-key".to_string(),
1831            "GLBX.MDP3".to_string(),
1832            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1833            true,
1834            None,
1835            None,
1836        )
1837        .unwrap()
1838    }
1839
1840    #[rstest]
1841    #[case(Some(10))]
1842    #[case(None)]
1843    fn test_backoff_initialization(#[case] reconnect_timeout_mins: Option<u64>) {
1844        let handler = create_test_handler(reconnect_timeout_mins);
1845
1846        assert_eq!(handler.reconnect_timeout_mins, reconnect_timeout_mins);
1847        assert!(handler.subscriptions.is_empty());
1848        assert!(handler.buffered_commands.is_empty());
1849    }
1850
1851    #[rstest]
1852    fn test_subscription_with_and_without_start() {
1853        let start_time = datetime!(2024-01-01 00:00:00 UTC);
1854        let sub_with_start = Subscription::builder()
1855            .symbols("ES.FUT")
1856            .schema(databento::dbn::Schema::Mbp1)
1857            .start(start_time)
1858            .build();
1859
1860        let mut sub_without_start = sub_with_start.clone();
1861        sub_without_start.start = None;
1862
1863        assert!(sub_with_start.start.is_some());
1864        assert!(sub_without_start.start.is_none());
1865        assert_eq!(sub_with_start.schema, sub_without_start.schema);
1866        assert_eq!(sub_with_start.symbols, sub_without_start.symbols);
1867    }
1868
1869    #[rstest]
1870    fn test_handler_initialization_state() {
1871        let handler = create_test_handler(Some(10));
1872
1873        assert!(!handler.replay);
1874        assert_eq!(handler.dataset, "GLBX.MDP3");
1875        assert_eq!(handler.credential.api_key(), "test_key");
1876        assert!(handler.subscriptions.is_empty());
1877        assert!(handler.buffered_commands.is_empty());
1878    }
1879
1880    #[rstest]
1881    fn test_handler_with_no_timeout() {
1882        let handler = create_test_handler(None);
1883
1884        assert_eq!(handler.reconnect_timeout_mins, None);
1885        assert!(!handler.replay);
1886    }
1887
1888    #[rstest]
1889    fn test_handler_with_zero_timeout() {
1890        let handler = create_test_handler(Some(0));
1891
1892        assert_eq!(handler.reconnect_timeout_mins, Some(0));
1893        assert!(!handler.replay);
1894    }
1895
1896    #[rstest]
1897    fn test_subscribe_uses_explicit_parent_stype() {
1898        let mut client = create_test_client();
1899
1900        client
1901            .subscribe(
1902                "definition".to_string(),
1903                vec![InstrumentId::from("ES.FUT.GLBX")],
1904                None,
1905                None,
1906                None,
1907                Some("parent".to_string()),
1908            )
1909            .unwrap();
1910
1911        let command = client.cmd_rx.as_mut().unwrap().try_recv().unwrap();
1912        match command {
1913            HandlerCommand::Subscribe(sub) => {
1914                assert_eq!(sub.schema, dbn::Schema::Definition);
1915                assert_eq!(sub.stype_in, dbn::SType::Parent);
1916                assert_eq!(sub.symbols.to_api_string(), "ES.FUT");
1917            }
1918            other => panic!("expected HandlerCommand::Subscribe, was {other:?}"),
1919        }
1920    }
1921
1922    #[rstest]
1923    fn test_subscribe_rejects_invalid_stype() {
1924        let mut client = create_test_client();
1925
1926        let err = client
1927            .subscribe(
1928                "definition".to_string(),
1929                vec![InstrumentId::from("ES.FUT.GLBX")],
1930                None,
1931                None,
1932                None,
1933                Some("not-a-stype".to_string()),
1934            )
1935            .unwrap_err();
1936
1937        assert!(err.to_string().contains("not-a-stype"));
1938        assert!(!is_command_send_error(&err));
1939        assert!(matches!(
1940            client.cmd_rx.as_mut().unwrap().try_recv(),
1941            Err(tokio::sync::mpsc::error::TryRecvError::Empty)
1942        ));
1943    }
1944
1945    #[rstest]
1946    fn test_subscribe_classifies_command_send_errors() {
1947        let mut client = create_test_client();
1948        client.cmd_rx = None;
1949
1950        let err = client
1951            .subscribe(
1952                "definition".to_string(),
1953                vec![InstrumentId::from("ES.FUT.GLBX")],
1954                None,
1955                None,
1956                None,
1957                Some("parent".to_string()),
1958            )
1959            .unwrap_err();
1960
1961        assert!(is_command_send_error(&err));
1962    }
1963
1964    #[rstest]
1965    fn test_close_after_handler_exit_marks_closed() {
1966        let mut client = create_test_client();
1967        let (handler, _msg_rx) = client.start().unwrap();
1968        drop(handler);
1969
1970        client.close().unwrap();
1971
1972        assert!(!client.is_running());
1973        assert!(client.is_closed());
1974    }
1975
1976    fn test_delta(instrument_id: InstrumentId, ts_event: u64) -> OrderBookDelta {
1977        OrderBookDelta::clear(instrument_id, 0, ts_event.into(), 0.into())
1978    }
1979
1980    #[rstest]
1981    fn test_mbo_delta_without_f_last_buffers() {
1982        let instrument_id = InstrumentId::from("ESM4.GLBX");
1983        let delta = test_delta(instrument_id, 1_000_000_000);
1984        let mut buffering_start = None;
1985        let mut buffered = AHashMap::new();
1986
1987        let result = process_mbo_delta(delta, 0, &mut buffering_start, &mut buffered);
1988
1989        assert!(result.is_none());
1990        assert_eq!(buffered[&instrument_id].len(), 1);
1991    }
1992
1993    #[rstest]
1994    fn test_mbo_single_f_last_emits_without_buffering() {
1995        let instrument_id = InstrumentId::from("ESM4.GLBX");
1996        let ts_event = 1_000_000_000;
1997        let mut delta = test_delta(instrument_id, ts_event);
1998        delta.flags = 128;
1999        delta.sequence = 42;
2000        let mut buffering_start = None;
2001        let mut buffered = AHashMap::new();
2002
2003        let result = process_mbo_delta(delta, delta.flags, &mut buffering_start, &mut buffered);
2004
2005        let emitted = result.expect("single F_LAST delta should emit");
2006        assert_eq!(emitted.instrument_id, instrument_id);
2007        assert_eq!(emitted.deltas.len(), 1);
2008        assert_eq!(emitted.flags, 128);
2009        assert_eq!(emitted.sequence, 42);
2010        assert_eq!(emitted.ts_event, UnixNanos::from(ts_event));
2011        assert_eq!(emitted.deltas[0].instrument_id, instrument_id);
2012        assert_eq!(emitted.deltas[0].flags, 128);
2013        assert_eq!(emitted.deltas[0].sequence, 42);
2014        assert_eq!(emitted.deltas[0].ts_event, UnixNanos::from(ts_event));
2015        assert!(buffering_start.is_none());
2016        assert!(buffered.is_empty());
2017    }
2018
2019    #[rstest]
2020    fn test_mbo_delta_with_f_last_emits() {
2021        let instrument_id = InstrumentId::from("ESM4.GLBX");
2022        let mut buffering_start = None;
2023        let mut buffered = AHashMap::new();
2024
2025        let _ = process_mbo_delta(
2026            test_delta(instrument_id, 1_000_000_000),
2027            0,
2028            &mut buffering_start,
2029            &mut buffered,
2030        );
2031
2032        let result = process_mbo_delta(
2033            test_delta(instrument_id, 2_000_000_000),
2034            128, // F_LAST
2035            &mut buffering_start,
2036            &mut buffered,
2037        );
2038
2039        assert!(result.is_some());
2040        assert_eq!(result.unwrap().deltas.len(), 2);
2041        assert!(buffered.is_empty());
2042    }
2043
2044    #[rstest]
2045    fn test_mbo_snapshot_with_f_last_buffers() {
2046        let instrument_id = InstrumentId::from("ESM4.GLBX");
2047        let mut buffering_start = None;
2048        let mut buffered = AHashMap::new();
2049
2050        let result = process_mbo_delta(
2051            test_delta(instrument_id, 1_000_000_000),
2052            128 | 32, // F_LAST | F_SNAPSHOT
2053            &mut buffering_start,
2054            &mut buffered,
2055        );
2056
2057        assert!(result.is_none());
2058        assert_eq!(buffered[&instrument_id].len(), 1);
2059    }
2060
2061    #[rstest]
2062    fn test_mbo_replay_buffers_until_past_start() {
2063        let instrument_id = InstrumentId::from("ESM4.GLBX");
2064        let start_ns = 5_000_000_000u64;
2065        let mut buffering_start = Some(start_ns.into());
2066        let mut buffered = AHashMap::new();
2067
2068        let result = process_mbo_delta(
2069            test_delta(instrument_id, 4_000_000_000),
2070            128, // F_LAST
2071            &mut buffering_start,
2072            &mut buffered,
2073        );
2074        assert!(result.is_none());
2075
2076        let result = process_mbo_delta(
2077            test_delta(instrument_id, 5_000_000_000),
2078            128,
2079            &mut buffering_start,
2080            &mut buffered,
2081        );
2082        assert!(result.is_none());
2083
2084        // Delta past start: emits and clears buffering_start
2085        let result = process_mbo_delta(
2086            test_delta(instrument_id, 6_000_000_000),
2087            128,
2088            &mut buffering_start,
2089            &mut buffered,
2090        );
2091        assert!(result.is_some());
2092        assert!(buffering_start.is_none());
2093    }
2094
2095    #[rstest]
2096    fn test_mbo_multiple_deltas_accumulated() {
2097        let instrument_id = InstrumentId::from("ESM4.GLBX");
2098        let mut buffering_start = None;
2099        let mut buffered = AHashMap::new();
2100
2101        for i in 0..5 {
2102            process_mbo_delta(
2103                test_delta(instrument_id, 1_000_000_000 + i),
2104                0,
2105                &mut buffering_start,
2106                &mut buffered,
2107            );
2108        }
2109
2110        let result = process_mbo_delta(
2111            test_delta(instrument_id, 2_000_000_000),
2112            128,
2113            &mut buffering_start,
2114            &mut buffered,
2115        );
2116
2117        assert!(result.is_some());
2118        assert_eq!(result.unwrap().deltas.len(), 6);
2119    }
2120
2121    #[rstest]
2122    fn test_mbo_multi_instrument_isolation() {
2123        let id_a = InstrumentId::from("ESM4.GLBX");
2124        let id_b = InstrumentId::from("NQM4.GLBX");
2125        let mut buffering_start = None;
2126        let mut buffered = AHashMap::new();
2127
2128        process_mbo_delta(
2129            test_delta(id_a, 1_000_000_000),
2130            0,
2131            &mut buffering_start,
2132            &mut buffered,
2133        );
2134        process_mbo_delta(
2135            test_delta(id_b, 1_000_000_000),
2136            0,
2137            &mut buffering_start,
2138            &mut buffered,
2139        );
2140
2141        // F_LAST for A: only A's deltas emitted, B remains
2142        let result = process_mbo_delta(
2143            test_delta(id_a, 2_000_000_000),
2144            128,
2145            &mut buffering_start,
2146            &mut buffered,
2147        );
2148
2149        assert!(result.is_some());
2150        assert_eq!(result.unwrap().instrument_id, id_a);
2151        assert!(buffered.contains_key(&id_b));
2152        assert!(!buffered.contains_key(&id_a));
2153    }
2154
2155    mod property_tests {
2156        use proptest::prelude::*;
2157        use rstest::rstest;
2158
2159        use super::*;
2160
2161        proptest! {
2162            #[rstest]
2163            fn mbo_buffering_conserves_deltas(
2164                num_non_last in 0usize..=20,
2165            ) {
2166                let instrument_id = InstrumentId::from("ESM4.GLBX");
2167                let mut buffering_start = None;
2168                let mut buffered = AHashMap::new();
2169                let total = num_non_last + 1;
2170
2171                for i in 0..num_non_last {
2172                    let result = process_mbo_delta(
2173                        test_delta(instrument_id, 1_000_000_000 + i as u64),
2174                        0, // No F_LAST
2175                        &mut buffering_start,
2176                        &mut buffered,
2177                    );
2178                    prop_assert!(result.is_none());
2179                }
2180
2181                let result = process_mbo_delta(
2182                    test_delta(instrument_id, 2_000_000_000),
2183                    128, // F_LAST
2184                    &mut buffering_start,
2185                    &mut buffered,
2186                );
2187
2188                prop_assert!(result.is_some());
2189                let emitted = result.unwrap();
2190                prop_assert_eq!(emitted.deltas.len(), total);
2191                prop_assert!(buffered.is_empty());
2192            }
2193
2194            #[rstest]
2195            fn mbo_snapshots_never_emit(
2196                num_snapshots in 1usize..=20,
2197            ) {
2198                let instrument_id = InstrumentId::from("ESM4.GLBX");
2199                let mut buffering_start = None;
2200                let mut buffered = AHashMap::new();
2201
2202                for i in 0..num_snapshots {
2203                    let result = process_mbo_delta(
2204                        test_delta(instrument_id, 1_000_000_000 + i as u64),
2205                        128 | 32, // F_LAST | F_SNAPSHOT
2206                        &mut buffering_start,
2207                        &mut buffered,
2208                    );
2209                    prop_assert!(result.is_none());
2210                }
2211
2212                prop_assert_eq!(buffered[&instrument_id].len(), num_snapshots);
2213            }
2214
2215            #[rstest]
2216            fn mbo_replay_delays_emission(
2217                start_offset in 1u64..=100,
2218                num_before in 1usize..=10,
2219            ) {
2220                let instrument_id = InstrumentId::from("ESM4.GLBX");
2221                let start_ns = 1_000_000_000u64 * start_offset;
2222                let mut buffering_start = Some(start_ns.into());
2223                let mut buffered = AHashMap::new();
2224
2225                for i in 0..num_before {
2226                    let ts = start_ns - (num_before as u64 - i as u64);
2227                    let result = process_mbo_delta(
2228                        test_delta(instrument_id, ts),
2229                        128, // F_LAST
2230                        &mut buffering_start,
2231                        &mut buffered,
2232                    );
2233                    prop_assert!(result.is_none());
2234                }
2235
2236                let result = process_mbo_delta(
2237                    test_delta(instrument_id, start_ns + 1),
2238                    128,
2239                    &mut buffering_start,
2240                    &mut buffered,
2241                );
2242
2243                prop_assert!(result.is_some());
2244                prop_assert!(buffering_start.is_none());
2245                let total = num_before + 1;
2246                prop_assert_eq!(result.unwrap().deltas.len(), total);
2247            }
2248        }
2249    }
2250
2251    #[rstest]
2252    #[case(2, "CLZ29-CLF30.XNYM")]
2253    #[case(1, "CLZ9-CLF0.XNYM")]
2254    fn test_live_subscription_preserves_native_symbols(
2255        #[case] digits: u8,
2256        #[case] id: InstrumentId,
2257    ) {
2258        let mut client = create_test_client().with_futures_year_digits(digits.try_into().unwrap());
2259        client
2260            .subscribe(
2261                "mbp-1".to_string(),
2262                vec![id],
2263                None,
2264                None,
2265                Some(vec![Some(3)]),
2266                None,
2267            )
2268            .unwrap();
2269        let rx = client.cmd_rx.as_mut().unwrap();
2270        assert!(
2271            matches!(rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 3) if symbol == Symbol::from("CLZ9-CLF0"))
2272        );
2273
2274        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
2275            panic!("expected subscription");
2276        };
2277
2278        let expected = Subscription::builder()
2279            .schema(dbn::Schema::Mbp1)
2280            .symbols(vec!["CLZ9-CLF0".to_string()])
2281            .build();
2282        assert_eq!(subscription, expected);
2283        assert_eq!(
2284            client.symbol_venue_map.get_cloned(&id.symbol),
2285            Some(Venue::from("XNYM"))
2286        );
2287        assert_eq!(
2288            rx.try_recv().unwrap_err(),
2289            tokio::sync::mpsc::error::TryRecvError::Empty
2290        );
2291    }
2292
2293    #[rstest]
2294    fn test_futures_cache_uses_requested_venue_after_year_conversion() {
2295        let mut handler = create_test_handler(None);
2296        handler.publisher_venue_map.insert(1, Venue::from("GLBX"));
2297        handler
2298            .symbol_venue_map
2299            .insert(Symbol::from("CLF26"), Venue::from("XNYM"));
2300        let mut mapping = dbn::PitSymbolMap::new();
2301        mapping.inner_mut().insert(42, "CLF6".to_string());
2302        let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
2303        msg.hd.publisher_id = 1;
2304        msg.hd.instrument_id = 42;
2305        msg.ts_recv = 1_766_102_400_000_000_000;
2306        let mut ids = AHashMap::new();
2307        handler
2308            .cache_futures_instrument_id(&dbn::RecordRef::from(&msg), &mapping, &mut ids)
2309            .unwrap();
2310        assert_eq!(
2311            ids.get(&42).copied().map(CachedInstrumentId::id),
2312            Some(InstrumentId::from("CLF26.XNYM"))
2313        );
2314    }
2315
2316    #[rstest]
2317    #[case(Some(2), Some(5), Some(4), 5)]
2318    #[case(Some(2), None, Some(5), 5)]
2319    #[case(Some(5), Some(2), None, 2)]
2320    #[case(Some(5), None, Some(2), 2)]
2321    #[case(Some(3), None, None, 3)]
2322    #[case(None, None, None, 2)]
2323    fn test_explicit_precision_precedes_definitions(
2324        #[case] definition: Option<u8>,
2325        #[case] mapped: Option<u8>,
2326        #[case] native: Option<u8>,
2327        #[case] expected: u8,
2328    ) {
2329        let definitions = AHashMap::from_iter(definition.map(|precision| (42, precision)));
2330        let subscriptions = AHashMap::from_iter(mapped.map(|precision| (42, precision)));
2331        let overrides =
2332            AHashMap::from_iter(native.map(|precision| (Symbol::from("CLF6"), precision)));
2333
2334        let actual =
2335            resolve_price_precision(42, Some("CLF6"), &definitions, &subscriptions, &overrides);
2336
2337        assert_eq!(actual, expected);
2338    }
2339
2340    #[rstest]
2341    fn test_futures_precision_override_uses_native_symbol() {
2342        let mut handler = create_test_handler(None);
2343        handler.publisher_venue_map.insert(1, Venue::from("GLBX"));
2344        let mut mapping = dbn::PitSymbolMap::new();
2345        mapping.inner_mut().insert(42, "CLF6".to_string());
2346        let mut msg = dbn::TradeMsg::default();
2347        msg.hd.publisher_id = 1;
2348        msg.hd.instrument_id = 42;
2349        msg.hd.ts_event = 1_766_102_400_000_000_000;
2350        msg.ts_recv = msg.hd.ts_event;
2351        msg.price = 75_125_000_000;
2352        msg.size = 7;
2353        msg.side = b'A' as std::ffi::c_char;
2354        let record = dbn::RecordRef::from(&msg);
2355        let mut ids = AHashMap::new();
2356        handler
2357            .cache_futures_instrument_id(&record, &mapping, &mut ids)
2358            .unwrap();
2359        let overrides = AHashMap::from_iter([(Symbol::from("CLF6"), 3)]);
2360        let (data, extra) = handle_record(
2361            record,
2362            &mapping,
2363            &handler.publisher_venue_map,
2364            &handler.symbol_venue_map,
2365            &mut ids,
2366            &AHashMap::new(),
2367            &AHashMap::new(),
2368            &overrides,
2369            1_766_102_400_000_000_001.into(),
2370            &HashSet::new(),
2371            true,
2372        )
2373        .unwrap();
2374
2375        let Some(Data::Trade(trade)) = data else {
2376            panic!("expected trade");
2377        };
2378
2379        assert_eq!(trade.instrument_id, InstrumentId::from("CLF26.GLBX"));
2380        assert_eq!(trade.price.precision, 3);
2381        assert_eq!(trade.price, Price::from("75.125"));
2382        assert_eq!(trade.size, Quantity::from(7));
2383        assert_eq!(trade.ts_event, msg.hd.ts_event);
2384        assert_eq!(trade.ts_init, 1_766_102_400_000_000_001);
2385        assert!(extra.is_none());
2386    }
2387}