Skip to main content

nautilus_databento/
live.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Databento live feed handler.
17//!
18//! The feed handler runs a single async task per dataset. It receives
19//! [`HandlerCommand`] messages over an unbounded channel and streams decoded
20//! market data back as [`DatabentoMessage`]s on an unbounded tokio channel.
21//!
22//! The inner loop uses `tokio::select!` to concurrently await the next record
23//! from the Databento gateway and the next command from the engine, giving
24//! near-zero idle CPU and immediate command responsiveness.
25//!
26//! Heartbeat detection is delegated to the upstream `databento` client, which
27//! returns `Error::HeartbeatTimeout` when no data arrives within
28//! `heartbeat_interval + 5 s` (default 35 s). The handler treats this as a
29//! connection error and enters the reconnection backoff loop.
30
31use std::{
32    fmt::{Debug, Display},
33    path::PathBuf,
34    str::FromStr,
35    sync::Arc,
36    time::Duration,
37};
38
39use ahash::{AHashMap, HashSet, HashSetExt};
40use databento::{
41    dbn::{self, PitSymbolMap, SymbolIndex},
42    live::Subscription,
43};
44use indexmap::IndexMap;
45use nautilus_core::{
46    AtomicMap, UnixNanos, consts::NAUTILUS_USER_AGENT, string::secret::SecretString,
47    time::get_atomic_clock_realtime,
48};
49use nautilus_model::{
50    data::{Data, InstrumentStatus, OrderBookDelta, OrderBookDeltas},
51    enums::RecordFlag,
52    identifiers::{InstrumentId, Symbol, Venue, futures::FuturesYearDigits},
53    instruments::{Instrument, InstrumentAny},
54    types::Currency,
55};
56use nautilus_network::backoff::ExponentialBackoff;
57use time::OffsetDateTime;
58
59use super::{
60    decode::{
61        decode_imbalance_msg, decode_statistics_msg, decode_status_msg, is_supported_stat_type,
62    },
63    types::{DatabentoImbalance, DatabentoStatistics, SubscriptionAckEvent},
64};
65use crate::{
66    common::{Credential, build_publisher_venue_map, load_publishers},
67    decode::{decode_instrument_def_msg, decode_record},
68    symbology::{
69        check_consistent_symbology, dataset_uses_futures_symbology, infer_symbology_type,
70        nautilus_symbol, raw_symbol, resolve_futures_instrument_id,
71    },
72    types::PublisherId,
73};
74
75#[derive(Debug)]
76pub enum HandlerCommand {
77    Subscribe(Subscription),
78    SetPricePrecision(Symbol, u8),
79    Start,
80    Close,
81}
82
83#[derive(Debug)]
84pub enum DatabentoMessage {
85    Data(Data),
86    Instrument(Box<InstrumentAny>),
87    Status(InstrumentStatus),
88    Imbalance(DatabentoImbalance),
89    Statistics(DatabentoStatistics),
90    SubscriptionAck(SubscriptionAckEvent),
91    Error(anyhow::Error),
92    Close,
93}
94
95#[cfg_attr(
96    feature = "python",
97    pyo3::pyclass(module = "nautilus_trader.adapters.databento")
98)]
99#[cfg_attr(
100    feature = "python",
101    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.adapters.databento")
102)]
103pub struct DatabentoLiveClient {
104    futures_year_digits: FuturesYearDigits,
105    share_class_space: bool,
106    credential: Credential,
107    pub dataset: String,
108    is_running: bool,
109    is_closed: bool,
110    cmd_tx: tokio::sync::mpsc::UnboundedSender<HandlerCommand>,
111    cmd_rx: Option<tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>>,
112    publisher_venue_map: IndexMap<PublisherId, Venue>,
113    symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
114    use_exchange_as_venue: bool,
115    bars_timestamp_on_close: bool,
116    reconnect_timeout_mins: Option<u64>,
117}
118
119impl Debug for DatabentoLiveClient {
120    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
121        f.debug_struct(stringify!(DatabentoLiveClient))
122            .field("credential", &self.credential)
123            .field("dataset", &self.dataset)
124            .field("is_running", &self.is_running)
125            .field("is_closed", &self.is_closed)
126            .finish()
127    }
128}
129
130impl DatabentoLiveClient {
131    /// Creates a new [`DatabentoLiveClient`] instance.
132    ///
133    /// # Errors
134    ///
135    /// Returns an error if reading or parsing the publishers file fails.
136    pub fn new(
137        key: String,
138        dataset: String,
139        publishers_filepath: PathBuf,
140        use_exchange_as_venue: bool,
141        bars_timestamp_on_close: Option<bool>,
142        reconnect_timeout_mins: Option<i64>,
143    ) -> anyhow::Result<Self> {
144        let publishers = load_publishers(publishers_filepath)?;
145        let publisher_venue_map = build_publisher_venue_map(&publishers);
146
147        let (cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
148
149        let reconnect_timeout_mins = reconnect_timeout_mins
150            .and_then(|mins| if mins >= 0 { Some(mins as u64) } else { None });
151
152        Ok(Self {
153            futures_year_digits: FuturesYearDigits::default(),
154            share_class_space: true,
155            credential: Credential::new(key),
156            dataset,
157            cmd_tx,
158            cmd_rx: Some(cmd_rx),
159            is_running: false,
160            is_closed: false,
161            publisher_venue_map,
162            symbol_venue_map: Arc::new(AtomicMap::new()),
163            use_exchange_as_venue,
164            bars_timestamp_on_close: bars_timestamp_on_close.unwrap_or(true),
165            reconnect_timeout_mins,
166        })
167    }
168
169    /// Selects one- or two-digit years for futures instrument IDs.
170    #[must_use]
171    pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
172        self.futures_year_digits = futures_year_digits;
173        self
174    }
175
176    /// Spells a share class with a space in instrument IDs (`BRK.B` -> `BRK B`).
177    #[must_use]
178    pub fn with_share_class_space(mut self, share_class_space: bool) -> Self {
179        self.share_class_space = share_class_space;
180        self
181    }
182
183    #[must_use]
184    pub const fn is_running(&self) -> bool {
185        self.is_running
186    }
187
188    #[must_use]
189    pub const fn is_closed(&self) -> bool {
190        self.is_closed
191    }
192
193    /// Subscribes to Databento live data for the requested instruments.
194    ///
195    /// # Errors
196    ///
197    /// Returns an error if symbology, schema, timestamp, or precision inputs are invalid,
198    /// or if the command cannot be sent to the feed handler.
199    #[expect(clippy::needless_pass_by_value)]
200    pub fn subscribe(
201        &mut self,
202        schema: String,
203        instrument_ids: Vec<InstrumentId>,
204        start: Option<u64>,
205        snapshot: Option<bool>,
206        price_precisions: Option<Vec<Option<u8>>>,
207        stype_in: Option<String>,
208    ) -> anyhow::Result<()> {
209        if let Some(precisions) = &price_precisions
210            && precisions.len() != instrument_ids.len()
211        {
212            anyhow::bail!(
213                "`price_precisions` length ({}) must match `instrument_ids` length ({})",
214                precisions.len(),
215                instrument_ids.len()
216            );
217        }
218
219        let symbols: Vec<String> = instrument_ids
220            .iter()
221            .map(|id| {
222                raw_symbol(
223                    id.symbol.as_str(),
224                    &self.dataset,
225                    self.futures_year_digits,
226                    self.share_class_space,
227                )
228            })
229            .collect();
230        let first_symbol = symbols
231            .first()
232            .ok_or_else(|| anyhow::anyhow!("No symbols provided"))?;
233        let stype_in = match stype_in {
234            Some(stype_in) => dbn::SType::from_str(&stype_in)?,
235            None => infer_symbology_type(first_symbol),
236        };
237        let symbols: Vec<&str> = symbols.iter().map(String::as_str).collect();
238        check_consistent_symbology(symbols.as_slice())?;
239        let mut sub = Subscription::builder()
240            .symbols(symbols)
241            .schema(dbn::Schema::from_str(&schema)?)
242            .stype_in(stype_in)
243            .build();
244
245        if let Some(start) = start {
246            sub.start = Some(OffsetDateTime::from_unix_timestamp_nanos(i128::from(
247                start,
248            ))?);
249        }
250        sub.use_snapshot = snapshot.unwrap_or(false);
251
252        self.symbol_venue_map.rcu(|m| {
253            for id in &instrument_ids {
254                m.entry(id.symbol).or_insert(id.venue);
255            }
256        });
257
258        if let Some(precisions) = price_precisions {
259            for (instrument_id, precision) in instrument_ids.iter().zip(precisions) {
260                if let Some(precision) = precision {
261                    let raw = raw_symbol(
262                        instrument_id.symbol.as_str(),
263                        &self.dataset,
264                        self.futures_year_digits,
265                        self.share_class_space,
266                    );
267                    self.send_command(HandlerCommand::SetPricePrecision(
268                        Symbol::from(raw.as_str()),
269                        precision,
270                    ))?;
271                }
272            }
273        }
274
275        self.send_command(HandlerCommand::Subscribe(sub))
276    }
277
278    /// Starts the live feed handler and returns its message receiver.
279    ///
280    /// # Errors
281    ///
282    /// Returns an error if the client is already closed, already running, or cannot start.
283    pub fn start(
284        &mut self,
285    ) -> anyhow::Result<(
286        DatabentoFeedHandler,
287        tokio::sync::mpsc::UnboundedReceiver<DatabentoMessage>,
288    )> {
289        if self.is_closed {
290            anyhow::bail!("Client already closed");
291        }
292
293        if self.is_running {
294            anyhow::bail!("Client already running");
295        }
296
297        log::debug!("Starting client");
298
299        let (msg_tx, msg_rx) = tokio::sync::mpsc::unbounded_channel::<DatabentoMessage>();
300        let cmd_rx = self
301            .cmd_rx
302            .take()
303            .ok_or_else(|| anyhow::anyhow!("Command receiver already taken"))?;
304
305        let feed_handler = DatabentoFeedHandler::new(
306            self.credential.clone(),
307            self.dataset.clone(),
308            cmd_rx,
309            msg_tx,
310            self.publisher_venue_map.clone(),
311            self.symbol_venue_map.clone(),
312            self.use_exchange_as_venue,
313            self.bars_timestamp_on_close,
314            self.reconnect_timeout_mins,
315        )
316        .with_futures_year_digits(self.futures_year_digits)
317        .with_share_class_space(self.share_class_space);
318
319        self.send_command(HandlerCommand::Start)?;
320        self.is_running = true;
321
322        Ok((feed_handler, msg_rx))
323    }
324
325    /// Closes the live client.
326    ///
327    /// # Errors
328    ///
329    /// Returns an error if the client was never started, is already closed, or cannot send
330    /// the close command to the feed handler.
331    pub fn close(&mut self) -> anyhow::Result<()> {
332        if !self.is_running {
333            anyhow::bail!("Client never started");
334        }
335
336        if self.is_closed {
337            anyhow::bail!("Client already closed");
338        }
339
340        log::debug!("Closing client");
341
342        if !self.cmd_tx.is_closed() {
343            self.send_command(HandlerCommand::Close)?;
344        }
345
346        self.is_running = false;
347        self.is_closed = true;
348
349        Ok(())
350    }
351
352    fn send_command(&self, cmd: HandlerCommand) -> anyhow::Result<()> {
353        self.cmd_tx.send(cmd).map_err(|e| {
354            anyhow::Error::new(CommandSendError {
355                details: e.to_string(),
356            })
357        })
358    }
359}
360
361#[cfg(any(test, feature = "python"))]
362pub(crate) fn is_command_send_error(error: &anyhow::Error) -> bool {
363    error.is::<CommandSendError>()
364}
365
366#[derive(Debug)]
367struct CommandSendError {
368    details: String,
369}
370
371impl Display for CommandSendError {
372    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
373        write!(
374            f,
375            "Failed to send command to Databento feed handler: {}",
376            self.details
377        )
378    }
379}
380
381impl std::error::Error for CommandSendError {}
382
383/// Handles a raw TCP data feed from the Databento LSG for a single dataset.
384///
385/// [`HandlerCommand`] messages are received synchronously across a channel,
386/// decoded records are sent asynchronously on a tokio channel as [`DatabentoMessage`]s
387/// back to a message processing task.
388///
389/// # Crash Policy
390///
391/// This handler intentionally avoids applying downstream backpressure to the
392/// live feed. If decoded output cannot be drained, memory pressure is the hard
393/// failure mode instead of arbitrary queue limits or delayed market data.
394pub struct DatabentoFeedHandler {
395    futures_year_digits: FuturesYearDigits,
396    share_class_space: bool,
397    credential: Credential,
398    dataset: String,
399    cmd_rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
400    msg_tx: tokio::sync::mpsc::UnboundedSender<DatabentoMessage>,
401    publisher_venue_map: IndexMap<PublisherId, Venue>,
402    symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
403    replay: bool,
404    use_exchange_as_venue: bool,
405    bars_timestamp_on_close: bool,
406    reconnect_timeout_mins: Option<u64>,
407    backoff: ExponentialBackoff,
408    subscriptions: Vec<Subscription>,
409    buffered_commands: Vec<HandlerCommand>,
410    price_precision_overrides: AHashMap<Symbol, u8>,
411    gateway_addr: Option<String>,
412    success_threshold: Duration,
413}
414
415impl Debug for DatabentoFeedHandler {
416    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
417        f.debug_struct(stringify!(DatabentoFeedHandler))
418            .field("credential", &self.credential)
419            .field("dataset", &self.dataset)
420            .field("replay", &self.replay)
421            .field("reconnect_timeout_mins", &self.reconnect_timeout_mins)
422            .field("subscriptions", &self.subscriptions.len())
423            .finish()
424    }
425}
426
427impl DatabentoFeedHandler {
428    /// Selects one- or two-digit years for futures instrument IDs.
429    #[must_use]
430    pub fn with_futures_year_digits(mut self, futures_year_digits: FuturesYearDigits) -> Self {
431        self.futures_year_digits = futures_year_digits;
432        self
433    }
434
435    /// Spells a share class with a space in instrument IDs (`BRK.B` -> `BRK B`).
436    #[must_use]
437    pub fn with_share_class_space(mut self, share_class_space: bool) -> Self {
438        self.share_class_space = share_class_space;
439        self
440    }
441
442    /// Creates a new [`DatabentoFeedHandler`] instance.
443    ///
444    /// # Panics
445    ///
446    /// Panics if exponential backoff creation fails (should never happen with valid hardcoded parameters).
447    #[must_use]
448    #[expect(clippy::too_many_arguments)]
449    pub fn new(
450        credential: Credential,
451        dataset: String,
452        rx: tokio::sync::mpsc::UnboundedReceiver<HandlerCommand>,
453        tx: tokio::sync::mpsc::UnboundedSender<DatabentoMessage>,
454        publisher_venue_map: IndexMap<PublisherId, Venue>,
455        symbol_venue_map: Arc<AtomicMap<Symbol, Venue>>,
456        use_exchange_as_venue: bool,
457        bars_timestamp_on_close: bool,
458        reconnect_timeout_mins: Option<u64>,
459    ) -> Self {
460        // Choose max delay based on timeout configuration:
461        // - With timeout: 60s max (quick recovery to reconnect within window)
462        // - Without timeout (None): 600s max (patient recovery, respectful of infrastructure)
463        let delay_max = if reconnect_timeout_mins.is_some() {
464            Duration::from_secs(60)
465        } else {
466            Duration::from_secs(600)
467        };
468
469        let backoff = ExponentialBackoff::new(Duration::from_secs(1), delay_max, 2.0, 1000, false)
470            .expect("hardcoded backoff parameters are valid");
471
472        Self {
473            futures_year_digits: FuturesYearDigits::default(),
474            share_class_space: true,
475            credential,
476            dataset,
477            cmd_rx: rx,
478            msg_tx: tx,
479            publisher_venue_map,
480            symbol_venue_map,
481            replay: false,
482            use_exchange_as_venue,
483            bars_timestamp_on_close,
484            reconnect_timeout_mins,
485            backoff,
486            subscriptions: Vec::new(),
487            buffered_commands: Vec::new(),
488            price_precision_overrides: AHashMap::new(),
489            gateway_addr: None,
490            success_threshold: Duration::from_secs(60),
491        }
492    }
493
494    /// Sets a custom gateway address, overriding the default Databento LSG endpoint.
495    #[must_use]
496    pub fn with_gateway_addr(mut self, addr: String) -> Self {
497        self.gateway_addr = Some(addr);
498        self
499    }
500
501    /// Sets the duration a session must run before it counts as successful.
502    ///
503    /// A successful session resets the reconnection backoff cycle.
504    /// Defaults to 60 seconds.
505    #[must_use]
506    pub fn with_success_threshold(mut self, threshold: Duration) -> Self {
507        self.success_threshold = threshold;
508        self
509    }
510
511    /// Runs the feed handler main loop, processing commands and streaming market data.
512    ///
513    /// Establishes a connection to the Databento LSG, subscribes to requested data feeds,
514    /// and continuously processes incoming market data messages until shutdown.
515    ///
516    /// Implements automatic reconnection with exponential backoff (1s to 60s with jitter).
517    /// Each successful session resets the reconnection cycle, giving the next disconnect
518    /// a fresh timeout window. Gives up after `reconnect_timeout_mins` if configured.
519    ///
520    /// # Errors
521    ///
522    /// Returns an error if any client operation or message handling fails.
523    pub async fn run(&mut self) -> anyhow::Result<()> {
524        log::debug!("Running feed handler");
525
526        let mut reconnect_start: Option<tokio::time::Instant> = None;
527        let mut attempt = 0;
528
529        loop {
530            attempt += 1;
531
532            match self.run_session(attempt).await {
533                Ok(ran_successfully) => {
534                    if ran_successfully {
535                        log::info!("Resetting reconnection cycle after successful session");
536                        reconnect_start = None;
537                        attempt = 0;
538                        self.backoff.reset();
539                    } else {
540                        log::debug!("Session ended normally");
541                        break Ok(());
542                    }
543                }
544                Err(e) => {
545                    let cycle_start = reconnect_start.get_or_insert_with(tokio::time::Instant::now);
546
547                    if let Some(timeout_mins) = self.reconnect_timeout_mins {
548                        let elapsed = cycle_start.elapsed();
549                        let timeout = Duration::from_mins(timeout_mins);
550
551                        if elapsed >= timeout {
552                            log::error!("Giving up reconnection after {timeout_mins} minutes");
553                            self.send_msg(DatabentoMessage::Error(anyhow::anyhow!(
554                                "Reconnection timeout after {timeout_mins} minutes: {e}"
555                            )));
556                            break Err(e);
557                        }
558                    }
559
560                    let delay = self.backoff.next_duration();
561
562                    log::warn!(
563                        "Connection lost (attempt {}): {}. Reconnecting in {}s...",
564                        attempt,
565                        e,
566                        delay.as_secs()
567                    );
568
569                    let sleep = tokio::time::sleep(delay);
570                    tokio::pin!(sleep);
571
572                    loop {
573                        tokio::select! {
574                            () = &mut sleep => break,
575                            cmd = self.cmd_rx.recv() => {
576                                match cmd {
577                                    Some(HandlerCommand::Close) => {
578                                        log::debug!("Close received during backoff");
579                                        return Ok(());
580                                    }
581                                    None => {
582                                        log::debug!("Command channel closed during backoff");
583                                        return Ok(());
584                                    }
585                                    Some(cmd) => {
586                                        log::debug!("Buffering command received during backoff: {cmd:?}");
587                                        self.buffered_commands.push(cmd);
588                                    }
589                                }
590                            }
591                        }
592                    }
593                }
594            }
595        }
596    }
597
598    /// Runs a single session, handling connection, subscriptions, and data streaming.
599    ///
600    /// Returns `Ok(bool)` where the bool indicates if the session ran successfully
601    /// for a meaningful duration (true) or was intentionally closed (false).
602    ///
603    /// # Errors
604    ///
605    /// Returns an error if connection fails, subscription fails, or data streaming encounters an error.
606    async fn run_session(&mut self, attempt: usize) -> anyhow::Result<bool> {
607        if attempt > 1 {
608            log::info!("Reconnecting (attempt {attempt})...");
609        }
610
611        let session_start = tokio::time::Instant::now();
612        let clock = get_atomic_clock_realtime();
613        let mut symbol_map = PitSymbolMap::new();
614        let mut instrument_id_map: AHashMap<u32, CachedInstrumentId> = AHashMap::new();
615        let mut instrument_def_price_precision_map: AHashMap<u32, u8> = AHashMap::new();
616        let mut subscription_price_precision_map: AHashMap<u32, u8> = AHashMap::new();
617
618        let mut buffering_start = None;
619        let mut buffered_deltas: AHashMap<InstrumentId, Vec<OrderBookDelta>> = AHashMap::new();
620        let mut initialized_books = HashSet::new();
621        let mut trade_sources: AHashMap<InstrumentId, u8> = AHashMap::new();
622        let timeout = Duration::from_secs(5); // Hardcoded timeout for now
623
624        let gateway_addr = self.gateway_addr.clone();
625        let api_key = SecretString::from(self.credential.api_key().to_owned());
626        let dataset = self.dataset.clone();
627
628        let result = tokio::time::timeout(timeout, async move {
629            let base = databento::LiveClient::builder();
630            let base = if let Some(addr) = gateway_addr {
631                base.addr(addr).await?
632            } else {
633                base
634            };
635            base.user_agent_extension(NAUTILUS_USER_AGENT.into())
636                .key(api_key.expose_secret().to_owned())?
637                .dataset(dataset)
638                .build()
639                .await
640        })
641        .await?;
642
643        let mut client = match result {
644            Ok(client) => {
645                if attempt > 1 {
646                    log::info!("Reconnected successfully");
647                } else {
648                    log::info!("Connected");
649                }
650                client
651            }
652            Err(e) => {
653                anyhow::bail!("Failed to connect to Databento LSG: {e}");
654            }
655        };
656
657        // Process any commands buffered during reconnection backoff
658        let mut start_buffered = false;
659
660        if !self.buffered_commands.is_empty() {
661            log::debug!(
662                "Processing {} buffered commands",
663                self.buffered_commands.len()
664            );
665
666            for cmd in self.buffered_commands.drain(..) {
667                match cmd {
668                    HandlerCommand::Subscribe(sub) => {
669                        if !self.replay && sub.start.is_some() {
670                            self.replay = true;
671                        }
672                        self.subscriptions.push(sub);
673                    }
674                    HandlerCommand::SetPricePrecision(symbol, precision) => {
675                        self.price_precision_overrides.insert(symbol, precision);
676                    }
677                    HandlerCommand::Start => {
678                        start_buffered = true;
679                    }
680                    HandlerCommand::Close => {
681                        log::warn!("Close command was buffered, shutting down");
682                        return Ok(false);
683                    }
684                }
685            }
686        }
687
688        let mut running = false;
689
690        if !self.subscriptions.is_empty() {
691            log::info!(
692                "Resubscribing to {} subscriptions",
693                self.subscriptions.len()
694            );
695
696            // Anchors remain only when the first connection failed, so this session replays
697            let replay = self.subscriptions.iter().any(|sub| sub.start.is_some());
698
699            for sub in self.subscriptions.clone() {
700                client.subscribe(sub).await?;
701            }
702            // Strip start timestamps after successful subscription to avoid replaying history on future reconnects
703            for sub in &mut self.subscriptions {
704                sub.start = None;
705            }
706
707            buffering_start = replay.then(|| clock.get_time_ns());
708            client.start().await?;
709            running = true;
710            log::info!("Resubscription complete");
711        } else if start_buffered {
712            log::debug!("Starting session from buffered Start command");
713            buffering_start = if self.replay {
714                Some(clock.get_time_ns())
715            } else {
716                None
717            };
718            client.start().await?;
719            running = true;
720        }
721
722        loop {
723            if self.msg_tx.is_closed() {
724                log::debug!("Message channel was closed: stopping");
725                return Ok(false);
726            }
727
728            // Wait for either a command or a record. When the session has not
729            // started yet (`!running`), only commands are awaited. Once running,
730            // `next_record` is cancel-safe so `tokio::select!` can safely
731            // race both futures.
732            if !running {
733                match self.cmd_rx.recv().await {
734                    Some(HandlerCommand::Subscribe(sub)) => {
735                        log::debug!("Received command: Subscribe");
736
737                        if !self.replay && sub.start.is_some() {
738                            self.replay = true;
739                        }
740                        client.subscribe(sub.clone()).await?;
741                        let mut sub_for_reconnect = sub;
742                        sub_for_reconnect.start = None;
743                        self.subscriptions.push(sub_for_reconnect);
744                        continue;
745                    }
746                    Some(HandlerCommand::SetPricePrecision(symbol, precision)) => {
747                        log::debug!(
748                            "Received command: SetPricePrecision for {symbol} to {precision}"
749                        );
750                        self.price_precision_overrides.insert(symbol, precision);
751                        continue;
752                    }
753                    Some(HandlerCommand::Start) => {
754                        log::debug!("Received command: Start");
755                        buffering_start = if self.replay {
756                            Some(clock.get_time_ns())
757                        } else {
758                            None
759                        };
760                        client.start().await?;
761                        running = true;
762                        continue;
763                    }
764                    Some(HandlerCommand::Close) => {
765                        self.send_close_msg();
766                        return Ok(false);
767                    }
768                    None => {
769                        log::debug!("Command channel disconnected");
770                        return Ok(false);
771                    }
772                }
773            }
774
775            let record_opt = tokio::select! {
776                cmd = self.cmd_rx.recv() =>
777                match cmd {
778                    Some(HandlerCommand::Subscribe(sub)) => {
779                        log::debug!("Received command: Subscribe");
780
781                        if sub.start.is_some() {
782                            self.replay = true;
783                            log::error!(
784                                "Ignoring `start` on {} subscribe, session already running, Databento drops replay anchors sent after session start",
785                                self.dataset,
786                            );
787                        }
788                        client.subscribe(sub.clone()).await?;
789                        let mut sub_for_reconnect = sub;
790                        sub_for_reconnect.start = None;
791                        self.subscriptions.push(sub_for_reconnect);
792                        continue;
793                    }
794                    Some(HandlerCommand::SetPricePrecision(symbol, precision)) => {
795                        log::debug!(
796                            "Received command: SetPricePrecision for {symbol} to {precision}"
797                        );
798                        self.price_precision_overrides.insert(symbol, precision);
799                        continue;
800                    }
801                    Some(HandlerCommand::Start) => {
802                        log::warn!("Received Start command but session already running");
803                        continue;
804                    }
805                    Some(HandlerCommand::Close) => {
806                        self.send_close_msg();
807                        client.close().await?;
808                        log::debug!("Closed inner client");
809                        return Ok(false);
810                    }
811                    None => {
812                        log::debug!("Command channel disconnected");
813                        return Ok(false);
814                    }
815                },
816                result = client.next_record() => result,
817            };
818
819            let record = match record_opt {
820                Ok(Some(record)) => record,
821                Ok(None) => {
822                    if session_start.elapsed() >= self.success_threshold {
823                        log::debug!("Session ended after successful run");
824                        return Ok(true);
825                    }
826                    anyhow::bail!("Session ended by gateway");
827                }
828                Err(e) => {
829                    if session_start.elapsed() >= self.success_threshold {
830                        log::debug!("Connection error after successful run: {e}");
831                        return Ok(true);
832                    }
833                    anyhow::bail!("Connection error: {e}");
834                }
835            };
836
837            let ts_init = clock.get_time_ns();
838
839            if let Err(e) =
840                self.cache_futures_instrument_id(&record, &symbol_map, &mut instrument_id_map)
841            {
842                log::error!("Error resolving futures instrument ID: {e}");
843                continue;
844            }
845
846            // Decode record
847            if let Some(msg) = record.get::<dbn::ErrorMsg>() {
848                handle_error_msg(msg);
849            } else if let Some(msg) = record.get::<dbn::SystemMsg>() {
850                if let Some(ack) = handle_system_msg(msg, ts_init) {
851                    self.send_msg(DatabentoMessage::SubscriptionAck(ack));
852                }
853            } else if let Some(msg) = record.get::<dbn::SymbolMappingMsg>() {
854                instrument_def_price_precision_map.remove(&msg.hd.instrument_id);
855                update_price_precision_map_with_symbol_mapping_msg(
856                    msg,
857                    &self.price_precision_overrides,
858                    &mut subscription_price_precision_map,
859                )?;
860                handle_symbol_mapping_msg(
861                    msg,
862                    &mut symbol_map,
863                    &self.symbol_venue_map,
864                    &mut instrument_id_map,
865                )?;
866            } else if let Some(msg) = record.get::<dbn::InstrumentDefMsg>() {
867                let maybe_data = self.handle_instrument_def_msg(
868                    msg,
869                    &record,
870                    &symbol_map,
871                    &mut instrument_id_map,
872                    ts_init,
873                )?;
874
875                if let Some(data) = maybe_data {
876                    instrument_def_price_precision_map
877                        .insert(msg.hd.instrument_id, data.price_precision());
878                    self.send_msg(DatabentoMessage::Instrument(Box::new(data)));
879                }
880            } else if let Some(msg) = record.get::<dbn::StatusMsg>() {
881                let data = handle_status_msg(
882                    msg,
883                    &record,
884                    &symbol_map,
885                    &self.publisher_venue_map,
886                    &self.symbol_venue_map,
887                    &mut instrument_id_map,
888                    ts_init,
889                    self.share_class_space,
890                )?;
891                self.send_msg(DatabentoMessage::Status(data));
892            } else if let Some(msg) = record.get::<dbn::ImbalanceMsg>() {
893                handle_imbalance_msg(
894                    msg,
895                    &record,
896                    &symbol_map,
897                    &self.publisher_venue_map,
898                    &self.symbol_venue_map,
899                    &mut instrument_id_map,
900                    &instrument_def_price_precision_map,
901                    &subscription_price_precision_map,
902                    &self.price_precision_overrides,
903                    ts_init,
904                    self.share_class_space,
905                )
906                .map_or_else(
907                    |e| log::error!("Error decoding imbalance: {e}"),
908                    |data| self.send_msg(DatabentoMessage::Imbalance(data)),
909                );
910            } else if let Some(msg) = record.get::<dbn::StatMsg>() {
911                let maybe_data = handle_statistics_msg(
912                    msg,
913                    &record,
914                    &symbol_map,
915                    &self.publisher_venue_map,
916                    &self.symbol_venue_map,
917                    &mut instrument_id_map,
918                    &instrument_def_price_precision_map,
919                    &subscription_price_precision_map,
920                    &self.price_precision_overrides,
921                    ts_init,
922                    self.share_class_space,
923                )?;
924
925                if let Some(data) = maybe_data {
926                    self.send_msg(DatabentoMessage::Statistics(data));
927                }
928            } else {
929                // Decode a generic record with possible errors
930                let res = handle_record(
931                    record,
932                    &symbol_map,
933                    &self.publisher_venue_map,
934                    &self.symbol_venue_map,
935                    &mut instrument_id_map,
936                    &instrument_def_price_precision_map,
937                    &subscription_price_precision_map,
938                    &self.price_precision_overrides,
939                    ts_init,
940                    &initialized_books,
941                    self.bars_timestamp_on_close,
942                    self.share_class_space,
943                );
944                let (mut data1, data2) = match res {
945                    Ok(decoded) => decoded,
946                    Err(e) => {
947                        log::error!("Error decoding record: {e}");
948
949                        // Preserve raw MBO event boundaries when the payload is rejected
950                        (None, None)
951                    }
952                };
953
954                if let Some(msg) = record.get::<dbn::MboMsg>() {
955                    if let Some(Data::BookDelta(delta)) = &data1 {
956                        initialized_books.insert(delta.instrument_id);
957
958                        log::trace!(
959                            "Buffering delta: {} {buffering_start:?} flags={}",
960                            delta.ts_event,
961                            msg.flags.raw(),
962                        );
963
964                        match process_mbo_delta(
965                            *delta,
966                            msg.flags.raw(),
967                            &mut buffering_start,
968                            &mut buffered_deltas,
969                        ) {
970                            Some(deltas) => data1 = Some(Data::BookDeltas(Box::new(deltas))),
971                            None => continue,
972                        }
973                    } else {
974                        // Records that decode to no delta (`Action::Fill`
975                        // attribution, `Action::None` status, or trades) may
976                        // still carry the match-event boundary: honor the raw
977                        // `F_LAST` flag or a buffered partial event is
978                        // stranded and merged into the next event (e.g. a
979                        // non-terminal 'C' followed by 'N' | F_LAST).
980                        let deltas = instrument_id_map.get(&msg.hd.instrument_id).and_then(|id| {
981                            flush_mbo_event_boundary(
982                                id.id(),
983                                msg.ts_recv.into(),
984                                msg.flags.raw(),
985                                &mut buffering_start,
986                                &mut buffered_deltas,
987                            )
988                        });
989
990                        if let Some(deltas) = deltas {
991                            self.send_msg(DatabentoMessage::Data(Data::BookDeltas(Box::new(
992                                deltas,
993                            ))));
994                        }
995
996                        // Replayed trades stay out of the live stream
997                        if buffering_start.is_some_and(|start| msg.ts_recv <= start) {
998                            continue;
999                        }
1000                    }
1001                }
1002
1003                let rtype = record.header().rtype;
1004
1005                for data in [data1, data2].into_iter().flatten() {
1006                    if claim_trade_source(&data, rtype, &mut trade_sources) {
1007                        self.send_msg(DatabentoMessage::Data(data));
1008                    }
1009                }
1010            }
1011        }
1012    }
1013
1014    /// Handles instrument definition messages and decodes them into Nautilus instruments.
1015    ///
1016    /// # Errors
1017    ///
1018    /// Returns an error if instrument decoding fails.
1019    fn handle_instrument_def_msg(
1020        &self,
1021        msg: &dbn::InstrumentDefMsg,
1022        record: &dbn::RecordRef,
1023        symbol_map: &PitSymbolMap,
1024        instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1025        ts_init: UnixNanos,
1026    ) -> anyhow::Result<Option<InstrumentAny>> {
1027        instrument_id_map.remove(&msg.hd.instrument_id);
1028        let exchange = if self.use_exchange_as_venue {
1029            msg.exchange()?
1030        } else {
1031            ""
1032        };
1033
1034        if !exchange.is_empty() {
1035            update_instrument_id_map_with_exchange(
1036                symbol_map,
1037                &self.symbol_venue_map,
1038                instrument_id_map,
1039                msg.hd.instrument_id,
1040                exchange,
1041                self.share_class_space,
1042            )?;
1043        }
1044
1045        let instrument_id = update_instrument_id_map(
1046            record,
1047            symbol_map,
1048            &self.publisher_venue_map,
1049            &self.symbol_venue_map,
1050            instrument_id_map,
1051            self.share_class_space,
1052        )?;
1053        let mut instrument_id = resolve_futures_instrument_id(
1054            record,
1055            instrument_id,
1056            self.futures_year_digits,
1057            &self.symbol_venue_map.load(),
1058            None,
1059        )?;
1060
1061        if exchange.is_empty()
1062            && let Some(venue) = self.symbol_venue_map.get_cloned(&instrument_id.symbol)
1063        {
1064            instrument_id.venue = venue;
1065        }
1066
1067        let instrument = decode_instrument_def_msg(msg, instrument_id, Some(ts_init), None)?;
1068
1069        let cached = if instrument.is_some()
1070            && self.futures_year_digits == FuturesYearDigits::Two
1071            && dataset_uses_futures_symbology(&self.dataset)
1072        {
1073            if exchange.is_empty() {
1074                CachedInstrumentId::Definition(instrument_id)
1075            } else {
1076                CachedInstrumentId::Exchange(instrument_id)
1077            }
1078        } else {
1079            CachedInstrumentId::Inferred(instrument_id)
1080        };
1081
1082        instrument_id_map.insert(record.header().instrument_id, cached);
1083        Ok(instrument)
1084    }
1085
1086    fn cache_futures_instrument_id(
1087        &self,
1088        record: &dbn::RecordRef,
1089        symbol_map: &PitSymbolMap,
1090        instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1091    ) -> anyhow::Result<()> {
1092        if self.futures_year_digits != FuturesYearDigits::Two
1093            || !dataset_uses_futures_symbology(&self.dataset)
1094            || record.get::<dbn::InstrumentDefMsg>().is_some()
1095            || record.get::<dbn::SymbolMappingMsg>().is_some()
1096            || instrument_id_map.contains_key(&record.header().instrument_id)
1097            || symbol_map.get_for_rec(record).is_none()
1098        {
1099            return Ok(());
1100        }
1101
1102        let instrument_id = update_instrument_id_map(
1103            record,
1104            symbol_map,
1105            &self.publisher_venue_map,
1106            &self.symbol_venue_map,
1107            instrument_id_map,
1108            self.share_class_space,
1109        )?;
1110        let mut instrument_id = resolve_futures_instrument_id(
1111            record,
1112            instrument_id,
1113            self.futures_year_digits,
1114            &self.symbol_venue_map.load(),
1115            None,
1116        )?;
1117
1118        if let Some(venue) = self.symbol_venue_map.get_cloned(&instrument_id.symbol) {
1119            instrument_id.venue = venue;
1120        }
1121
1122        instrument_id_map.insert(
1123            record.header().instrument_id,
1124            CachedInstrumentId::Inferred(instrument_id),
1125        );
1126        Ok(())
1127    }
1128
1129    /// Sends a message to the message processing task.
1130    fn send_msg(&self, msg: DatabentoMessage) {
1131        log::trace!("Sending {msg:?}");
1132        match self.msg_tx.send(msg) {
1133            Ok(()) => {}
1134            Err(e) => log::debug!("Error sending message: {e}"),
1135        }
1136    }
1137
1138    fn send_close_msg(&self) {
1139        if let Err(e) = self.msg_tx.send(DatabentoMessage::Close) {
1140            log::debug!("Could not send close message: {e}");
1141        }
1142    }
1143}
1144
1145#[derive(Debug, Clone, Copy)]
1146enum CachedInstrumentId {
1147    Inferred(InstrumentId),
1148    Definition(InstrumentId),
1149    Exchange(InstrumentId),
1150}
1151
1152/// Handles Databento error messages by logging them.
1153fn handle_error_msg(msg: &dbn::ErrorMsg) {
1154    log::error!("{msg:?}");
1155}
1156
1157/// Handles Databento system messages, returning a subscription ack event if applicable.
1158fn handle_system_msg(msg: &dbn::SystemMsg, ts_received: UnixNanos) -> Option<SubscriptionAckEvent> {
1159    match msg.code() {
1160        Ok(dbn::SystemCode::SubscriptionAck) => {
1161            let message = msg.msg().unwrap_or("<invalid utf-8>");
1162            log::debug!("Subscription acknowledged: {message}");
1163
1164            let schema = parse_ack_message(message);
1165
1166            Some(SubscriptionAckEvent {
1167                schema,
1168                message: message.to_string(),
1169                ts_received,
1170            })
1171        }
1172        Ok(dbn::SystemCode::Heartbeat) => {
1173            log::trace!("Heartbeat received");
1174            None
1175        }
1176        Ok(dbn::SystemCode::SlowReaderWarning) => {
1177            let message = msg.msg().unwrap_or("<invalid utf-8>");
1178            log::warn!("Slow reader warning: {message}");
1179            None
1180        }
1181        Ok(dbn::SystemCode::ReplayCompleted) => {
1182            let message = msg.msg().unwrap_or("<invalid utf-8>");
1183            log::debug!("Replay completed: {message}");
1184            None
1185        }
1186        _ => {
1187            log::debug!("{msg:?}");
1188            None
1189        }
1190    }
1191}
1192
1193/// Parses a subscription ack message to extract the schema.
1194fn parse_ack_message(message: &str) -> String {
1195    // Format: "Subscription request N for <schema> data succeeded"
1196    message
1197        .strip_circumfix("Subscription request ", " data succeeded")
1198        .and_then(|rest| rest.split_once(" for "))
1199        .map_or_default(|(_, schema)| schema.trim().to_string())
1200}
1201
1202/// Handles symbol mapping messages and updates the instrument ID map.
1203///
1204/// # Errors
1205///
1206/// Returns an error if symbol mapping fails.
1207fn handle_symbol_mapping_msg(
1208    msg: &dbn::SymbolMappingMsg,
1209    symbol_map: &mut PitSymbolMap,
1210    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1211    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1212) -> anyhow::Result<()> {
1213    let changed =
1214        symbol_map.get(msg.hd.instrument_id).map(String::as_str) != Some(msg.stype_out_symbol()?);
1215    symbol_map
1216        .on_symbol_mapping(msg)
1217        .map_err(|e| anyhow::anyhow!("on_symbol_mapping failed for {msg:?}: {e}"))?;
1218
1219    match instrument_id_map.get(&msg.hd.instrument_id).copied() {
1220        Some(CachedInstrumentId::Definition(mut id)) if !changed => {
1221            if let Some(venue) = symbol_venue_map.get_cloned(&id.symbol) {
1222                id.venue = venue;
1223            }
1224
1225            instrument_id_map.insert(msg.hd.instrument_id, CachedInstrumentId::Definition(id));
1226        }
1227        Some(CachedInstrumentId::Exchange(_)) if !changed => {}
1228        _ => {
1229            instrument_id_map.remove(&msg.hd.instrument_id);
1230        }
1231    }
1232
1233    Ok(())
1234}
1235
1236fn update_price_precision_map_with_symbol_mapping_msg(
1237    msg: &dbn::SymbolMappingMsg,
1238    price_precision_overrides: &AHashMap<Symbol, u8>,
1239    subscription_price_precision_map: &mut AHashMap<u32, u8>,
1240) -> anyhow::Result<()> {
1241    subscription_price_precision_map.remove(&msg.hd.instrument_id);
1242
1243    if price_precision_overrides.is_empty() {
1244        return Ok(());
1245    }
1246
1247    let stype_in_symbol = msg
1248        .stype_in_symbol()
1249        .map_err(|e| anyhow::anyhow!("Error decoding `stype_in_symbol`: {e}"))?;
1250    let stype_out_symbol = msg
1251        .stype_out_symbol()
1252        .map_err(|e| anyhow::anyhow!("Error decoding `stype_out_symbol`: {e}"))?;
1253
1254    let price_precision = [stype_in_symbol, stype_out_symbol]
1255        .into_iter()
1256        .find_map(|symbol| {
1257            price_precision_overrides
1258                .get(&Symbol::from_str_unchecked(symbol))
1259                .copied()
1260        });
1261
1262    if let Some(price_precision) = price_precision {
1263        subscription_price_precision_map.insert(msg.hd.instrument_id, price_precision);
1264    }
1265
1266    Ok(())
1267}
1268
1269/// Updates the instrument ID map using exchange information from the symbol map.
1270fn update_instrument_id_map_with_exchange(
1271    symbol_map: &PitSymbolMap,
1272    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1273    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1274    raw_instrument_id: u32,
1275    exchange: &str,
1276    share_class_space: bool,
1277) -> anyhow::Result<InstrumentId> {
1278    let raw_symbol = symbol_map.get(raw_instrument_id).ok_or_else(|| {
1279        anyhow::anyhow!("Cannot resolve raw_symbol for instrument_id {raw_instrument_id}")
1280    })?;
1281    let symbol = nautilus_symbol(raw_symbol, share_class_space);
1282    let venue = Venue::from_code(exchange)
1283        .map_err(|e| anyhow::anyhow!("Invalid venue code '{exchange}': {e}"))?;
1284    let instrument_id = InstrumentId::new(symbol, venue);
1285    symbol_venue_map.rcu(|m| {
1286        m.entry(symbol).or_insert(venue);
1287    });
1288
1289    instrument_id_map.insert(
1290        raw_instrument_id,
1291        CachedInstrumentId::Exchange(instrument_id),
1292    );
1293    Ok(instrument_id)
1294}
1295
1296fn update_instrument_id_map(
1297    record: &dbn::RecordRef,
1298    symbol_map: &PitSymbolMap,
1299    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1300    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1301    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1302    share_class_space: bool,
1303) -> anyhow::Result<InstrumentId> {
1304    let header = record.header();
1305
1306    // Check if instrument ID is already in the map
1307    if let Some(&instrument_id) = instrument_id_map.get(&header.instrument_id) {
1308        return Ok(instrument_id.id());
1309    }
1310
1311    let raw_symbol = symbol_map.get_for_rec(record).ok_or_else(|| {
1312        anyhow::anyhow!(
1313            "Cannot resolve `raw_symbol` from `symbol_map` for instrument_id {}",
1314            header.instrument_id
1315        )
1316    })?;
1317
1318    let symbol = nautilus_symbol(raw_symbol, share_class_space);
1319
1320    let publisher_id = header.publisher_id;
1321    let venue = if let Some(venue) = symbol_venue_map.get_cloned(&symbol) {
1322        venue
1323    } else {
1324        let venue = publisher_venue_map
1325            .get(&publisher_id)
1326            .ok_or_else(|| anyhow::anyhow!("No venue found for `publisher_id` {publisher_id}"))?;
1327        *venue
1328    };
1329    let instrument_id = InstrumentId::new(symbol, venue);
1330
1331    instrument_id_map.insert(
1332        header.instrument_id,
1333        CachedInstrumentId::Inferred(instrument_id),
1334    );
1335    Ok(instrument_id)
1336}
1337
1338impl CachedInstrumentId {
1339    fn id(self) -> InstrumentId {
1340        match self {
1341            Self::Inferred(id) | Self::Definition(id) | Self::Exchange(id) => id,
1342        }
1343    }
1344}
1345
1346#[expect(clippy::too_many_arguments)]
1347fn handle_status_msg(
1348    msg: &dbn::StatusMsg,
1349    record: &dbn::RecordRef,
1350    symbol_map: &PitSymbolMap,
1351    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1352    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1353    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1354    ts_init: UnixNanos,
1355    share_class_space: bool,
1356) -> anyhow::Result<InstrumentStatus> {
1357    let instrument_id = update_instrument_id_map(
1358        record,
1359        symbol_map,
1360        publisher_venue_map,
1361        symbol_venue_map,
1362        instrument_id_map,
1363        share_class_space,
1364    )?;
1365
1366    decode_status_msg(msg, instrument_id, Some(ts_init))
1367}
1368
1369#[expect(clippy::too_many_arguments)]
1370fn handle_imbalance_msg(
1371    msg: &dbn::ImbalanceMsg,
1372    record: &dbn::RecordRef,
1373    symbol_map: &PitSymbolMap,
1374    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1375    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1376    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1377    instrument_def_price_precision_map: &AHashMap<u32, u8>,
1378    subscription_price_precision_map: &AHashMap<u32, u8>,
1379    price_precision_overrides: &AHashMap<Symbol, u8>,
1380    ts_init: UnixNanos,
1381    share_class_space: bool,
1382) -> anyhow::Result<DatabentoImbalance> {
1383    let instrument_id = update_instrument_id_map(
1384        record,
1385        symbol_map,
1386        publisher_venue_map,
1387        symbol_venue_map,
1388        instrument_id_map,
1389        share_class_space,
1390    )?;
1391
1392    let price_precision = resolve_price_precision(
1393        msg.hd.instrument_id,
1394        symbol_map.get_for_rec(record).map(String::as_str),
1395        instrument_def_price_precision_map,
1396        subscription_price_precision_map,
1397        price_precision_overrides,
1398    );
1399
1400    decode_imbalance_msg(msg, instrument_id, price_precision, Some(ts_init))
1401}
1402
1403#[expect(clippy::too_many_arguments)]
1404fn handle_statistics_msg(
1405    msg: &dbn::StatMsg,
1406    record: &dbn::RecordRef,
1407    symbol_map: &PitSymbolMap,
1408    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1409    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1410    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1411    instrument_def_price_precision_map: &AHashMap<u32, u8>,
1412    subscription_price_precision_map: &AHashMap<u32, u8>,
1413    price_precision_overrides: &AHashMap<Symbol, u8>,
1414    ts_init: UnixNanos,
1415    share_class_space: bool,
1416) -> anyhow::Result<Option<DatabentoStatistics>> {
1417    // Precheck before symbol resolution so unmodeled types skip cleanly
1418    if !is_supported_stat_type(msg.stat_type) {
1419        log::warn!("Skipping unsupported `stat_type` {}", msg.stat_type);
1420        return Ok(None);
1421    }
1422
1423    let instrument_id = update_instrument_id_map(
1424        record,
1425        symbol_map,
1426        publisher_venue_map,
1427        symbol_venue_map,
1428        instrument_id_map,
1429        share_class_space,
1430    )?;
1431
1432    let price_precision = resolve_price_precision(
1433        msg.hd.instrument_id,
1434        symbol_map.get_for_rec(record).map(String::as_str),
1435        instrument_def_price_precision_map,
1436        subscription_price_precision_map,
1437        price_precision_overrides,
1438    );
1439
1440    decode_statistics_msg(msg, instrument_id, price_precision, Some(ts_init))
1441}
1442
1443#[expect(clippy::too_many_arguments)]
1444fn handle_record(
1445    record: dbn::RecordRef,
1446    symbol_map: &PitSymbolMap,
1447    publisher_venue_map: &IndexMap<PublisherId, Venue>,
1448    symbol_venue_map: &AtomicMap<Symbol, Venue>,
1449    instrument_id_map: &mut AHashMap<u32, CachedInstrumentId>,
1450    instrument_def_price_precision_map: &AHashMap<u32, u8>,
1451    subscription_price_precision_map: &AHashMap<u32, u8>,
1452    price_precision_overrides: &AHashMap<Symbol, u8>,
1453    ts_init: UnixNanos,
1454    initialized_books: &HashSet<InstrumentId>,
1455    bars_timestamp_on_close: bool,
1456    share_class_space: bool,
1457) -> anyhow::Result<(Option<Data>, Option<Data>)> {
1458    let instrument_id = update_instrument_id_map(
1459        &record,
1460        symbol_map,
1461        publisher_venue_map,
1462        symbol_venue_map,
1463        instrument_id_map,
1464        share_class_space,
1465    )?;
1466
1467    let price_precision = resolve_price_precision(
1468        record.header().instrument_id,
1469        symbol_map.get_for_rec(&record).map(String::as_str),
1470        instrument_def_price_precision_map,
1471        subscription_price_precision_map,
1472        price_precision_overrides,
1473    );
1474
1475    // For MBP-1 and quote-based schemas, always include trades since they're integral to the data
1476    // For MBO, only include trades after the book is initialized to maintain consistency
1477    let include_trades = if record.get::<dbn::Mbp1Msg>().is_some()
1478        || record.get::<dbn::TbboMsg>().is_some()
1479        || record.get::<dbn::Cmbp1Msg>().is_some()
1480    {
1481        true // These schemas include trade information directly
1482    } else {
1483        initialized_books.contains(&instrument_id) // MBO requires initialized book
1484    };
1485
1486    decode_record(
1487        &record,
1488        instrument_id,
1489        price_precision,
1490        Some(ts_init),
1491        include_trades,
1492        bars_timestamp_on_close,
1493    )
1494}
1495
1496/// Claims the instrument's trade source for a trade decoded from a record of `rtype`, returning
1497/// whether `data` should be sent.
1498///
1499/// The `trades`, `mbp-1`, `tbbo`, `cmbp-1`, `tcbbo`, and `mbo` schemas all carry the same venue
1500/// trades, so an instrument subscribed on several of them would receive each trade once per schema.
1501/// The first record type to deliver a trade for an instrument carries its trades for the session,
1502/// and trades decoded from any other record type are dropped. A later subscription never takes
1503/// over, since its stream starts at an unknown point and a handover could drop or repeat trades.
1504/// Data other than trades always passes.
1505fn claim_trade_source(
1506    data: &Data,
1507    rtype: u8,
1508    trade_sources: &mut AHashMap<InstrumentId, u8>,
1509) -> bool {
1510    match data {
1511        Data::Trade(trade) => *trade_sources.entry(trade.instrument_id).or_insert(rtype) == rtype,
1512        _ => true,
1513    }
1514}
1515
1516fn resolve_price_precision(
1517    record_instrument_id: u32,
1518    raw_symbol: Option<&str>,
1519    instrument_def_price_precision_map: &AHashMap<u32, u8>,
1520    subscription_price_precision_map: &AHashMap<u32, u8>,
1521    price_precision_overrides: &AHashMap<Symbol, u8>,
1522) -> u8 {
1523    subscription_price_precision_map
1524        .get(&record_instrument_id)
1525        .copied()
1526        .or_else(|| {
1527            raw_symbol.and_then(|symbol| {
1528                price_precision_overrides
1529                    .get(&Symbol::from_str_unchecked(symbol))
1530                    .copied()
1531            })
1532        })
1533        .or_else(|| {
1534            instrument_def_price_precision_map
1535                .get(&record_instrument_id)
1536                .copied()
1537        })
1538        .unwrap_or(Currency::USD().precision)
1539}
1540
1541/// Processes an MBO delta through the buffering state machine.
1542///
1543/// Returns `Some(deltas)` when a complete batch is ready to emit (non-snapshot
1544/// F_LAST with replay buffering complete), or `None` when still accumulating.
1545fn process_mbo_delta(
1546    delta: OrderBookDelta,
1547    flags: u8,
1548    buffering_start: &mut Option<UnixNanos>,
1549    buffered_deltas: &mut AHashMap<InstrumentId, Vec<OrderBookDelta>>,
1550) -> Option<OrderBookDeltas> {
1551    let is_last = RecordFlag::F_LAST.matches(flags);
1552    let is_snapshot = RecordFlag::F_SNAPSHOT.matches(flags);
1553
1554    // Most live MBO events are single non-snapshot deltas, avoid map churn on that path
1555    if is_last
1556        && !is_snapshot
1557        && buffering_start.is_none()
1558        && !buffered_deltas.contains_key(&delta.instrument_id)
1559    {
1560        let deltas = OrderBookDeltas::new(delta.instrument_id, vec![delta]);
1561        return Some(deltas);
1562    }
1563
1564    let buffer = buffered_deltas.entry(delta.instrument_id).or_default();
1565    buffer.push(delta);
1566
1567    if is_snapshot {
1568        return None;
1569    }
1570
1571    flush_mbo_event_boundary(
1572        delta.instrument_id,
1573        delta.ts_event,
1574        flags,
1575        buffering_start,
1576        buffered_deltas,
1577    )
1578}
1579
1580/// Flushes the buffered deltas for `instrument_id` when `flags` marks a
1581/// non-snapshot match-event boundary (`F_LAST`).
1582///
1583/// Databento documents that records which decode to no delta (`Action::Fill`,
1584/// `Action::None`) may carry `F_LAST`, so the boundary must be processed
1585/// independently of the decoded payload or a buffered partial event is
1586/// stranded (follow-up to #4445).
1587fn flush_mbo_event_boundary(
1588    instrument_id: InstrumentId,
1589    ts_event: UnixNanos,
1590    flags: u8,
1591    buffering_start: &mut Option<UnixNanos>,
1592    buffered_deltas: &mut AHashMap<InstrumentId, Vec<OrderBookDelta>>,
1593) -> Option<OrderBookDeltas> {
1594    if !RecordFlag::F_LAST.matches(flags) || RecordFlag::F_SNAPSHOT.matches(flags) {
1595        return None;
1596    }
1597
1598    let buffer = buffered_deltas.get(&instrument_id)?;
1599
1600    if buffer.is_empty() {
1601        return None;
1602    }
1603
1604    if let Some(start_ns) = *buffering_start {
1605        if ts_event <= start_ns {
1606            return None;
1607        }
1608        *buffering_start = None;
1609    }
1610
1611    let buffer = buffered_deltas.remove(&instrument_id)?;
1612    let deltas = OrderBookDeltas::new(instrument_id, buffer);
1613    Some(deltas)
1614}
1615
1616#[cfg(test)]
1617mod tests {
1618    use std::path::PathBuf;
1619
1620    use databento::{dbn::enums::rtype, live::Subscription};
1621    use indexmap::IndexMap;
1622    use nautilus_model::types::{Price, Quantity};
1623    use rstest::*;
1624    use time::macros::datetime;
1625
1626    use super::*;
1627
1628    fn stub_delta(instrument_id: InstrumentId, ts: u64) -> OrderBookDelta {
1629        use nautilus_model::{
1630            data::BookOrder,
1631            enums::{BookAction, OrderSide},
1632            types::{Price, Quantity},
1633        };
1634
1635        OrderBookDelta::new(
1636            instrument_id,
1637            BookAction::Delete,
1638            BookOrder::new(
1639                OrderSide::Sell,
1640                Price::from("100.00"),
1641                Quantity::from("5"),
1642                42,
1643            ),
1644            0, // non-terminal: no F_LAST
1645            1,
1646            ts.into(),
1647            ts.into(),
1648        )
1649    }
1650
1651    #[rstest]
1652    fn test_boundary_flag_on_recordless_message_flushes_buffered_event() {
1653        // A non-terminal 'C' delta buffers; the event terminates on an 'N'
1654        // record carrying F_LAST which decodes to no delta - the raw flag
1655        // must still flush the buffer (follow-up to #4445).
1656        let instrument_id = InstrumentId::from("TEST.GLBX");
1657        let mut buffering_start = None;
1658        let mut buffered = AHashMap::new();
1659
1660        let buffered_result = process_mbo_delta(
1661            stub_delta(instrument_id, 1),
1662            0,
1663            &mut buffering_start,
1664            &mut buffered,
1665        );
1666        assert!(buffered_result.is_none());
1667
1668        let flushed = flush_mbo_event_boundary(
1669            instrument_id,
1670            2.into(),
1671            RecordFlag::F_LAST as u8,
1672            &mut buffering_start,
1673            &mut buffered,
1674        );
1675        assert!(flushed.is_some());
1676        assert!(buffered.is_empty());
1677    }
1678
1679    #[rstest]
1680    fn test_boundary_flag_with_empty_buffer_is_noop() {
1681        let instrument_id = InstrumentId::from("TEST.GLBX");
1682        let start = UnixNanos::from(1);
1683        let mut buffering_start = Some(start);
1684        let mut buffered: AHashMap<InstrumentId, Vec<OrderBookDelta>> = AHashMap::new();
1685        buffered.insert(instrument_id, Vec::new());
1686
1687        let flushed = flush_mbo_event_boundary(
1688            instrument_id,
1689            2.into(),
1690            RecordFlag::F_LAST as u8,
1691            &mut buffering_start,
1692            &mut buffered,
1693        );
1694
1695        assert!(flushed.is_none());
1696        assert_eq!(buffering_start, Some(start));
1697    }
1698
1699    #[rstest]
1700    fn test_empty_boundary_preserves_replay_gate_for_buffered_instrument() {
1701        let empty_id = InstrumentId::from("EMPTY.GLBX");
1702        let buffered_id = InstrumentId::from("BUFFERED.GLBX");
1703        let start = UnixNanos::from(10);
1704        let mut buffering_start = Some(start);
1705        let mut buffered = AHashMap::new();
1706        let _ = process_mbo_delta(
1707            stub_delta(buffered_id, 1),
1708            0,
1709            &mut buffering_start,
1710            &mut buffered,
1711        );
1712
1713        let empty_boundary = flush_mbo_event_boundary(
1714            empty_id,
1715            11.into(),
1716            RecordFlag::F_LAST as u8,
1717            &mut buffering_start,
1718            &mut buffered,
1719        );
1720        let buffered_boundary = flush_mbo_event_boundary(
1721            buffered_id,
1722            5.into(),
1723            RecordFlag::F_LAST as u8,
1724            &mut buffering_start,
1725            &mut buffered,
1726        );
1727
1728        assert!(empty_boundary.is_none());
1729        assert!(buffered_boundary.is_none());
1730        assert_eq!(buffering_start, Some(start));
1731        assert!(buffered.contains_key(&buffered_id));
1732    }
1733
1734    #[rstest]
1735    fn test_boundary_flag_respects_replay_buffering_gate() {
1736        let instrument_id = InstrumentId::from("TEST.GLBX");
1737        let mut buffering_start = Some(UnixNanos::from(10));
1738        let mut buffered = AHashMap::new();
1739        let _ = process_mbo_delta(
1740            stub_delta(instrument_id, 1),
1741            0,
1742            &mut buffering_start,
1743            &mut buffered,
1744        );
1745
1746        // Boundary inside the replay gate: keep buffering
1747        let flushed = flush_mbo_event_boundary(
1748            instrument_id,
1749            5.into(),
1750            RecordFlag::F_LAST as u8,
1751            &mut buffering_start,
1752            &mut buffered,
1753        );
1754        assert!(flushed.is_none());
1755        assert!(!buffered.is_empty());
1756
1757        // Boundary past the gate: flush and clear the gate
1758        let flushed = flush_mbo_event_boundary(
1759            instrument_id,
1760            11.into(),
1761            RecordFlag::F_LAST as u8,
1762            &mut buffering_start,
1763            &mut buffered,
1764        );
1765        assert!(flushed.is_some());
1766        assert!(buffering_start.is_none());
1767    }
1768
1769    #[rstest]
1770    fn test_boundary_snapshot_flag_never_flushes() {
1771        let instrument_id = InstrumentId::from("TEST.GLBX");
1772        let mut buffering_start = None;
1773        let mut buffered = AHashMap::new();
1774        let _ = process_mbo_delta(
1775            stub_delta(instrument_id, 1),
1776            0,
1777            &mut buffering_start,
1778            &mut buffered,
1779        );
1780
1781        let flags = RecordFlag::F_LAST as u8 | RecordFlag::F_SNAPSHOT as u8;
1782        let flushed = flush_mbo_event_boundary(
1783            instrument_id,
1784            2.into(),
1785            flags,
1786            &mut buffering_start,
1787            &mut buffered,
1788        );
1789        assert!(flushed.is_none());
1790    }
1791
1792    fn stub_trade(instrument_id: InstrumentId) -> Data {
1793        use nautilus_model::{
1794            data::TradeTick,
1795            enums::AggressorSide,
1796            identifiers::TradeId,
1797            types::{Price, Quantity},
1798        };
1799
1800        Data::Trade(TradeTick::new(
1801            instrument_id,
1802            Price::from("100.00"),
1803            Quantity::from("5"),
1804            AggressorSide::Buy,
1805            TradeId::new("1"),
1806            1.into(),
1807            1.into(),
1808        ))
1809    }
1810
1811    #[rstest]
1812    fn test_trade_source_is_first_record_type_to_deliver_a_trade() {
1813        let trade = stub_trade(InstrumentId::from("TEST.GLBX"));
1814        let mut trade_sources = AHashMap::new();
1815
1816        let sent: Vec<bool> = [rtype::MBP_1, rtype::MBP_0, rtype::MBO, rtype::MBP_1]
1817            .into_iter()
1818            .map(|rtype| claim_trade_source(&trade, rtype, &mut trade_sources))
1819            .collect();
1820
1821        assert_eq!(sent, [true, false, false, true]);
1822    }
1823
1824    #[rstest]
1825    fn test_trade_source_is_kept_per_instrument() {
1826        let es_trade = stub_trade(InstrumentId::from("ESM4.GLBX"));
1827        let nq_trade = stub_trade(InstrumentId::from("NQM4.GLBX"));
1828        let mut trade_sources = AHashMap::new();
1829
1830        let sent: Vec<bool> = [
1831            (&es_trade, rtype::MBP_0),
1832            (&nq_trade, rtype::MBO),
1833            (&es_trade, rtype::MBO),
1834            (&nq_trade, rtype::MBP_0),
1835        ]
1836        .into_iter()
1837        .map(|(trade, rtype)| claim_trade_source(trade, rtype, &mut trade_sources))
1838        .collect();
1839
1840        assert_eq!(sent, [true, true, false, false]);
1841    }
1842
1843    #[rstest]
1844    fn test_trade_source_passes_data_other_than_trades() {
1845        let instrument_id = InstrumentId::from("TEST.GLBX");
1846        let mut trade_sources = AHashMap::new();
1847        let _ = claim_trade_source(&stub_trade(instrument_id), rtype::MBP_0, &mut trade_sources);
1848        let delta = Data::BookDelta(stub_delta(instrument_id, 1));
1849
1850        let sent = claim_trade_source(&delta, rtype::MBO, &mut trade_sources);
1851
1852        assert!(sent);
1853    }
1854
1855    fn create_test_handler(reconnect_timeout_mins: Option<u64>) -> DatabentoFeedHandler {
1856        let (_cmd_tx, cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1857        let (msg_tx, _msg_rx) = tokio::sync::mpsc::unbounded_channel();
1858
1859        DatabentoFeedHandler::new(
1860            Credential::new("test_key"),
1861            "GLBX.MDP3".to_string(),
1862            cmd_rx,
1863            msg_tx,
1864            IndexMap::new(),
1865            Arc::new(AtomicMap::new()),
1866            false,
1867            false,
1868            reconnect_timeout_mins,
1869        )
1870    }
1871
1872    fn create_test_client() -> DatabentoLiveClient {
1873        DatabentoLiveClient::new(
1874            "test-api-key".to_string(),
1875            "GLBX.MDP3".to_string(),
1876            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("publishers.json"),
1877            true,
1878            None,
1879            None,
1880        )
1881        .unwrap()
1882    }
1883
1884    #[rstest]
1885    #[case(Some(10))]
1886    #[case(None)]
1887    fn test_backoff_initialization(#[case] reconnect_timeout_mins: Option<u64>) {
1888        let handler = create_test_handler(reconnect_timeout_mins);
1889
1890        assert_eq!(handler.reconnect_timeout_mins, reconnect_timeout_mins);
1891        assert!(handler.subscriptions.is_empty());
1892        assert!(handler.buffered_commands.is_empty());
1893    }
1894
1895    #[rstest]
1896    fn test_subscription_with_and_without_start() {
1897        let start_time = datetime!(2024-01-01 00:00:00 UTC);
1898        let sub_with_start = Subscription::builder()
1899            .symbols("ES.FUT")
1900            .schema(databento::dbn::Schema::Mbp1)
1901            .start(start_time)
1902            .build();
1903
1904        let mut sub_without_start = sub_with_start.clone();
1905        sub_without_start.start = None;
1906
1907        assert!(sub_with_start.start.is_some());
1908        assert!(sub_without_start.start.is_none());
1909        assert_eq!(sub_with_start.schema, sub_without_start.schema);
1910        assert_eq!(sub_with_start.symbols, sub_without_start.symbols);
1911    }
1912
1913    #[rstest]
1914    fn test_handler_initialization_state() {
1915        let handler = create_test_handler(Some(10));
1916
1917        assert!(!handler.replay);
1918        assert_eq!(handler.dataset, "GLBX.MDP3");
1919        assert_eq!(handler.credential.api_key(), "test_key");
1920        assert!(handler.subscriptions.is_empty());
1921        assert!(handler.buffered_commands.is_empty());
1922    }
1923
1924    #[rstest]
1925    fn test_handler_with_no_timeout() {
1926        let handler = create_test_handler(None);
1927
1928        assert_eq!(handler.reconnect_timeout_mins, None);
1929        assert!(!handler.replay);
1930    }
1931
1932    #[rstest]
1933    fn test_handler_with_zero_timeout() {
1934        let handler = create_test_handler(Some(0));
1935
1936        assert_eq!(handler.reconnect_timeout_mins, Some(0));
1937        assert!(!handler.replay);
1938    }
1939
1940    #[rstest]
1941    fn test_subscribe_uses_explicit_parent_stype() {
1942        let mut client = create_test_client();
1943
1944        client
1945            .subscribe(
1946                "definition".to_string(),
1947                vec![InstrumentId::from("ES.FUT.GLBX")],
1948                None,
1949                None,
1950                None,
1951                Some("parent".to_string()),
1952            )
1953            .unwrap();
1954
1955        let command = client.cmd_rx.as_mut().unwrap().try_recv().unwrap();
1956        match command {
1957            HandlerCommand::Subscribe(sub) => {
1958                assert_eq!(sub.schema, dbn::Schema::Definition);
1959                assert_eq!(sub.stype_in, dbn::SType::Parent);
1960                assert_eq!(sub.symbols.to_api_string(), "ES.FUT");
1961            }
1962            other => panic!("expected HandlerCommand::Subscribe, was {other:?}"),
1963        }
1964    }
1965
1966    #[rstest]
1967    fn test_subscribe_rejects_invalid_stype() {
1968        let mut client = create_test_client();
1969
1970        let err = client
1971            .subscribe(
1972                "definition".to_string(),
1973                vec![InstrumentId::from("ES.FUT.GLBX")],
1974                None,
1975                None,
1976                None,
1977                Some("not-a-stype".to_string()),
1978            )
1979            .unwrap_err();
1980
1981        assert!(err.to_string().contains("not-a-stype"));
1982        assert!(!is_command_send_error(&err));
1983        assert!(matches!(
1984            client.cmd_rx.as_mut().unwrap().try_recv(),
1985            Err(tokio::sync::mpsc::error::TryRecvError::Empty)
1986        ));
1987    }
1988
1989    #[rstest]
1990    fn test_subscribe_classifies_command_send_errors() {
1991        let mut client = create_test_client();
1992        client.cmd_rx = None;
1993
1994        let err = client
1995            .subscribe(
1996                "definition".to_string(),
1997                vec![InstrumentId::from("ES.FUT.GLBX")],
1998                None,
1999                None,
2000                None,
2001                Some("parent".to_string()),
2002            )
2003            .unwrap_err();
2004
2005        assert!(is_command_send_error(&err));
2006    }
2007
2008    #[rstest]
2009    fn test_close_after_handler_exit_marks_closed() {
2010        let mut client = create_test_client();
2011        let (handler, _msg_rx) = client.start().unwrap();
2012        drop(handler);
2013
2014        client.close().unwrap();
2015
2016        assert!(!client.is_running());
2017        assert!(client.is_closed());
2018    }
2019
2020    fn test_delta(instrument_id: InstrumentId, ts_event: u64) -> OrderBookDelta {
2021        OrderBookDelta::clear(instrument_id, 0, ts_event.into(), 0.into())
2022    }
2023
2024    #[rstest]
2025    fn test_mbo_delta_without_f_last_buffers() {
2026        let instrument_id = InstrumentId::from("ESM4.GLBX");
2027        let delta = test_delta(instrument_id, 1_000_000_000);
2028        let mut buffering_start = None;
2029        let mut buffered = AHashMap::new();
2030
2031        let result = process_mbo_delta(delta, 0, &mut buffering_start, &mut buffered);
2032
2033        assert!(result.is_none());
2034        assert_eq!(buffered[&instrument_id].len(), 1);
2035    }
2036
2037    #[rstest]
2038    fn test_mbo_single_f_last_emits_without_buffering() {
2039        let instrument_id = InstrumentId::from("ESM4.GLBX");
2040        let ts_event = 1_000_000_000;
2041        let mut delta = test_delta(instrument_id, ts_event);
2042        delta.flags = 128;
2043        delta.sequence = 42;
2044        let mut buffering_start = None;
2045        let mut buffered = AHashMap::new();
2046
2047        let result = process_mbo_delta(delta, delta.flags, &mut buffering_start, &mut buffered);
2048
2049        let emitted = result.expect("single F_LAST delta should emit");
2050        assert_eq!(emitted.instrument_id, instrument_id);
2051        assert_eq!(emitted.deltas.len(), 1);
2052        assert_eq!(emitted.flags, 128);
2053        assert_eq!(emitted.sequence, 42);
2054        assert_eq!(emitted.ts_event, UnixNanos::from(ts_event));
2055        assert_eq!(emitted.deltas[0].instrument_id, instrument_id);
2056        assert_eq!(emitted.deltas[0].flags, 128);
2057        assert_eq!(emitted.deltas[0].sequence, 42);
2058        assert_eq!(emitted.deltas[0].ts_event, UnixNanos::from(ts_event));
2059        assert!(buffering_start.is_none());
2060        assert!(buffered.is_empty());
2061    }
2062
2063    #[rstest]
2064    fn test_mbo_delta_with_f_last_emits() {
2065        let instrument_id = InstrumentId::from("ESM4.GLBX");
2066        let mut buffering_start = None;
2067        let mut buffered = AHashMap::new();
2068
2069        let _ = process_mbo_delta(
2070            test_delta(instrument_id, 1_000_000_000),
2071            0,
2072            &mut buffering_start,
2073            &mut buffered,
2074        );
2075
2076        let result = process_mbo_delta(
2077            test_delta(instrument_id, 2_000_000_000),
2078            128, // F_LAST
2079            &mut buffering_start,
2080            &mut buffered,
2081        );
2082
2083        assert!(result.is_some());
2084        assert_eq!(result.unwrap().deltas.len(), 2);
2085        assert!(buffered.is_empty());
2086    }
2087
2088    #[rstest]
2089    fn test_mbo_snapshot_with_f_last_buffers() {
2090        let instrument_id = InstrumentId::from("ESM4.GLBX");
2091        let mut buffering_start = None;
2092        let mut buffered = AHashMap::new();
2093
2094        let result = process_mbo_delta(
2095            test_delta(instrument_id, 1_000_000_000),
2096            128 | 32, // F_LAST | F_SNAPSHOT
2097            &mut buffering_start,
2098            &mut buffered,
2099        );
2100
2101        assert!(result.is_none());
2102        assert_eq!(buffered[&instrument_id].len(), 1);
2103    }
2104
2105    #[rstest]
2106    fn test_mbo_replay_buffers_until_past_start() {
2107        let instrument_id = InstrumentId::from("ESM4.GLBX");
2108        let start_ns = 5_000_000_000u64;
2109        let mut buffering_start = Some(start_ns.into());
2110        let mut buffered = AHashMap::new();
2111
2112        let result = process_mbo_delta(
2113            test_delta(instrument_id, 4_000_000_000),
2114            128, // F_LAST
2115            &mut buffering_start,
2116            &mut buffered,
2117        );
2118        assert!(result.is_none());
2119
2120        let result = process_mbo_delta(
2121            test_delta(instrument_id, 5_000_000_000),
2122            128,
2123            &mut buffering_start,
2124            &mut buffered,
2125        );
2126        assert!(result.is_none());
2127
2128        // Delta past start: emits and clears buffering_start
2129        let result = process_mbo_delta(
2130            test_delta(instrument_id, 6_000_000_000),
2131            128,
2132            &mut buffering_start,
2133            &mut buffered,
2134        );
2135        assert!(result.is_some());
2136        assert!(buffering_start.is_none());
2137    }
2138
2139    #[rstest]
2140    fn test_mbo_multiple_deltas_accumulated() {
2141        let instrument_id = InstrumentId::from("ESM4.GLBX");
2142        let mut buffering_start = None;
2143        let mut buffered = AHashMap::new();
2144
2145        for i in 0..5 {
2146            process_mbo_delta(
2147                test_delta(instrument_id, 1_000_000_000 + i),
2148                0,
2149                &mut buffering_start,
2150                &mut buffered,
2151            );
2152        }
2153
2154        let result = process_mbo_delta(
2155            test_delta(instrument_id, 2_000_000_000),
2156            128,
2157            &mut buffering_start,
2158            &mut buffered,
2159        );
2160
2161        assert!(result.is_some());
2162        assert_eq!(result.unwrap().deltas.len(), 6);
2163    }
2164
2165    #[rstest]
2166    fn test_mbo_multi_instrument_isolation() {
2167        let id_a = InstrumentId::from("ESM4.GLBX");
2168        let id_b = InstrumentId::from("NQM4.GLBX");
2169        let mut buffering_start = None;
2170        let mut buffered = AHashMap::new();
2171
2172        process_mbo_delta(
2173            test_delta(id_a, 1_000_000_000),
2174            0,
2175            &mut buffering_start,
2176            &mut buffered,
2177        );
2178        process_mbo_delta(
2179            test_delta(id_b, 1_000_000_000),
2180            0,
2181            &mut buffering_start,
2182            &mut buffered,
2183        );
2184
2185        // F_LAST for A: only A's deltas emitted, B remains
2186        let result = process_mbo_delta(
2187            test_delta(id_a, 2_000_000_000),
2188            128,
2189            &mut buffering_start,
2190            &mut buffered,
2191        );
2192
2193        assert!(result.is_some());
2194        assert_eq!(result.unwrap().instrument_id, id_a);
2195        assert!(buffered.contains_key(&id_b));
2196        assert!(!buffered.contains_key(&id_a));
2197    }
2198
2199    mod property_tests {
2200        use proptest::prelude::*;
2201        use rstest::rstest;
2202
2203        use super::*;
2204
2205        proptest! {
2206            #[rstest]
2207            fn mbo_buffering_conserves_deltas(
2208                num_non_last in 0usize..=20,
2209            ) {
2210                let instrument_id = InstrumentId::from("ESM4.GLBX");
2211                let mut buffering_start = None;
2212                let mut buffered = AHashMap::new();
2213                let total = num_non_last + 1;
2214
2215                for i in 0..num_non_last {
2216                    let result = process_mbo_delta(
2217                        test_delta(instrument_id, 1_000_000_000 + i as u64),
2218                        0, // No F_LAST
2219                        &mut buffering_start,
2220                        &mut buffered,
2221                    );
2222                    prop_assert!(result.is_none());
2223                }
2224
2225                let result = process_mbo_delta(
2226                    test_delta(instrument_id, 2_000_000_000),
2227                    128, // F_LAST
2228                    &mut buffering_start,
2229                    &mut buffered,
2230                );
2231
2232                prop_assert!(result.is_some());
2233                let emitted = result.unwrap();
2234                prop_assert_eq!(emitted.deltas.len(), total);
2235                prop_assert!(buffered.is_empty());
2236            }
2237
2238            #[rstest]
2239            fn mbo_snapshots_never_emit(
2240                num_snapshots in 1usize..=20,
2241            ) {
2242                let instrument_id = InstrumentId::from("ESM4.GLBX");
2243                let mut buffering_start = None;
2244                let mut buffered = AHashMap::new();
2245
2246                for i in 0..num_snapshots {
2247                    let result = process_mbo_delta(
2248                        test_delta(instrument_id, 1_000_000_000 + i as u64),
2249                        128 | 32, // F_LAST | F_SNAPSHOT
2250                        &mut buffering_start,
2251                        &mut buffered,
2252                    );
2253                    prop_assert!(result.is_none());
2254                }
2255
2256                prop_assert_eq!(buffered[&instrument_id].len(), num_snapshots);
2257            }
2258
2259            #[rstest]
2260            fn mbo_replay_delays_emission(
2261                start_offset in 1u64..=100,
2262                num_before in 1usize..=10,
2263            ) {
2264                let instrument_id = InstrumentId::from("ESM4.GLBX");
2265                let start_ns = 1_000_000_000u64 * start_offset;
2266                let mut buffering_start = Some(start_ns.into());
2267                let mut buffered = AHashMap::new();
2268
2269                for i in 0..num_before {
2270                    let ts = start_ns - (num_before as u64 - i as u64);
2271                    let result = process_mbo_delta(
2272                        test_delta(instrument_id, ts),
2273                        128, // F_LAST
2274                        &mut buffering_start,
2275                        &mut buffered,
2276                    );
2277                    prop_assert!(result.is_none());
2278                }
2279
2280                let result = process_mbo_delta(
2281                    test_delta(instrument_id, start_ns + 1),
2282                    128,
2283                    &mut buffering_start,
2284                    &mut buffered,
2285                );
2286
2287                prop_assert!(result.is_some());
2288                prop_assert!(buffering_start.is_none());
2289                let total = num_before + 1;
2290                prop_assert_eq!(result.unwrap().deltas.len(), total);
2291            }
2292        }
2293    }
2294
2295    #[rstest]
2296    #[case(2, "CLZ29-CLF30.XNYM")]
2297    #[case(1, "CLZ9-CLF0.XNYM")]
2298    fn test_live_subscription_preserves_native_symbols(
2299        #[case] digits: u8,
2300        #[case] id: InstrumentId,
2301    ) {
2302        let mut client = create_test_client().with_futures_year_digits(digits.try_into().unwrap());
2303        client
2304            .subscribe(
2305                "mbp-1".to_string(),
2306                vec![id],
2307                None,
2308                None,
2309                Some(vec![Some(3)]),
2310                None,
2311            )
2312            .unwrap();
2313        let rx = client.cmd_rx.as_mut().unwrap();
2314        assert!(
2315            matches!(rx.try_recv().unwrap(), HandlerCommand::SetPricePrecision(symbol, 3) if symbol == Symbol::from("CLZ9-CLF0"))
2316        );
2317
2318        let HandlerCommand::Subscribe(subscription) = rx.try_recv().unwrap() else {
2319            panic!("expected subscription");
2320        };
2321
2322        let expected = Subscription::builder()
2323            .schema(dbn::Schema::Mbp1)
2324            .symbols(vec!["CLZ9-CLF0".to_string()])
2325            .build();
2326        assert_eq!(subscription, expected);
2327        assert_eq!(
2328            client.symbol_venue_map.get_cloned(&id.symbol),
2329            Some(Venue::from("XNYM"))
2330        );
2331        assert_eq!(
2332            rx.try_recv().unwrap_err(),
2333            tokio::sync::mpsc::error::TryRecvError::Empty
2334        );
2335    }
2336
2337    #[rstest]
2338    fn test_futures_cache_uses_requested_venue_after_year_conversion() {
2339        let mut handler = create_test_handler(None);
2340        handler.publisher_venue_map.insert(1, Venue::from("GLBX"));
2341        handler
2342            .symbol_venue_map
2343            .insert(Symbol::from("CLF26"), Venue::from("XNYM"));
2344        let mut mapping = dbn::PitSymbolMap::new();
2345        mapping.inner_mut().insert(42, "CLF6".to_string());
2346        let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
2347        msg.hd.publisher_id = 1;
2348        msg.hd.instrument_id = 42;
2349        msg.ts_recv = 1_766_102_400_000_000_000;
2350        let mut ids = AHashMap::new();
2351        handler
2352            .cache_futures_instrument_id(&dbn::RecordRef::from(&msg), &mapping, &mut ids)
2353            .unwrap();
2354        assert_eq!(
2355            ids.get(&42).copied().map(CachedInstrumentId::id),
2356            Some(InstrumentId::from("CLF26.XNYM"))
2357        );
2358    }
2359
2360    #[rstest]
2361    #[case(Some(2), Some(5), Some(4), 5)]
2362    #[case(Some(2), None, Some(5), 5)]
2363    #[case(Some(5), Some(2), None, 2)]
2364    #[case(Some(5), None, Some(2), 2)]
2365    #[case(Some(3), None, None, 3)]
2366    #[case(None, None, None, 2)]
2367    fn test_explicit_precision_precedes_definitions(
2368        #[case] definition: Option<u8>,
2369        #[case] mapped: Option<u8>,
2370        #[case] native: Option<u8>,
2371        #[case] expected: u8,
2372    ) {
2373        let definitions = AHashMap::from_iter(definition.map(|precision| (42, precision)));
2374        let subscriptions = AHashMap::from_iter(mapped.map(|precision| (42, precision)));
2375        let overrides =
2376            AHashMap::from_iter(native.map(|precision| (Symbol::from("CLF6"), precision)));
2377
2378        let actual =
2379            resolve_price_precision(42, Some("CLF6"), &definitions, &subscriptions, &overrides);
2380
2381        assert_eq!(actual, expected);
2382    }
2383
2384    #[rstest]
2385    #[case(true, "BRK B.EQUS")]
2386    #[case(false, "BRK.B.EQUS")]
2387    fn test_handle_record_spells_share_class_with_space(
2388        #[case] share_class_space: bool,
2389        #[case] expected: &str,
2390    ) {
2391        let handler = create_test_handler(None);
2392        let mut publisher_venue_map = IndexMap::new();
2393        publisher_venue_map.insert(1, Venue::from("EQUS"));
2394        let mut mapping = dbn::PitSymbolMap::new();
2395        mapping.inner_mut().insert(42, "BRK.B".to_string());
2396        let mut msg = dbn::TradeMsg::default();
2397        msg.hd.publisher_id = 1;
2398        msg.hd.instrument_id = 42;
2399        msg.hd.ts_event = 1_766_102_400_000_000_000;
2400        msg.ts_recv = msg.hd.ts_event;
2401        msg.price = 480_250_000_000;
2402        msg.size = 10;
2403        msg.side = b'A' as std::ffi::c_char;
2404        let record = dbn::RecordRef::from(&msg);
2405        let mut ids = AHashMap::new();
2406
2407        let (data, _) = handle_record(
2408            record,
2409            &mapping,
2410            &publisher_venue_map,
2411            &handler.symbol_venue_map,
2412            &mut ids,
2413            &AHashMap::new(),
2414            &AHashMap::new(),
2415            &AHashMap::new(),
2416            1_766_102_400_000_000_001.into(),
2417            &HashSet::new(),
2418            true,
2419            share_class_space,
2420        )
2421        .unwrap();
2422
2423        let Some(Data::Trade(trade)) = data else {
2424            panic!("expected trade");
2425        };
2426        assert_eq!(trade.instrument_id, InstrumentId::from(expected));
2427    }
2428
2429    #[rstest]
2430    fn test_futures_precision_override_uses_native_symbol() {
2431        let mut handler = create_test_handler(None);
2432        handler.publisher_venue_map.insert(1, Venue::from("GLBX"));
2433        let mut mapping = dbn::PitSymbolMap::new();
2434        mapping.inner_mut().insert(42, "CLF6".to_string());
2435        let mut msg = dbn::TradeMsg::default();
2436        msg.hd.publisher_id = 1;
2437        msg.hd.instrument_id = 42;
2438        msg.hd.ts_event = 1_766_102_400_000_000_000;
2439        msg.ts_recv = msg.hd.ts_event;
2440        msg.price = 75_125_000_000;
2441        msg.size = 7;
2442        msg.side = b'A' as std::ffi::c_char;
2443        let record = dbn::RecordRef::from(&msg);
2444        let mut ids = AHashMap::new();
2445        handler
2446            .cache_futures_instrument_id(&record, &mapping, &mut ids)
2447            .unwrap();
2448        let overrides = AHashMap::from_iter([(Symbol::from("CLF6"), 3)]);
2449        let (data, extra) = handle_record(
2450            record,
2451            &mapping,
2452            &handler.publisher_venue_map,
2453            &handler.symbol_venue_map,
2454            &mut ids,
2455            &AHashMap::new(),
2456            &AHashMap::new(),
2457            &overrides,
2458            1_766_102_400_000_000_001.into(),
2459            &HashSet::new(),
2460            true,
2461            false,
2462        )
2463        .unwrap();
2464
2465        let Some(Data::Trade(trade)) = data else {
2466            panic!("expected trade");
2467        };
2468
2469        assert_eq!(trade.instrument_id, InstrumentId::from("CLF26.GLBX"));
2470        assert_eq!(trade.price.precision, 3);
2471        assert_eq!(trade.price, Price::from("75.125"));
2472        assert_eq!(trade.size, Quantity::from(7));
2473        assert_eq!(trade.ts_event, msg.hd.ts_event);
2474        assert_eq!(trade.ts_init, 1_766_102_400_000_000_001);
2475        assert!(extra.is_none());
2476    }
2477}