Skip to main content

nautilus_backtest/
node.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Provides a [`BacktestNode`] that orchestrates catalog-driven backtests.
17
18use std::{fmt::Debug, iter::Peekable, mem};
19
20use ahash::{AHashMap, AHashSet};
21use nautilus_core::{Params, UnixNanos};
22use nautilus_data::replay::{DataIterator, ReplayBatch, ReplayPoll, ReplaySource};
23use nautilus_model::{
24    data::{Data, DataBatch, HasTsInit, NautilusDataType, OrderBookDelta, OrderBookDeltas},
25    enums::{BookType, OtoTriggerMode, RecordFlag},
26    identifiers::{InstrumentId, Venue},
27    types::Money,
28};
29use nautilus_persistence::{
30    backend::default_catalog_factories,
31    catalog::{
32        factory::{self as catalog_factory, CatalogFactoryRegistry},
33        traits::{CatalogInstrumentQuery, CatalogQuery, DataCatalog},
34    },
35};
36
37use crate::{
38    config::{BacktestDataConfig, BacktestRunConfig, SimulatedVenueConfig},
39    engine::BacktestEngine,
40    result::BacktestResult,
41};
42
43/// Orchestrates catalog-driven backtests from run configurations.
44///
45/// `BacktestNode` connects the catalog with [`BacktestEngine`] to load
46/// historical data and run backtests. Supports both oneshot and streaming modes.
47#[cfg_attr(
48    feature = "python",
49    pyo3::pyclass(module = "nautilus_trader.backtest", unsendable)
50)]
51#[cfg_attr(
52    feature = "python",
53    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.backtest")
54)]
55pub struct BacktestNode {
56    configs: Vec<BacktestRunConfig>,
57    catalog_factories: CatalogFactoryRegistry,
58    engines: AHashMap<String, BacktestEngine>,
59    disposed: bool,
60}
61
62impl BacktestNode {
63    /// Creates a [`BacktestNode`] with the default catalog factory registry.
64    ///
65    /// Validates that configs are non-empty and internally consistent:
66    /// - All data config instrument venues must have a matching venue config.
67    /// - L2/L3 book types require order book data in the data configs.
68    /// - Data config time ranges must be valid (start <= end).
69    ///
70    /// # Errors
71    ///
72    /// Returns an error if `configs` is empty or validation fails.
73    pub fn new(configs: Vec<BacktestRunConfig>) -> anyhow::Result<Self> {
74        Self::builder(configs).build()
75    }
76
77    /// Creates a builder with the default catalog factory registry.
78    ///
79    /// Validates the run configurations when [`BacktestNodeBuilder::build`] creates the node.
80    #[must_use]
81    pub fn builder(configs: Vec<BacktestRunConfig>) -> BacktestNodeBuilder {
82        let catalog_factories = default_catalog_factories();
83
84        BacktestNodeBuilder {
85            configs,
86            catalog_factories,
87        }
88    }
89
90    /// Returns the run configurations.
91    #[must_use]
92    pub fn configs(&self) -> &[BacktestRunConfig] {
93        &self.configs
94    }
95
96    /// Builds backtest engines from the run configurations.
97    ///
98    /// For each config, creates a [`BacktestEngine`], adds venues, and loads
99    /// instruments from the catalog. If building a config fails with
100    /// [`BacktestRunConfig::raise_exception`] disabled, logs the error and skips that config;
101    /// successful return does not guarantee an engine for every config.
102    /// A disposed node cannot be built again; create a new node instead.
103    ///
104    /// # Errors
105    ///
106    /// Returns an error if:
107    /// - This node has been disposed.
108    /// - Building an engine from a config fails and
109    ///   [`BacktestRunConfig::raise_exception`] is enabled for that config.
110    pub fn build(&mut self) -> anyhow::Result<()> {
111        self.ensure_not_disposed()?;
112
113        for config in &self.configs {
114            if self.engines.contains_key(config.id()) {
115                continue;
116            }
117
118            match build_engine(config, &self.catalog_factories) {
119                Ok(engine) => {
120                    self.engines.insert(config.id().to_string(), engine);
121                }
122                Err(e) if config.raise_exception() => return Err(e),
123                Err(e) => {
124                    log::error!("Error building backtest '{}': {e:#}", config.id());
125                }
126            }
127        }
128
129        Ok(())
130    }
131
132    /// Returns a mutable reference to the engine for the given run config ID.
133    #[must_use]
134    pub fn get_engine_mut(&mut self, id: &str) -> Option<&mut BacktestEngine> {
135        self.engines.get_mut(id)
136    }
137
138    /// Returns a reference to the engine for the given run config ID.
139    #[must_use]
140    pub fn get_engine(&self, id: &str) -> Option<&BacktestEngine> {
141        self.engines.get(id)
142    }
143
144    /// Returns all created backtest engines.
145    #[must_use]
146    pub fn get_engines(&self) -> Vec<&BacktestEngine> {
147        self.engines.values().collect()
148    }
149
150    /// Runs all configured backtests and returns results.
151    ///
152    /// Automatically calls [`build()`](Self::build) if engines have not been created yet.
153    /// For each run config, loads data from the catalog and runs the engine.
154    /// Supports both oneshot (`chunk_size = None`) and streaming modes.
155    /// Configs without a built engine are skipped. If a run fails with
156    /// [`BacktestRunConfig::raise_exception`] disabled, logs the error, clears its loaded data,
157    /// leaves the engine undisposed, and omits its result. A streaming run that fails to load
158    /// data after replaying earlier chunks ends its engine first, stopping the trader and engines.
159    /// A node disposed by a completed run or by [`dispose()`](Self::dispose)
160    /// cannot run again; create a new node instead.
161    ///
162    /// # Errors
163    ///
164    /// Returns an error if:
165    /// - This node has been disposed.
166    /// - Building, data loading, or engine execution fails and
167    ///   [`BacktestRunConfig::raise_exception`] is enabled for the run config.
168    pub fn run(&mut self) -> anyhow::Result<Vec<BacktestResult>> {
169        self.ensure_not_disposed()?;
170
171        // Auto-build if not already done
172        if self.engines.is_empty() {
173            self.build()?;
174        }
175
176        let mut results = Vec::new();
177
178        for config in &self.configs {
179            let Some(engine) = self.engines.get_mut(config.id()) else {
180                continue;
181            };
182
183            let run_result = match config.chunk_size() {
184                None => run_oneshot(engine, config, &self.catalog_factories),
185                Some(chunk_size) => {
186                    run_streaming(engine, config, chunk_size, &self.catalog_factories)
187                }
188            };
189
190            if let Err(e) = run_result {
191                if config.raise_exception() {
192                    return Err(e);
193                }
194
195                log::error!("Error running backtest '{}': {e:#}", config.id());
196                engine.clear_data();
197                continue;
198            }
199
200            results.push(engine.get_result());
201
202            if config.dispose_on_completion() {
203                engine.dispose();
204                self.disposed = true;
205            } else {
206                engine.clear_data();
207            }
208        }
209
210        Ok(results)
211    }
212
213    /// Creates a catalog from a data config using the default factory registry.
214    ///
215    /// # Errors
216    ///
217    /// Returns an error if the catalog cannot be created from the URI.
218    pub fn load_catalog(config: &BacktestDataConfig) -> anyhow::Result<DataCatalog> {
219        create_catalog(config, &default_catalog_factories())
220    }
221
222    /// Loads data for a specific data config using the default catalog factory registry.
223    ///
224    /// # Errors
225    ///
226    /// Returns an error if catalog creation or data querying fails.
227    pub fn load_data_config(
228        config: &BacktestDataConfig,
229        start: Option<UnixNanos>,
230        end: Option<UnixNanos>,
231    ) -> anyhow::Result<Vec<Data>> {
232        load_data(config, start, end, &default_catalog_factories())
233    }
234
235    /// Disposes all engines and releases resources.
236    /// Subsequent calls to [`run()`](Self::run) or [`build()`](Self::build)
237    /// return an error; create a new node to run again.
238    pub fn dispose(&mut self) {
239        self.disposed = true;
240
241        for engine in self.engines.values_mut() {
242            engine.dispose();
243        }
244        self.engines.clear();
245    }
246
247    fn ensure_not_disposed(&self) -> anyhow::Result<()> {
248        anyhow::ensure!(
249            !self.disposed,
250            "BacktestNode has been disposed; create a new BacktestNode to run again"
251        );
252        Ok(())
253    }
254}
255
256impl Debug for BacktestNode {
257    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
258        f.debug_struct(stringify!(BacktestNode))
259            .field("configs", &self.configs)
260            .field("catalog_factories", &self.catalog_factories.keys())
261            .field("engines", &self.engines)
262            .field("disposed", &self.disposed)
263            .finish()
264    }
265}
266
267/// Builder for constructing a [`BacktestNode`] with a fluent API.
268///
269/// Engines are initialized separately by [`BacktestNode::build`] or [`BacktestNode::run`].
270pub struct BacktestNodeBuilder {
271    configs: Vec<BacktestRunConfig>,
272    catalog_factories: CatalogFactoryRegistry,
273}
274
275impl BacktestNodeBuilder {
276    /// Replaces the catalog factory registry for instrument and replay data loading.
277    ///
278    /// Use [`nautilus_persistence::backend::extend_catalog_factories`] to retain built-in backends
279    /// alongside external factories. Both oneshot and streaming runs use this registry.
280    #[must_use]
281    pub fn with_catalog_factories(mut self, catalog_factories: CatalogFactoryRegistry) -> Self {
282        self.catalog_factories = catalog_factories;
283        self
284    }
285
286    /// Validates the run configurations and creates the node without initializing its engines.
287    ///
288    /// # Errors
289    ///
290    /// Returns an error if `configs` is empty or validation fails.
291    pub fn build(self) -> anyhow::Result<BacktestNode> {
292        anyhow::ensure!(
293            !self.configs.is_empty(),
294            "At least one run config is required"
295        );
296        validate_configs(&self.configs)?;
297        Ok(BacktestNode {
298            configs: self.configs,
299            catalog_factories: self.catalog_factories,
300            engines: AHashMap::new(),
301            disposed: false,
302        })
303    }
304}
305
306impl Debug for BacktestNodeBuilder {
307    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
308        f.debug_struct(stringify!(BacktestNodeBuilder))
309            .field("configs", &self.configs)
310            .field("catalog_factories", &self.catalog_factories.keys())
311            .finish()
312    }
313}
314
315fn build_engine(
316    config: &BacktestRunConfig,
317    catalog_factories: &CatalogFactoryRegistry,
318) -> anyhow::Result<BacktestEngine> {
319    let engine_config = config.engine().clone();
320    let mut engine = BacktestEngine::new(engine_config)?;
321
322    for venue_config in config.venues() {
323        let starting_balances: Vec<Money> = venue_config
324            .starting_balances()
325            .iter()
326            .map(|s| s.parse::<Money>())
327            .collect::<Result<Vec<_>, _>>()
328            .map_err(|e| anyhow::anyhow!("Invalid starting balance: {e}"))?;
329
330        let default_leverage = venue_config.default_leverage();
331        let leverages = venue_config.leverages().cloned().unwrap_or_default();
332        let fill_model = venue_config
333            .fill_model()
334            .cloned()
335            .unwrap_or_default()
336            .into();
337
338        let fee_model = venue_config
339            .fee_model()
340            .cloned()
341            .ok_or_else(|| {
342                anyhow::anyhow!(
343                    "BacktestVenueConfig for '{}' requires an explicit fee_model, including an explicit zero-fee model",
344                    venue_config.name()
345                )
346            })?
347            .into();
348
349        let margin_model = venue_config.margin_model().cloned().map(Into::into);
350
351        let latency_model = venue_config.latency_model().cloned().map(Into::into);
352
353        let modules = venue_config
354            .modules()
355            .iter()
356            .cloned()
357            .map(Into::into)
358            .collect();
359
360        let sim_config = SimulatedVenueConfig::builder()
361            .venue(Venue::from(venue_config.name().as_str()))
362            .oms_type(venue_config.oms_type())
363            .account_type(venue_config.account_type())
364            .book_type(venue_config.book_type())
365            .starting_balances(starting_balances)
366            .maybe_base_currency(venue_config.base_currency())
367            .maybe_default_leverage(default_leverage)
368            .leverages(leverages)
369            .fill_model(fill_model)
370            .fee_model(fee_model)
371            .maybe_margin_model(margin_model)
372            .maybe_latency_model(latency_model)
373            .modules(modules)
374            .routing(venue_config.routing())
375            .reject_stop_orders(venue_config.reject_stop_orders())
376            .support_gtd_orders(venue_config.support_gtd_orders())
377            .support_contingent_orders(venue_config.support_contingent_orders())
378            .use_position_ids(venue_config.use_position_ids())
379            .use_random_ids(venue_config.use_random_ids())
380            .use_reduce_only(venue_config.use_reduce_only())
381            .use_market_order_acks(venue_config.use_market_order_acks())
382            .bar_execution(venue_config.bar_execution())
383            .bar_adaptive_high_low_ordering(venue_config.bar_adaptive_high_low_ordering())
384            .trade_execution(venue_config.trade_execution())
385            .liquidity_consumption(venue_config.liquidity_consumption())
386            .allow_cash_borrowing(venue_config.allow_cash_borrowing())
387            .frozen_account(venue_config.frozen_account())
388            .queue_position(venue_config.queue_position())
389            .order_priority_reset_policy(venue_config.order_priority_reset_policy())
390            .oto_full_trigger(venue_config.oto_trigger_mode() == OtoTriggerMode::Full)
391            .price_protection_points(venue_config.price_protection_points())
392            .liquidation_enabled(venue_config.liquidation_enabled())
393            .liquidation_trigger_ratio(venue_config.liquidation_trigger_ratio())
394            .liquidation_cancel_open_orders(venue_config.liquidation_cancel_open_orders())
395            .build()?;
396        engine.add_venue(sim_config)?;
397    }
398
399    for data_config in config.data() {
400        let mut catalog = create_catalog(data_config, catalog_factories)?;
401        let instr_ids: Vec<InstrumentId> = data_config.get_instrument_ids()?;
402        let filter: Option<Vec<String>> = if instr_ids.is_empty() {
403            None
404        } else {
405            Some(instr_ids.iter().map(ToString::to_string).collect())
406        };
407
408        let instruments =
409            catalog.instruments(&CatalogInstrumentQuery::new().with_instrument_ids(filter))?;
410
411        if !instr_ids.is_empty() && instruments.is_empty() {
412            let ids: Vec<String> = instr_ids.iter().map(ToString::to_string).collect();
413            anyhow::bail!(
414                "No instruments found in catalog for requested IDs: [{}]",
415                ids.join(", ")
416            );
417        }
418
419        for instrument in instruments {
420            engine.add_instrument(&instrument)?;
421        }
422    }
423
424    Ok(engine)
425}
426
427fn validate_configs(configs: &[BacktestRunConfig]) -> anyhow::Result<()> {
428    // Kernel initialization sets a thread-local MessageBus that can only be
429    // initialized once per thread, so multiple engines cannot coexist
430    anyhow::ensure!(
431        configs.len() <= 1,
432        "Only one run config per BacktestNode is supported \
433         (kernel MessageBus is a thread-local singleton)"
434    );
435
436    let mut seen_ids = AHashSet::new();
437
438    for config in configs {
439        anyhow::ensure!(
440            seen_ids.insert(config.id()),
441            "Duplicate run config ID '{}'",
442            config.id()
443        );
444
445        let venue_names: Vec<String> = config
446            .venues()
447            .iter()
448            .map(|v| v.name().to_string())
449            .collect();
450
451        for data_config in config.data() {
452            if let (Some(start), Some(end)) = (data_config.start_time(), data_config.end_time()) {
453                anyhow::ensure!(
454                    start <= end,
455                    "Data config start_time ({start}) must be <= end_time ({end})"
456                );
457            }
458
459            for instrument_id in data_config.get_instrument_ids()? {
460                let venue = instrument_id.venue.to_string();
461                anyhow::ensure!(
462                    venue_names.contains(&venue),
463                    "No venue config found for venue '{venue}' (required by instrument {instrument_id})"
464                );
465            }
466        }
467
468        for venue_config in config.venues() {
469            let needs_book_data = matches!(
470                venue_config.book_type(),
471                BookType::L2_MBP | BookType::L3_MBO
472            );
473
474            if needs_book_data {
475                let venue_name = venue_config.name().to_string();
476                let has_book_data = config.data().iter().any(|dc| {
477                    let is_book_type = matches!(
478                        dc.data_type(),
479                        NautilusDataType::OrderBookDelta | NautilusDataType::OrderBookDepth
480                    );
481
482                    if !is_book_type {
483                        return false;
484                    }
485
486                    // Unfiltered config (no instrument filter) covers all venues
487                    let ids = dc.get_instrument_ids().unwrap_or_default();
488                    ids.is_empty() || ids.iter().any(|id| id.venue.to_string() == venue_name)
489                });
490                anyhow::ensure!(
491                    has_book_data,
492                    "Venue '{venue_name}' has book_type {:?} but no order book data configured",
493                    venue_config.book_type()
494                );
495            }
496        }
497    }
498    Ok(())
499}
500
501fn run_oneshot(
502    engine: &mut BacktestEngine,
503    config: &BacktestRunConfig,
504    catalog_factories: &CatalogFactoryRegistry,
505) -> anyhow::Result<()> {
506    for data_config in config.data() {
507        let data = load_data(data_config, config.start(), config.end(), catalog_factories)?;
508        if data.is_empty() {
509            log::warn!("No data found for config: {:?}", data_config.data_type());
510            continue;
511        }
512        engine.add_data(data, data_config.client_id(), false, false)?;
513    }
514
515    engine.sort_data();
516    engine.run(
517        config.start(),
518        config.end(),
519        Some(config.id().to_string()),
520        false,
521    )
522}
523
524fn run_streaming(
525    engine: &mut BacktestEngine,
526    config: &BacktestRunConfig,
527    chunk_size: usize,
528    catalog_factories: &CatalogFactoryRegistry,
529) -> anyhow::Result<()> {
530    let data_configs = config.data();
531
532    // Equal timestamps and incomplete book frames can extend beyond the target chunk size
533    let mut catalogs = data_configs
534        .iter()
535        .map(|config| create_catalog(config, catalog_factories))
536        .collect::<anyhow::Result<Vec<_>>>()?;
537    let mut streams = Vec::with_capacity(catalogs.len());
538
539    for (index, (catalog, data_config)) in catalogs.iter_mut().zip(data_configs).enumerate() {
540        let result: Box<dyn Iterator<Item = anyhow::Result<ReplayBatch>>> = if data_config
541            .batch_deltas()
542            && *data_config.data_type() == NautilusDataType::OrderBookDelta
543        {
544            let rows = dispatch_query(catalog, data_config, config.start(), config.end())?;
545
546            Box::new(rows.map(|item| {
547                item.map(|data| match data {
548                    Data::BookDeltas(deltas) => ReplayBatch::from_book_deltas(deltas),
549                    _ => unreachable!("delta query yields assembled book events"),
550                })
551            }))
552        } else {
553            let batches = dispatch_batch_query(catalog, data_config, config.start(), config.end())?;
554            Box::new(batches.map(|item| item.map(ReplayBatch::Typed)))
555        };
556
557        let mut stream = result.peekable();
558
559        match stream.peek() {
560            Some(Ok(_)) => streams.push((
561                format!("config {index} ({:?})", data_config.data_type()),
562                stream,
563            )),
564            // Surface a failed query in config order, before opening later ones
565            Some(Err(_)) => {
566                stream.next().transpose()?;
567            }
568            None => log::warn!("No data found for config: {:?}", data_config.data_type()),
569        }
570    }
571
572    stream_chunks(
573        engine,
574        config,
575        merge_streams(streams)?.peekable(),
576        chunk_size,
577    )
578}
579
580fn merge_streams<I: Iterator<Item = anyhow::Result<ReplayBatch>> + 'static>(
581    streams: Vec<(String, I)>,
582) -> anyhow::Result<impl Iterator<Item = anyhow::Result<Data>>> {
583    let mut iterator = DataIterator::new();
584    for (name, stream) in streams {
585        iterator.add_source(&name, Box::new(BatchSource(stream)), true)?;
586    }
587
588    let mut failed = false;
589    Ok(std::iter::from_fn(move || {
590        if failed {
591            return None;
592        }
593
594        match iterator.next_stream_item() {
595            Ok(ReplayPoll::Ready((data, _))) => Some(Ok(data)),
596            Ok(ReplayPoll::Complete) => None,
597            Err(e) => {
598                failed = true;
599                Some(Err(e))
600            }
601            Ok(ReplayPoll::Pending { .. }) => {
602                failed = true;
603                Some(Err(anyhow::anyhow!(
604                    "Synchronous replay source returned Pending"
605                )))
606            }
607        }
608    }))
609}
610
611struct BatchSource<I>(I);
612
613impl<I: Iterator<Item = anyhow::Result<ReplayBatch>>> ReplaySource for BatchSource<I> {
614    fn poll_batch(&mut self) -> anyhow::Result<ReplayPoll<ReplayBatch>> {
615        self.0.next().transpose().map(|batch| match batch {
616            Some(batch) => ReplayPoll::Ready(batch),
617            None => ReplayPoll::Complete,
618        })
619    }
620}
621
622// Feeds data from an iterator to the engine in timestamp-aligned chunks.
623// Each chunk contains up to `chunk_size` events, extended to include all
624// events sharing the boundary timestamp so timers flush correctly.
625fn stream_chunks<I: Iterator<Item = anyhow::Result<Data>>>(
626    engine: &mut BacktestEngine,
627    config: &BacktestRunConfig,
628    mut iter: Peekable<I>,
629    chunk_size: usize,
630) -> anyhow::Result<()> {
631    if iter.peek().is_none() {
632        return engine.end();
633    }
634
635    let mut next_start = config.start();
636    let mut started = false;
637
638    loop {
639        let chunk = take_aligned_chunk(&mut iter, chunk_size)
640            .map_err(|e| end_after_load_failure(engine, started, e))?;
641        if chunk.is_empty() {
642            break;
643        }
644
645        let is_last = iter.peek().is_none();
646        let end = if is_last {
647            config.end()
648        } else {
649            chunk.last().map(HasTsInit::ts_init)
650        };
651
652        engine
653            .add_data(chunk, None, false, true)
654            .map_err(|e| end_after_load_failure(engine, started, e))?;
655        engine.run(next_start, end, Some(config.id().to_string()), true)?;
656        engine.clear_data();
657        started = true;
658
659        // A shutdown request during the chunk already triggered end() inside
660        // engine.run(); stop loading further chunks so later data is not processed
661        if engine.kernel().is_shutdown_requested() {
662            return Ok(());
663        }
664
665        // Carry forward the end timestamp so the next chunk's run_impl
666        // sets clocks contiguously and processes gap timers correctly
667        next_start = end;
668    }
669
670    engine.end()
671}
672
673// Ends an engine that earlier chunks started, so a failure to load later data still stops the
674// trader and engines; the load failure stays the reported error
675fn end_after_load_failure(
676    engine: &mut BacktestEngine,
677    started: bool,
678    error: anyhow::Error,
679) -> anyhow::Error {
680    if started && let Err(e) = engine.end() {
681        log::error!("Failed to end backtest after a data load failure: {e:#}");
682    }
683    error
684}
685
686// Takes up to `chunk_size` items, then extends to include all remaining
687// items sharing the boundary timestamp to avoid splitting same-ts events.
688fn take_aligned_chunk<I: Iterator<Item = anyhow::Result<Data>>>(
689    iter: &mut Peekable<I>,
690    chunk_size: usize,
691) -> anyhow::Result<Vec<Data>> {
692    let mut chunk = Vec::with_capacity(chunk_size);
693
694    for _ in 0..chunk_size {
695        match iter.next() {
696            Some(item) => chunk.push(item?),
697            None => return Ok(chunk),
698        }
699    }
700
701    if let Some(boundary_ts) = chunk.last().map(HasTsInit::ts_init) {
702        // A failing item ends the extension and surfaces on the next chunk
703        while let Some(item) = iter.next_if(|item| {
704            item.as_ref()
705                .is_ok_and(|data| data.ts_init() == boundary_ts)
706        }) {
707            chunk.push(item?);
708        }
709    }
710
711    Ok(chunk)
712}
713
714fn create_catalog(
715    config: &BacktestDataConfig,
716    catalog_factories: &CatalogFactoryRegistry,
717) -> anyhow::Result<DataCatalog> {
718    let catalog_config = config.catalog();
719    catalog_factory::create_catalog(
720        catalog_config.catalog_backend(),
721        &catalog_config.connect_config(),
722        catalog_factories,
723    )
724}
725
726fn load_data(
727    config: &BacktestDataConfig,
728    run_start: Option<UnixNanos>,
729    run_end: Option<UnixNanos>,
730    catalog_factories: &CatalogFactoryRegistry,
731) -> anyhow::Result<Vec<Data>> {
732    let mut catalog = create_catalog(config, catalog_factories)?;
733    let result = dispatch_query(&mut catalog, config, run_start, run_end)?;
734    result.collect::<Result<Vec<_>, _>>()
735}
736
737fn dispatch_query(
738    catalog: &mut DataCatalog,
739    config: &BacktestDataConfig,
740    start: Option<UnixNanos>,
741    end: Option<UnixNanos>,
742) -> anyhow::Result<Box<dyn Iterator<Item = anyhow::Result<Data>>>> {
743    let batches = dispatch_batch_query(catalog, config, start, end)?;
744
745    let rows = batches
746        .map(|batch| batch.map(|batch| batch.to_data_vec_for_compat()))
747        .flat_map(|batch| match batch {
748            Ok(rows) => rows.into_iter().map(Ok).collect::<Vec<_>>(),
749            Err(e) => vec![Err(e)],
750        });
751
752    if config.batch_deltas() && *config.data_type() == NautilusDataType::OrderBookDelta {
753        return Ok(Box::new(BookDeltasBatcher::new(rows)));
754    }
755
756    Ok(Box::new(rows))
757}
758
759fn dispatch_batch_query(
760    catalog: &mut DataCatalog,
761    config: &BacktestDataConfig,
762    start: Option<UnixNanos>,
763    end: Option<UnixNanos>,
764) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DataBatch>> + use<>> {
765    catalog.reset_session();
766    let mut query = CatalogQuery::new(config.data_type().clone())
767        .with_identifiers(config.query_identifiers())
768        .with_range(
769            max_opt(config.start_time(), start),
770            min_opt(config.end_time(), end),
771        )
772        .with_where_clause(config.filter_expr().map(str::to_string));
773
774    let params = Params::new(
775        [(
776            "optimize_file_loading".to_string(),
777            config.optimize_file_loading().into(),
778        )]
779        .into(),
780    );
781    query.params = Some(params);
782    let mut session = catalog.query_batch_session(&query, None)?;
783    let mut failed = false;
784    Ok(std::iter::from_fn(move || {
785        if failed {
786            return None;
787        }
788
789        loop {
790            match session.next_batch() {
791                Ok(Some(batch)) if batch.is_empty() => {}
792                Ok(batch) => return batch.map(Ok),
793                Err(e) => {
794                    failed = true;
795                    return Some(Err(e));
796                }
797            }
798        }
799    }))
800}
801
802fn max_opt(a: Option<UnixNanos>, b: Option<UnixNanos>) -> Option<UnixNanos> {
803    match (a, b) {
804        (Some(a), Some(b)) => Some(a.max(b)),
805        (Some(a), None) => Some(a),
806        (None, Some(b)) => Some(b),
807        (None, None) => None,
808    }
809}
810
811fn min_opt(a: Option<UnixNanos>, b: Option<UnixNanos>) -> Option<UnixNanos> {
812    match (a, b) {
813        (Some(a), Some(b)) => Some(a.min(b)),
814        (Some(a), None) => Some(a),
815        (None, Some(b)) => Some(b),
816        (None, None) => None,
817    }
818}
819
820// Regroups flat catalog deltas into `OrderBookDeltas` closed by `F_LAST`, so the engine applies
821// each book event at once. Each instrument keeps its own pending group because a multi-instrument
822// query interleaves rows by `ts_init`; a group replays at its closing delta, preserving that order.
823struct BookDeltasBatcher<I> {
824    rows: I,
825    pending: AHashMap<InstrumentId, Vec<OrderBookDelta>>,
826    finished: bool,
827}
828
829impl<I> BookDeltasBatcher<I> {
830    fn new(rows: I) -> Self {
831        Self {
832            rows,
833            pending: AHashMap::new(),
834            finished: false,
835        }
836    }
837}
838
839impl<I: Iterator<Item = anyhow::Result<Data>>> Iterator for BookDeltasBatcher<I> {
840    type Item = anyhow::Result<Data>;
841
842    fn next(&mut self) -> Option<Self::Item> {
843        if self.finished {
844            return None;
845        }
846
847        loop {
848            match self.rows.next() {
849                Some(Ok(Data::BookDelta(delta))) => {
850                    let group = self.pending.entry(delta.instrument_id).or_default();
851                    group.push(delta);
852
853                    if RecordFlag::F_LAST.matches(delta.flags) {
854                        let deltas = mem::replace(group, Vec::with_capacity(group.len()));
855                        let deltas = OrderBookDeltas::new_checked(delta.instrument_id, deltas);
856                        return Some(deltas.map(|deltas| Data::BookDeltas(Box::new(deltas))));
857                    }
858                }
859                Some(item) => {
860                    // A failed query must not flush its pending groups as complete events
861                    if item.is_err() {
862                        self.finished = true;
863                    }
864                    return Some(item);
865                }
866                None => {
867                    self.finished = true;
868                    return unterminated_deltas_error(&self.pending).map(Err);
869                }
870            }
871        }
872    }
873}
874
875fn unterminated_deltas_error(
876    pending: &AHashMap<InstrumentId, Vec<OrderBookDelta>>,
877) -> Option<anyhow::Error> {
878    let mut groups: Vec<_> = pending
879        .iter()
880        .filter_map(|(instrument_id, group)| {
881            let first = group.first()?.ts_init;
882            let last = group.last()?.ts_init;
883            Some((*instrument_id, group.len(), first, last))
884        })
885        .collect();
886
887    if groups.is_empty() {
888        return None;
889    }
890
891    groups.sort_unstable_by_key(|(instrument_id, ..)| *instrument_id);
892    let details = groups
893        .iter()
894        .map(|(instrument_id, count, first, last)| {
895            format!("{instrument_id} ({count} pending, ts_init {first} to {last})")
896        })
897        .collect::<Vec<_>>()
898        .join(", ");
899
900    Some(anyhow::anyhow!(
901        "Order book deltas end without an `F_LAST` delta for {details}; \
902         set `batch_deltas` to false to replay individual deltas"
903    ))
904}
905
906#[cfg(test)]
907mod tests {
908    use nautilus_common::component::Component;
909    use nautilus_execution::models::fee::{FeeModelAny, MakerTakerFeeModel};
910    #[cfg(feature = "defi")]
911    use nautilus_model::defi::{
912        DefiData,
913        data::block::BlockPosition,
914        pool_analysis::snapshot::{PoolAnalytics, PoolSnapshot, PoolState},
915    };
916    use nautilus_model::{
917        data::{BookOrder, QuoteTick, TradeTick},
918        enums::{AccountType, AggressorSide, BookAction, OmsType, OrderSide},
919        identifiers::{InstrumentId, TradeId},
920        types::{Price, Quantity},
921    };
922    #[cfg(feature = "python")]
923    use pyo3::{ffi::c_str, prelude::*, types::PyDict};
924    use rstest::rstest;
925
926    use super::*;
927    use crate::{
928        config::{
929            BacktestEngineConfig, BacktestVenueConfig, MAX_BACKTEST_CHUNK_SIZE,
930            SimulatedVenueConfig,
931        },
932        modules::{
933            AccountAdjustmentOutcome, ExchangeContext, SimulationModule, SimulationModuleHandle,
934            SimulationModuleResult,
935        },
936    };
937    #[cfg(feature = "python")]
938    use crate::{
939        modules::SimulationModuleAny,
940        python::modules::{PySimulationModule, PythonSimulationModule},
941    };
942
943    fn quote(ts_init: u64) -> Data {
944        Data::Quote(QuoteTick::new(
945            InstrumentId::from("EUR/USD.SIM"),
946            Price::from("1.0001"),
947            Price::from("1.0002"),
948            Quantity::from("100"),
949            Quantity::from("100"),
950            UnixNanos::from(ts_init),
951            UnixNanos::from(ts_init),
952        ))
953    }
954
955    fn trade(ts_init: u64) -> Data {
956        Data::Trade(TradeTick::new(
957            InstrumentId::from("EUR/USD.SIM"),
958            Price::from("1.0001"),
959            Quantity::from("100"),
960            AggressorSide::Buy,
961            TradeId::from("T-1"),
962            UnixNanos::from(ts_init),
963            UnixNanos::from(ts_init),
964        ))
965    }
966
967    fn stream_failure() -> anyhow::Error {
968        anyhow::anyhow!("injected stream failure")
969    }
970
971    fn row_batch(data: &Data) -> ReplayBatch {
972        let batch = match data {
973            Data::Quote(quote) => DataBatch::Quote(vec![*quote].into()),
974            Data::Trade(trade) => DataBatch::Trade(vec![*trade].into()),
975            _ => panic!("expected a quote or trade fixture"),
976        };
977
978        ReplayBatch::Typed(batch)
979    }
980
981    fn merge_rows<I: Iterator<Item = anyhow::Result<Data>> + 'static>(
982        streams: Vec<Peekable<I>>,
983    ) -> impl Iterator<Item = anyhow::Result<Data>> {
984        let batches = streams
985            .into_iter()
986            .map(|stream| stream.map(|item| item.map(|data| row_batch(&data))))
987            .collect();
988        merge_batches(batches).expect("initial batches must load")
989    }
990
991    fn merge_batches<I: Iterator<Item = anyhow::Result<ReplayBatch>> + 'static>(
992        streams: Vec<I>,
993    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<Data>>> {
994        merge_streams(
995            streams
996                .into_iter()
997                .enumerate()
998                .map(|(index, stream)| (index.to_string(), stream))
999                .collect(),
1000        )
1001    }
1002
1003    #[derive(Debug)]
1004    struct FailingDiagnosticsModule;
1005
1006    impl SimulationModule for FailingDiagnosticsModule {
1007        fn pre_process(&self, _data: &Data) -> anyhow::Result<()> {
1008            Ok(())
1009        }
1010
1011        fn process(
1012            &self,
1013            _ts_now: UnixNanos,
1014            _ctx: &ExchangeContext,
1015        ) -> anyhow::Result<SimulationModuleResult> {
1016            Ok(SimulationModuleResult::NotReady)
1017        }
1018
1019        fn acknowledge(&self, _outcomes: &[AccountAdjustmentOutcome]) -> anyhow::Result<()> {
1020            Ok(())
1021        }
1022
1023        fn log_diagnostics(&self) -> anyhow::Result<()> {
1024            anyhow::bail!("diagnostics failure")
1025        }
1026
1027        fn reset(&self) -> anyhow::Result<()> {
1028            Ok(())
1029        }
1030    }
1031
1032    fn book_delta(instrument_id: &str, flags: u8, sequence: u64, ts_init: u64) -> OrderBookDelta {
1033        OrderBookDelta::new(
1034            InstrumentId::from(instrument_id),
1035            BookAction::Add,
1036            BookOrder::new(
1037                OrderSide::Buy,
1038                Price::from("1.0000"),
1039                Quantity::from("100"),
1040                sequence,
1041            ),
1042            flags,
1043            sequence,
1044            UnixNanos::from(ts_init - 1),
1045            UnixNanos::from(ts_init),
1046        )
1047    }
1048
1049    fn batch_book_deltas(deltas: &[OrderBookDelta]) -> Vec<Data> {
1050        BookDeltasBatcher::new(deltas.iter().map(|delta| Ok(Data::BookDelta(*delta))))
1051            .collect::<anyhow::Result<_>>()
1052            .expect("the batcher must not fail")
1053    }
1054
1055    type BookDeltasFields = (
1056        InstrumentId,
1057        Vec<OrderBookDelta>,
1058        u8,
1059        u64,
1060        UnixNanos,
1061        UnixNanos,
1062    );
1063
1064    // Lists every field because `OrderBookDeltas` equality compares only instrument and sequence
1065    fn book_deltas_fields(data: &Data) -> BookDeltasFields {
1066        let Data::BookDeltas(deltas) = data else {
1067            panic!("expected `OrderBookDeltas`, was {data:?}");
1068        };
1069        (
1070            deltas.instrument_id,
1071            deltas.deltas.clone(),
1072            deltas.flags,
1073            deltas.sequence,
1074            deltas.ts_event,
1075            deltas.ts_init,
1076        )
1077    }
1078
1079    #[rstest]
1080    fn merge_streams_orders_items_across_streams_by_ts_init() {
1081        let streams = vec![
1082            vec![Ok(quote(1)), Ok(quote(3)), Ok(quote(3))]
1083                .into_iter()
1084                .peekable(),
1085            vec![Ok(trade(2)), Ok(trade(3))].into_iter().peekable(),
1086            vec![].into_iter().peekable(),
1087        ];
1088
1089        let merged: Vec<(u64, bool)> = merge_rows(streams)
1090            .map(|item| item.expect("the merged stream must not fail"))
1091            .map(|data| (data.ts_init().as_u64(), matches!(data, Data::Trade(_))))
1092            .collect();
1093
1094        assert_eq!(
1095            merged,
1096            vec![(1, false), (2, true), (3, false), (3, false), (3, true)]
1097        );
1098    }
1099
1100    #[rstest]
1101    fn merge_streams_leaves_its_streams_undrained() {
1102        // Unbounded streams, so a merge that materialized its input would never return
1103        let ok_quote: fn(u64) -> anyhow::Result<Data> = |ts_init| Ok(quote(ts_init));
1104        let evens = (0u64..).step_by(2).map(ok_quote);
1105        let odds = (1u64..).step_by(2).map(ok_quote);
1106
1107        let merged: Vec<u64> = merge_rows(vec![evens.peekable(), odds.peekable()])
1108            .take(4)
1109            .map(|item| item.expect("the merged stream must not fail").ts_init())
1110            .map(|ts_init| ts_init.as_u64())
1111            .collect();
1112
1113        assert_eq!(merged, vec![0, 1, 2, 3]);
1114    }
1115
1116    #[rstest]
1117    fn merge_streams_reports_a_stream_failure() {
1118        let streams = vec![
1119            vec![Ok(quote(1)), Err(stream_failure())]
1120                .into_iter()
1121                .peekable(),
1122            vec![Ok(quote(2))].into_iter().peekable(),
1123        ];
1124        let mut merged = merge_rows(streams);
1125
1126        let first = merged.next().expect("the first item must be present");
1127        let second = merged.next().expect("the failure must be yielded");
1128
1129        assert_eq!(
1130            first.expect("the first item must not fail").ts_init(),
1131            UnixNanos::from(1)
1132        );
1133        assert_eq!(
1134            second
1135                .expect_err("a failed stream must not read as exhaustion")
1136                .to_string(),
1137            "injected stream failure"
1138        );
1139    }
1140
1141    #[rstest]
1142    fn merge_streams_reports_initial_failure_before_consumption() {
1143        let streams = vec![vec![Err(stream_failure())].into_iter()];
1144
1145        let Err(failure) = merge_batches(streams) else {
1146            panic!("initial source failure must be reported");
1147        };
1148
1149        assert_eq!(failure.to_string(), "injected stream failure");
1150    }
1151
1152    #[rstest]
1153    fn merge_streams_keeps_config_order_across_typed_batch_refills() {
1154        let batch = |times: &[u64]| {
1155            let quotes = times
1156                .iter()
1157                .map(|&ts| QuoteTick {
1158                    ts_init: UnixNanos::from(ts),
1159                    ..QuoteTick::default()
1160                })
1161                .collect::<Vec<_>>();
1162
1163            Ok(ReplayBatch::Typed(DataBatch::Quote(quotes.into())))
1164        };
1165
1166        let streams = vec![
1167            vec![batch(&[1, 3]), batch(&[3, 4])].into_iter(),
1168            vec![Ok(row_batch(&trade(2))), Ok(row_batch(&trade(3)))].into_iter(),
1169        ];
1170
1171        let merged: Vec<_> = merge_batches(streams)
1172            .unwrap()
1173            .map(|item| {
1174                let data = item.unwrap();
1175                (data.ts_init().as_u64(), matches!(data, Data::Trade(_)))
1176            })
1177            .collect();
1178
1179        assert_eq!(
1180            merged,
1181            vec![
1182                (1, false),
1183                (2, true),
1184                (3, false),
1185                (3, false),
1186                (3, true),
1187                (4, false)
1188            ]
1189        );
1190    }
1191
1192    #[cfg(feature = "defi")]
1193    #[rstest]
1194    fn merge_streams_orders_full_width_defi_positions_before_config_priority() {
1195        let batch = |id: &str, block, transaction_index, log_index| {
1196            let position =
1197                BlockPosition::new(block, "0xabc".to_string(), transaction_index, log_index);
1198
1199            let snapshot = PoolSnapshot::new(
1200                InstrumentId::from(id),
1201                PoolState::default(),
1202                Vec::new(),
1203                Vec::new(),
1204                PoolAnalytics::default(),
1205                position,
1206                UnixNanos::from(10),
1207                UnixNanos::from(10),
1208            );
1209            let data = vec![DefiData::PoolSnapshot(snapshot)];
1210            ReplayBatch::Typed(DataBatch::Defi(data.into()))
1211        };
1212
1213        let streams = vec![
1214            vec![Ok(batch("A.B", u64::MAX, u32::MAX, u32::MAX))].into_iter(),
1215            vec![
1216                Ok(batch("C.D", u64::MAX - 1, u32::MAX, u32::MAX)),
1217                Ok(batch("C.D", u64::MAX, u32::MAX - 1, u32::MAX)),
1218                Ok(batch("C.D", u64::MAX, u32::MAX, u32::MAX - 1)),
1219            ]
1220            .into_iter(),
1221        ];
1222
1223        let merged: Vec<_> = merge_batches(streams)
1224            .unwrap()
1225            .map(|item| {
1226                let data = item.unwrap();
1227
1228                let Data::Defi(defi) = data else {
1229                    panic!("expected DeFi data");
1230                };
1231
1232                (defi.instrument_id(), defi.ts_init(), defi.block_position())
1233            })
1234            .collect();
1235
1236        assert_eq!(
1237            merged,
1238            vec![
1239                (
1240                    InstrumentId::from("C.D"),
1241                    UnixNanos::from(10),
1242                    (u64::MAX - 1, u32::MAX, u32::MAX)
1243                ),
1244                (
1245                    InstrumentId::from("C.D"),
1246                    UnixNanos::from(10),
1247                    (u64::MAX, u32::MAX - 1, u32::MAX)
1248                ),
1249                (
1250                    InstrumentId::from("C.D"),
1251                    UnixNanos::from(10),
1252                    (u64::MAX, u32::MAX, u32::MAX - 1)
1253                ),
1254                (
1255                    InstrumentId::from("A.B"),
1256                    UnixNanos::from(10),
1257                    (u64::MAX, u32::MAX, u32::MAX)
1258                ),
1259            ]
1260        );
1261    }
1262
1263    #[rstest]
1264    fn merge_streams_returns_selected_row_before_a_refill_failure() {
1265        let streams = vec![vec![Ok(row_batch(&quote(1))), Err(stream_failure())].into_iter()];
1266        let mut merged = merge_batches(streams).unwrap();
1267        let first = merged.next().unwrap().unwrap();
1268        let failure = merged.next().unwrap().unwrap_err();
1269
1270        assert_eq!(first, quote(1));
1271        assert_eq!(failure.to_string(), "injected stream failure");
1272        assert!(merged.next().is_none());
1273    }
1274
1275    #[rstest]
1276    fn merge_streams_rejects_a_backward_source_before_emitting_other_streams() {
1277        let streams = vec![
1278            vec![Ok(row_batch(&quote(3))), Ok(row_batch(&quote(2)))].into_iter(),
1279            vec![Ok(row_batch(&trade(4)))].into_iter(),
1280        ];
1281        let mut merged = merge_batches(streams).unwrap();
1282        let first = merged.next().unwrap().unwrap();
1283        let failure = merged.next().unwrap().unwrap_err();
1284
1285        assert_eq!(first, quote(3));
1286        assert!(
1287            failure
1288                .to_string()
1289                .contains("moved backward across batches")
1290        );
1291        assert!(merged.next().is_none());
1292    }
1293
1294    #[rstest]
1295    #[case(false)]
1296    #[case(true)]
1297    fn merge_streams_reports_original_config_after_empty_sources(#[case] include_empty: bool) {
1298        let mut streams = Vec::new();
1299        if include_empty {
1300            streams.push(("config 0 (TradeTick)".to_string(), Vec::new().into_iter()));
1301        }
1302
1303        streams.push((
1304            "config 3 (QuoteTick)".to_string(),
1305            vec![Ok(row_batch(&quote(3))), Ok(row_batch(&quote(2)))].into_iter(),
1306        ));
1307        let mut merged = merge_streams(streams).unwrap();
1308        let first = merged.next().unwrap().unwrap();
1309        let failure = merged.next().unwrap().unwrap_err();
1310
1311        assert_eq!(first, quote(3));
1312        assert!(
1313            failure
1314                .to_string()
1315                .starts_with("Replay stream 'config 3 (QuoteTick)' moved backward across batches:")
1316        );
1317        assert!(merged.next().is_none());
1318    }
1319
1320    #[rstest]
1321    fn take_aligned_chunk_reports_a_stream_failure() {
1322        let mut iter = vec![Ok(quote(1)), Err(stream_failure())]
1323            .into_iter()
1324            .peekable();
1325
1326        let chunk = take_aligned_chunk(&mut iter, 4);
1327
1328        assert_eq!(
1329            chunk
1330                .expect_err("a failed stream must not read as a short chunk")
1331                .to_string(),
1332            "injected stream failure"
1333        );
1334    }
1335
1336    #[rstest]
1337    fn take_aligned_chunk_reports_a_failure_found_at_the_boundary() {
1338        let mut iter = vec![Ok(quote(1)), Err(stream_failure()), Ok(quote(1))]
1339            .into_iter()
1340            .peekable();
1341
1342        let first = take_aligned_chunk(&mut iter, 1).expect("the first chunk must be complete");
1343        let second = take_aligned_chunk(&mut iter, 1);
1344
1345        assert_eq!(first.len(), 1);
1346        assert_eq!(first[0].ts_init(), UnixNanos::from(1));
1347        assert_eq!(
1348            second
1349                .expect_err("the failure must survive the boundary extension")
1350                .to_string(),
1351            "injected stream failure"
1352        );
1353    }
1354
1355    #[rstest]
1356    fn take_aligned_chunk_extends_past_the_boundary_for_equal_timestamps() {
1357        let mut iter = vec![Ok(quote(1)), Ok(quote(1)), Ok(quote(2))]
1358            .into_iter()
1359            .peekable();
1360
1361        let chunk = take_aligned_chunk(&mut iter, 1).expect("the chunk must be complete");
1362
1363        assert_eq!(chunk.len(), 2);
1364        assert_eq!(chunk[0].ts_init(), UnixNanos::from(1));
1365        assert_eq!(chunk[1].ts_init(), UnixNanos::from(1));
1366    }
1367
1368    #[rstest]
1369    fn take_aligned_chunk_reserves_maximum_supported_capacity() {
1370        let mut iter = vec![Ok(quote(1))].into_iter().peekable();
1371
1372        let chunk = take_aligned_chunk(&mut iter, MAX_BACKTEST_CHUNK_SIZE).unwrap();
1373
1374        assert_eq!(chunk.len(), 1);
1375        assert!(chunk.capacity() >= MAX_BACKTEST_CHUNK_SIZE);
1376        assert_eq!(chunk[0].ts_init(), UnixNanos::from(1));
1377    }
1378
1379    #[rstest]
1380    fn book_deltas_batcher_closes_groups_only_on_f_last() {
1381        let last = RecordFlag::F_LAST as u8;
1382        let d1 = book_delta("EUR/USD.SIM", 0, 1, 10);
1383        let d2 = book_delta("EUR/USD.SIM", last, 2, 10);
1384        let d3 = book_delta("EUR/USD.SIM", last, 3, 10);
1385        let d4 = book_delta("EUR/USD.SIM", 0, 4, 20);
1386        let d5 = book_delta("EUR/USD.SIM", last, 5, 30);
1387
1388        let batches = batch_book_deltas(&[d1, d2, d3, d4, d5]);
1389
1390        // Same-timestamp events stay apart, and an event spanning timestamps replays at its close
1391        let instrument_id = InstrumentId::from("EUR/USD.SIM");
1392        assert_eq!(
1393            batches.iter().map(book_deltas_fields).collect::<Vec<_>>(),
1394            vec![
1395                (
1396                    instrument_id,
1397                    vec![d1, d2],
1398                    last,
1399                    2,
1400                    UnixNanos::from(9),
1401                    UnixNanos::from(10),
1402                ),
1403                (
1404                    instrument_id,
1405                    vec![d3],
1406                    last,
1407                    3,
1408                    UnixNanos::from(9),
1409                    UnixNanos::from(10),
1410                ),
1411                (
1412                    instrument_id,
1413                    vec![d4, d5],
1414                    last,
1415                    5,
1416                    UnixNanos::from(29),
1417                    UnixNanos::from(30),
1418                ),
1419            ]
1420        );
1421    }
1422
1423    #[rstest]
1424    fn book_deltas_batcher_keeps_interleaved_instruments_apart() {
1425        let last = RecordFlag::F_LAST as u8;
1426        let aud1 = book_delta("AUD/USD.SIM", 0, 1, 10);
1427        let eur1 = book_delta("EUR/USD.SIM", 0, 2, 10);
1428        let eur2 = book_delta("EUR/USD.SIM", last, 3, 10);
1429        let aud2 = book_delta("AUD/USD.SIM", last, 4, 20);
1430
1431        let batches = batch_book_deltas(&[aud1, eur1, eur2, aud2]);
1432
1433        // The completed EUR/USD event is not held behind the pending AUD/USD event
1434        assert_eq!(
1435            batches.iter().map(book_deltas_fields).collect::<Vec<_>>(),
1436            vec![
1437                (
1438                    InstrumentId::from("EUR/USD.SIM"),
1439                    vec![eur1, eur2],
1440                    last,
1441                    3,
1442                    UnixNanos::from(9),
1443                    UnixNanos::from(10),
1444                ),
1445                (
1446                    InstrumentId::from("AUD/USD.SIM"),
1447                    vec![aud1, aud2],
1448                    last,
1449                    4,
1450                    UnixNanos::from(19),
1451                    UnixNanos::from(20),
1452                ),
1453            ]
1454        );
1455    }
1456
1457    #[rstest]
1458    fn book_deltas_batcher_takes_flags_from_closing_delta() {
1459        let instrument_id = InstrumentId::from("EUR/USD.SIM");
1460        let clear =
1461            OrderBookDelta::clear(instrument_id, 1, UnixNanos::from(9), UnixNanos::from(10));
1462        let add = book_delta("EUR/USD.SIM", RecordFlag::F_MBP as u8, 2, 10);
1463        let close = book_delta("EUR/USD.SIM", RecordFlag::F_LAST as u8, 3, 10);
1464
1465        let batches = batch_book_deltas(&[clear, add, close]);
1466
1467        // A snapshot clear does not close the event, and earlier flags are not merged in
1468        assert_eq!(
1469            batches.iter().map(book_deltas_fields).collect::<Vec<_>>(),
1470            vec![(
1471                instrument_id,
1472                vec![clear, add, close],
1473                RecordFlag::F_LAST as u8,
1474                3,
1475                UnixNanos::from(9),
1476                UnixNanos::from(10),
1477            )]
1478        );
1479    }
1480
1481    #[rstest]
1482    fn book_deltas_batcher_yields_nothing_for_empty_input() {
1483        let batches = batch_book_deltas(&[]);
1484
1485        assert!(batches.is_empty());
1486    }
1487
1488    #[rstest]
1489    fn book_deltas_batcher_rejects_deltas_left_without_f_last() {
1490        let last = RecordFlag::F_LAST as u8;
1491        let d1 = book_delta("EUR/USD.SIM", last, 1, 10);
1492        let d2 = book_delta("EUR/USD.SIM", 0, 2, 20);
1493        let d3 = book_delta("EUR/USD.SIM", 0, 3, 30);
1494        let d4 = book_delta("AUD/USD.SIM", 0, 4, 40);
1495        let rows = [d1, d2, d3, d4].map(|delta| Ok(Data::BookDelta(delta)));
1496        let mut batcher = BookDeltasBatcher::new(rows.into_iter());
1497
1498        let first = batcher.next().expect("the complete event must be yielded");
1499        let second = batcher.next().expect("the pending deltas must be reported");
1500        let third = batcher.next();
1501
1502        assert_eq!(
1503            book_deltas_fields(&first.expect("the complete event must not fail")),
1504            (
1505                InstrumentId::from("EUR/USD.SIM"),
1506                vec![d1],
1507                last,
1508                1,
1509                UnixNanos::from(9),
1510                UnixNanos::from(10),
1511            )
1512        );
1513        assert_eq!(
1514            second
1515                .expect_err("pending deltas must not replay as an event")
1516                .to_string(),
1517            "Order book deltas end without an `F_LAST` delta for \
1518             AUD/USD.SIM (1 pending, ts_init 40 to 40), \
1519             EUR/USD.SIM (2 pending, ts_init 20 to 30); \
1520             set `batch_deltas` to false to replay individual deltas"
1521        );
1522        assert!(third.is_none());
1523    }
1524
1525    #[rstest]
1526    fn book_deltas_batcher_reports_a_query_failure_without_flushing() {
1527        let pending = book_delta("EUR/USD.SIM", 0, 1, 10);
1528        let after_failure = book_delta("EUR/USD.SIM", RecordFlag::F_LAST as u8, 2, 20);
1529        let rows = vec![
1530            Ok(Data::BookDelta(pending)),
1531            Err(stream_failure()),
1532            Ok(Data::BookDelta(after_failure)),
1533        ];
1534
1535        let items: Vec<anyhow::Result<Data>> = BookDeltasBatcher::new(rows.into_iter()).collect();
1536
1537        assert_eq!(items.len(), 1);
1538        assert_eq!(
1539            items[0]
1540                .as_ref()
1541                .expect_err("the query failure must be yielded")
1542                .to_string(),
1543            "injected stream failure"
1544        );
1545    }
1546
1547    #[rstest]
1548    fn end_after_load_failure_reports_the_load_failure_when_ending_fails() {
1549        let mut engine = BacktestEngine::new(BacktestEngineConfig::default()).unwrap();
1550        let venue = SimulatedVenueConfig::builder()
1551            .venue(Venue::from("SIM"))
1552            .oms_type(OmsType::Netting)
1553            .account_type(AccountType::Margin)
1554            .book_type(BookType::L1_MBP)
1555            .starting_balances(vec![Money::from("1_000_000 USD")])
1556            .modules(vec![SimulationModuleHandle::new(FailingDiagnosticsModule)])
1557            .fee_model(FeeModelAny::MakerTaker(MakerTakerFeeModel::zero()).into())
1558            .build()
1559            .unwrap();
1560        engine.add_venue(venue).unwrap();
1561        engine.run(None, None, None, true).unwrap();
1562
1563        let error = end_after_load_failure(&mut engine, true, stream_failure());
1564
1565        // Ending fails on the module diagnostics after the trader stops
1566        assert_eq!(error.to_string(), "injected stream failure");
1567        assert!(engine.kernel().trader.borrow().is_stopped());
1568        assert!(engine.get_result().run_finished.is_some());
1569    }
1570
1571    #[cfg(feature = "python")]
1572    #[rstest]
1573    fn build_engine_accepts_python_module_from_node_config() {
1574        Python::initialize();
1575
1576        Python::attach(|py| {
1577            let locals = PyDict::new(py);
1578            locals
1579                .set_item("SimulationModule", py.get_type::<PySimulationModule>())
1580                .unwrap();
1581            let module = py
1582                .eval(
1583                    c_str!(
1584                        "type('NodeSimulationModule', (SimulationModule,), {\
1585                            'process': lambda self, ts_now, context: \
1586                                (setattr(self, 'calls', self.calls + 1), [])[1]\
1587                        })()"
1588                    ),
1589                    None,
1590                    Some(&locals),
1591                )
1592                .unwrap();
1593            module.setattr("calls", 0).unwrap();
1594
1595            let venue = BacktestVenueConfig::builder()
1596                .name("SIM")
1597                .oms_type(OmsType::Netting)
1598                .account_type(AccountType::Margin)
1599                .book_type(BookType::L1_MBP)
1600                .starting_balances(vec!["1000 USD".to_string()])
1601                .fee_model(FeeModelAny::MakerTaker(MakerTakerFeeModel::zero()))
1602                .modules(vec![SimulationModuleAny::Python(
1603                    PythonSimulationModule::new(module.clone().unbind()),
1604                )])
1605                .build()
1606                .unwrap();
1607            let config = BacktestRunConfig::builder()
1608                .venues(vec![venue])
1609                .data(Vec::new())
1610                .build()
1611                .unwrap();
1612            let mut engine = build_engine(&config, &default_catalog_factories()).unwrap();
1613
1614            engine.run(None, None, None, false).unwrap();
1615
1616            assert_eq!(
1617                module.getattr("calls").unwrap().extract::<u32>().unwrap(),
1618                1
1619            );
1620        });
1621    }
1622
1623    #[rstest]
1624    fn test_build_engine_rejects_venue_without_fee_model() {
1625        let venue = BacktestVenueConfig::builder()
1626            .name("SIM")
1627            .oms_type(OmsType::Netting)
1628            .account_type(AccountType::Margin)
1629            .book_type(BookType::L1_MBP)
1630            .starting_balances(vec!["1_000_000 USD".to_string()])
1631            .build()
1632            .unwrap();
1633        let config = BacktestRunConfig::builder()
1634            .venues(vec![venue])
1635            .data(Vec::new())
1636            .build()
1637            .unwrap();
1638        let err = build_engine(&config, &default_catalog_factories()).unwrap_err();
1639        assert!(
1640            err.to_string().contains("explicit fee_model"),
1641            "unexpected error: {err}"
1642        );
1643    }
1644}