Skip to main content

nautilus_backtest/
node.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Provides a [`BacktestNode`] that orchestrates catalog-driven backtests.
17
18use std::{fmt::Debug, iter::Peekable, mem};
19
20use ahash::AHashMap;
21use nautilus_core::{Params, UnixNanos};
22use nautilus_data::replay::{DataIterator, ReplayBatch, ReplayPoll, ReplaySource};
23use nautilus_model::{
24    data::{Data, DataBatch, HasTsInit, NautilusDataType, OrderBookDelta, OrderBookDeltas},
25    enums::{BookType, RecordFlag},
26    identifiers::InstrumentId,
27};
28use nautilus_persistence::{
29    backend::default_catalog_factories,
30    catalog::{
31        factory::{self as catalog_factory, CatalogFactoryRegistry},
32        traits::{CatalogInstrumentQuery, CatalogQuery, DataCatalog},
33    },
34};
35
36use crate::{
37    config::{BacktestDataConfig, BacktestRunConfig},
38    engine::BacktestEngine,
39    result::BacktestResult,
40};
41
42/// Orchestrates catalog-driven backtests from run configurations.
43///
44/// `BacktestNode` connects the catalog with [`BacktestEngine`] to load
45/// historical data and run backtests. Supports both oneshot and streaming modes.
46#[cfg_attr(
47    feature = "python",
48    pyo3::pyclass(module = "nautilus_trader.backtest", unsendable)
49)]
50#[cfg_attr(
51    feature = "python",
52    pyo3_stub_gen::derive::gen_stub_pyclass(module = "nautilus_trader.backtest")
53)]
54pub struct BacktestNode {
55    configs: Vec<BacktestRunConfig>,
56    catalog_factories: CatalogFactoryRegistry,
57    engines: AHashMap<String, BacktestEngine>,
58    disposed: bool,
59}
60
61impl BacktestNode {
62    /// Creates a [`BacktestNode`] with the default catalog factory registry.
63    ///
64    /// Validates that configs are non-empty and internally consistent:
65    /// - All data config instrument venues must have a matching venue config.
66    /// - L2/L3 book types require order book data in the data configs.
67    /// - Data config time ranges must be valid (start <= end).
68    ///
69    /// # Errors
70    ///
71    /// Returns an error if `configs` is empty or validation fails.
72    pub fn new(configs: Vec<BacktestRunConfig>) -> anyhow::Result<Self> {
73        Self::builder(configs).build()
74    }
75
76    /// Creates a builder with the default catalog factory registry.
77    ///
78    /// Validates the run configurations when [`BacktestNodeBuilder::build`] creates the node.
79    #[must_use]
80    pub fn builder(configs: Vec<BacktestRunConfig>) -> BacktestNodeBuilder {
81        let catalog_factories = default_catalog_factories();
82
83        BacktestNodeBuilder {
84            configs,
85            catalog_factories,
86        }
87    }
88
89    /// Returns the run configurations.
90    #[must_use]
91    pub fn configs(&self) -> &[BacktestRunConfig] {
92        &self.configs
93    }
94
95    /// Builds backtest engines from the run configurations.
96    ///
97    /// For each config, creates a [`BacktestEngine`], adds venues, and loads
98    /// instruments from the catalog. If building a config fails with
99    /// [`BacktestRunConfig::raise_exception`] disabled, logs the error and skips that config;
100    /// successful return does not guarantee an engine for every config.
101    /// A disposed node cannot be built again; create a new node instead.
102    ///
103    /// # Errors
104    ///
105    /// Returns an error if:
106    /// - This node has been disposed.
107    /// - Building an engine from a config fails and
108    ///   [`BacktestRunConfig::raise_exception`] is enabled for that config.
109    pub fn build(&mut self) -> anyhow::Result<()> {
110        self.ensure_not_disposed()?;
111
112        for config in &self.configs {
113            if self.engines.contains_key(config.id()) {
114                continue;
115            }
116
117            match build_engine(config, &self.catalog_factories) {
118                Ok(engine) => {
119                    self.engines.insert(config.id().to_string(), engine);
120                }
121                Err(e) if config.raise_exception() => return Err(e),
122                Err(e) => {
123                    log::error!("Error building backtest '{}': {e:#}", config.id());
124                }
125            }
126        }
127
128        Ok(())
129    }
130
131    /// Returns a mutable reference to the engine for the given run config ID.
132    #[must_use]
133    pub fn get_engine_mut(&mut self, id: &str) -> Option<&mut BacktestEngine> {
134        self.engines.get_mut(id)
135    }
136
137    /// Returns a reference to the engine for the given run config ID.
138    #[must_use]
139    pub fn get_engine(&self, id: &str) -> Option<&BacktestEngine> {
140        self.engines.get(id)
141    }
142
143    /// Returns all created backtest engines.
144    #[must_use]
145    pub fn get_engines(&self) -> Vec<&BacktestEngine> {
146        self.engines.values().collect()
147    }
148
149    /// Runs all configured backtests and returns results.
150    ///
151    /// Automatically calls [`build()`](Self::build) if engines have not been created yet.
152    /// For each run config, loads data from the catalog and runs the engine.
153    /// Supports both oneshot (`chunk_size = None`) and streaming modes.
154    /// Configs without a built engine are skipped. If a run fails with
155    /// [`BacktestRunConfig::raise_exception`] disabled, logs the error, clears its loaded data,
156    /// leaves the engine undisposed, and omits its result. A streaming run that fails to load
157    /// data after replaying earlier chunks ends its engine first, stopping the trader and engines.
158    /// A node disposed by a completed run or by [`dispose()`](Self::dispose)
159    /// cannot run again; create a new node instead.
160    ///
161    /// # Errors
162    ///
163    /// Returns an error if:
164    /// - This node has been disposed.
165    /// - Building, data loading, or engine execution fails and
166    ///   [`BacktestRunConfig::raise_exception`] is enabled for the run config.
167    pub fn run(&mut self) -> anyhow::Result<Vec<BacktestResult>> {
168        self.ensure_not_disposed()?;
169
170        // Auto-build if not already done
171        if self.engines.is_empty() {
172            self.build()?;
173        }
174
175        let mut results = Vec::new();
176
177        for config in &self.configs {
178            let Some(engine) = self.engines.get_mut(config.id()) else {
179                continue;
180            };
181
182            let run_result = match config.chunk_size() {
183                None => run_oneshot(engine, config, &self.catalog_factories),
184                Some(chunk_size) => {
185                    run_streaming(engine, config, chunk_size, &self.catalog_factories)
186                }
187            };
188
189            if let Err(e) = run_result {
190                if config.raise_exception() {
191                    return Err(e);
192                }
193
194                log::error!("Error running backtest '{}': {e:#}", config.id());
195                engine.clear_data();
196                continue;
197            }
198
199            results.push(engine.get_result());
200
201            if config.dispose_on_completion() {
202                engine.dispose();
203                self.disposed = true;
204            } else {
205                engine.clear_data();
206            }
207        }
208
209        Ok(results)
210    }
211
212    /// Creates a catalog from a data config using the default factory registry.
213    ///
214    /// # Errors
215    ///
216    /// Returns an error if the catalog cannot be created from the URI.
217    pub fn load_catalog(config: &BacktestDataConfig) -> anyhow::Result<DataCatalog> {
218        create_catalog(config, &default_catalog_factories())
219    }
220
221    /// Loads data for a specific data config using the default catalog factory registry.
222    ///
223    /// # Errors
224    ///
225    /// Returns an error if catalog creation or data querying fails.
226    pub fn load_data_config(
227        config: &BacktestDataConfig,
228        start: Option<UnixNanos>,
229        end: Option<UnixNanos>,
230    ) -> anyhow::Result<Vec<Data>> {
231        load_data(config, start, end, &default_catalog_factories())
232    }
233
234    /// Disposes all engines and releases resources.
235    /// Subsequent calls to [`run()`](Self::run) or [`build()`](Self::build)
236    /// return an error; create a new node to run again.
237    pub fn dispose(&mut self) {
238        self.disposed = true;
239
240        for engine in self.engines.values_mut() {
241            engine.dispose();
242        }
243        self.engines.clear();
244    }
245
246    fn ensure_not_disposed(&self) -> anyhow::Result<()> {
247        anyhow::ensure!(
248            !self.disposed,
249            "BacktestNode has been disposed; create a new BacktestNode to run again"
250        );
251        Ok(())
252    }
253}
254
255impl Debug for BacktestNode {
256    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
257        f.debug_struct(stringify!(BacktestNode))
258            .field("configs", &self.configs)
259            .field("catalog_factories", &self.catalog_factories.keys())
260            .field("engines", &self.engines)
261            .field("disposed", &self.disposed)
262            .finish()
263    }
264}
265
266/// Builder for constructing a [`BacktestNode`] with a fluent API.
267///
268/// Engines are initialized separately by [`BacktestNode::build`] or [`BacktestNode::run`].
269pub struct BacktestNodeBuilder {
270    configs: Vec<BacktestRunConfig>,
271    catalog_factories: CatalogFactoryRegistry,
272}
273
274impl BacktestNodeBuilder {
275    /// Replaces the catalog factory registry for instrument and replay data loading.
276    ///
277    /// Use [`nautilus_persistence::backend::extend_catalog_factories`] to retain built-in backends
278    /// alongside external factories. Both oneshot and streaming runs use this registry.
279    #[must_use]
280    pub fn with_catalog_factories(mut self, catalog_factories: CatalogFactoryRegistry) -> Self {
281        self.catalog_factories = catalog_factories;
282        self
283    }
284
285    /// Validates the run configurations and creates the node without initializing its engines.
286    ///
287    /// # Errors
288    ///
289    /// Returns an error if `configs` is empty or validation fails.
290    pub fn build(self) -> anyhow::Result<BacktestNode> {
291        anyhow::ensure!(
292            !self.configs.is_empty(),
293            "At least one run config is required"
294        );
295        validate_configs(&self.configs)?;
296        Ok(BacktestNode {
297            configs: self.configs,
298            catalog_factories: self.catalog_factories,
299            engines: AHashMap::new(),
300            disposed: false,
301        })
302    }
303}
304
305impl Debug for BacktestNodeBuilder {
306    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
307        f.debug_struct(stringify!(BacktestNodeBuilder))
308            .field("configs", &self.configs)
309            .field("catalog_factories", &self.catalog_factories.keys())
310            .finish()
311    }
312}
313
314fn build_engine(
315    config: &BacktestRunConfig,
316    catalog_factories: &CatalogFactoryRegistry,
317) -> anyhow::Result<BacktestEngine> {
318    let engine_config = config.engine().clone();
319    let mut engine = BacktestEngine::new(engine_config)?;
320
321    for venue_config in config.venues() {
322        let sim_config = venue_config.to_simulated()?;
323        engine.add_venue(sim_config)?;
324    }
325
326    add_catalog_instruments(&mut engine, config.data(), catalog_factories)?;
327    Ok(engine)
328}
329
330fn add_catalog_instruments(
331    engine: &mut BacktestEngine,
332    configs: &[BacktestDataConfig],
333    catalog_factories: &CatalogFactoryRegistry,
334) -> anyhow::Result<()> {
335    for data_config in configs {
336        let mut catalog = create_catalog(data_config, catalog_factories)?;
337        let instr_ids: Vec<InstrumentId> = data_config.get_instrument_ids()?;
338        let filter: Option<Vec<String>> = if instr_ids.is_empty() {
339            None
340        } else {
341            Some(instr_ids.iter().map(ToString::to_string).collect())
342        };
343
344        let instruments =
345            catalog.instruments(&CatalogInstrumentQuery::new().with_instrument_ids(filter))?;
346
347        if !instr_ids.is_empty() && instruments.is_empty() {
348            let ids: Vec<String> = instr_ids.iter().map(ToString::to_string).collect();
349            anyhow::bail!(
350                "No instruments found in catalog for requested IDs: [{}]",
351                ids.join(", ")
352            );
353        }
354
355        for instrument in instruments {
356            engine.add_instrument(&instrument)?;
357        }
358    }
359
360    Ok(())
361}
362
363fn validate_configs(configs: &[BacktestRunConfig]) -> anyhow::Result<()> {
364    // Kernel initialization sets a thread-local MessageBus that can only be
365    // initialized once per thread, so multiple engines cannot coexist
366    anyhow::ensure!(
367        configs.len() <= 1,
368        "Only one run config per BacktestNode is supported \
369         (kernel MessageBus is a thread-local singleton)"
370    );
371
372    for config in configs {
373        validate_data_configs(config)?;
374        validate_venue_data(config)?;
375    }
376
377    Ok(())
378}
379
380fn validate_data_configs(config: &BacktestRunConfig) -> anyhow::Result<()> {
381    let venue_names: Vec<String> = config
382        .venues()
383        .iter()
384        .map(|v| v.name().to_string())
385        .collect();
386
387    for data_config in config.data() {
388        if let (Some(start), Some(end)) = (data_config.start_time(), data_config.end_time()) {
389            anyhow::ensure!(
390                start <= end,
391                "Data config start_time ({start}) must be <= end_time ({end})"
392            );
393        }
394
395        for instrument_id in data_config.get_instrument_ids()? {
396            let venue = instrument_id.venue.to_string();
397            anyhow::ensure!(
398                venue_names.contains(&venue),
399                "No venue config found for venue '{venue}' (required by instrument {instrument_id})"
400            );
401        }
402    }
403
404    Ok(())
405}
406
407fn validate_venue_data(config: &BacktestRunConfig) -> anyhow::Result<()> {
408    for venue_config in config.venues() {
409        let needs_book_data = matches!(
410            venue_config.book_type(),
411            BookType::L2_MBP | BookType::L3_MBO
412        );
413
414        if !needs_book_data {
415            continue;
416        }
417
418        let venue_name = venue_config.name().to_string();
419
420        let has_book_data = config.data().iter().any(|dc| {
421            let is_book_type = matches!(
422                dc.data_type(),
423                NautilusDataType::OrderBookDelta | NautilusDataType::OrderBookDepth
424            );
425
426            if !is_book_type {
427                return false;
428            }
429
430            // Unfiltered config (no instrument filter) covers all venues
431            let ids = dc.get_instrument_ids().unwrap_or_default();
432            ids.is_empty() || ids.iter().any(|id| id.venue.to_string() == venue_name)
433        });
434
435        anyhow::ensure!(
436            has_book_data,
437            "Venue '{venue_name}' has book_type {:?} but no order book data configured",
438            venue_config.book_type()
439        );
440    }
441    Ok(())
442}
443
444fn run_oneshot(
445    engine: &mut BacktestEngine,
446    config: &BacktestRunConfig,
447    catalog_factories: &CatalogFactoryRegistry,
448) -> anyhow::Result<()> {
449    for data_config in config.data() {
450        let data = load_data(data_config, config.start(), config.end(), catalog_factories)?;
451        if data.is_empty() {
452            log::warn!("No data found for config: {:?}", data_config.data_type());
453            continue;
454        }
455        engine.add_data(data, data_config.client_id(), false, false)?;
456    }
457
458    engine.sort_data();
459    engine.run(
460        config.start(),
461        config.end(),
462        Some(config.id().to_string()),
463        false,
464    )
465}
466
467fn run_streaming(
468    engine: &mut BacktestEngine,
469    config: &BacktestRunConfig,
470    chunk_size: usize,
471    catalog_factories: &CatalogFactoryRegistry,
472) -> anyhow::Result<()> {
473    let data_configs = config.data();
474
475    // Equal timestamps and incomplete book frames can extend beyond the target chunk size
476    let mut catalogs = data_configs
477        .iter()
478        .map(|config| create_catalog(config, catalog_factories))
479        .collect::<anyhow::Result<Vec<_>>>()?;
480    let mut streams = Vec::with_capacity(catalogs.len());
481
482    for (index, (catalog, data_config)) in catalogs.iter_mut().zip(data_configs).enumerate() {
483        let result: Box<dyn Iterator<Item = anyhow::Result<ReplayBatch>>> = if data_config
484            .batch_deltas()
485            && *data_config.data_type() == NautilusDataType::OrderBookDelta
486        {
487            let rows = dispatch_query(catalog, data_config, config.start(), config.end())?;
488
489            Box::new(rows.map(|item| {
490                item.map(|data| match data {
491                    Data::BookDeltas(deltas) => ReplayBatch::from_book_deltas(deltas),
492                    _ => unreachable!("delta query yields assembled book events"),
493                })
494            }))
495        } else {
496            let batches = dispatch_batch_query(catalog, data_config, config.start(), config.end())?;
497            Box::new(batches.map(|item| item.map(ReplayBatch::Typed)))
498        };
499
500        let mut stream = result.peekable();
501
502        match stream.peek() {
503            Some(Ok(_)) => streams.push((
504                format!("config {index} ({:?})", data_config.data_type()),
505                stream,
506            )),
507            // Surface a failed query in config order, before opening later ones
508            Some(Err(_)) => {
509                stream.next().transpose()?;
510            }
511            None => log::warn!("No data found for config: {:?}", data_config.data_type()),
512        }
513    }
514
515    stream_chunks(
516        engine,
517        config,
518        merge_streams(streams)?.peekable(),
519        chunk_size,
520    )
521}
522
523fn merge_streams<I: Iterator<Item = anyhow::Result<ReplayBatch>> + 'static>(
524    streams: Vec<(String, I)>,
525) -> anyhow::Result<impl Iterator<Item = anyhow::Result<Data>>> {
526    let mut iterator = DataIterator::new();
527    for (name, stream) in streams {
528        iterator.add_source(&name, Box::new(BatchSource(stream)), true)?;
529    }
530
531    let mut failed = false;
532    Ok(std::iter::from_fn(move || {
533        if failed {
534            return None;
535        }
536
537        match iterator.next_stream_item() {
538            Ok(ReplayPoll::Ready((data, _))) => Some(Ok(data)),
539            Ok(ReplayPoll::Complete) => None,
540            Err(e) => {
541                failed = true;
542                Some(Err(e))
543            }
544            Ok(ReplayPoll::Pending { .. }) => {
545                failed = true;
546                Some(Err(anyhow::anyhow!(
547                    "Synchronous replay source returned Pending"
548                )))
549            }
550        }
551    }))
552}
553
554struct BatchSource<I>(I);
555
556impl<I: Iterator<Item = anyhow::Result<ReplayBatch>>> ReplaySource for BatchSource<I> {
557    fn poll_batch(&mut self) -> anyhow::Result<ReplayPoll<ReplayBatch>> {
558        self.0.next().transpose().map(|batch| match batch {
559            Some(batch) => ReplayPoll::Ready(batch),
560            None => ReplayPoll::Complete,
561        })
562    }
563}
564
565// Feeds data from an iterator to the engine in timestamp-aligned chunks.
566// Each chunk contains up to `chunk_size` events, extended to include all
567// events sharing the boundary timestamp so timers flush correctly.
568fn stream_chunks<I: Iterator<Item = anyhow::Result<Data>>>(
569    engine: &mut BacktestEngine,
570    config: &BacktestRunConfig,
571    mut iter: Peekable<I>,
572    chunk_size: usize,
573) -> anyhow::Result<()> {
574    if iter.peek().is_none() {
575        return engine.end();
576    }
577
578    let mut next_start = config.start();
579    let mut started = false;
580
581    loop {
582        let chunk = take_aligned_chunk(&mut iter, chunk_size)
583            .map_err(|e| end_after_load_failure(engine, started, e))?;
584        if chunk.is_empty() {
585            break;
586        }
587
588        let is_last = iter.peek().is_none();
589        let end = if is_last {
590            config.end()
591        } else {
592            chunk.last().map(HasTsInit::ts_init)
593        };
594
595        engine
596            .add_data(chunk, None, false, true)
597            .map_err(|e| end_after_load_failure(engine, started, e))?;
598        engine.run(next_start, end, Some(config.id().to_string()), true)?;
599        engine.clear_data();
600        started = true;
601
602        // A shutdown request during the chunk already triggered end() inside
603        // engine.run(); stop loading further chunks so later data is not processed
604        if engine.kernel().is_shutdown_requested() {
605            return Ok(());
606        }
607
608        // Carry forward the end timestamp so the next chunk's run_impl
609        // sets clocks contiguously and processes gap timers correctly
610        next_start = end;
611    }
612
613    engine.end()
614}
615
616// Ends an engine that earlier chunks started, so a failure to load later data still stops the
617// trader and engines; the load failure stays the reported error
618fn end_after_load_failure(
619    engine: &mut BacktestEngine,
620    started: bool,
621    error: anyhow::Error,
622) -> anyhow::Error {
623    if started && let Err(e) = engine.end() {
624        log::error!("Failed to end backtest after a data load failure: {e:#}");
625    }
626    error
627}
628
629// Takes up to `chunk_size` items, then extends to include all remaining
630// items sharing the boundary timestamp to avoid splitting same-ts events.
631fn take_aligned_chunk<I: Iterator<Item = anyhow::Result<Data>>>(
632    iter: &mut Peekable<I>,
633    chunk_size: usize,
634) -> anyhow::Result<Vec<Data>> {
635    let mut chunk = Vec::with_capacity(chunk_size);
636
637    for _ in 0..chunk_size {
638        match iter.next() {
639            Some(item) => chunk.push(item?),
640            None => return Ok(chunk),
641        }
642    }
643
644    if let Some(boundary_ts) = chunk.last().map(HasTsInit::ts_init) {
645        // A failing item ends the extension and surfaces on the next chunk
646        while let Some(item) = iter.next_if(|item| {
647            item.as_ref()
648                .is_ok_and(|data| data.ts_init() == boundary_ts)
649        }) {
650            chunk.push(item?);
651        }
652    }
653
654    Ok(chunk)
655}
656
657fn create_catalog(
658    config: &BacktestDataConfig,
659    catalog_factories: &CatalogFactoryRegistry,
660) -> anyhow::Result<DataCatalog> {
661    let catalog_config = config.catalog();
662    catalog_factory::create_catalog(
663        catalog_config.catalog_backend(),
664        &catalog_config.connect_config(),
665        catalog_factories,
666    )
667}
668
669fn load_data(
670    config: &BacktestDataConfig,
671    run_start: Option<UnixNanos>,
672    run_end: Option<UnixNanos>,
673    catalog_factories: &CatalogFactoryRegistry,
674) -> anyhow::Result<Vec<Data>> {
675    let mut catalog = create_catalog(config, catalog_factories)?;
676    let result = dispatch_query(&mut catalog, config, run_start, run_end)?;
677    result.collect::<Result<Vec<_>, _>>()
678}
679
680fn dispatch_query(
681    catalog: &mut DataCatalog,
682    config: &BacktestDataConfig,
683    start: Option<UnixNanos>,
684    end: Option<UnixNanos>,
685) -> anyhow::Result<Box<dyn Iterator<Item = anyhow::Result<Data>>>> {
686    let batches = dispatch_batch_query(catalog, config, start, end)?;
687
688    let rows = batches
689        .map(|batch| batch.map(|batch| batch.to_data_vec_for_compat()))
690        .flat_map(|batch| match batch {
691            Ok(rows) => rows.into_iter().map(Ok).collect::<Vec<_>>(),
692            Err(e) => vec![Err(e)],
693        });
694
695    if config.batch_deltas() && *config.data_type() == NautilusDataType::OrderBookDelta {
696        return Ok(Box::new(BookDeltasBatcher::new(rows)));
697    }
698
699    Ok(Box::new(rows))
700}
701
702fn dispatch_batch_query(
703    catalog: &mut DataCatalog,
704    config: &BacktestDataConfig,
705    start: Option<UnixNanos>,
706    end: Option<UnixNanos>,
707) -> anyhow::Result<impl Iterator<Item = anyhow::Result<DataBatch>> + use<>> {
708    catalog.reset_session();
709    let mut query = CatalogQuery::new(config.data_type().clone())
710        .with_identifiers(config.query_identifiers())
711        .with_range(
712            max_opt(config.start_time(), start),
713            min_opt(config.end_time(), end),
714        )
715        .with_where_clause(config.filter_expr().map(str::to_string));
716
717    let params = Params::new(
718        [(
719            "optimize_file_loading".to_string(),
720            config.optimize_file_loading().into(),
721        )]
722        .into(),
723    );
724    query.params = Some(params);
725    let mut session = catalog.query_batch_session(&query, None)?;
726    let mut failed = false;
727    Ok(std::iter::from_fn(move || {
728        if failed {
729            return None;
730        }
731
732        loop {
733            match session.next_batch() {
734                Ok(Some(batch)) if batch.is_empty() => {}
735                Ok(batch) => return batch.map(Ok),
736                Err(e) => {
737                    failed = true;
738                    return Some(Err(e));
739                }
740            }
741        }
742    }))
743}
744
745fn max_opt(a: Option<UnixNanos>, b: Option<UnixNanos>) -> Option<UnixNanos> {
746    match (a, b) {
747        (Some(a), Some(b)) => Some(a.max(b)),
748        (Some(a), None) => Some(a),
749        (None, Some(b)) => Some(b),
750        (None, None) => None,
751    }
752}
753
754fn min_opt(a: Option<UnixNanos>, b: Option<UnixNanos>) -> Option<UnixNanos> {
755    match (a, b) {
756        (Some(a), Some(b)) => Some(a.min(b)),
757        (Some(a), None) => Some(a),
758        (None, Some(b)) => Some(b),
759        (None, None) => None,
760    }
761}
762
763// Regroups flat catalog deltas into `OrderBookDeltas` closed by `F_LAST`, so the engine applies
764// each book event at once. Each instrument keeps its own pending group because a multi-instrument
765// query interleaves rows by `ts_init`; a group replays at its closing delta, preserving that order.
766struct BookDeltasBatcher<I> {
767    rows: I,
768    pending: AHashMap<InstrumentId, Vec<OrderBookDelta>>,
769    finished: bool,
770}
771
772impl<I> BookDeltasBatcher<I> {
773    fn new(rows: I) -> Self {
774        Self {
775            rows,
776            pending: AHashMap::new(),
777            finished: false,
778        }
779    }
780}
781
782impl<I: Iterator<Item = anyhow::Result<Data>>> Iterator for BookDeltasBatcher<I> {
783    type Item = anyhow::Result<Data>;
784
785    fn next(&mut self) -> Option<Self::Item> {
786        if self.finished {
787            return None;
788        }
789
790        loop {
791            match self.rows.next() {
792                Some(Ok(Data::BookDelta(delta))) => {
793                    let group = self.pending.entry(delta.instrument_id).or_default();
794                    group.push(delta);
795
796                    if RecordFlag::F_LAST.matches(delta.flags) {
797                        let deltas = mem::replace(group, Vec::with_capacity(group.len()));
798                        let deltas = OrderBookDeltas::new_checked(delta.instrument_id, deltas);
799                        return Some(deltas.map(|deltas| Data::BookDeltas(Box::new(deltas))));
800                    }
801                }
802                Some(item) => {
803                    // A failed query must not flush its pending groups as complete events
804                    if item.is_err() {
805                        self.finished = true;
806                    }
807                    return Some(item);
808                }
809                None => {
810                    self.finished = true;
811                    return unterminated_deltas_error(&self.pending).map(Err);
812                }
813            }
814        }
815    }
816}
817
818fn unterminated_deltas_error(
819    pending: &AHashMap<InstrumentId, Vec<OrderBookDelta>>,
820) -> Option<anyhow::Error> {
821    let mut groups: Vec<_> = pending
822        .iter()
823        .filter_map(|(instrument_id, group)| {
824            let first = group.first()?.ts_init;
825            let last = group.last()?.ts_init;
826            Some((*instrument_id, group.len(), first, last))
827        })
828        .collect();
829
830    if groups.is_empty() {
831        return None;
832    }
833
834    groups.sort_unstable_by_key(|(instrument_id, ..)| *instrument_id);
835    let details = groups
836        .iter()
837        .map(|(instrument_id, count, first, last)| {
838            format!("{instrument_id} ({count} pending, ts_init {first} to {last})")
839        })
840        .collect::<Vec<_>>()
841        .join(", ");
842
843    Some(anyhow::anyhow!(
844        "Order book deltas end without an `F_LAST` delta for {details}; \
845         set `batch_deltas` to false to replay individual deltas"
846    ))
847}
848
849#[cfg(test)]
850mod tests {
851    use nautilus_common::component::Component;
852    use nautilus_execution::models::fee::{FeeModelAny, MakerTakerFeeModel};
853    #[cfg(feature = "defi")]
854    use nautilus_model::defi::{
855        DefiData,
856        data::block::BlockPosition,
857        pool_analysis::snapshot::{PoolAnalytics, PoolSnapshot, PoolState},
858    };
859    use nautilus_model::{
860        data::{BookOrder, QuoteTick, TradeTick},
861        enums::{AccountType, AggressorSide, BookAction, OmsType, OrderSide},
862        identifiers::{InstrumentId, TradeId, Venue},
863        types::{Money, Price, Quantity},
864    };
865    #[cfg(feature = "python")]
866    use pyo3::{ffi::c_str, prelude::*, types::PyDict};
867    use rstest::rstest;
868
869    use super::*;
870    use crate::{
871        config::{
872            BacktestEngineConfig, BacktestVenueConfig, MAX_BACKTEST_CHUNK_SIZE,
873            SimulatedVenueConfig,
874        },
875        modules::{
876            AccountAdjustmentOutcome, ExchangeContext, SimulationModule, SimulationModuleHandle,
877            SimulationModuleResult,
878        },
879    };
880    #[cfg(feature = "python")]
881    use crate::{
882        modules::SimulationModuleAny,
883        python::modules::{PySimulationModule, PythonSimulationModule},
884    };
885
886    fn quote(ts_init: u64) -> Data {
887        Data::Quote(QuoteTick::new(
888            InstrumentId::from("EUR/USD.SIM"),
889            Price::from("1.0001"),
890            Price::from("1.0002"),
891            Quantity::from("100"),
892            Quantity::from("100"),
893            UnixNanos::from(ts_init),
894            UnixNanos::from(ts_init),
895        ))
896    }
897
898    fn trade(ts_init: u64) -> Data {
899        Data::Trade(TradeTick::new(
900            InstrumentId::from("EUR/USD.SIM"),
901            Price::from("1.0001"),
902            Quantity::from("100"),
903            AggressorSide::Buy,
904            TradeId::from("T-1"),
905            UnixNanos::from(ts_init),
906            UnixNanos::from(ts_init),
907        ))
908    }
909
910    fn stream_failure() -> anyhow::Error {
911        anyhow::anyhow!("injected stream failure")
912    }
913
914    fn row_batch(data: &Data) -> ReplayBatch {
915        let batch = match data {
916            Data::Quote(quote) => DataBatch::Quote(vec![*quote].into()),
917            Data::Trade(trade) => DataBatch::Trade(vec![*trade].into()),
918            _ => panic!("expected a quote or trade fixture"),
919        };
920
921        ReplayBatch::Typed(batch)
922    }
923
924    fn merge_rows<I: Iterator<Item = anyhow::Result<Data>> + 'static>(
925        streams: Vec<Peekable<I>>,
926    ) -> impl Iterator<Item = anyhow::Result<Data>> {
927        let batches = streams
928            .into_iter()
929            .map(|stream| stream.map(|item| item.map(|data| row_batch(&data))))
930            .collect();
931        merge_batches(batches).expect("initial batches must load")
932    }
933
934    fn merge_batches<I: Iterator<Item = anyhow::Result<ReplayBatch>> + 'static>(
935        streams: Vec<I>,
936    ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<Data>>> {
937        merge_streams(
938            streams
939                .into_iter()
940                .enumerate()
941                .map(|(index, stream)| (index.to_string(), stream))
942                .collect(),
943        )
944    }
945
946    #[derive(Debug)]
947    struct FailingDiagnosticsModule;
948
949    impl SimulationModule for FailingDiagnosticsModule {
950        fn pre_process(&self, _data: &Data) -> anyhow::Result<()> {
951            Ok(())
952        }
953
954        fn process(
955            &self,
956            _ts_now: UnixNanos,
957            _ctx: &ExchangeContext,
958        ) -> anyhow::Result<SimulationModuleResult> {
959            Ok(SimulationModuleResult::NotReady)
960        }
961
962        fn acknowledge(&self, _outcomes: &[AccountAdjustmentOutcome]) -> anyhow::Result<()> {
963            Ok(())
964        }
965
966        fn log_diagnostics(&self) -> anyhow::Result<()> {
967            anyhow::bail!("diagnostics failure")
968        }
969
970        fn reset(&self) -> anyhow::Result<()> {
971            Ok(())
972        }
973    }
974
975    fn book_delta(instrument_id: &str, flags: u8, sequence: u64, ts_init: u64) -> OrderBookDelta {
976        OrderBookDelta::new(
977            InstrumentId::from(instrument_id),
978            BookAction::Add,
979            BookOrder::new(
980                OrderSide::Buy,
981                Price::from("1.0000"),
982                Quantity::from("100"),
983                sequence,
984            ),
985            flags,
986            sequence,
987            UnixNanos::from(ts_init - 1),
988            UnixNanos::from(ts_init),
989        )
990    }
991
992    fn batch_book_deltas(deltas: &[OrderBookDelta]) -> Vec<Data> {
993        BookDeltasBatcher::new(deltas.iter().map(|delta| Ok(Data::BookDelta(*delta))))
994            .collect::<anyhow::Result<_>>()
995            .expect("the batcher must not fail")
996    }
997
998    type BookDeltasFields = (
999        InstrumentId,
1000        Vec<OrderBookDelta>,
1001        u8,
1002        u64,
1003        UnixNanos,
1004        UnixNanos,
1005    );
1006
1007    // Lists every field because `OrderBookDeltas` equality compares only instrument and sequence
1008    fn book_deltas_fields(data: &Data) -> BookDeltasFields {
1009        let Data::BookDeltas(deltas) = data else {
1010            panic!("expected `OrderBookDeltas`, was {data:?}");
1011        };
1012        (
1013            deltas.instrument_id,
1014            deltas.deltas.clone(),
1015            deltas.flags,
1016            deltas.sequence,
1017            deltas.ts_event,
1018            deltas.ts_init,
1019        )
1020    }
1021
1022    #[rstest]
1023    fn merge_streams_orders_items_across_streams_by_ts_init() {
1024        let streams = vec![
1025            vec![Ok(quote(1)), Ok(quote(3)), Ok(quote(3))]
1026                .into_iter()
1027                .peekable(),
1028            vec![Ok(trade(2)), Ok(trade(3))].into_iter().peekable(),
1029            vec![].into_iter().peekable(),
1030        ];
1031
1032        let merged: Vec<(u64, bool)> = merge_rows(streams)
1033            .map(|item| item.expect("the merged stream must not fail"))
1034            .map(|data| (data.ts_init().as_u64(), matches!(data, Data::Trade(_))))
1035            .collect();
1036
1037        assert_eq!(
1038            merged,
1039            vec![(1, false), (2, true), (3, false), (3, false), (3, true)]
1040        );
1041    }
1042
1043    #[rstest]
1044    fn merge_streams_leaves_its_streams_undrained() {
1045        // Unbounded streams, so a merge that materialized its input would never return
1046        let ok_quote: fn(u64) -> anyhow::Result<Data> = |ts_init| Ok(quote(ts_init));
1047        let evens = (0u64..).step_by(2).map(ok_quote);
1048        let odds = (1u64..).step_by(2).map(ok_quote);
1049
1050        let merged: Vec<u64> = merge_rows(vec![evens.peekable(), odds.peekable()])
1051            .take(4)
1052            .map(|item| item.expect("the merged stream must not fail").ts_init())
1053            .map(|ts_init| ts_init.as_u64())
1054            .collect();
1055
1056        assert_eq!(merged, vec![0, 1, 2, 3]);
1057    }
1058
1059    #[rstest]
1060    fn merge_streams_reports_a_stream_failure() {
1061        let streams = vec![
1062            vec![Ok(quote(1)), Err(stream_failure())]
1063                .into_iter()
1064                .peekable(),
1065            vec![Ok(quote(2))].into_iter().peekable(),
1066        ];
1067        let mut merged = merge_rows(streams);
1068
1069        let first = merged.next().expect("the first item must be present");
1070        let second = merged.next().expect("the failure must be yielded");
1071
1072        assert_eq!(
1073            first.expect("the first item must not fail").ts_init(),
1074            UnixNanos::from(1)
1075        );
1076        assert_eq!(
1077            second
1078                .expect_err("a failed stream must not read as exhaustion")
1079                .to_string(),
1080            "injected stream failure"
1081        );
1082    }
1083
1084    #[rstest]
1085    fn merge_streams_reports_initial_failure_before_consumption() {
1086        let streams = vec![vec![Err(stream_failure())].into_iter()];
1087
1088        let Err(failure) = merge_batches(streams) else {
1089            panic!("initial source failure must be reported");
1090        };
1091
1092        assert_eq!(failure.to_string(), "injected stream failure");
1093    }
1094
1095    #[rstest]
1096    fn merge_streams_keeps_config_order_across_typed_batch_refills() {
1097        let batch = |times: &[u64]| {
1098            let quotes = times
1099                .iter()
1100                .map(|&ts| QuoteTick {
1101                    ts_init: UnixNanos::from(ts),
1102                    ..QuoteTick::default()
1103                })
1104                .collect::<Vec<_>>();
1105
1106            Ok(ReplayBatch::Typed(DataBatch::Quote(quotes.into())))
1107        };
1108
1109        let streams = vec![
1110            vec![batch(&[1, 3]), batch(&[3, 4])].into_iter(),
1111            vec![Ok(row_batch(&trade(2))), Ok(row_batch(&trade(3)))].into_iter(),
1112        ];
1113
1114        let merged: Vec<_> = merge_batches(streams)
1115            .unwrap()
1116            .map(|item| {
1117                let data = item.unwrap();
1118                (data.ts_init().as_u64(), matches!(data, Data::Trade(_)))
1119            })
1120            .collect();
1121
1122        assert_eq!(
1123            merged,
1124            vec![
1125                (1, false),
1126                (2, true),
1127                (3, false),
1128                (3, false),
1129                (3, true),
1130                (4, false)
1131            ]
1132        );
1133    }
1134
1135    #[cfg(feature = "defi")]
1136    #[rstest]
1137    fn merge_streams_orders_full_width_defi_positions_before_config_priority() {
1138        let batch = |id: &str, block, transaction_index, log_index| {
1139            let position =
1140                BlockPosition::new(block, "0xabc".to_string(), transaction_index, log_index);
1141
1142            let snapshot = PoolSnapshot::new(
1143                InstrumentId::from(id),
1144                PoolState::default(),
1145                Vec::new(),
1146                Vec::new(),
1147                PoolAnalytics::default(),
1148                position,
1149                UnixNanos::from(10),
1150                UnixNanos::from(10),
1151            );
1152            let data = vec![DefiData::PoolSnapshot(snapshot)];
1153            ReplayBatch::Typed(DataBatch::Defi(data.into()))
1154        };
1155
1156        let streams = vec![
1157            vec![Ok(batch("A.B", u64::MAX, u32::MAX, u32::MAX))].into_iter(),
1158            vec![
1159                Ok(batch("C.D", u64::MAX - 1, u32::MAX, u32::MAX)),
1160                Ok(batch("C.D", u64::MAX, u32::MAX - 1, u32::MAX)),
1161                Ok(batch("C.D", u64::MAX, u32::MAX, u32::MAX - 1)),
1162            ]
1163            .into_iter(),
1164        ];
1165
1166        let merged: Vec<_> = merge_batches(streams)
1167            .unwrap()
1168            .map(|item| {
1169                let data = item.unwrap();
1170
1171                let Data::Defi(defi) = data else {
1172                    panic!("expected DeFi data");
1173                };
1174
1175                (defi.instrument_id(), defi.ts_init(), defi.block_position())
1176            })
1177            .collect();
1178
1179        assert_eq!(
1180            merged,
1181            vec![
1182                (
1183                    InstrumentId::from("C.D"),
1184                    UnixNanos::from(10),
1185                    (u64::MAX - 1, u32::MAX, u32::MAX)
1186                ),
1187                (
1188                    InstrumentId::from("C.D"),
1189                    UnixNanos::from(10),
1190                    (u64::MAX, u32::MAX - 1, u32::MAX)
1191                ),
1192                (
1193                    InstrumentId::from("C.D"),
1194                    UnixNanos::from(10),
1195                    (u64::MAX, u32::MAX, u32::MAX - 1)
1196                ),
1197                (
1198                    InstrumentId::from("A.B"),
1199                    UnixNanos::from(10),
1200                    (u64::MAX, u32::MAX, u32::MAX)
1201                ),
1202            ]
1203        );
1204    }
1205
1206    #[rstest]
1207    fn merge_streams_returns_selected_row_before_a_refill_failure() {
1208        let streams = vec![vec![Ok(row_batch(&quote(1))), Err(stream_failure())].into_iter()];
1209        let mut merged = merge_batches(streams).unwrap();
1210        let first = merged.next().unwrap().unwrap();
1211        let failure = merged.next().unwrap().unwrap_err();
1212
1213        assert_eq!(first, quote(1));
1214        assert_eq!(failure.to_string(), "injected stream failure");
1215        assert!(merged.next().is_none());
1216    }
1217
1218    #[rstest]
1219    fn merge_streams_rejects_a_backward_source_before_emitting_other_streams() {
1220        let streams = vec![
1221            vec![Ok(row_batch(&quote(3))), Ok(row_batch(&quote(2)))].into_iter(),
1222            vec![Ok(row_batch(&trade(4)))].into_iter(),
1223        ];
1224        let mut merged = merge_batches(streams).unwrap();
1225        let first = merged.next().unwrap().unwrap();
1226        let failure = merged.next().unwrap().unwrap_err();
1227
1228        assert_eq!(first, quote(3));
1229        assert!(
1230            failure
1231                .to_string()
1232                .contains("moved backward across batches")
1233        );
1234        assert!(merged.next().is_none());
1235    }
1236
1237    #[rstest]
1238    #[case(false)]
1239    #[case(true)]
1240    fn merge_streams_reports_original_config_after_empty_sources(#[case] include_empty: bool) {
1241        let mut streams = Vec::new();
1242        if include_empty {
1243            streams.push(("config 0 (TradeTick)".to_string(), Vec::new().into_iter()));
1244        }
1245
1246        streams.push((
1247            "config 3 (QuoteTick)".to_string(),
1248            vec![Ok(row_batch(&quote(3))), Ok(row_batch(&quote(2)))].into_iter(),
1249        ));
1250        let mut merged = merge_streams(streams).unwrap();
1251        let first = merged.next().unwrap().unwrap();
1252        let failure = merged.next().unwrap().unwrap_err();
1253
1254        assert_eq!(first, quote(3));
1255        assert!(
1256            failure
1257                .to_string()
1258                .starts_with("Replay stream 'config 3 (QuoteTick)' moved backward across batches:")
1259        );
1260        assert!(merged.next().is_none());
1261    }
1262
1263    #[rstest]
1264    fn take_aligned_chunk_reports_a_stream_failure() {
1265        let mut iter = vec![Ok(quote(1)), Err(stream_failure())]
1266            .into_iter()
1267            .peekable();
1268
1269        let chunk = take_aligned_chunk(&mut iter, 4);
1270
1271        assert_eq!(
1272            chunk
1273                .expect_err("a failed stream must not read as a short chunk")
1274                .to_string(),
1275            "injected stream failure"
1276        );
1277    }
1278
1279    #[rstest]
1280    fn take_aligned_chunk_reports_a_failure_found_at_the_boundary() {
1281        let mut iter = vec![Ok(quote(1)), Err(stream_failure()), Ok(quote(1))]
1282            .into_iter()
1283            .peekable();
1284
1285        let first = take_aligned_chunk(&mut iter, 1).expect("the first chunk must be complete");
1286        let second = take_aligned_chunk(&mut iter, 1);
1287
1288        assert_eq!(first.len(), 1);
1289        assert_eq!(first[0].ts_init(), UnixNanos::from(1));
1290        assert_eq!(
1291            second
1292                .expect_err("the failure must survive the boundary extension")
1293                .to_string(),
1294            "injected stream failure"
1295        );
1296    }
1297
1298    #[rstest]
1299    fn take_aligned_chunk_extends_past_the_boundary_for_equal_timestamps() {
1300        let mut iter = vec![Ok(quote(1)), Ok(quote(1)), Ok(quote(2))]
1301            .into_iter()
1302            .peekable();
1303
1304        let chunk = take_aligned_chunk(&mut iter, 1).expect("the chunk must be complete");
1305
1306        assert_eq!(chunk.len(), 2);
1307        assert_eq!(chunk[0].ts_init(), UnixNanos::from(1));
1308        assert_eq!(chunk[1].ts_init(), UnixNanos::from(1));
1309    }
1310
1311    #[rstest]
1312    fn take_aligned_chunk_reserves_maximum_supported_capacity() {
1313        let mut iter = vec![Ok(quote(1))].into_iter().peekable();
1314
1315        let chunk = take_aligned_chunk(&mut iter, MAX_BACKTEST_CHUNK_SIZE).unwrap();
1316
1317        assert_eq!(chunk.len(), 1);
1318        assert!(chunk.capacity() >= MAX_BACKTEST_CHUNK_SIZE);
1319        assert_eq!(chunk[0].ts_init(), UnixNanos::from(1));
1320    }
1321
1322    #[rstest]
1323    fn book_deltas_batcher_closes_groups_only_on_f_last() {
1324        let last = RecordFlag::F_LAST as u8;
1325        let d1 = book_delta("EUR/USD.SIM", 0, 1, 10);
1326        let d2 = book_delta("EUR/USD.SIM", last, 2, 10);
1327        let d3 = book_delta("EUR/USD.SIM", last, 3, 10);
1328        let d4 = book_delta("EUR/USD.SIM", 0, 4, 20);
1329        let d5 = book_delta("EUR/USD.SIM", last, 5, 30);
1330
1331        let batches = batch_book_deltas(&[d1, d2, d3, d4, d5]);
1332
1333        // Same-timestamp events stay apart, and an event spanning timestamps replays at its close
1334        let instrument_id = InstrumentId::from("EUR/USD.SIM");
1335        assert_eq!(
1336            batches.iter().map(book_deltas_fields).collect::<Vec<_>>(),
1337            vec![
1338                (
1339                    instrument_id,
1340                    vec![d1, d2],
1341                    last,
1342                    2,
1343                    UnixNanos::from(9),
1344                    UnixNanos::from(10),
1345                ),
1346                (
1347                    instrument_id,
1348                    vec![d3],
1349                    last,
1350                    3,
1351                    UnixNanos::from(9),
1352                    UnixNanos::from(10),
1353                ),
1354                (
1355                    instrument_id,
1356                    vec![d4, d5],
1357                    last,
1358                    5,
1359                    UnixNanos::from(29),
1360                    UnixNanos::from(30),
1361                ),
1362            ]
1363        );
1364    }
1365
1366    #[rstest]
1367    fn book_deltas_batcher_keeps_interleaved_instruments_apart() {
1368        let last = RecordFlag::F_LAST as u8;
1369        let aud1 = book_delta("AUD/USD.SIM", 0, 1, 10);
1370        let eur1 = book_delta("EUR/USD.SIM", 0, 2, 10);
1371        let eur2 = book_delta("EUR/USD.SIM", last, 3, 10);
1372        let aud2 = book_delta("AUD/USD.SIM", last, 4, 20);
1373
1374        let batches = batch_book_deltas(&[aud1, eur1, eur2, aud2]);
1375
1376        // The completed EUR/USD event is not held behind the pending AUD/USD event
1377        assert_eq!(
1378            batches.iter().map(book_deltas_fields).collect::<Vec<_>>(),
1379            vec![
1380                (
1381                    InstrumentId::from("EUR/USD.SIM"),
1382                    vec![eur1, eur2],
1383                    last,
1384                    3,
1385                    UnixNanos::from(9),
1386                    UnixNanos::from(10),
1387                ),
1388                (
1389                    InstrumentId::from("AUD/USD.SIM"),
1390                    vec![aud1, aud2],
1391                    last,
1392                    4,
1393                    UnixNanos::from(19),
1394                    UnixNanos::from(20),
1395                ),
1396            ]
1397        );
1398    }
1399
1400    #[rstest]
1401    fn book_deltas_batcher_takes_flags_from_closing_delta() {
1402        let instrument_id = InstrumentId::from("EUR/USD.SIM");
1403        let clear =
1404            OrderBookDelta::clear(instrument_id, 1, UnixNanos::from(9), UnixNanos::from(10));
1405        let add = book_delta("EUR/USD.SIM", RecordFlag::F_MBP as u8, 2, 10);
1406        let close = book_delta("EUR/USD.SIM", RecordFlag::F_LAST as u8, 3, 10);
1407
1408        let batches = batch_book_deltas(&[clear, add, close]);
1409
1410        // A snapshot clear does not close the event, and earlier flags are not merged in
1411        assert_eq!(
1412            batches.iter().map(book_deltas_fields).collect::<Vec<_>>(),
1413            vec![(
1414                instrument_id,
1415                vec![clear, add, close],
1416                RecordFlag::F_LAST as u8,
1417                3,
1418                UnixNanos::from(9),
1419                UnixNanos::from(10),
1420            )]
1421        );
1422    }
1423
1424    #[rstest]
1425    fn book_deltas_batcher_yields_nothing_for_empty_input() {
1426        let batches = batch_book_deltas(&[]);
1427
1428        assert!(batches.is_empty());
1429    }
1430
1431    #[rstest]
1432    fn book_deltas_batcher_rejects_deltas_left_without_f_last() {
1433        let last = RecordFlag::F_LAST as u8;
1434        let d1 = book_delta("EUR/USD.SIM", last, 1, 10);
1435        let d2 = book_delta("EUR/USD.SIM", 0, 2, 20);
1436        let d3 = book_delta("EUR/USD.SIM", 0, 3, 30);
1437        let d4 = book_delta("AUD/USD.SIM", 0, 4, 40);
1438        let rows = [d1, d2, d3, d4].map(|delta| Ok(Data::BookDelta(delta)));
1439        let mut batcher = BookDeltasBatcher::new(rows.into_iter());
1440
1441        let first = batcher.next().expect("the complete event must be yielded");
1442        let second = batcher.next().expect("the pending deltas must be reported");
1443        let third = batcher.next();
1444
1445        assert_eq!(
1446            book_deltas_fields(&first.expect("the complete event must not fail")),
1447            (
1448                InstrumentId::from("EUR/USD.SIM"),
1449                vec![d1],
1450                last,
1451                1,
1452                UnixNanos::from(9),
1453                UnixNanos::from(10),
1454            )
1455        );
1456        assert_eq!(
1457            second
1458                .expect_err("pending deltas must not replay as an event")
1459                .to_string(),
1460            "Order book deltas end without an `F_LAST` delta for \
1461             AUD/USD.SIM (1 pending, ts_init 40 to 40), \
1462             EUR/USD.SIM (2 pending, ts_init 20 to 30); \
1463             set `batch_deltas` to false to replay individual deltas"
1464        );
1465        assert!(third.is_none());
1466    }
1467
1468    #[rstest]
1469    fn book_deltas_batcher_reports_a_query_failure_without_flushing() {
1470        let pending = book_delta("EUR/USD.SIM", 0, 1, 10);
1471        let after_failure = book_delta("EUR/USD.SIM", RecordFlag::F_LAST as u8, 2, 20);
1472        let rows = vec![
1473            Ok(Data::BookDelta(pending)),
1474            Err(stream_failure()),
1475            Ok(Data::BookDelta(after_failure)),
1476        ];
1477
1478        let items: Vec<anyhow::Result<Data>> = BookDeltasBatcher::new(rows.into_iter()).collect();
1479
1480        assert_eq!(items.len(), 1);
1481        assert_eq!(
1482            items[0]
1483                .as_ref()
1484                .expect_err("the query failure must be yielded")
1485                .to_string(),
1486            "injected stream failure"
1487        );
1488    }
1489
1490    #[rstest]
1491    fn end_after_load_failure_reports_the_load_failure_when_ending_fails() {
1492        let mut engine = BacktestEngine::new(BacktestEngineConfig::default()).unwrap();
1493        let venue = SimulatedVenueConfig::builder()
1494            .venue(Venue::from("SIM"))
1495            .oms_type(OmsType::Netting)
1496            .account_type(AccountType::Margin)
1497            .book_type(BookType::L1_MBP)
1498            .starting_balances(vec![Money::from("1_000_000 USD")])
1499            .modules(vec![SimulationModuleHandle::new(FailingDiagnosticsModule)])
1500            .fee_model(FeeModelAny::MakerTaker(MakerTakerFeeModel::zero()).into())
1501            .build()
1502            .unwrap();
1503        engine.add_venue(venue).unwrap();
1504        engine.run(None, None, None, true).unwrap();
1505
1506        let error = end_after_load_failure(&mut engine, true, stream_failure());
1507
1508        // Ending fails on the module diagnostics after the trader stops
1509        assert_eq!(error.to_string(), "injected stream failure");
1510        assert!(engine.kernel().trader.borrow().is_stopped());
1511        assert!(engine.get_result().run_finished.is_some());
1512    }
1513
1514    #[cfg(feature = "python")]
1515    #[rstest]
1516    fn build_engine_accepts_python_module_from_node_config() {
1517        Python::initialize();
1518
1519        Python::attach(|py| {
1520            let locals = PyDict::new(py);
1521            locals
1522                .set_item("SimulationModule", py.get_type::<PySimulationModule>())
1523                .unwrap();
1524            let module = py
1525                .eval(
1526                    c_str!(
1527                        "type('NodeSimulationModule', (SimulationModule,), {\
1528                            'process': lambda self, ts_now, context: \
1529                                (setattr(self, 'calls', self.calls + 1), [])[1]\
1530                        })()"
1531                    ),
1532                    None,
1533                    Some(&locals),
1534                )
1535                .unwrap();
1536            module.setattr("calls", 0).unwrap();
1537
1538            let venue = BacktestVenueConfig::builder()
1539                .name("SIM")
1540                .oms_type(OmsType::Netting)
1541                .account_type(AccountType::Margin)
1542                .book_type(BookType::L1_MBP)
1543                .starting_balances(vec!["1000 USD".to_string()])
1544                .fee_model(FeeModelAny::MakerTaker(MakerTakerFeeModel::zero()))
1545                .modules(vec![SimulationModuleAny::Python(
1546                    PythonSimulationModule::new(module.clone().unbind()),
1547                )])
1548                .build()
1549                .unwrap();
1550            let config = BacktestRunConfig::builder()
1551                .venues(vec![venue])
1552                .data(Vec::new())
1553                .build()
1554                .unwrap();
1555            let mut engine = build_engine(&config, &default_catalog_factories()).unwrap();
1556
1557            engine.run(None, None, None, false).unwrap();
1558
1559            assert_eq!(
1560                module.getattr("calls").unwrap().extract::<u32>().unwrap(),
1561                1
1562            );
1563        });
1564    }
1565
1566    #[rstest]
1567    fn test_build_engine_rejects_venue_without_fee_model() {
1568        let venue = BacktestVenueConfig::builder()
1569            .name("SIM")
1570            .oms_type(OmsType::Netting)
1571            .account_type(AccountType::Margin)
1572            .book_type(BookType::L1_MBP)
1573            .starting_balances(vec!["1_000_000 USD".to_string()])
1574            .build()
1575            .unwrap();
1576        let config = BacktestRunConfig::builder()
1577            .venues(vec![venue])
1578            .data(Vec::new())
1579            .build()
1580            .unwrap();
1581        let err = build_engine(&config, &default_catalog_factories()).unwrap_err();
1582        assert!(
1583            err.to_string().contains("explicit fee_model"),
1584            "unexpected error: {err}"
1585        );
1586    }
1587}