1#[cfg(feature = "streaming")]
38use std::collections::HashSet;
39use std::{
40 cell::{Cell, Ref, RefCell},
41 fmt::Debug,
42 rc::Rc,
43 time::Duration,
44};
45
46use nautilus_common::{
47 cache::{Cache, CacheConfig, database::CacheDatabaseAdapter},
48 clock::Clock,
49 component::Component,
50 enums::{ComponentState, Environment},
51 logging::{
52 arm_shutdown_on_error, disarm_shutdown_on_error, headers, init_logging,
53 logger::{LogGuard, LoggerConfig},
54 try_drain_shutdown_on_error_trigger,
55 },
56 messages::system::ShutdownSystem,
57 msgbus::{
58 self, MessageBus, MessagingSwitchboard, ShareableMessageHandler, get_message_bus,
59 set_message_bus,
60 },
61};
62use nautilus_core::{UUID4, UnixNanos};
63use nautilus_data::engine::DataEngine;
64use nautilus_execution::{
65 engine::ExecutionEngine,
66 order_emulator::{adapter::OrderEmulatorAdapter, emulator::OrderEmulator},
67};
68use nautilus_model::identifiers::{ClientId, TraderId};
69#[cfg(feature = "streaming")]
70use nautilus_persistence::{
71 backend::{default_catalog_factories, default_writer_factories},
72 catalog::factory::create_catalog,
73 common::paths::{create_local_directory, environment_directory},
74 config::{DataCatalogConfig, StreamingConfig},
75 writer::{
76 factory::{WriterConnectConfig, create_writer, replace_existing_writer_data},
77 feather::WriterClock,
78 filter::WriterRecordFilter,
79 subscription::StreamingSinkSubscription,
80 },
81};
82use nautilus_portfolio::portfolio::Portfolio;
83use nautilus_risk::engine::RiskEngine;
84use ustr::Ustr;
85
86use crate::{
87 builder::NautilusKernelBuilder,
88 clock_factory::ClockFactory,
89 config::NautilusKernelConfig,
90 event_store::{EventStoreFactory, KernelEventStore, RegisteredComponents},
91 trader::Trader,
92};
93
94pub struct NautilusKernel {
98 pub name: String,
100 pub instance_id: UUID4,
102 pub machine_id: String,
104 pub config: Box<dyn NautilusKernelConfig>,
106 pub cache: Rc<RefCell<Cache>>,
108 pub clock: Rc<RefCell<dyn Clock>>,
110 pub portfolio: Rc<RefCell<Portfolio>>,
112 pub log_guard: LogGuard,
114 pub data_engine: Rc<RefCell<DataEngine>>,
116 pub risk_engine: Rc<RefCell<RiskEngine>>,
118 pub exec_engine: Rc<RefCell<ExecutionEngine>>,
120 pub order_emulator: OrderEmulatorAdapter,
122 pub trader: Rc<RefCell<Trader>>,
124 pub ts_created: UnixNanos,
126 pub ts_started: Option<UnixNanos>,
128 pub ts_shutdown: Option<UnixNanos>,
130 shutdown_requested: Rc<Cell<bool>>,
131 event_store: Option<Box<dyn KernelEventStore>>,
132 event_store_replay: bool,
133 state_save_armed: bool,
134 #[cfg(feature = "streaming")]
135 streaming_writer: Option<(String, StreamingSinkSubscription)>,
136}
137
138impl Debug for NautilusKernel {
139 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
140 f.debug_struct(stringify!(NautilusKernel))
141 .field("name", &self.name)
142 .field("instance_id", &self.instance_id)
143 .field("machine_id", &self.machine_id)
144 .field("environment", &self.config.environment())
145 .finish_non_exhaustive()
146 }
147}
148
149#[derive(Default)]
151pub struct NautilusKernelDependencies {
152 clock_factory: Option<ClockFactory>,
153 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
154 event_store_factory: Option<EventStoreFactory>,
155}
156
157impl Debug for NautilusKernelDependencies {
158 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
159 f.debug_struct(stringify!(NautilusKernelDependencies))
160 .field("clock_factory", &self.clock_factory.is_some())
161 .field("cache_database", &self.cache_database.is_some())
162 .field("event_store_factory", &self.event_store_factory.is_some())
163 .finish()
164 }
165}
166
167impl NautilusKernelDependencies {
168 #[must_use]
170 pub fn with_clock_factory(mut self, clock_factory: Option<ClockFactory>) -> Self {
171 self.clock_factory = clock_factory;
172 self
173 }
174
175 #[must_use]
177 pub fn with_cache_database(
178 mut self,
179 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
180 ) -> Self {
181 self.cache_database = cache_database;
182 self
183 }
184
185 #[must_use]
187 pub fn with_event_store_factory(
188 mut self,
189 event_store_factory: Option<EventStoreFactory>,
190 ) -> Self {
191 self.event_store_factory = event_store_factory;
192 self
193 }
194}
195
196impl NautilusKernel {
197 #[cfg(feature = "streaming")]
198 fn setup_streaming_writer(
199 environment: Environment,
200 instance_id: UUID4,
201 config: &StreamingConfig,
202 clock: &Rc<RefCell<dyn Clock>>,
203 ) -> anyhow::Result<(String, StreamingSinkSubscription)> {
204 config.validate()?;
205
206 let writer_path = config.writer_path.trim_end_matches('/');
207 let uri = format!(
208 "{writer_path}/{}/{instance_id}",
209 environment_directory(environment)
210 );
211 let catalog = config
212 .catalog
213 .as_ref()
214 .map(DataCatalogConfig::connect_config);
215 let mut connect = WriterConnectConfig::new(&uri, catalog);
216 connect.rotation_config = config.rotation_config.to_writer_rotation_config()?;
217 connect.flush_interval_ms = Some(config.flush_interval_ms);
218 connect.params.clone_from(&config.params);
219 connect.record_filter = Self::writer_record_filter(config);
220 connect.promotion_interval_ms = config.promotion_interval_ms;
221 connect.promote_on_close = config.promote_on_close;
222 connect.delete_feather_after_promotion = config.delete_feather_after_promotion;
223 connect.use_ts_event_for_ts_init = config.use_ts_event_for_ts_init;
224
225 if config.replace_existing {
226 if let Some(catalog) = &config.catalog
229 && let Some(catalog_connect) = &connect.catalog
230 {
231 create_local_directory(&catalog_connect.uri)?;
232 create_catalog(
233 catalog.catalog_backend(),
234 catalog_connect,
235 &default_catalog_factories(),
236 )?;
237 }
238
239 replace_existing_writer_data(&connect)?;
240 }
241
242 let (writer_clock, bridge) = WriterClock::from_shared_clock(clock);
243 let sink = create_writer(
244 &config.writer_backend(),
245 &connect,
246 writer_clock,
247 &default_writer_factories(),
248 )?;
249 let subscription = StreamingSinkSubscription::subscribe_named(
250 Rc::new(RefCell::new(sink)),
251 bridge.map(|atomic| (Rc::clone(clock), atomic)),
252 uri.clone(),
253 );
254
255 log::info!("Writing data and events to {uri}");
256 Ok((uri, subscription))
257 }
258
259 #[cfg(feature = "streaming")]
260 fn writer_record_filter(config: &StreamingConfig) -> Option<WriterRecordFilter> {
261 let mut filter = config
262 .record_types
263 .clone()
264 .map_or_default(WriterRecordFilter::from_record_types);
265
266 if let Some(data_types) = &config.data_types {
267 for data_type in data_types {
268 filter.insert(data_type.clone(), None);
269 }
270 }
271
272 if let Some(instrument_types) = &config.instrument_types {
273 for instrument_type in instrument_types {
274 filter.insert_instrument_type(*instrument_type);
275 }
276 }
277
278 if let Some(entries) = &config.record_filters {
279 for entry in entries {
280 filter.insert(entry.record_type, entry.identifiers.clone());
281 }
282 }
283
284 (!filter.is_empty()).then_some(filter)
285 }
286
287 #[must_use]
289 pub const fn builder(
290 name: String,
291 trader_id: TraderId,
292 environment: Environment,
293 ) -> NautilusKernelBuilder {
294 NautilusKernelBuilder::new(name, trader_id, environment)
295 }
296
297 pub fn new<T: NautilusKernelConfig + 'static>(name: String, config: T) -> anyhow::Result<Self> {
303 Self::new_with(name, config, None, None)
304 }
305
306 pub fn new_with_cache_database<T: NautilusKernelConfig + 'static>(
317 name: String,
318 config: T,
319 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
320 ) -> anyhow::Result<Self> {
321 Self::new_with(name, config, cache_database, None)
322 }
323
324 pub fn new_with<T: NautilusKernelConfig + 'static>(
336 name: String,
337 config: T,
338 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
339 event_store_factory: Option<EventStoreFactory>,
340 ) -> anyhow::Result<Self> {
341 Self::new_with_dependencies(
342 name,
343 config,
344 NautilusKernelDependencies::default()
345 .with_cache_database(cache_database)
346 .with_event_store_factory(event_store_factory),
347 )
348 }
349
350 #[expect(
356 clippy::too_many_lines,
357 reason = "kernel construction keeps initialization order and ownership visible in one place"
358 )]
359 pub fn new_with_dependencies<T: NautilusKernelConfig + 'static>(
360 name: String,
361 config: T,
362 dependencies: NautilusKernelDependencies,
363 ) -> anyhow::Result<Self> {
364 let NautilusKernelDependencies {
365 clock_factory,
366 cache_database,
367 event_store_factory,
368 } = dependencies;
369 let instance_id = config.instance_id().unwrap_or_default();
370 let machine_id = Self::determine_machine_id()?;
371
372 let logger_config = config.logging();
373 let log_guard = Self::initialize_logging(config.trader_id(), instance_id, logger_config)?;
374 headers::log_header(
375 config.trader_id(),
376 &machine_id,
377 instance_id,
378 Ustr::from(&name),
379 );
380
381 log::info!("Building system kernel");
382
383 let clock_factory =
384 clock_factory.unwrap_or_else(|| ClockFactory::for_environment(config.environment()));
385 let clock = clock_factory.clock();
386 let event_store = match event_store_factory {
387 Some(factory) => Some(factory(instance_id, clock.clone())?),
388 None => None,
389 };
390 let cache = Self::initialize_cache(config.cache(), cache_database);
391
392 let msgbus = Rc::new(RefCell::new(MessageBus::new(
393 config.trader_id(),
394 instance_id,
395 Some(name.clone()),
396 None,
397 )));
398 set_message_bus(msgbus);
399
400 if let Some(config) = config.msgbus()
401 && let Some(filter) = config.types_filter
402 {
403 get_message_bus().borrow_mut().set_types_filter(filter);
404 }
405
406 let portfolio = Rc::new(RefCell::new(Portfolio::new(
407 clock.clone(),
408 cache.clone(),
409 config.portfolio(),
410 )));
411
412 let risk_engine = RiskEngine::new(
413 config.risk_engine().unwrap_or_default(),
414 portfolio.borrow().clone_shallow(),
415 clock.clone(),
416 cache.clone(),
417 );
418 let risk_engine = Rc::new(RefCell::new(risk_engine));
419
420 let exec_engine = ExecutionEngine::new(clock.clone(), cache.clone(), config.exec_engine());
421 let exec_engine = Rc::new(RefCell::new(exec_engine));
422
423 let order_emulator = OrderEmulatorAdapter::new(clock.clone(), cache.clone());
424
425 let data_engine = DataEngine::new(clock.clone(), cache.clone(), config.data_engine());
426 #[cfg(feature = "streaming")]
427 let mut data_engine = data_engine;
428 #[cfg(feature = "streaming")]
429 {
430 let catalog_factories = default_catalog_factories();
431 let mut unnamed_index = 0;
432 let mut catalog_names = HashSet::new();
433
434 for catalog_config in config.catalogs() {
435 let name = catalog_config.name().map_or_else(
436 || {
437 let name = format!("catalog_{unnamed_index}");
438 unnamed_index += 1;
439 name
440 },
441 str::to_owned,
442 );
443 anyhow::ensure!(
444 catalog_names.insert(name.clone()),
445 "Duplicate data catalog name '{name}'",
446 );
447
448 let catalog = create_catalog(
449 catalog_config.catalog_backend(),
450 &catalog_config.connect_config(),
451 &catalog_factories,
452 )
453 .map_err(|e| {
454 anyhow::anyhow!(
455 "Failed to create data catalog from '{}': {e:#}",
456 catalog_config.path()
457 )
458 })?;
459 data_engine.register_catalog(catalog, Some(&name));
460 }
461 }
462 let data_engine = Rc::new(RefCell::new(data_engine));
463
464 DataEngine::register_msgbus_handlers(&data_engine);
465 RiskEngine::register_msgbus_handlers(&risk_engine);
466 ExecutionEngine::register_msgbus_handlers(&exec_engine);
467 OrderEmulator::register_msgbus_handlers(&order_emulator.emulator());
468
469 let shutdown_requested = Rc::new(Cell::new(false));
470 Self::register_shutdown_handler(config.trader_id(), shutdown_requested.clone());
471
472 let trader = Rc::new(RefCell::new(Trader::new(
473 config.trader_id(),
474 instance_id,
475 config.environment(),
476 clock_factory,
477 cache.clone(),
478 portfolio.clone(),
479 )));
480
481 let ts_created = clock.borrow().timestamp_ns();
482
483 #[cfg(feature = "streaming")]
484 let streaming_writer = config
485 .streaming()
486 .map(|streaming| {
487 Self::setup_streaming_writer(config.environment(), instance_id, &streaming, &clock)
488 })
489 .transpose()?;
490
491 Ok(Self {
492 name,
493 instance_id,
494 machine_id,
495 event_store,
496 config: Box::new(config),
497 cache,
498 clock,
499 portfolio,
500 log_guard,
501 data_engine,
502 risk_engine,
503 exec_engine,
504 order_emulator,
505 trader,
506 ts_created,
507 ts_started: None,
508 ts_shutdown: None,
509 shutdown_requested,
510 event_store_replay: false,
511 state_save_armed: false,
512 #[cfg(feature = "streaming")]
513 streaming_writer,
514 })
515 }
516
517 fn register_shutdown_handler(trader_id: TraderId, shutdown_requested: Rc<Cell<bool>>) {
518 let handler = ShareableMessageHandler::from_typed(move |cmd: &ShutdownSystem| {
519 if cmd.trader_id != trader_id {
520 log::warn!("Received {cmd} not for this trader {trader_id}, ignoring");
521 return;
522 }
523
524 if shutdown_requested.get() {
525 log::debug!("Shutdown already requested, ignoring {cmd}");
526 return;
527 }
528
529 log::info!("Received {cmd}, requesting shutdown");
530 shutdown_requested.set(true);
531 });
532 let topic = MessagingSwitchboard::shutdown_system_topic();
533 msgbus::subscribe_any(topic.into(), handler, None);
534 }
535
536 fn determine_machine_id() -> anyhow::Result<String> {
537 sysinfo::System::host_name()
538 .map_err(|e| anyhow::anyhow!("Failed to determine hostname: {e}"))
539 }
540
541 fn initialize_logging(
542 trader_id: TraderId,
543 instance_id: UUID4,
544 config: LoggerConfig,
545 ) -> anyhow::Result<LogGuard> {
546 #[cfg(feature = "tracing-bridge")]
547 let use_tracing = config.use_tracing;
548
549 let file_config = config.file_config.clone().unwrap_or_default();
550 let log_guard = match init_logging(trader_id, instance_id, config, file_config) {
551 Ok(guard) => guard,
552 Err(e) => {
553 if e.downcast_ref::<log::SetLoggerError>().is_some() {
558 if let Some(guard) = LogGuard::new() {
559 guard
560 } else {
561 return Err(e.context(
562 "A non-Nautilus logger is already registered; \
563 cannot initialize Nautilus logging",
564 ));
565 }
566 } else {
567 return Err(e);
568 }
569 }
570 };
571
572 #[cfg(feature = "tracing-bridge")]
574 if use_tracing && !nautilus_common::logging::bridge::tracing_is_initialized() {
575 nautilus_common::logging::bridge::init_tracing()?;
576 }
577
578 Ok(log_guard)
579 }
580
581 fn initialize_cache(
582 cache_config: Option<CacheConfig>,
583 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
584 ) -> Rc<RefCell<Cache>> {
585 let cache_config = cache_config.unwrap_or_default();
586 let cache = Cache::new(Some(cache_config), cache_database);
587
588 Rc::new(RefCell::new(cache))
589 }
590
591 fn cancel_timers(&self) {
592 self.clock.borrow_mut().cancel_timers();
593 }
594
595 #[must_use]
596 pub fn generate_timestamp_ns(&self) -> UnixNanos {
597 self.clock.borrow().timestamp_ns()
598 }
599
600 #[must_use]
602 pub fn environment(&self) -> Environment {
603 self.config.environment()
604 }
605
606 #[must_use]
608 pub const fn name(&self) -> &str {
609 self.name.as_str()
610 }
611
612 #[must_use]
614 pub fn trader_id(&self) -> TraderId {
615 self.config.trader_id()
616 }
617
618 #[must_use]
620 pub fn machine_id(&self) -> &str {
621 &self.machine_id
622 }
623
624 #[must_use]
626 pub const fn instance_id(&self) -> UUID4 {
627 self.instance_id
628 }
629
630 #[must_use]
632 pub fn delay_post_stop(&self) -> Duration {
633 self.config.delay_post_stop()
634 }
635
636 #[must_use]
638 pub const fn ts_created(&self) -> UnixNanos {
639 self.ts_created
640 }
641
642 #[must_use]
644 pub const fn ts_started(&self) -> Option<UnixNanos> {
645 self.ts_started
646 }
647
648 #[must_use]
650 pub const fn ts_shutdown(&self) -> Option<UnixNanos> {
651 self.ts_shutdown
652 }
653
654 #[must_use]
658 pub fn is_shutdown_requested(&self) -> bool {
659 self.drain_shutdown_on_error_trigger();
660 self.shutdown_requested.get()
661 }
662
663 pub fn reset_shutdown_flag(&self) {
668 self.shutdown_requested.set(false);
669 }
670
671 #[must_use]
674 pub fn shutdown_flag(&self) -> Rc<Cell<bool>> {
675 self.shutdown_requested.clone()
676 }
677
678 fn drain_shutdown_on_error_trigger(&self) {
679 try_drain_shutdown_on_error_trigger(|trigger| {
680 let command = ShutdownSystem::new(
681 self.config.trader_id(),
682 trigger.component,
683 Some(format!(
684 "Error log received from {}: {}",
685 trigger.component, trigger.message
686 )),
687 UUID4::new(),
688 trigger.timestamp,
689 None,
690 );
691
692 msgbus::try_publish_any(
693 MessagingSwitchboard::shutdown_system_topic(),
694 command.as_any(),
695 )
696 });
697 }
698
699 #[must_use]
701 pub fn load_state(&self) -> bool {
702 self.config.load_state()
703 }
704
705 #[must_use]
707 pub fn save_state(&self) -> bool {
708 self.config.save_state()
709 }
710
711 #[must_use]
713 pub fn clock(&self) -> Rc<RefCell<dyn Clock>> {
714 self.clock.clone()
715 }
716
717 #[must_use]
719 pub fn cache(&self) -> Rc<RefCell<Cache>> {
720 self.cache.clone()
721 }
722
723 #[must_use]
725 pub fn portfolio(&self) -> Ref<'_, Portfolio> {
726 self.portfolio.borrow()
727 }
728
729 #[must_use]
731 pub fn data_engine(&self) -> Ref<'_, DataEngine> {
732 self.data_engine.borrow()
733 }
734
735 #[must_use]
737 pub const fn risk_engine(&self) -> &Rc<RefCell<RiskEngine>> {
738 &self.risk_engine
739 }
740
741 #[must_use]
743 pub const fn exec_engine(&self) -> &Rc<RefCell<ExecutionEngine>> {
744 &self.exec_engine
745 }
746
747 #[must_use]
749 pub fn trader(&self) -> &Rc<RefCell<Trader>> {
750 &self.trader
751 }
752
753 pub fn start(&mut self) {
755 arm_shutdown_on_error(self.config.shutdown_on_error());
756 log::info!("Starting");
757
758 self.event_store_replay = false;
759
760 if let Some(event_store) = self.event_store.as_deref_mut() {
761 self.exec_engine.borrow_mut().set_snapshot_anchorer(None);
762
763 let components = Self::collect_registered_components(&self.trader);
764 let environment = self.config.environment();
765 let event_store_replay_configured = event_store.is_event_store_replay_configured();
766
767 if event_store_replay_configured && !self.config.load_state() {
768 log::error!("Event-store replay requires load_state=true");
769 return;
770 }
771
772 if self.config.load_state()
773 && let Err(e) =
774 event_store.restore_parent_cache(self.instance_id, &mut self.cache.borrow_mut())
775 {
776 log::error!("Failed to restore cache from event-store replay source: {e}");
777 return;
778 }
779
780 if let Err(e) = event_store.open(self.instance_id, &components, environment) {
781 log::error!("Failed to open event-store run: {e}");
782 return;
783 }
784
785 let anchorer = event_store.snapshot_anchorer();
786 self.exec_engine
787 .borrow_mut()
788 .set_snapshot_anchorer(anchorer);
789 self.event_store_replay = event_store_replay_configured;
790 }
791
792 if self.event_store_replay {
793 log::info!(
794 "Event-store replay loaded; skipping engines, clients, trader startup, and live reconciliation",
795 );
796 self.ts_started = Some(self.clock.borrow().timestamp_ns());
797 log::info!("Started");
798 return;
799 }
800
801 self.start_engines();
802
803 let trader_state = self.trader.borrow().state();
805 if trader_state == ComponentState::PreInitialized {
806 log::info!("Initializing trader");
807
808 if let Err(e) = self.trader.borrow_mut().initialize() {
809 log::error!("Error initializing trader: {e:?}");
810 return;
811 }
812 }
813
814 self.ts_started = Some(self.clock.borrow().timestamp_ns());
817 log::info!("Started");
818 }
819
820 fn collect_registered_components(trader: &Rc<RefCell<Trader>>) -> RegisteredComponents {
821 let trader = trader.borrow();
822 let mut components = RegisteredComponents::default();
823 for actor_id in trader.actor_ids() {
824 components
825 .actors
826 .insert(actor_id.to_string(), String::new());
827 }
828
829 for strategy_id in trader.strategy_ids() {
830 components
831 .strategies
832 .insert(strategy_id.to_string(), String::new());
833 }
834
835 for algo_id in trader.exec_algorithm_ids() {
836 components
837 .algorithms
838 .insert(algo_id.to_string(), String::new());
839 }
840 components
841 }
842
843 #[expect(
845 clippy::unused_async,
846 reason = "keeps the public async kernel API shape stable"
847 )]
848 pub async fn start_async(&mut self) {
849 self.start();
850 }
851
852 pub fn start_trader(&mut self) -> anyhow::Result<()> {
861 log::info!("Starting trader...");
862
863 let load_state = self.config.load_state();
864 let save_state = self.config.save_state();
865
866 if (load_state || save_state) && !self.cache.borrow().has_backing() {
867 log::warn!(
868 "Cache has no database backing, load_state={load_state} and save_state={save_state} will have no effect"
869 );
870 }
871
872 if load_state {
873 Trader::load_state(&self.trader)
874 .map_err(|e| anyhow::anyhow!("Failed to load actor and strategy state: {e:#}"))?;
875 }
876
877 self.state_save_armed = save_state;
878 self.order_emulator.start();
879
880 if let Err(start_err) = Trader::start_with_component_callbacks(&self.trader) {
881 let stop_result = self.stop_trader_after_start_failure();
882 self.order_emulator.stop();
883 let save_result = self.save_trader_state();
884
885 let mut errors = vec![format!("Failed to start trader: {start_err}")];
886 if let Err(e) = stop_result {
887 errors.push(format!("failed to stop partial trader start: {e}"));
888 }
889
890 if let Err(e) = save_result {
891 errors.push(format!("failed to save partial trader state: {e}"));
892 }
893 anyhow::bail!("{}", errors.join("; "));
894 }
895
896 log::info!("Trader started");
897 Ok(())
898 }
899
900 pub fn stop_trader(&mut self) {
906 disarm_shutdown_on_error();
907
908 if !self.trader.borrow().is_running() {
909 return;
910 }
911
912 log::info!("Stopping trader...");
913
914 if let Err(e) = self.trader.borrow_mut().stop() {
915 log::error!("Error stopping trader: {e}");
916 }
917 }
918
919 pub fn stop_trader_after_start_failure(&mut self) -> anyhow::Result<()> {
925 disarm_shutdown_on_error();
926
927 if !matches!(
928 self.trader.borrow().state(),
929 ComponentState::Starting | ComponentState::Running
930 ) {
931 return Ok(());
932 }
933
934 log::info!("Stopping trader immediately...");
935 self.trader.borrow_mut().stop_after_start_failure()
936 }
937
938 #[allow(unknown_lints)]
951 #[expect(
952 clippy::unused_async,
953 clippy::unused_async_trait_impl,
954 reason = "keeps the public async kernel API shape stable"
955 )]
956 pub async fn finalize_stop(&mut self) -> anyhow::Result<()> {
957 disarm_shutdown_on_error();
958
959 let save_result = self.save_trader_state();
962 self.portfolio.borrow_mut().finalize_equity_curve();
963 self.stop_engines();
964 self.cancel_timers();
965
966 let ts_shutdown = self.clock.borrow().timestamp_ns();
967
968 if let Some(event_store) = self.event_store.as_deref_mut() {
969 self.exec_engine.borrow_mut().set_snapshot_anchorer(None);
970 event_store.seal(ts_shutdown);
971 }
972 self.ts_shutdown = Some(ts_shutdown);
973 log::info!("Stopped");
974 save_result?;
975 self.flush_streaming_writer()
976 }
977
978 pub fn save_trader_state(&mut self) -> anyhow::Result<()> {
986 if !std::mem::take(&mut self.state_save_armed) {
987 return Ok(());
988 }
989
990 Trader::save_state(&self.trader)
991 }
992
993 #[must_use]
999 pub fn event_store(&self) -> Option<&dyn KernelEventStore> {
1000 self.event_store.as_deref()
1001 }
1002
1003 #[must_use]
1005 pub fn is_event_store_replay(&self) -> bool {
1006 self.event_store_replay
1007 }
1008
1009 #[must_use]
1011 pub fn is_event_store_replay_configured(&self) -> bool {
1012 self.event_store
1013 .as_deref()
1014 .is_some_and(KernelEventStore::is_event_store_replay_configured)
1015 }
1016
1017 pub fn reset(&mut self) {
1019 disarm_shutdown_on_error();
1020 log::info!("Resetting");
1021
1022 if let Err(e) = self.trader.borrow_mut().reset() {
1023 log::error!("Error resetting trader: {e:?}");
1024 }
1025
1026 self.data_engine.borrow_mut().reset();
1027 self.exec_engine.borrow_mut().reset();
1028 self.risk_engine.borrow_mut().reset();
1029 self.order_emulator.reset();
1030 self.portfolio.borrow_mut().reset();
1031
1032 self.ts_started = None;
1033 self.ts_shutdown = None;
1034 self.state_save_armed = false;
1035
1036 log::info!("Reset");
1037 }
1038
1039 pub fn dispose(&mut self) {
1041 disarm_shutdown_on_error();
1042 log::info!("Disposing");
1043
1044 let trader_state = self.trader.borrow().state();
1045 match trader_state {
1046 ComponentState::Running => self.stop_trader(),
1047 ComponentState::Starting => {
1048 if let Err(e) = self.stop_trader_after_start_failure() {
1049 log::error!("Error stopping partial trader start during disposal: {e:?}");
1050 }
1051 }
1052 _ => {}
1053 }
1054
1055 if let Err(e) = self.save_trader_state() {
1056 log::error!("Error saving trader state during disposal: {e:?}");
1057 }
1058
1059 {
1060 let mut trader = self.trader.borrow_mut();
1061 if trader.state() == ComponentState::PreInitialized
1062 && let Err(e) = trader.initialize()
1063 {
1064 log::error!("Error initializing trader for disposal: {e:?}");
1065 }
1066
1067 if !trader.is_disposed()
1068 && let Err(e) = trader.dispose()
1069 {
1070 log::error!("Error disposing trader: {e:?}");
1071 }
1072 }
1073
1074 self.stop_engines();
1075 self.portfolio.borrow_mut().reset();
1076 self.cancel_timers();
1077
1078 if let Some(event_store) = self.event_store.as_deref_mut() {
1083 self.exec_engine.borrow_mut().set_snapshot_anchorer(None);
1084 let ts_dispose = self.clock.borrow().timestamp_ns();
1085 event_store.seal(ts_dispose);
1086 }
1087
1088 if let Err(e) = self.close_streaming_writer() {
1089 log::warn!("{e}");
1090 }
1091
1092 self.data_engine.borrow_mut().dispose();
1093 self.exec_engine.borrow_mut().dispose();
1094 self.risk_engine.borrow_mut().dispose();
1095 self.order_emulator.dispose();
1096 self.cache.borrow_mut().dispose();
1097 get_message_bus().borrow_mut().dispose();
1098
1099 log::info!("Disposed");
1100 }
1101
1102 pub fn reopen_streaming_writer(&mut self) -> anyhow::Result<()> {
1111 #[cfg(feature = "streaming")]
1112 if self.streaming_writer.is_none()
1113 && let Some(mut config) = self.config.streaming()
1114 {
1115 config.replace_existing = false;
1116 self.streaming_writer = Some(Self::setup_streaming_writer(
1117 self.config.environment(),
1118 self.instance_id,
1119 &config,
1120 &self.clock,
1121 )?);
1122 }
1123
1124 Ok(())
1125 }
1126
1127 #[cfg(feature = "streaming")]
1130 fn flush_streaming_writer(&self) -> anyhow::Result<()> {
1131 match &self.streaming_writer {
1132 Some((destination, writer)) => writer.flush().map_err(|e| {
1133 anyhow::anyhow!("Failed to flush streaming writer at {destination}: {e}")
1134 }),
1135 None => Ok(()),
1136 }
1137 }
1138
1139 #[cfg(not(feature = "streaming"))]
1140 #[expect(
1141 clippy::unnecessary_wraps,
1142 clippy::unused_self,
1143 reason = "matches the streaming variant's signature"
1144 )]
1145 fn flush_streaming_writer(&self) -> anyhow::Result<()> {
1146 Ok(())
1147 }
1148
1149 #[cfg(feature = "streaming")]
1156 pub fn close_streaming_writer(&mut self) -> anyhow::Result<()> {
1157 match self.streaming_writer.take() {
1158 Some((destination, writer)) => writer.close().map_err(|e| {
1159 anyhow::anyhow!("Failed to close streaming writer at {destination}: {e}")
1160 }),
1161 None => Ok(()),
1162 }
1163 }
1164
1165 #[cfg(not(feature = "streaming"))]
1172 pub fn close_streaming_writer(&mut self) -> anyhow::Result<()> {
1173 Ok(())
1174 }
1175
1176 fn start_engines(&self) {
1178 self.data_engine.borrow_mut().start();
1179 self.exec_engine.borrow_mut().start();
1180 self.risk_engine.borrow_mut().start();
1181 }
1182
1183 fn stop_engines(&self) {
1185 self.data_engine.borrow_mut().stop();
1186 self.exec_engine.borrow_mut().stop();
1187 self.risk_engine.borrow_mut().stop();
1188 self.order_emulator.stop();
1189 }
1190
1191 #[expect(clippy::await_holding_refcell_ref)] pub async fn connect_data_clients(&mut self) -> anyhow::Result<()> {
1201 log::info!("Connecting data clients...");
1202 self.data_engine.borrow_mut().connect().await
1203 }
1204
1205 #[expect(clippy::await_holding_refcell_ref)] pub async fn connect_exec_clients(&mut self) -> anyhow::Result<()> {
1215 log::info!("Connecting execution clients...");
1216 self.exec_engine.borrow_mut().connect().await
1217 }
1218
1219 #[expect(clippy::await_holding_refcell_ref)] pub async fn disconnect_clients(&mut self) -> anyhow::Result<()> {
1226 log::info!("Disconnecting clients...");
1227 let mut data_engine = self.data_engine.borrow_mut();
1228 let mut exec_engine = self.exec_engine.borrow_mut();
1229 let (data_result, exec_result) =
1230 futures::join!(data_engine.disconnect(), exec_engine.disconnect());
1231
1232 match (data_result, exec_result) {
1233 (Ok(()), Ok(())) => Ok(()),
1234 (Err(data_err), Ok(())) => Err(data_err),
1235 (Ok(()), Err(exec_err)) => Err(exec_err),
1236 (Err(data_err), Err(exec_err)) => anyhow::bail!(
1237 "Failed to disconnect data clients: {data_err}; failed to disconnect execution \
1238 clients: {exec_err}"
1239 ),
1240 }
1241 }
1242
1243 #[must_use]
1245 pub fn check_engines_connected(&self) -> bool {
1246 self.data_engine.borrow().check_connected() && self.exec_engine.borrow().check_connected()
1247 }
1248
1249 #[must_use]
1251 pub fn check_engines_disconnected(&self) -> bool {
1252 self.data_engine.borrow().check_disconnected()
1253 && self.exec_engine.borrow().check_disconnected()
1254 }
1255
1256 #[must_use]
1258 pub fn data_client_connection_status(&self) -> Vec<(ClientId, bool)> {
1259 self.data_engine.borrow().client_connection_status()
1260 }
1261
1262 #[must_use]
1264 pub fn exec_client_connection_status(&self) -> Vec<(ClientId, bool)> {
1265 self.exec_engine.borrow().client_connection_status()
1266 }
1267}
1268
1269#[cfg(all(test, feature = "python"))]
1270mod tests {
1271 use nautilus_common::messages::system::ShutdownSystem;
1272 use nautilus_core::UUID4;
1273 use rstest::*;
1274 use ustr::Ustr;
1275
1276 use super::*;
1277 use crate::builder::NautilusKernelBuilder;
1278
1279 #[rstest]
1280 fn test_shutdown_system_sets_kernel_flag() {
1281 let kernel = NautilusKernelBuilder::default().build().unwrap();
1282 assert!(!kernel.is_shutdown_requested());
1283
1284 let command = ShutdownSystem::new(
1285 kernel.trader_id(),
1286 Ustr::from("TestComponent"),
1287 Some("unit test".to_string()),
1288 UUID4::new(),
1289 kernel.generate_timestamp_ns(),
1290 None, );
1292
1293 msgbus::publish_any(
1294 MessagingSwitchboard::shutdown_system_topic(),
1295 command.as_any(),
1296 );
1297 assert!(kernel.is_shutdown_requested());
1298
1299 kernel.reset_shutdown_flag();
1300 assert!(!kernel.is_shutdown_requested());
1301 }
1302
1303 #[rstest]
1304 fn test_shutdown_system_idempotent() {
1305 let kernel = NautilusKernelBuilder::default().build().unwrap();
1306
1307 let make_cmd = || {
1308 ShutdownSystem::new(
1309 kernel.trader_id(),
1310 Ustr::from("TestComponent"),
1311 None,
1312 UUID4::new(),
1313 kernel.generate_timestamp_ns(),
1314 None, )
1316 };
1317
1318 let topic = MessagingSwitchboard::shutdown_system_topic();
1319 msgbus::publish_any(topic, make_cmd().as_any());
1320 assert!(kernel.is_shutdown_requested());
1321
1322 msgbus::publish_any(topic, make_cmd().as_any());
1323 assert!(kernel.is_shutdown_requested());
1324
1325 kernel.reset_shutdown_flag();
1326 assert!(!kernel.is_shutdown_requested());
1327
1328 msgbus::publish_any(topic, make_cmd().as_any());
1329 assert!(kernel.is_shutdown_requested());
1330 }
1331
1332 #[rstest]
1333 fn test_shutdown_system_ignores_other_trader() {
1334 let kernel = NautilusKernelBuilder::default().build().unwrap();
1335
1336 let command = ShutdownSystem::new(
1337 TraderId::from("OTHER-TRADER"),
1338 Ustr::from("TestComponent"),
1339 None,
1340 UUID4::new(),
1341 kernel.generate_timestamp_ns(),
1342 None, );
1344
1345 msgbus::publish_any(
1346 MessagingSwitchboard::shutdown_system_topic(),
1347 command.as_any(),
1348 );
1349 assert!(!kernel.is_shutdown_requested());
1350 }
1351}
1352
1353#[cfg(all(test, feature = "streaming"))]
1354mod streaming_tests {
1355 use std::{cell::RefCell, rc::Rc, sync::Arc};
1356
1357 use nautilus_common::{
1358 clock::VirtualClock,
1359 messages::data::{
1360 DataCommand, DataResponse, DataResponseEnvelope, RequestCommand, RequestQuotes,
1361 },
1362 msgbus::{self, MStr, TypedHandler},
1363 };
1364 use nautilus_core::Params;
1365 use nautilus_model::{
1366 data::{CustomData, DataType, NautilusDataType, QuoteTick},
1367 identifiers::InstrumentId,
1368 types::{Price, Quantity},
1369 };
1370 use nautilus_persistence::{
1371 backend::parquet::catalog::ParquetDataCatalog, config::DataCatalogConfig,
1372 test_data::RustTestCustomData,
1373 };
1374 use nautilus_serialization::ensure_custom_data_registered;
1375 use rstest::rstest;
1376 use tempfile::tempdir;
1377
1378 use super::*;
1379 use crate::config::{KernelConfig, RotationConfig, StreamingConfig};
1380
1381 #[rstest]
1382 fn test_configured_catalog_serves_builtin_quotes() {
1383 let directory = tempdir().unwrap();
1384 let instrument_id = InstrumentId::from("AUD/USD.SIM");
1385 let quotes = vec![
1386 QuoteTick::new(
1387 instrument_id,
1388 Price::from("1.00001"),
1389 Price::from("1.00003"),
1390 Quantity::from("100_000"),
1391 Quantity::from("200_000"),
1392 UnixNanos::from(1),
1393 UnixNanos::from(1),
1394 ),
1395 QuoteTick::new(
1396 instrument_id,
1397 Price::from("1.00002"),
1398 Price::from("1.00004"),
1399 Quantity::from("300_000"),
1400 Quantity::from("400_000"),
1401 UnixNanos::from(2),
1402 UnixNanos::from(2),
1403 ),
1404 ];
1405 let catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1406 catalog.write_to_parquet("es, None, None, None).unwrap();
1407 let config = KernelConfig {
1408 catalogs: vec![
1409 DataCatalogConfig::new(
1410 directory.path().to_string_lossy().into_owned(),
1411 Some("file".to_string()),
1412 None,
1413 )
1414 .with_name(Some("history".to_string())),
1415 ],
1416 ..KernelConfig::default()
1417 };
1418 let mut kernel = NautilusKernel::new("CatalogQueryTest".to_string(), config).unwrap();
1419 kernel
1420 .clock
1421 .borrow_mut()
1422 .as_any_mut()
1423 .downcast_mut::<VirtualClock>()
1424 .unwrap()
1425 .set_time(UnixNanos::from(3));
1426 let request_id = UUID4::new();
1427 let received = Rc::new(RefCell::new(Vec::new()));
1428 let received_quotes = received.clone();
1429 msgbus::register_response_handler(
1430 &request_id,
1431 TypedHandler::from(move |envelope: &DataResponseEnvelope| {
1432 let Some(DataResponse::Quotes(response)) = envelope.response() else {
1433 panic!("expected quotes");
1434 };
1435
1436 *received_quotes.borrow_mut() = response.data.to_vec();
1437 }),
1438 );
1439 let request = RequestQuotes::new(
1440 instrument_id,
1441 Some(UnixNanos::from(1).to_datetime_utc()),
1442 Some(UnixNanos::from(2).to_datetime_utc()),
1443 None,
1444 None,
1445 request_id,
1446 UnixNanos::from(3),
1447 None,
1448 );
1449
1450 kernel
1451 .data_engine
1452 .borrow_mut()
1453 .execute(DataCommand::Request(RequestCommand::Quotes(request)));
1454
1455 assert_eq!(*received.borrow(), quotes);
1456 assert_eq!(kernel.data_engine.borrow().request_count(), 1);
1457 assert_eq!(kernel.data_engine.borrow().response_count(), 1);
1458 kernel.dispose();
1459 }
1460
1461 #[rstest]
1462 fn test_configured_catalog_error_names_rejected_param() {
1463 let directory = tempdir().unwrap();
1464 let path = directory.path().to_string_lossy().into_owned();
1465 let params = Params::new([("batch_size".to_string(), serde_json::json!(1024))].into());
1466
1467 let config = KernelConfig {
1468 catalogs: vec![
1469 DataCatalogConfig::new(path.clone(), Some("file".to_string()), None)
1470 .with_params(Some(params)),
1471 ],
1472 ..KernelConfig::default()
1473 };
1474
1475 let error = NautilusKernel::new("CatalogParamTest".to_string(), config)
1476 .err()
1477 .unwrap();
1478
1479 assert_eq!(
1480 error.to_string(),
1481 format!(
1482 "Failed to create data catalog from '{path}': Unknown Parquet catalog param \
1483 'batch_size': this catalog takes no params"
1484 )
1485 );
1486 }
1487
1488 #[rstest]
1489 fn test_configured_streaming_excludes_custom_data() {
1490 ensure_custom_data_registered::<RustTestCustomData>();
1491
1492 let directory = tempdir().unwrap();
1493 let instance_id = UUID4::new();
1494 let mut streaming = StreamingConfig::new(
1495 directory.path().to_string_lossy().into_owned(),
1496 None,
1497 1_000,
1498 false,
1499 RotationConfig::NoRotation,
1500 );
1501 streaming.data_types = Some(vec![NautilusDataType::QuoteTick]);
1502 let config = KernelConfig {
1503 instance_id: Some(instance_id),
1504 streaming: Some(streaming),
1505 ..KernelConfig::default()
1506 };
1507 let mut kernel = NautilusKernel::new("BuiltInStreamingTest".to_string(), config).unwrap();
1508 let instrument_id = InstrumentId::from("RUST.TEST");
1509 let custom = CustomData::new(
1510 Arc::new(RustTestCustomData {
1511 instrument_id,
1512 value: 1.23,
1513 flag: true,
1514 ts_event: UnixNanos::from(1_000),
1515 ts_init: UnixNanos::from(1_000),
1516 }),
1517 DataType::new("RustTestCustomData", None, Some(instrument_id.to_string())),
1518 );
1519
1520 msgbus::publish_any(MStr::from("data.custom"), &custom);
1521 kernel.close_streaming_writer().unwrap();
1522
1523 let custom_path = directory
1524 .path()
1525 .join("backtest")
1526 .join(instance_id.to_string())
1527 .join("data/custom");
1528 assert!(!custom_path.exists());
1529 kernel.dispose();
1530 }
1531
1532 #[rstest]
1533 #[case::promotion_without_catalog(
1534 None,
1535 "promotion_interval_ms requires catalog: promotion needs a catalog"
1536 )]
1537 #[case::catalog_param(
1538 Some(serde_json::json!({"params": {"batch_size": 1024}})),
1539 "Unknown Parquet catalog param 'batch_size': this catalog takes no params"
1540 )]
1541 #[case::zero_batch_size(
1542 Some(serde_json::json!({"batch_size": 0})),
1543 "invalid batch_size: must be a positive number of rows; omit the field for the \
1544 backend default"
1545 )]
1546 fn test_invalid_streaming_config_fails_before_replacing_files(
1547 #[case] catalog_fields: Option<serde_json::Value>,
1548 #[case] expected: &str,
1549 ) {
1550 let directory = tempdir().unwrap();
1551 let instance_id = UUID4::new();
1552 let run_directory = directory
1553 .path()
1554 .join("backtest")
1555 .join(instance_id.to_string());
1556 std::fs::create_dir_all(&run_directory).unwrap();
1557 let existing = run_directory.join("existing.feather");
1558 std::fs::write(&existing, b"preserve").unwrap();
1559
1560 let catalog_path = directory.path().join("catalog");
1561
1562 let catalog = catalog_fields.map(|mut fields| {
1564 fields["path"] = serde_json::json!(catalog_path.to_string_lossy());
1565 serde_json::from_value::<DataCatalogConfig>(fields).unwrap()
1566 });
1567
1568 let mut streaming = StreamingConfig::new(
1569 directory.path().to_string_lossy().into_owned(),
1570 catalog,
1571 1_000,
1572 true,
1573 RotationConfig::NoRotation,
1574 );
1575 streaming.promotion_interval_ms = Some(1_000);
1576 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1577
1578 let error = NautilusKernel::setup_streaming_writer(
1579 Environment::Backtest,
1580 instance_id,
1581 &streaming,
1582 &clock,
1583 )
1584 .err()
1585 .unwrap();
1586
1587 assert_eq!(error.to_string(), expected);
1588 assert_eq!(std::fs::read(&existing).unwrap(), b"preserve");
1589 }
1590
1591 #[rstest]
1592 fn test_streaming_replace_existing_creates_missing_catalog_directory() {
1593 let directory = tempdir().unwrap();
1594 let instance_id = UUID4::new();
1595 let run_directory = directory
1596 .path()
1597 .join("backtest")
1598 .join(instance_id.to_string());
1599 std::fs::create_dir_all(&run_directory).unwrap();
1600 let existing = run_directory.join("existing.feather");
1601 std::fs::write(&existing, b"replace").unwrap();
1602 let catalog_path = directory.path().join("catalog");
1603
1604 let streaming = StreamingConfig::new(
1605 directory.path().to_string_lossy().into_owned(),
1606 Some(DataCatalogConfig::new(
1607 catalog_path.to_string_lossy().into_owned(),
1608 None,
1609 None,
1610 )),
1611 1_000,
1612 true,
1613 RotationConfig::NoRotation,
1614 );
1615 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1616
1617 let (_, subscription) = NautilusKernel::setup_streaming_writer(
1618 Environment::Backtest,
1619 instance_id,
1620 &streaming,
1621 &clock,
1622 )
1623 .unwrap();
1624 subscription.close().unwrap();
1625
1626 assert!(!existing.exists());
1627 assert!(catalog_path.is_dir());
1628 }
1629
1630 #[rstest]
1631 fn test_configured_streaming_promotes_into_separate_catalog() {
1632 let writer_directory = tempdir().unwrap();
1633 let catalog_directory = tempdir().unwrap();
1634 let catalog_path = catalog_directory.path().join("catalog");
1635 let instance_id = UUID4::new();
1636
1637 let mut streaming = StreamingConfig::new(
1638 writer_directory.path().to_string_lossy().into_owned(),
1639 Some(DataCatalogConfig::new(
1640 catalog_path.to_string_lossy().into_owned(),
1641 None,
1642 None,
1643 )),
1644 1_000,
1645 false,
1646 RotationConfig::NoRotation,
1647 );
1648 streaming.data_types = Some(vec![NautilusDataType::QuoteTick]);
1649
1650 let config = KernelConfig {
1651 instance_id: Some(instance_id),
1652 streaming: Some(streaming),
1653 ..KernelConfig::default()
1654 };
1655
1656 let mut kernel =
1657 NautilusKernel::new("SeparateCatalogStreamingTest".to_string(), config).unwrap();
1658
1659 let quote = QuoteTick::new(
1660 InstrumentId::from("AUD/USD.SIM"),
1661 Price::from("1.00001"),
1662 Price::from("1.00003"),
1663 Quantity::from("100_000"),
1664 Quantity::from("200_000"),
1665 UnixNanos::from(1),
1666 UnixNanos::from(1),
1667 );
1668
1669 msgbus::publish_quote(MStr::from("data.quotes.SIM.AUD/USD"), "e);
1670 kernel.close_streaming_writer().unwrap();
1671
1672 let mut catalog = ParquetDataCatalog::new(&catalog_path, None, None, None, None);
1673 let promoted = catalog
1674 .query::<QuoteTick>(None, None, None, None, None, true)
1675 .unwrap();
1676 let run_directory = writer_directory
1677 .path()
1678 .join("backtest")
1679 .join(instance_id.to_string());
1680 assert_eq!(promoted, vec![quote]);
1681 assert!(run_directory.join("quotes").is_dir());
1682 assert!(!catalog_path.join("backtest").exists());
1683 kernel.dispose();
1684 }
1685}
1686
1687#[cfg(test)]
1688mod lifecycle_tests {
1689 use futures::FutureExt;
1690 use indexmap::IndexMap;
1691 use nautilus_common::{
1692 actor::registry::get_actor_unchecked,
1693 cache::Cache,
1694 clock::VirtualClock,
1695 messages::data::{DataCommand, SubscribeCommand, UnsubscribeCommand},
1696 msgbus::stubs::{TypedIntoMessageSavingHandler, get_typed_into_message_saving_handler},
1697 };
1698 use nautilus_execution::engine::SnapshotAnchorer;
1699 use nautilus_model::{
1700 enums::{OrderSide, OrderStatus, OrderType, TriggerType},
1701 identifiers::{ActorId, ClientOrderId, StrategyId},
1702 instruments::{
1703 CryptoPerpetual, Instrument, InstrumentAny, stubs::crypto_perpetual_ethusdt,
1704 },
1705 orders::{Order, OrderAny, OrderTestBuilder},
1706 types::{Price, Quantity},
1707 };
1708 use nautilus_testkit::{
1709 cache::TestCacheDatabaseControl,
1710 components::{StateActor, StateStrategy},
1711 };
1712 use rstest::rstest;
1713 use ustr::Ustr;
1714
1715 use super::*;
1716 use crate::{
1717 builder::NautilusKernelBuilder,
1718 event_store::{KernelEventStore, RegisteredComponents},
1719 };
1720
1721 #[derive(Debug)]
1722 struct RecordingEventStore {
1723 control: TestCacheDatabaseControl,
1724 opened: bool,
1725 }
1726
1727 impl KernelEventStore for RecordingEventStore {
1728 fn restore_parent_cache(
1729 &mut self,
1730 _instance_id: UUID4,
1731 _cache: &mut Cache,
1732 ) -> anyhow::Result<()> {
1733 self.control.record("event_store.restore");
1734 Ok(())
1735 }
1736
1737 fn open(
1738 &mut self,
1739 _instance_id: UUID4,
1740 _components: &RegisteredComponents,
1741 _environment: Environment,
1742 ) -> anyhow::Result<()> {
1743 self.control.record("event_store.open");
1744 self.opened = true;
1745 Ok(())
1746 }
1747
1748 fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
1749 None
1750 }
1751
1752 fn seal(&mut self, _ts_init: UnixNanos) {
1753 if self.opened {
1754 self.control.record("event_store.seal");
1755 self.opened = false;
1756 }
1757 }
1758
1759 fn run_id(&self) -> Option<&str> {
1760 None
1761 }
1762
1763 fn parent_run_id(&self) -> Option<&str> {
1764 None
1765 }
1766
1767 fn is_halted(&self) -> bool {
1768 false
1769 }
1770 }
1771
1772 fn state(key: &str, value: &[u8]) -> IndexMap<String, Vec<u8>> {
1773 IndexMap::from([(key.to_string(), value.to_vec())])
1774 }
1775
1776 fn finalize(kernel: &mut NautilusKernel) -> anyhow::Result<()> {
1777 kernel
1778 .finalize_stop()
1779 .now_or_never()
1780 .expect("kernel finalization must not yield")
1781 }
1782
1783 fn add_state_components(
1784 kernel: &NautilusKernel,
1785 control: &TestCacheDatabaseControl,
1786 actor: StateActor,
1787 strategy: StateStrategy,
1788 ) {
1789 kernel.trader.borrow_mut().add_actor(actor).unwrap();
1790 kernel.trader.borrow_mut().add_strategy(strategy).unwrap();
1791 control.record("components.registered");
1792 }
1793
1794 fn create_stop_market_order(instrument: &CryptoPerpetual, client_order_id: &str) -> OrderAny {
1795 OrderTestBuilder::new(OrderType::StopMarket)
1796 .instrument_id(instrument.id())
1797 .client_order_id(ClientOrderId::from(client_order_id))
1798 .side(OrderSide::Buy)
1799 .trigger_price(Price::from("5100.00"))
1800 .quantity(Quantity::from(1))
1801 .emulation_trigger(TriggerType::BidAsk)
1802 .build()
1803 }
1804
1805 fn register_data_command_handler(id: &str) -> TypedIntoMessageSavingHandler<DataCommand> {
1806 let (handler, saving_handler) =
1807 get_typed_into_message_saving_handler::<DataCommand>(Some(Ustr::from(id)));
1808 msgbus::register_data_command_endpoint(
1809 MessagingSwitchboard::data_engine_queue_execute(),
1810 handler,
1811 );
1812 saving_handler
1813 }
1814
1815 #[rstest]
1816 fn test_state_persistence_orders_restore_load_start_stop_save_seal_and_dispose() {
1817 let actor_id = ActorId::from("STATE-ACTOR");
1818 let strategy_id = StrategyId::from("STATE-STRATEGY-001");
1819 let actor_load = state("actor-loaded", b"actor-load-value");
1820 let strategy_load = state("strategy-loaded", b"strategy-load-value");
1821 let actor_save = state("actor-saved", b"actor-save-value");
1822 let strategy_save = state("strategy-saved", b"strategy-save-value");
1823 let (database, control) = TestCacheDatabaseControl::create();
1824 control.set_actor_state(actor_id, &actor_load);
1825 control.set_strategy_state(strategy_id, &strategy_load);
1826
1827 let event_store_control = control.clone();
1828 let mut kernel = NautilusKernelBuilder::default()
1829 .with_cache_database(Box::new(database))
1830 .with_event_store(move |_instance_id, _clock| {
1831 Ok(Box::new(RecordingEventStore {
1832 control: event_store_control,
1833 opened: false,
1834 }))
1835 })
1836 .build()
1837 .unwrap();
1838
1839 let actor = StateActor::new(actor_id, control.clone(), actor_save.clone());
1840 let strategy = StateStrategy::new(strategy_id, control.clone(), strategy_save.clone());
1841 add_state_components(&kernel, &control, actor, strategy);
1842
1843 kernel.start();
1844 kernel.start_trader().unwrap();
1845
1846 let actor_state = get_actor_unchecked::<StateActor>(&actor_id.inner())
1847 .state_load()
1848 .cloned();
1849 let strategy_state = get_actor_unchecked::<StateStrategy>(&strategy_id.inner())
1850 .state_load()
1851 .cloned();
1852 assert_eq!(actor_state, Some(actor_load));
1853 assert_eq!(strategy_state, Some(strategy_load));
1854
1855 kernel.stop_trader();
1856 kernel.stop_trader();
1857 finalize(&mut kernel).unwrap();
1858 finalize(&mut kernel).unwrap();
1859 kernel.dispose();
1860
1861 assert_eq!(
1862 control.events(),
1863 vec![
1864 "components.registered",
1865 "event_store.restore",
1866 "event_store.open",
1867 "actor.load:STATE-ACTOR",
1868 "actor.on_load",
1869 "strategy.load:STATE-STRATEGY-001",
1870 "strategy.on_load",
1871 "actor.on_start",
1872 "strategy.on_start",
1873 "actor.on_stop",
1874 "strategy.on_stop",
1875 "actor.on_save",
1876 "actor.update:STATE-ACTOR",
1877 "strategy.on_save",
1878 "strategy.update:STATE-STRATEGY-001",
1879 "event_store.seal",
1880 "database.close",
1881 ]
1882 );
1883 assert_eq!(control.actor_state(&actor_id), Some(actor_save));
1884 assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
1885 }
1886
1887 #[rstest]
1888 fn test_state_persistence_skips_callbacks_without_cache_backing() {
1889 let actor_id = ActorId::from("NO-BACKING-ACTOR");
1890 let strategy_id = StrategyId::from("NO-BACKING-STRATEGY-001");
1891 let control = TestCacheDatabaseControl::default();
1892 let mut kernel = NautilusKernelBuilder::default().build().unwrap();
1893 let actor = StateActor::new(actor_id, control.clone(), state("actor", b"save"));
1894 let strategy = StateStrategy::new(strategy_id, control.clone(), state("strategy", b"save"));
1895 add_state_components(&kernel, &control, actor, strategy);
1896
1897 kernel.start();
1898 kernel.start_trader().unwrap();
1899 kernel.stop_trader();
1900 finalize(&mut kernel).unwrap();
1901 kernel.dispose();
1902
1903 assert_eq!(
1904 control.events(),
1905 vec![
1906 "components.registered",
1907 "actor.on_start",
1908 "strategy.on_start",
1909 "actor.on_stop",
1910 "strategy.on_stop",
1911 ]
1912 );
1913 }
1914
1915 #[rstest]
1916 fn test_state_persistence_skips_empty_load_and_persists_empty_save() {
1917 let actor_id = ActorId::from("EMPTY-STATE-ACTOR");
1918 let strategy_id = StrategyId::from("EMPTY-STATE-STRATEGY-001");
1919 let (database, control) = TestCacheDatabaseControl::create();
1920 let mut kernel = NautilusKernelBuilder::default()
1921 .with_cache_database(Box::new(database))
1922 .build()
1923 .unwrap();
1924 let actor = StateActor::new(actor_id, control.clone(), IndexMap::new());
1925 let strategy = StateStrategy::new(strategy_id, control.clone(), IndexMap::new());
1926 add_state_components(&kernel, &control, actor, strategy);
1927
1928 kernel.start();
1929 kernel.start_trader().unwrap();
1930 kernel.stop_trader();
1931 finalize(&mut kernel).unwrap();
1932
1933 assert_eq!(
1934 control.events(),
1935 vec![
1936 "components.registered",
1937 "actor.load:EMPTY-STATE-ACTOR",
1938 "strategy.load:EMPTY-STATE-STRATEGY-001",
1939 "actor.on_start",
1940 "strategy.on_start",
1941 "actor.on_stop",
1942 "strategy.on_stop",
1943 "actor.on_save",
1944 "actor.update:EMPTY-STATE-ACTOR",
1945 "strategy.on_save",
1946 "strategy.update:EMPTY-STATE-STRATEGY-001",
1947 ]
1948 );
1949 assert_eq!(control.actor_state(&actor_id), Some(IndexMap::new()));
1950 assert_eq!(control.strategy_state(&strategy_id), Some(IndexMap::new()));
1951 kernel.dispose();
1952 }
1953
1954 #[rstest]
1955 fn test_state_save_reports_all_callback_errors_and_continues_shutdown() {
1956 let actor_id = ActorId::from("FAIL-SAVE-ACTOR");
1957 let strategy_id = StrategyId::from("FAIL-SAVE-STRATEGY-001");
1958 let (database, control) = TestCacheDatabaseControl::create();
1959 let event_store_control = control.clone();
1960 let mut kernel = NautilusKernelBuilder::default()
1961 .with_cache_database(Box::new(database))
1962 .with_event_store(move |_instance_id, _clock| {
1963 Ok(Box::new(RecordingEventStore {
1964 control: event_store_control,
1965 opened: false,
1966 }))
1967 })
1968 .build()
1969 .unwrap();
1970 let actor = StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save();
1971 let strategy =
1972 StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save();
1973 add_state_components(&kernel, &control, actor, strategy);
1974
1975 kernel.start();
1976 kernel.start_trader().unwrap();
1977 kernel.stop_trader();
1978 let expected_shutdown = kernel.clock.borrow().timestamp_ns();
1979 let error = finalize(&mut kernel).unwrap_err();
1980 kernel.dispose();
1981
1982 assert_eq!(
1983 error.to_string(),
1984 "Failed to save component state: actor FAIL-SAVE-ACTOR callback: test actor on_save \
1985 failure; strategy FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
1986 );
1987 assert_eq!(kernel.ts_shutdown, Some(expected_shutdown));
1988 assert_eq!(
1989 control.events(),
1990 vec![
1991 "components.registered",
1992 "event_store.restore",
1993 "event_store.open",
1994 "actor.load:FAIL-SAVE-ACTOR",
1995 "strategy.load:FAIL-SAVE-STRATEGY-001",
1996 "actor.on_start",
1997 "strategy.on_start",
1998 "actor.on_stop",
1999 "strategy.on_stop",
2000 "actor.on_save",
2001 "strategy.on_save",
2002 "event_store.seal",
2003 "database.close",
2004 ]
2005 );
2006 }
2007
2008 #[rstest]
2009 fn test_state_load_callback_failure_prevents_start_and_save() {
2010 let actor_id = ActorId::from("FAIL-LOAD-ACTOR");
2011 let strategy_id = StrategyId::from("FAIL-LOAD-STRATEGY-001");
2012 let (database, control) = TestCacheDatabaseControl::create();
2013 control.set_actor_state(actor_id, &state("actor", b"load"));
2014 control.set_strategy_state(strategy_id, &state("strategy", b"load"));
2015 let mut kernel = NautilusKernelBuilder::default()
2016 .with_cache_database(Box::new(database))
2017 .build()
2018 .unwrap();
2019 let actor = StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_load();
2020 let strategy = StateStrategy::new(strategy_id, control.clone(), IndexMap::new());
2021 add_state_components(&kernel, &control, actor, strategy);
2022
2023 kernel.start();
2024 let error = kernel.start_trader().unwrap_err();
2025 kernel.dispose();
2026
2027 assert_eq!(
2028 error.to_string(),
2029 "Failed to load actor and strategy state: Failed to restore actor FAIL-LOAD-ACTOR \
2030 state: test actor on_load failure"
2031 );
2032 assert_eq!(
2033 control.events(),
2034 vec![
2035 "components.registered",
2036 "actor.load:FAIL-LOAD-ACTOR",
2037 "actor.on_load",
2038 "database.close",
2039 ]
2040 );
2041 }
2042
2043 #[rstest]
2044 fn test_state_save_persistence_errors_do_not_fail_finalization() {
2045 let actor_id = ActorId::from("FAIL-UPDATE-ACTOR");
2046 let strategy_id = StrategyId::from("FAIL-UPDATE-STRATEGY-001");
2047 let (database, control) = TestCacheDatabaseControl::create();
2048 control.set_fail_update_actor(true);
2049 control.set_fail_update_strategy(true);
2050 let mut kernel = NautilusKernelBuilder::default()
2051 .with_cache_database(Box::new(database))
2052 .build()
2053 .unwrap();
2054 let actor = StateActor::new(actor_id, control.clone(), state("actor", b"save"));
2055 let strategy = StateStrategy::new(strategy_id, control.clone(), state("strategy", b"save"));
2056 add_state_components(&kernel, &control, actor, strategy);
2057
2058 kernel.start();
2059 kernel.start_trader().unwrap();
2060 kernel.stop_trader();
2061 finalize(&mut kernel).unwrap();
2062 kernel.dispose();
2063
2064 assert_eq!(
2065 control.events(),
2066 vec![
2067 "components.registered",
2068 "actor.load:FAIL-UPDATE-ACTOR",
2069 "strategy.load:FAIL-UPDATE-STRATEGY-001",
2070 "actor.on_start",
2071 "strategy.on_start",
2072 "actor.on_stop",
2073 "strategy.on_stop",
2074 "actor.on_save",
2075 "actor.update:FAIL-UPDATE-ACTOR",
2076 "strategy.on_save",
2077 "strategy.update:FAIL-UPDATE-STRATEGY-001",
2078 "database.close",
2079 ]
2080 );
2081 }
2082
2083 #[rstest]
2084 fn test_partial_startup_stops_and_saves_once() {
2085 let actor_id = ActorId::from("PARTIAL-ACTOR");
2086 let strategy_id = StrategyId::from("PARTIAL-STRATEGY-001");
2087 let (database, control) = TestCacheDatabaseControl::create();
2088 let mut kernel = NautilusKernelBuilder::default()
2089 .with_cache_database(Box::new(database))
2090 .build()
2091 .unwrap();
2092 let actor = StateActor::new(actor_id, control.clone(), state("actor", b"partial"));
2093 let strategy =
2094 StateStrategy::new(strategy_id, control.clone(), state("strategy", b"partial"))
2095 .with_fail_start();
2096 add_state_components(&kernel, &control, actor, strategy);
2097
2098 kernel.start();
2099 let error = kernel.start_trader().unwrap_err();
2100 kernel.dispose();
2101
2102 assert_eq!(
2103 error.to_string(),
2104 "Failed to start trader: test strategy on_start failure"
2105 );
2106 assert_eq!(
2107 control.events(),
2108 vec![
2109 "components.registered",
2110 "actor.load:PARTIAL-ACTOR",
2111 "strategy.load:PARTIAL-STRATEGY-001",
2112 "actor.on_start",
2113 "strategy.on_start",
2114 "actor.on_stop",
2115 "strategy.on_stop",
2116 "actor.on_save",
2117 "actor.update:PARTIAL-ACTOR",
2118 "strategy.on_save",
2119 "strategy.update:PARTIAL-STRATEGY-001",
2120 "database.close",
2121 ]
2122 );
2123 assert_eq!(
2124 control.actor_state(&actor_id),
2125 Some(state("actor", b"partial"))
2126 );
2127 assert_eq!(
2128 control.strategy_state(&strategy_id),
2129 Some(state("strategy", b"partial"))
2130 );
2131 }
2132
2133 #[rstest]
2134 fn test_forced_dispose_stops_and_saves_once() {
2135 let actor_id = ActorId::from("FORCED-ACTOR");
2136 let strategy_id = StrategyId::from("FORCED-STRATEGY-001");
2137 let (database, control) = TestCacheDatabaseControl::create();
2138 let mut kernel = NautilusKernelBuilder::default()
2139 .with_cache_database(Box::new(database))
2140 .build()
2141 .unwrap();
2142 let actor = StateActor::new(actor_id, control.clone(), state("actor", b"forced"));
2143 let strategy =
2144 StateStrategy::new(strategy_id, control.clone(), state("strategy", b"forced"));
2145 add_state_components(&kernel, &control, actor, strategy);
2146
2147 kernel.start();
2148 kernel.start_trader().unwrap();
2149 kernel.dispose();
2150
2151 assert_eq!(
2152 control.events(),
2153 vec![
2154 "components.registered",
2155 "actor.load:FORCED-ACTOR",
2156 "strategy.load:FORCED-STRATEGY-001",
2157 "actor.on_start",
2158 "strategy.on_start",
2159 "actor.on_stop",
2160 "strategy.on_stop",
2161 "actor.on_save",
2162 "actor.update:FORCED-ACTOR",
2163 "strategy.on_save",
2164 "strategy.update:FORCED-STRATEGY-001",
2165 "database.close",
2166 ]
2167 );
2168 }
2169
2170 #[rstest]
2171 fn test_start_trader_starts_order_emulator_for_cached_emulated_orders() {
2172 let mut kernel = NautilusKernelBuilder::default().build().unwrap();
2173 let data_commands = register_data_command_handler("DataEngine.queue_execute.kernel_start");
2174 let instrument = crypto_perpetual_ethusdt();
2175 let instrument_id = instrument.id();
2176 let first_order = create_stop_market_order(&instrument, "O-KERNEL-001");
2177 let second_order = create_stop_market_order(&instrument, "O-KERNEL-002");
2178 let first_client_order_id = first_order.client_order_id();
2179 let second_client_order_id = second_order.client_order_id();
2180 kernel
2181 .cache
2182 .borrow_mut()
2183 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
2184 .unwrap();
2185 kernel
2186 .cache
2187 .borrow_mut()
2188 .add_order(first_order, None, None, false)
2189 .unwrap();
2190 kernel
2191 .cache
2192 .borrow_mut()
2193 .add_order(second_order, None, None, false)
2194 .unwrap();
2195
2196 kernel.start();
2197 assert!(
2198 kernel
2199 .order_emulator
2200 .get_emulator()
2201 .get_matching_core(&instrument_id)
2202 .is_none()
2203 );
2204 kernel.start_trader().unwrap();
2205
2206 let commands = data_commands.get_messages();
2207 let cache = kernel.cache.borrow();
2208 let first_status = cache.order(&first_client_order_id).unwrap().status();
2209 let second_status = cache.order(&second_client_order_id).unwrap().status();
2210 drop(cache);
2211 let emulator = kernel.order_emulator.get_emulator();
2212 assert!(emulator.get_matching_core(&instrument_id).is_some());
2213 assert_eq!(emulator.subscribed_quotes(), vec![instrument_id]);
2214 assert_eq!(first_status, OrderStatus::Emulated);
2215 assert_eq!(second_status, OrderStatus::Emulated);
2216 assert!(commands.iter().any(|command| matches!(
2217 command,
2218 DataCommand::Subscribe(SubscribeCommand::Quotes(command))
2219 if command.instrument_ids == [instrument_id]
2220 )));
2221
2222 data_commands.clear();
2223 drop(emulator);
2224 kernel.stop_trader();
2225 kernel.dispose();
2226
2227 let commands = data_commands.get_messages();
2228 let emulator = kernel.order_emulator.get_emulator();
2229 assert!(emulator.subscribed_quotes().is_empty());
2230 assert!(emulator.get_matching_core(&instrument_id).is_none());
2231 assert!(commands.iter().any(|command| matches!(
2232 command,
2233 DataCommand::Unsubscribe(UnsubscribeCommand::Quotes(command))
2234 if command.instrument_ids == [instrument_id]
2235 )));
2236 }
2237
2238 #[rstest]
2239 fn test_reset_resets_order_emulator_state() {
2240 let mut kernel = NautilusKernelBuilder::default().build().unwrap();
2241 let data_commands = register_data_command_handler("DataEngine.queue_execute.kernel_reset");
2242 let instrument = crypto_perpetual_ethusdt();
2243 let instrument_id = instrument.id();
2244 let order = create_stop_market_order(&instrument, "O-KERNEL-RESET-001");
2245 kernel
2246 .cache
2247 .borrow_mut()
2248 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
2249 .unwrap();
2250 kernel
2251 .cache
2252 .borrow_mut()
2253 .add_order(order, None, None, false)
2254 .unwrap();
2255
2256 kernel.start();
2257 kernel.start_trader().unwrap();
2258 assert!(
2259 kernel
2260 .order_emulator
2261 .get_emulator()
2262 .get_matching_core(&instrument_id)
2263 .is_some()
2264 );
2265 kernel.stop_trader();
2266 data_commands.clear();
2267
2268 kernel.reset();
2269
2270 let commands = data_commands.get_messages();
2271 let emulator = kernel.order_emulator.get_emulator();
2272 assert!(emulator.subscribed_quotes().is_empty());
2273 assert!(emulator.get_matching_core(&instrument_id).is_none());
2274 assert!(commands.iter().any(|command| matches!(
2275 command,
2276 DataCommand::Unsubscribe(UnsubscribeCommand::Quotes(command))
2277 if command.instrument_ids == [instrument_id]
2278 )));
2279
2280 drop(emulator);
2281 kernel.dispose();
2282 }
2283
2284 #[rstest]
2285 fn test_start_after_reset_skips_trader_initialize() {
2286 let mut kernel = NautilusKernelBuilder::default().build().unwrap();
2287 kernel.start();
2288 kernel.start_trader().unwrap();
2289 kernel.stop_trader();
2290 kernel.reset();
2291 let restart_ns = UnixNanos::from(1_700_000_000_000_000_000);
2292 kernel
2293 .clock
2294 .borrow_mut()
2295 .as_any_mut()
2296 .downcast_mut::<VirtualClock>()
2297 .unwrap()
2298 .set_time(restart_ns);
2299
2300 kernel.start();
2301 let ts_started = kernel.ts_started();
2302 kernel.start_trader().unwrap();
2303 let trader_state = kernel.trader.borrow().state();
2304 kernel.stop_trader();
2305 kernel.dispose();
2306
2307 assert_eq!(ts_started, Some(restart_ns));
2308 assert_eq!(trader_state, ComponentState::Running);
2309 }
2310}