1#[cfg(feature = "streaming")]
38use std::collections::HashSet;
39use std::{
40 cell::{Cell, Ref, RefCell},
41 fmt::Debug,
42 rc::Rc,
43 time::Duration,
44};
45
46use nautilus_common::{
47 cache::{Cache, CacheConfig, database::CacheDatabaseAdapter},
48 clock::Clock,
49 component::Component,
50 enums::{ComponentState, Environment},
51 logging::{
52 arm_shutdown_on_error, disarm_shutdown_on_error, headers, init_logging,
53 logger::{LogGuard, LoggerConfig},
54 try_drain_shutdown_on_error_trigger,
55 },
56 messages::system::ShutdownSystem,
57 msgbus::{
58 self, MessageBus, MessagingSwitchboard, ShareableMessageHandler, get_message_bus,
59 set_message_bus,
60 },
61};
62use nautilus_core::{UUID4, UnixNanos};
63use nautilus_data::engine::DataEngine;
64use nautilus_execution::{
65 engine::ExecutionEngine,
66 order_emulator::{adapter::OrderEmulatorAdapter, emulator::OrderEmulator},
67};
68use nautilus_model::identifiers::{ClientId, TraderId};
69#[cfg(feature = "streaming")]
70use nautilus_persistence::{
71 backend::{default_catalog_factories, default_writer_factories},
72 catalog::factory::create_catalog,
73 common::paths::{create_local_directory, environment_directory},
74 config::{DataCatalogConfig, StreamingConfig},
75 writer::{
76 factory::{WriterConnectConfig, create_writer, replace_existing_writer_data},
77 feather::WriterClock,
78 filter::WriterRecordFilter,
79 subscription::StreamingSinkSubscription,
80 },
81};
82use nautilus_portfolio::portfolio::Portfolio;
83use nautilus_risk::engine::RiskEngine;
84use ustr::Ustr;
85
86use crate::{
87 builder::NautilusKernelBuilder,
88 clock_factory::ClockFactory,
89 config::NautilusKernelConfig,
90 event_store::{EventStoreFactory, KernelEventStore, RegisteredComponents},
91 trader::Trader,
92};
93
94pub struct NautilusKernel {
98 pub name: String,
100 pub instance_id: UUID4,
102 pub machine_id: String,
104 pub config: Box<dyn NautilusKernelConfig>,
106 pub cache: Rc<RefCell<Cache>>,
108 pub clock: Rc<RefCell<dyn Clock>>,
110 pub portfolio: Rc<RefCell<Portfolio>>,
112 pub log_guard: LogGuard,
114 pub data_engine: Rc<RefCell<DataEngine>>,
116 pub risk_engine: Rc<RefCell<RiskEngine>>,
118 pub exec_engine: Rc<RefCell<ExecutionEngine>>,
120 pub order_emulator: OrderEmulatorAdapter,
122 pub trader: Rc<RefCell<Trader>>,
124 pub ts_created: UnixNanos,
126 pub ts_started: Option<UnixNanos>,
128 pub ts_shutdown: Option<UnixNanos>,
130 shutdown_requested: Rc<Cell<bool>>,
131 event_store: Option<Box<dyn KernelEventStore>>,
132 event_store_replay: bool,
133 state_save_armed: bool,
134 #[cfg(feature = "streaming")]
135 streaming_writer: Option<(String, StreamingSinkSubscription)>,
136}
137
138impl Debug for NautilusKernel {
139 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
140 f.debug_struct(stringify!(NautilusKernel))
141 .field("name", &self.name)
142 .field("instance_id", &self.instance_id)
143 .field("machine_id", &self.machine_id)
144 .field("environment", &self.config.environment())
145 .finish_non_exhaustive()
146 }
147}
148
149#[derive(Default)]
151pub struct NautilusKernelDependencies {
152 clock_factory: Option<ClockFactory>,
153 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
154 event_store_factory: Option<EventStoreFactory>,
155}
156
157impl Debug for NautilusKernelDependencies {
158 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
159 f.debug_struct(stringify!(NautilusKernelDependencies))
160 .field("clock_factory", &self.clock_factory.is_some())
161 .field("cache_database", &self.cache_database.is_some())
162 .field("event_store_factory", &self.event_store_factory.is_some())
163 .finish()
164 }
165}
166
167impl NautilusKernelDependencies {
168 #[must_use]
170 pub fn with_clock_factory(mut self, clock_factory: Option<ClockFactory>) -> Self {
171 self.clock_factory = clock_factory;
172 self
173 }
174
175 #[must_use]
177 pub fn with_cache_database(
178 mut self,
179 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
180 ) -> Self {
181 self.cache_database = cache_database;
182 self
183 }
184
185 #[must_use]
187 pub fn with_event_store_factory(
188 mut self,
189 event_store_factory: Option<EventStoreFactory>,
190 ) -> Self {
191 self.event_store_factory = event_store_factory;
192 self
193 }
194}
195
196impl NautilusKernel {
197 #[cfg(feature = "streaming")]
198 fn setup_streaming_writer(
199 environment: Environment,
200 instance_id: UUID4,
201 config: &StreamingConfig,
202 clock: &Rc<RefCell<dyn Clock>>,
203 ) -> anyhow::Result<(String, StreamingSinkSubscription)> {
204 config.validate()?;
205
206 let writer_path = config.writer_path.trim_end_matches('/');
207 let uri = format!(
208 "{writer_path}/{}/{instance_id}",
209 environment_directory(environment)
210 );
211 let catalog = config
212 .catalog
213 .as_ref()
214 .map(DataCatalogConfig::connect_config);
215 let mut connect = WriterConnectConfig::new(&uri, catalog);
216 connect.rotation_config = config.rotation_config.to_writer_rotation_config()?;
217 connect.flush_interval_ms = Some(config.flush_interval_ms);
218 connect.params.clone_from(&config.params);
219 connect.record_filter = Self::writer_record_filter(config);
220 connect.promotion_interval_ms = config.promotion_interval_ms;
221 connect.promote_on_close = config.promote_on_close;
222 connect.delete_feather_after_promotion = config.delete_feather_after_promotion;
223 connect.use_ts_event_for_ts_init = config.use_ts_event_for_ts_init;
224
225 if config.replace_existing {
226 if let Some(catalog) = &config.catalog
229 && let Some(catalog_connect) = &connect.catalog
230 {
231 create_local_directory(&catalog_connect.uri)?;
232 create_catalog(
233 catalog.catalog_backend(),
234 catalog_connect,
235 &default_catalog_factories(),
236 )?;
237 }
238
239 replace_existing_writer_data(&connect)?;
240 }
241
242 let (writer_clock, bridge) = WriterClock::from_shared_clock(clock);
243 let sink = create_writer(
244 &config.writer_backend(),
245 &connect,
246 writer_clock,
247 &default_writer_factories(),
248 )?;
249 let subscription = StreamingSinkSubscription::subscribe_named(
250 Rc::new(RefCell::new(sink)),
251 bridge.map(|atomic| (Rc::clone(clock), atomic)),
252 uri.clone(),
253 );
254
255 log::info!("Writing data and events to {uri}");
256 Ok((uri, subscription))
257 }
258
259 #[cfg(feature = "streaming")]
260 fn writer_record_filter(config: &StreamingConfig) -> Option<WriterRecordFilter> {
261 let mut filter = config
262 .record_types
263 .clone()
264 .map(WriterRecordFilter::from_record_types)
265 .unwrap_or_default();
266
267 if let Some(data_types) = &config.data_types {
268 for data_type in data_types {
269 filter.insert(data_type.clone(), None);
270 }
271 }
272
273 if let Some(instrument_types) = &config.instrument_types {
274 for instrument_type in instrument_types {
275 filter.insert_instrument_type(*instrument_type);
276 }
277 }
278
279 if let Some(entries) = &config.record_filters {
280 for entry in entries {
281 filter.insert(entry.record_type, entry.identifiers.clone());
282 }
283 }
284
285 (!filter.is_empty()).then_some(filter)
286 }
287
288 #[must_use]
290 pub const fn builder(
291 name: String,
292 trader_id: TraderId,
293 environment: Environment,
294 ) -> NautilusKernelBuilder {
295 NautilusKernelBuilder::new(name, trader_id, environment)
296 }
297
298 pub fn new<T: NautilusKernelConfig + 'static>(name: String, config: T) -> anyhow::Result<Self> {
304 Self::new_with(name, config, None, None)
305 }
306
307 pub fn new_with_cache_database<T: NautilusKernelConfig + 'static>(
318 name: String,
319 config: T,
320 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
321 ) -> anyhow::Result<Self> {
322 Self::new_with(name, config, cache_database, None)
323 }
324
325 pub fn new_with<T: NautilusKernelConfig + 'static>(
337 name: String,
338 config: T,
339 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
340 event_store_factory: Option<EventStoreFactory>,
341 ) -> anyhow::Result<Self> {
342 Self::new_with_dependencies(
343 name,
344 config,
345 NautilusKernelDependencies::default()
346 .with_cache_database(cache_database)
347 .with_event_store_factory(event_store_factory),
348 )
349 }
350
351 #[expect(
357 clippy::too_many_lines,
358 reason = "kernel construction keeps initialization order and ownership visible in one place"
359 )]
360 pub fn new_with_dependencies<T: NautilusKernelConfig + 'static>(
361 name: String,
362 config: T,
363 dependencies: NautilusKernelDependencies,
364 ) -> anyhow::Result<Self> {
365 let NautilusKernelDependencies {
366 clock_factory,
367 cache_database,
368 event_store_factory,
369 } = dependencies;
370 let instance_id = config.instance_id().unwrap_or_default();
371 let machine_id = Self::determine_machine_id()?;
372
373 let logger_config = config.logging();
374 let log_guard = Self::initialize_logging(config.trader_id(), instance_id, logger_config)?;
375 headers::log_header(
376 config.trader_id(),
377 &machine_id,
378 instance_id,
379 Ustr::from(&name),
380 );
381
382 log::info!("Building system kernel");
383
384 let clock_factory =
385 clock_factory.unwrap_or_else(|| ClockFactory::for_environment(config.environment()));
386 let clock = clock_factory.clock();
387 let event_store = match event_store_factory {
388 Some(factory) => Some(factory(instance_id, clock.clone())?),
389 None => None,
390 };
391 let cache = Self::initialize_cache(config.cache(), cache_database);
392
393 let msgbus = Rc::new(RefCell::new(MessageBus::new(
394 config.trader_id(),
395 instance_id,
396 Some(name.clone()),
397 None,
398 )));
399 set_message_bus(msgbus);
400
401 if let Some(config) = config.msgbus()
402 && let Some(filter) = config.types_filter
403 {
404 get_message_bus().borrow_mut().set_types_filter(filter);
405 }
406
407 let portfolio = Rc::new(RefCell::new(Portfolio::new(
408 clock.clone(),
409 cache.clone(),
410 config.portfolio(),
411 )));
412
413 let risk_engine = RiskEngine::new(
414 config.risk_engine().unwrap_or_default(),
415 portfolio.borrow().clone_shallow(),
416 clock.clone(),
417 cache.clone(),
418 );
419 let risk_engine = Rc::new(RefCell::new(risk_engine));
420
421 let exec_engine = ExecutionEngine::new(clock.clone(), cache.clone(), config.exec_engine());
422 let exec_engine = Rc::new(RefCell::new(exec_engine));
423
424 let order_emulator = OrderEmulatorAdapter::new(clock.clone(), cache.clone());
425
426 let data_engine = DataEngine::new(clock.clone(), cache.clone(), config.data_engine());
427 #[cfg(feature = "streaming")]
428 let mut data_engine = data_engine;
429 #[cfg(feature = "streaming")]
430 {
431 let catalog_factories = default_catalog_factories();
432 let mut unnamed_index = 0;
433 let mut catalog_names = HashSet::new();
434
435 for catalog_config in config.catalogs() {
436 let name = catalog_config.name().map_or_else(
437 || {
438 let name = format!("catalog_{unnamed_index}");
439 unnamed_index += 1;
440 name
441 },
442 str::to_owned,
443 );
444 anyhow::ensure!(
445 catalog_names.insert(name.clone()),
446 "Duplicate data catalog name '{name}'",
447 );
448
449 let catalog = create_catalog(
450 catalog_config.catalog_backend(),
451 &catalog_config.connect_config(),
452 &catalog_factories,
453 )
454 .map_err(|e| {
455 anyhow::anyhow!(
456 "Failed to create data catalog from '{}': {e:#}",
457 catalog_config.path()
458 )
459 })?;
460 data_engine.register_catalog(catalog, Some(&name));
461 }
462 }
463 let data_engine = Rc::new(RefCell::new(data_engine));
464
465 DataEngine::register_msgbus_handlers(&data_engine);
466 RiskEngine::register_msgbus_handlers(&risk_engine);
467 ExecutionEngine::register_msgbus_handlers(&exec_engine);
468 OrderEmulator::register_msgbus_handlers(&order_emulator.emulator());
469
470 let shutdown_requested = Rc::new(Cell::new(false));
471 Self::register_shutdown_handler(config.trader_id(), shutdown_requested.clone());
472
473 let trader = Rc::new(RefCell::new(Trader::new(
474 config.trader_id(),
475 instance_id,
476 config.environment(),
477 clock_factory,
478 cache.clone(),
479 portfolio.clone(),
480 )));
481
482 let ts_created = clock.borrow().timestamp_ns();
483
484 #[cfg(feature = "streaming")]
485 let streaming_writer = config
486 .streaming()
487 .map(|streaming| {
488 Self::setup_streaming_writer(config.environment(), instance_id, &streaming, &clock)
489 })
490 .transpose()?;
491
492 Ok(Self {
493 name,
494 instance_id,
495 machine_id,
496 event_store,
497 config: Box::new(config),
498 cache,
499 clock,
500 portfolio,
501 log_guard,
502 data_engine,
503 risk_engine,
504 exec_engine,
505 order_emulator,
506 trader,
507 ts_created,
508 ts_started: None,
509 ts_shutdown: None,
510 shutdown_requested,
511 event_store_replay: false,
512 state_save_armed: false,
513 #[cfg(feature = "streaming")]
514 streaming_writer,
515 })
516 }
517
518 fn register_shutdown_handler(trader_id: TraderId, shutdown_requested: Rc<Cell<bool>>) {
519 let handler = ShareableMessageHandler::from_typed(move |cmd: &ShutdownSystem| {
520 if cmd.trader_id != trader_id {
521 log::warn!("Received {cmd} not for this trader {trader_id}, ignoring");
522 return;
523 }
524
525 if shutdown_requested.get() {
526 log::debug!("Shutdown already requested, ignoring {cmd}");
527 return;
528 }
529
530 log::info!("Received {cmd}, requesting shutdown");
531 shutdown_requested.set(true);
532 });
533 let topic = MessagingSwitchboard::shutdown_system_topic();
534 msgbus::subscribe_any(topic.into(), handler, None);
535 }
536
537 fn determine_machine_id() -> anyhow::Result<String> {
538 sysinfo::System::host_name()
539 .map_err(|e| anyhow::anyhow!("Failed to determine hostname: {e}"))
540 }
541
542 fn initialize_logging(
543 trader_id: TraderId,
544 instance_id: UUID4,
545 config: LoggerConfig,
546 ) -> anyhow::Result<LogGuard> {
547 #[cfg(feature = "tracing-bridge")]
548 let use_tracing = config.use_tracing;
549
550 let file_config = config.file_config.clone().unwrap_or_default();
551 let log_guard = match init_logging(trader_id, instance_id, config, file_config) {
552 Ok(guard) => guard,
553 Err(e) => {
554 if e.downcast_ref::<log::SetLoggerError>().is_some() {
559 if let Some(guard) = LogGuard::new() {
560 guard
561 } else {
562 return Err(e.context(
563 "A non-Nautilus logger is already registered; \
564 cannot initialize Nautilus logging",
565 ));
566 }
567 } else {
568 return Err(e);
569 }
570 }
571 };
572
573 #[cfg(feature = "tracing-bridge")]
575 if use_tracing && !nautilus_common::logging::bridge::tracing_is_initialized() {
576 nautilus_common::logging::bridge::init_tracing()?;
577 }
578
579 Ok(log_guard)
580 }
581
582 fn initialize_cache(
583 cache_config: Option<CacheConfig>,
584 cache_database: Option<Box<dyn CacheDatabaseAdapter>>,
585 ) -> Rc<RefCell<Cache>> {
586 let cache_config = cache_config.unwrap_or_default();
587 let cache = Cache::new(Some(cache_config), cache_database);
588
589 Rc::new(RefCell::new(cache))
590 }
591
592 fn cancel_timers(&self) {
593 self.clock.borrow_mut().cancel_timers();
594 }
595
596 #[must_use]
597 pub fn generate_timestamp_ns(&self) -> UnixNanos {
598 self.clock.borrow().timestamp_ns()
599 }
600
601 #[must_use]
603 pub fn environment(&self) -> Environment {
604 self.config.environment()
605 }
606
607 #[must_use]
609 pub const fn name(&self) -> &str {
610 self.name.as_str()
611 }
612
613 #[must_use]
615 pub fn trader_id(&self) -> TraderId {
616 self.config.trader_id()
617 }
618
619 #[must_use]
621 pub fn machine_id(&self) -> &str {
622 &self.machine_id
623 }
624
625 #[must_use]
627 pub const fn instance_id(&self) -> UUID4 {
628 self.instance_id
629 }
630
631 #[must_use]
633 pub fn delay_post_stop(&self) -> Duration {
634 self.config.delay_post_stop()
635 }
636
637 #[must_use]
639 pub const fn ts_created(&self) -> UnixNanos {
640 self.ts_created
641 }
642
643 #[must_use]
645 pub const fn ts_started(&self) -> Option<UnixNanos> {
646 self.ts_started
647 }
648
649 #[must_use]
651 pub const fn ts_shutdown(&self) -> Option<UnixNanos> {
652 self.ts_shutdown
653 }
654
655 #[must_use]
659 pub fn is_shutdown_requested(&self) -> bool {
660 self.drain_shutdown_on_error_trigger();
661 self.shutdown_requested.get()
662 }
663
664 pub fn reset_shutdown_flag(&self) {
669 self.shutdown_requested.set(false);
670 }
671
672 #[must_use]
675 pub fn shutdown_flag(&self) -> Rc<Cell<bool>> {
676 self.shutdown_requested.clone()
677 }
678
679 fn drain_shutdown_on_error_trigger(&self) {
680 try_drain_shutdown_on_error_trigger(|trigger| {
681 let command = ShutdownSystem::new(
682 self.config.trader_id(),
683 trigger.component,
684 Some(format!(
685 "Error log received from {}: {}",
686 trigger.component, trigger.message
687 )),
688 UUID4::new(),
689 trigger.timestamp,
690 None,
691 );
692
693 msgbus::try_publish_any(
694 MessagingSwitchboard::shutdown_system_topic(),
695 command.as_any(),
696 )
697 });
698 }
699
700 #[must_use]
702 pub fn load_state(&self) -> bool {
703 self.config.load_state()
704 }
705
706 #[must_use]
708 pub fn save_state(&self) -> bool {
709 self.config.save_state()
710 }
711
712 #[must_use]
714 pub fn clock(&self) -> Rc<RefCell<dyn Clock>> {
715 self.clock.clone()
716 }
717
718 #[must_use]
720 pub fn cache(&self) -> Rc<RefCell<Cache>> {
721 self.cache.clone()
722 }
723
724 #[must_use]
726 pub fn portfolio(&self) -> Ref<'_, Portfolio> {
727 self.portfolio.borrow()
728 }
729
730 #[must_use]
732 pub fn data_engine(&self) -> Ref<'_, DataEngine> {
733 self.data_engine.borrow()
734 }
735
736 #[must_use]
738 pub const fn risk_engine(&self) -> &Rc<RefCell<RiskEngine>> {
739 &self.risk_engine
740 }
741
742 #[must_use]
744 pub const fn exec_engine(&self) -> &Rc<RefCell<ExecutionEngine>> {
745 &self.exec_engine
746 }
747
748 #[must_use]
750 pub fn trader(&self) -> &Rc<RefCell<Trader>> {
751 &self.trader
752 }
753
754 pub fn start(&mut self) {
756 arm_shutdown_on_error(self.config.shutdown_on_error());
757 log::info!("Starting");
758
759 self.event_store_replay = false;
760
761 if let Some(event_store) = self.event_store.as_deref_mut() {
762 self.exec_engine.borrow_mut().set_snapshot_anchorer(None);
763
764 let components = Self::collect_registered_components(&self.trader);
765 let environment = self.config.environment();
766 let event_store_replay_configured = event_store.is_event_store_replay_configured();
767
768 if event_store_replay_configured && !self.config.load_state() {
769 log::error!("Event-store replay requires load_state=true");
770 return;
771 }
772
773 if self.config.load_state()
774 && let Err(e) =
775 event_store.restore_parent_cache(self.instance_id, &mut self.cache.borrow_mut())
776 {
777 log::error!("Failed to restore cache from event-store replay source: {e}");
778 return;
779 }
780
781 if let Err(e) = event_store.open(self.instance_id, &components, environment) {
782 log::error!("Failed to open event-store run: {e}");
783 return;
784 }
785
786 let anchorer = event_store.snapshot_anchorer();
787 self.exec_engine
788 .borrow_mut()
789 .set_snapshot_anchorer(anchorer);
790 self.event_store_replay = event_store_replay_configured;
791 }
792
793 if self.event_store_replay {
794 log::info!(
795 "Event-store replay loaded; skipping engines, clients, trader startup, and live reconciliation",
796 );
797 self.ts_started = Some(self.clock.borrow().timestamp_ns());
798 log::info!("Started");
799 return;
800 }
801
802 self.start_engines();
803
804 let trader_state = self.trader.borrow().state();
806 if trader_state == ComponentState::PreInitialized {
807 log::info!("Initializing trader");
808
809 if let Err(e) = self.trader.borrow_mut().initialize() {
810 log::error!("Error initializing trader: {e:?}");
811 return;
812 }
813 }
814
815 self.ts_started = Some(self.clock.borrow().timestamp_ns());
818 log::info!("Started");
819 }
820
821 fn collect_registered_components(trader: &Rc<RefCell<Trader>>) -> RegisteredComponents {
822 let trader = trader.borrow();
823 let mut components = RegisteredComponents::default();
824 for actor_id in trader.actor_ids() {
825 components
826 .actors
827 .insert(actor_id.to_string(), String::new());
828 }
829
830 for strategy_id in trader.strategy_ids() {
831 components
832 .strategies
833 .insert(strategy_id.to_string(), String::new());
834 }
835
836 for algo_id in trader.exec_algorithm_ids() {
837 components
838 .algorithms
839 .insert(algo_id.to_string(), String::new());
840 }
841 components
842 }
843
844 #[expect(
846 clippy::unused_async,
847 reason = "keeps the public async kernel API shape stable"
848 )]
849 pub async fn start_async(&mut self) {
850 self.start();
851 }
852
853 pub fn start_trader(&mut self) -> anyhow::Result<()> {
862 log::info!("Starting trader...");
863
864 let load_state = self.config.load_state();
865 let save_state = self.config.save_state();
866
867 if (load_state || save_state) && !self.cache.borrow().has_backing() {
868 log::warn!(
869 "Cache has no database backing, load_state={load_state} and save_state={save_state} will have no effect"
870 );
871 }
872
873 if load_state {
874 Trader::load_state(&self.trader)
875 .map_err(|e| anyhow::anyhow!("Failed to load actor and strategy state: {e:#}"))?;
876 }
877
878 self.state_save_armed = save_state;
879 self.order_emulator.start();
880
881 if let Err(start_err) = Trader::start_with_component_callbacks(&self.trader) {
882 let stop_result = self.stop_trader_after_start_failure();
883 self.order_emulator.stop();
884 let save_result = self.save_trader_state();
885
886 let mut errors = vec![format!("Failed to start trader: {start_err}")];
887 if let Err(e) = stop_result {
888 errors.push(format!("failed to stop partial trader start: {e}"));
889 }
890
891 if let Err(e) = save_result {
892 errors.push(format!("failed to save partial trader state: {e}"));
893 }
894 anyhow::bail!("{}", errors.join("; "));
895 }
896
897 log::info!("Trader started");
898 Ok(())
899 }
900
901 pub fn stop_trader(&mut self) {
907 disarm_shutdown_on_error();
908
909 if !self.trader.borrow().is_running() {
910 return;
911 }
912
913 log::info!("Stopping trader...");
914
915 if let Err(e) = self.trader.borrow_mut().stop() {
916 log::error!("Error stopping trader: {e}");
917 }
918 }
919
920 pub fn stop_trader_after_start_failure(&mut self) -> anyhow::Result<()> {
926 disarm_shutdown_on_error();
927
928 if !matches!(
929 self.trader.borrow().state(),
930 ComponentState::Starting | ComponentState::Running
931 ) {
932 return Ok(());
933 }
934
935 log::info!("Stopping trader immediately...");
936 self.trader.borrow_mut().stop_after_start_failure()
937 }
938
939 #[allow(unknown_lints)]
952 #[expect(
953 clippy::unused_async,
954 clippy::unused_async_trait_impl,
955 reason = "keeps the public async kernel API shape stable"
956 )]
957 pub async fn finalize_stop(&mut self) -> anyhow::Result<()> {
958 disarm_shutdown_on_error();
959
960 let save_result = self.save_trader_state();
963 self.portfolio.borrow_mut().finalize_equity_curve();
964 self.stop_engines();
965 self.cancel_timers();
966
967 let ts_shutdown = self.clock.borrow().timestamp_ns();
968
969 if let Some(event_store) = self.event_store.as_deref_mut() {
970 self.exec_engine.borrow_mut().set_snapshot_anchorer(None);
971 event_store.seal(ts_shutdown);
972 }
973 self.ts_shutdown = Some(ts_shutdown);
974 log::info!("Stopped");
975 save_result?;
976 self.flush_streaming_writer()
977 }
978
979 pub fn save_trader_state(&mut self) -> anyhow::Result<()> {
987 if !std::mem::take(&mut self.state_save_armed) {
988 return Ok(());
989 }
990
991 Trader::save_state(&self.trader)
992 }
993
994 #[must_use]
1000 pub fn event_store(&self) -> Option<&dyn KernelEventStore> {
1001 self.event_store.as_deref()
1002 }
1003
1004 #[must_use]
1006 pub fn is_event_store_replay(&self) -> bool {
1007 self.event_store_replay
1008 }
1009
1010 #[must_use]
1012 pub fn is_event_store_replay_configured(&self) -> bool {
1013 self.event_store
1014 .as_deref()
1015 .is_some_and(KernelEventStore::is_event_store_replay_configured)
1016 }
1017
1018 pub fn reset(&mut self) {
1020 disarm_shutdown_on_error();
1021 log::info!("Resetting");
1022
1023 if let Err(e) = self.trader.borrow_mut().reset() {
1024 log::error!("Error resetting trader: {e:?}");
1025 }
1026
1027 self.data_engine.borrow_mut().reset();
1028 self.exec_engine.borrow_mut().reset();
1029 self.risk_engine.borrow_mut().reset();
1030 self.order_emulator.reset();
1031 self.portfolio.borrow_mut().reset();
1032
1033 self.ts_started = None;
1034 self.ts_shutdown = None;
1035 self.state_save_armed = false;
1036
1037 log::info!("Reset");
1038 }
1039
1040 pub fn dispose(&mut self) {
1042 disarm_shutdown_on_error();
1043 log::info!("Disposing");
1044
1045 let trader_state = self.trader.borrow().state();
1046 match trader_state {
1047 ComponentState::Running => self.stop_trader(),
1048 ComponentState::Starting => {
1049 if let Err(e) = self.stop_trader_after_start_failure() {
1050 log::error!("Error stopping partial trader start during disposal: {e:?}");
1051 }
1052 }
1053 _ => {}
1054 }
1055
1056 if let Err(e) = self.save_trader_state() {
1057 log::error!("Error saving trader state during disposal: {e:?}");
1058 }
1059
1060 {
1061 let mut trader = self.trader.borrow_mut();
1062 if trader.state() == ComponentState::PreInitialized
1063 && let Err(e) = trader.initialize()
1064 {
1065 log::error!("Error initializing trader for disposal: {e:?}");
1066 }
1067
1068 if !trader.is_disposed()
1069 && let Err(e) = trader.dispose()
1070 {
1071 log::error!("Error disposing trader: {e:?}");
1072 }
1073 }
1074
1075 self.stop_engines();
1076 self.portfolio.borrow_mut().reset();
1077 self.cancel_timers();
1078
1079 if let Some(event_store) = self.event_store.as_deref_mut() {
1084 self.exec_engine.borrow_mut().set_snapshot_anchorer(None);
1085 let ts_dispose = self.clock.borrow().timestamp_ns();
1086 event_store.seal(ts_dispose);
1087 }
1088
1089 if let Err(e) = self.close_streaming_writer() {
1090 log::warn!("{e}");
1091 }
1092
1093 self.data_engine.borrow_mut().dispose();
1094 self.exec_engine.borrow_mut().dispose();
1095 self.risk_engine.borrow_mut().dispose();
1096 self.order_emulator.dispose();
1097 self.cache.borrow_mut().dispose();
1098 get_message_bus().borrow_mut().dispose();
1099
1100 log::info!("Disposed");
1101 }
1102
1103 pub fn reopen_streaming_writer(&mut self) -> anyhow::Result<()> {
1112 #[cfg(feature = "streaming")]
1113 if self.streaming_writer.is_none()
1114 && let Some(mut config) = self.config.streaming()
1115 {
1116 config.replace_existing = false;
1117 self.streaming_writer = Some(Self::setup_streaming_writer(
1118 self.config.environment(),
1119 self.instance_id,
1120 &config,
1121 &self.clock,
1122 )?);
1123 }
1124
1125 Ok(())
1126 }
1127
1128 #[cfg(feature = "streaming")]
1131 fn flush_streaming_writer(&self) -> anyhow::Result<()> {
1132 match &self.streaming_writer {
1133 Some((destination, writer)) => writer.flush().map_err(|e| {
1134 anyhow::anyhow!("Failed to flush streaming writer at {destination}: {e}")
1135 }),
1136 None => Ok(()),
1137 }
1138 }
1139
1140 #[cfg(not(feature = "streaming"))]
1141 #[expect(
1142 clippy::unnecessary_wraps,
1143 clippy::unused_self,
1144 reason = "matches the streaming variant's signature"
1145 )]
1146 fn flush_streaming_writer(&self) -> anyhow::Result<()> {
1147 Ok(())
1148 }
1149
1150 #[cfg(feature = "streaming")]
1157 pub fn close_streaming_writer(&mut self) -> anyhow::Result<()> {
1158 match self.streaming_writer.take() {
1159 Some((destination, writer)) => writer.close().map_err(|e| {
1160 anyhow::anyhow!("Failed to close streaming writer at {destination}: {e}")
1161 }),
1162 None => Ok(()),
1163 }
1164 }
1165
1166 #[cfg(not(feature = "streaming"))]
1173 pub fn close_streaming_writer(&mut self) -> anyhow::Result<()> {
1174 Ok(())
1175 }
1176
1177 fn start_engines(&self) {
1179 self.data_engine.borrow_mut().start();
1180 self.exec_engine.borrow_mut().start();
1181 self.risk_engine.borrow_mut().start();
1182 }
1183
1184 fn stop_engines(&self) {
1186 self.data_engine.borrow_mut().stop();
1187 self.exec_engine.borrow_mut().stop();
1188 self.risk_engine.borrow_mut().stop();
1189 self.order_emulator.stop();
1190 }
1191
1192 #[expect(clippy::await_holding_refcell_ref)] pub async fn connect_data_clients(&mut self) -> anyhow::Result<()> {
1202 log::info!("Connecting data clients...");
1203 self.data_engine.borrow_mut().connect().await
1204 }
1205
1206 #[expect(clippy::await_holding_refcell_ref)] pub async fn connect_exec_clients(&mut self) -> anyhow::Result<()> {
1216 log::info!("Connecting execution clients...");
1217 self.exec_engine.borrow_mut().connect().await
1218 }
1219
1220 #[expect(clippy::await_holding_refcell_ref)] pub async fn disconnect_clients(&mut self) -> anyhow::Result<()> {
1227 log::info!("Disconnecting clients...");
1228 let mut data_engine = self.data_engine.borrow_mut();
1229 let mut exec_engine = self.exec_engine.borrow_mut();
1230 let (data_result, exec_result) =
1231 futures::join!(data_engine.disconnect(), exec_engine.disconnect());
1232
1233 match (data_result, exec_result) {
1234 (Ok(()), Ok(())) => Ok(()),
1235 (Err(data_err), Ok(())) => Err(data_err),
1236 (Ok(()), Err(exec_err)) => Err(exec_err),
1237 (Err(data_err), Err(exec_err)) => anyhow::bail!(
1238 "Failed to disconnect data clients: {data_err}; failed to disconnect execution \
1239 clients: {exec_err}"
1240 ),
1241 }
1242 }
1243
1244 #[must_use]
1246 pub fn check_engines_connected(&self) -> bool {
1247 self.data_engine.borrow().check_connected() && self.exec_engine.borrow().check_connected()
1248 }
1249
1250 #[must_use]
1252 pub fn check_engines_disconnected(&self) -> bool {
1253 self.data_engine.borrow().check_disconnected()
1254 && self.exec_engine.borrow().check_disconnected()
1255 }
1256
1257 #[must_use]
1259 pub fn data_client_connection_status(&self) -> Vec<(ClientId, bool)> {
1260 self.data_engine.borrow().client_connection_status()
1261 }
1262
1263 #[must_use]
1265 pub fn exec_client_connection_status(&self) -> Vec<(ClientId, bool)> {
1266 self.exec_engine.borrow().client_connection_status()
1267 }
1268}
1269
1270#[cfg(all(test, feature = "python"))]
1271mod tests {
1272 use nautilus_common::messages::system::ShutdownSystem;
1273 use nautilus_core::UUID4;
1274 use rstest::*;
1275 use ustr::Ustr;
1276
1277 use super::*;
1278 use crate::builder::NautilusKernelBuilder;
1279
1280 #[rstest]
1281 fn test_shutdown_system_sets_kernel_flag() {
1282 let kernel = NautilusKernelBuilder::default().build().unwrap();
1283 assert!(!kernel.is_shutdown_requested());
1284
1285 let command = ShutdownSystem::new(
1286 kernel.trader_id(),
1287 Ustr::from("TestComponent"),
1288 Some("unit test".to_string()),
1289 UUID4::new(),
1290 kernel.generate_timestamp_ns(),
1291 None, );
1293
1294 msgbus::publish_any(
1295 MessagingSwitchboard::shutdown_system_topic(),
1296 command.as_any(),
1297 );
1298 assert!(kernel.is_shutdown_requested());
1299
1300 kernel.reset_shutdown_flag();
1301 assert!(!kernel.is_shutdown_requested());
1302 }
1303
1304 #[rstest]
1305 fn test_shutdown_system_idempotent() {
1306 let kernel = NautilusKernelBuilder::default().build().unwrap();
1307
1308 let make_cmd = || {
1309 ShutdownSystem::new(
1310 kernel.trader_id(),
1311 Ustr::from("TestComponent"),
1312 None,
1313 UUID4::new(),
1314 kernel.generate_timestamp_ns(),
1315 None, )
1317 };
1318
1319 let topic = MessagingSwitchboard::shutdown_system_topic();
1320 msgbus::publish_any(topic, make_cmd().as_any());
1321 assert!(kernel.is_shutdown_requested());
1322
1323 msgbus::publish_any(topic, make_cmd().as_any());
1324 assert!(kernel.is_shutdown_requested());
1325
1326 kernel.reset_shutdown_flag();
1327 assert!(!kernel.is_shutdown_requested());
1328
1329 msgbus::publish_any(topic, make_cmd().as_any());
1330 assert!(kernel.is_shutdown_requested());
1331 }
1332
1333 #[rstest]
1334 fn test_shutdown_system_ignores_other_trader() {
1335 let kernel = NautilusKernelBuilder::default().build().unwrap();
1336
1337 let command = ShutdownSystem::new(
1338 TraderId::from("OTHER-TRADER"),
1339 Ustr::from("TestComponent"),
1340 None,
1341 UUID4::new(),
1342 kernel.generate_timestamp_ns(),
1343 None, );
1345
1346 msgbus::publish_any(
1347 MessagingSwitchboard::shutdown_system_topic(),
1348 command.as_any(),
1349 );
1350 assert!(!kernel.is_shutdown_requested());
1351 }
1352}
1353
1354#[cfg(all(test, feature = "streaming"))]
1355mod streaming_tests {
1356 use std::{cell::RefCell, rc::Rc, sync::Arc};
1357
1358 use nautilus_common::{
1359 clock::VirtualClock,
1360 messages::data::{
1361 DataCommand, DataResponse, DataResponseEnvelope, RequestCommand, RequestQuotes,
1362 },
1363 msgbus::{self, MStr, TypedHandler},
1364 };
1365 use nautilus_core::Params;
1366 use nautilus_model::{
1367 data::{CustomData, DataType, NautilusDataType, QuoteTick},
1368 identifiers::InstrumentId,
1369 types::{Price, Quantity},
1370 };
1371 use nautilus_persistence::{
1372 backend::parquet::catalog::ParquetDataCatalog, config::DataCatalogConfig,
1373 test_data::RustTestCustomData,
1374 };
1375 use nautilus_serialization::ensure_custom_data_registered;
1376 use rstest::rstest;
1377 use tempfile::tempdir;
1378
1379 use super::*;
1380 use crate::config::{KernelConfig, RotationConfig, StreamingConfig};
1381
1382 #[rstest]
1383 fn test_configured_catalog_serves_builtin_quotes() {
1384 let directory = tempdir().unwrap();
1385 let instrument_id = InstrumentId::from("AUD/USD.SIM");
1386 let quotes = vec![
1387 QuoteTick::new(
1388 instrument_id,
1389 Price::from("1.00001"),
1390 Price::from("1.00003"),
1391 Quantity::from("100_000"),
1392 Quantity::from("200_000"),
1393 UnixNanos::from(1),
1394 UnixNanos::from(1),
1395 ),
1396 QuoteTick::new(
1397 instrument_id,
1398 Price::from("1.00002"),
1399 Price::from("1.00004"),
1400 Quantity::from("300_000"),
1401 Quantity::from("400_000"),
1402 UnixNanos::from(2),
1403 UnixNanos::from(2),
1404 ),
1405 ];
1406 let catalog = ParquetDataCatalog::new(directory.path(), None, None, None, None);
1407 catalog.write_to_parquet("es, None, None, None).unwrap();
1408 let config = KernelConfig {
1409 catalogs: vec![
1410 DataCatalogConfig::new(
1411 directory.path().to_string_lossy().into_owned(),
1412 Some("file".to_string()),
1413 None,
1414 )
1415 .with_name(Some("history".to_string())),
1416 ],
1417 ..KernelConfig::default()
1418 };
1419 let mut kernel = NautilusKernel::new("CatalogQueryTest".to_string(), config).unwrap();
1420 kernel
1421 .clock
1422 .borrow_mut()
1423 .as_any_mut()
1424 .downcast_mut::<VirtualClock>()
1425 .unwrap()
1426 .set_time(UnixNanos::from(3));
1427 let request_id = UUID4::new();
1428 let received = Rc::new(RefCell::new(Vec::new()));
1429 let received_quotes = received.clone();
1430 msgbus::register_response_handler(
1431 &request_id,
1432 TypedHandler::from(move |envelope: &DataResponseEnvelope| {
1433 let Some(DataResponse::Quotes(response)) = envelope.response() else {
1434 panic!("expected quotes");
1435 };
1436
1437 *received_quotes.borrow_mut() = response.data.to_vec();
1438 }),
1439 );
1440 let request = RequestQuotes::new(
1441 instrument_id,
1442 Some(UnixNanos::from(1).to_datetime_utc()),
1443 Some(UnixNanos::from(2).to_datetime_utc()),
1444 None,
1445 None,
1446 request_id,
1447 UnixNanos::from(3),
1448 None,
1449 );
1450
1451 kernel
1452 .data_engine
1453 .borrow_mut()
1454 .execute(DataCommand::Request(RequestCommand::Quotes(request)));
1455
1456 assert_eq!(*received.borrow(), quotes);
1457 assert_eq!(kernel.data_engine.borrow().request_count(), 1);
1458 assert_eq!(kernel.data_engine.borrow().response_count(), 1);
1459 kernel.dispose();
1460 }
1461
1462 #[rstest]
1463 fn test_configured_catalog_error_names_rejected_param() {
1464 let directory = tempdir().unwrap();
1465 let path = directory.path().to_string_lossy().into_owned();
1466 let params = Params::new([("batch_size".to_string(), serde_json::json!(1024))].into());
1467
1468 let config = KernelConfig {
1469 catalogs: vec![
1470 DataCatalogConfig::new(path.clone(), Some("file".to_string()), None)
1471 .with_params(Some(params)),
1472 ],
1473 ..KernelConfig::default()
1474 };
1475
1476 let error = NautilusKernel::new("CatalogParamTest".to_string(), config)
1477 .err()
1478 .unwrap();
1479
1480 assert_eq!(
1481 error.to_string(),
1482 format!(
1483 "Failed to create data catalog from '{path}': Unknown Parquet catalog param \
1484 'batch_size': this catalog takes no params"
1485 )
1486 );
1487 }
1488
1489 #[rstest]
1490 fn test_configured_streaming_excludes_custom_data() {
1491 ensure_custom_data_registered::<RustTestCustomData>();
1492
1493 let directory = tempdir().unwrap();
1494 let instance_id = UUID4::new();
1495 let mut streaming = StreamingConfig::new(
1496 directory.path().to_string_lossy().into_owned(),
1497 None,
1498 1_000,
1499 false,
1500 RotationConfig::NoRotation,
1501 );
1502 streaming.data_types = Some(vec![NautilusDataType::QuoteTick]);
1503 let config = KernelConfig {
1504 instance_id: Some(instance_id),
1505 streaming: Some(streaming),
1506 ..KernelConfig::default()
1507 };
1508 let mut kernel = NautilusKernel::new("BuiltInStreamingTest".to_string(), config).unwrap();
1509 let instrument_id = InstrumentId::from("RUST.TEST");
1510 let custom = CustomData::new(
1511 Arc::new(RustTestCustomData {
1512 instrument_id,
1513 value: 1.23,
1514 flag: true,
1515 ts_event: UnixNanos::from(1_000),
1516 ts_init: UnixNanos::from(1_000),
1517 }),
1518 DataType::new("RustTestCustomData", None, Some(instrument_id.to_string())),
1519 );
1520
1521 msgbus::publish_any(MStr::from("data.custom"), &custom);
1522 kernel.close_streaming_writer().unwrap();
1523
1524 let custom_path = directory
1525 .path()
1526 .join("backtest")
1527 .join(instance_id.to_string())
1528 .join("data/custom");
1529 assert!(!custom_path.exists());
1530 kernel.dispose();
1531 }
1532
1533 #[rstest]
1534 #[case::promotion_without_catalog(
1535 None,
1536 "promotion_interval_ms requires catalog: promotion needs a catalog"
1537 )]
1538 #[case::catalog_param(
1539 Some(serde_json::json!({"params": {"batch_size": 1024}})),
1540 "Unknown Parquet catalog param 'batch_size': this catalog takes no params"
1541 )]
1542 #[case::zero_batch_size(
1543 Some(serde_json::json!({"batch_size": 0})),
1544 "invalid batch_size: must be a positive number of rows; omit the field for the \
1545 backend default"
1546 )]
1547 fn test_invalid_streaming_config_fails_before_replacing_files(
1548 #[case] catalog_fields: Option<serde_json::Value>,
1549 #[case] expected: &str,
1550 ) {
1551 let directory = tempdir().unwrap();
1552 let instance_id = UUID4::new();
1553 let run_directory = directory
1554 .path()
1555 .join("backtest")
1556 .join(instance_id.to_string());
1557 std::fs::create_dir_all(&run_directory).unwrap();
1558 let existing = run_directory.join("existing.feather");
1559 std::fs::write(&existing, b"preserve").unwrap();
1560
1561 let catalog_path = directory.path().join("catalog");
1562
1563 let catalog = catalog_fields.map(|mut fields| {
1565 fields["path"] = serde_json::json!(catalog_path.to_string_lossy());
1566 serde_json::from_value::<DataCatalogConfig>(fields).unwrap()
1567 });
1568
1569 let mut streaming = StreamingConfig::new(
1570 directory.path().to_string_lossy().into_owned(),
1571 catalog,
1572 1_000,
1573 true,
1574 RotationConfig::NoRotation,
1575 );
1576 streaming.promotion_interval_ms = Some(1_000);
1577 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1578
1579 let error = NautilusKernel::setup_streaming_writer(
1580 Environment::Backtest,
1581 instance_id,
1582 &streaming,
1583 &clock,
1584 )
1585 .err()
1586 .unwrap();
1587
1588 assert_eq!(error.to_string(), expected);
1589 assert_eq!(std::fs::read(&existing).unwrap(), b"preserve");
1590 }
1591
1592 #[rstest]
1593 fn test_streaming_replace_existing_creates_missing_catalog_directory() {
1594 let directory = tempdir().unwrap();
1595 let instance_id = UUID4::new();
1596 let run_directory = directory
1597 .path()
1598 .join("backtest")
1599 .join(instance_id.to_string());
1600 std::fs::create_dir_all(&run_directory).unwrap();
1601 let existing = run_directory.join("existing.feather");
1602 std::fs::write(&existing, b"replace").unwrap();
1603 let catalog_path = directory.path().join("catalog");
1604
1605 let streaming = StreamingConfig::new(
1606 directory.path().to_string_lossy().into_owned(),
1607 Some(DataCatalogConfig::new(
1608 catalog_path.to_string_lossy().into_owned(),
1609 None,
1610 None,
1611 )),
1612 1_000,
1613 true,
1614 RotationConfig::NoRotation,
1615 );
1616 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1617
1618 let (_, subscription) = NautilusKernel::setup_streaming_writer(
1619 Environment::Backtest,
1620 instance_id,
1621 &streaming,
1622 &clock,
1623 )
1624 .unwrap();
1625 subscription.close().unwrap();
1626
1627 assert!(!existing.exists());
1628 assert!(catalog_path.is_dir());
1629 }
1630
1631 #[rstest]
1632 fn test_configured_streaming_promotes_into_separate_catalog() {
1633 let writer_directory = tempdir().unwrap();
1634 let catalog_directory = tempdir().unwrap();
1635 let catalog_path = catalog_directory.path().join("catalog");
1636 let instance_id = UUID4::new();
1637
1638 let mut streaming = StreamingConfig::new(
1639 writer_directory.path().to_string_lossy().into_owned(),
1640 Some(DataCatalogConfig::new(
1641 catalog_path.to_string_lossy().into_owned(),
1642 None,
1643 None,
1644 )),
1645 1_000,
1646 false,
1647 RotationConfig::NoRotation,
1648 );
1649 streaming.data_types = Some(vec![NautilusDataType::QuoteTick]);
1650
1651 let config = KernelConfig {
1652 instance_id: Some(instance_id),
1653 streaming: Some(streaming),
1654 ..KernelConfig::default()
1655 };
1656
1657 let mut kernel =
1658 NautilusKernel::new("SeparateCatalogStreamingTest".to_string(), config).unwrap();
1659
1660 let quote = QuoteTick::new(
1661 InstrumentId::from("AUD/USD.SIM"),
1662 Price::from("1.00001"),
1663 Price::from("1.00003"),
1664 Quantity::from("100_000"),
1665 Quantity::from("200_000"),
1666 UnixNanos::from(1),
1667 UnixNanos::from(1),
1668 );
1669
1670 msgbus::publish_quote(MStr::from("data.quotes.SIM.AUD/USD"), "e);
1671 kernel.close_streaming_writer().unwrap();
1672
1673 let mut catalog = ParquetDataCatalog::new(&catalog_path, None, None, None, None);
1674 let promoted = catalog
1675 .query::<QuoteTick>(None, None, None, None, None, true)
1676 .unwrap();
1677 let run_directory = writer_directory
1678 .path()
1679 .join("backtest")
1680 .join(instance_id.to_string());
1681 assert_eq!(promoted, vec![quote]);
1682 assert!(run_directory.join("quotes").is_dir());
1683 assert!(!catalog_path.join("backtest").exists());
1684 kernel.dispose();
1685 }
1686}
1687
1688#[cfg(test)]
1689mod lifecycle_tests {
1690 use futures::FutureExt;
1691 use indexmap::IndexMap;
1692 use nautilus_common::{
1693 actor::registry::get_actor_unchecked,
1694 cache::Cache,
1695 clock::VirtualClock,
1696 messages::data::{DataCommand, SubscribeCommand, UnsubscribeCommand},
1697 msgbus::stubs::{TypedIntoMessageSavingHandler, get_typed_into_message_saving_handler},
1698 };
1699 use nautilus_execution::engine::SnapshotAnchorer;
1700 use nautilus_model::{
1701 enums::{OrderSide, OrderStatus, OrderType, TriggerType},
1702 identifiers::{ActorId, ClientOrderId, StrategyId},
1703 instruments::{
1704 CryptoPerpetual, Instrument, InstrumentAny, stubs::crypto_perpetual_ethusdt,
1705 },
1706 orders::{Order, OrderAny, OrderTestBuilder},
1707 types::{Price, Quantity},
1708 };
1709 use nautilus_testkit::{
1710 cache::TestCacheDatabaseControl,
1711 components::{StateActor, StateStrategy},
1712 };
1713 use rstest::rstest;
1714 use ustr::Ustr;
1715
1716 use super::*;
1717 use crate::{
1718 builder::NautilusKernelBuilder,
1719 event_store::{KernelEventStore, RegisteredComponents},
1720 };
1721
1722 #[derive(Debug)]
1723 struct RecordingEventStore {
1724 control: TestCacheDatabaseControl,
1725 opened: bool,
1726 }
1727
1728 impl KernelEventStore for RecordingEventStore {
1729 fn restore_parent_cache(
1730 &mut self,
1731 _instance_id: UUID4,
1732 _cache: &mut Cache,
1733 ) -> anyhow::Result<()> {
1734 self.control.record("event_store.restore");
1735 Ok(())
1736 }
1737
1738 fn open(
1739 &mut self,
1740 _instance_id: UUID4,
1741 _components: &RegisteredComponents,
1742 _environment: Environment,
1743 ) -> anyhow::Result<()> {
1744 self.control.record("event_store.open");
1745 self.opened = true;
1746 Ok(())
1747 }
1748
1749 fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
1750 None
1751 }
1752
1753 fn seal(&mut self, _ts_init: UnixNanos) {
1754 if self.opened {
1755 self.control.record("event_store.seal");
1756 self.opened = false;
1757 }
1758 }
1759
1760 fn run_id(&self) -> Option<&str> {
1761 None
1762 }
1763
1764 fn parent_run_id(&self) -> Option<&str> {
1765 None
1766 }
1767
1768 fn is_halted(&self) -> bool {
1769 false
1770 }
1771 }
1772
1773 fn state(key: &str, value: &[u8]) -> IndexMap<String, Vec<u8>> {
1774 IndexMap::from([(key.to_string(), value.to_vec())])
1775 }
1776
1777 fn finalize(kernel: &mut NautilusKernel) -> anyhow::Result<()> {
1778 kernel
1779 .finalize_stop()
1780 .now_or_never()
1781 .expect("kernel finalization must not yield")
1782 }
1783
1784 fn add_state_components(
1785 kernel: &NautilusKernel,
1786 control: &TestCacheDatabaseControl,
1787 actor: StateActor,
1788 strategy: StateStrategy,
1789 ) {
1790 kernel.trader.borrow_mut().add_actor(actor).unwrap();
1791 kernel.trader.borrow_mut().add_strategy(strategy).unwrap();
1792 control.record("components.registered");
1793 }
1794
1795 fn create_stop_market_order(instrument: &CryptoPerpetual, client_order_id: &str) -> OrderAny {
1796 OrderTestBuilder::new(OrderType::StopMarket)
1797 .instrument_id(instrument.id())
1798 .client_order_id(ClientOrderId::from(client_order_id))
1799 .side(OrderSide::Buy)
1800 .trigger_price(Price::from("5100.00"))
1801 .quantity(Quantity::from(1))
1802 .emulation_trigger(TriggerType::BidAsk)
1803 .build()
1804 }
1805
1806 fn register_data_command_handler(id: &str) -> TypedIntoMessageSavingHandler<DataCommand> {
1807 let (handler, saving_handler) =
1808 get_typed_into_message_saving_handler::<DataCommand>(Some(Ustr::from(id)));
1809 msgbus::register_data_command_endpoint(
1810 MessagingSwitchboard::data_engine_queue_execute(),
1811 handler,
1812 );
1813 saving_handler
1814 }
1815
1816 #[rstest]
1817 fn test_state_persistence_orders_restore_load_start_stop_save_seal_and_dispose() {
1818 let actor_id = ActorId::from("STATE-ACTOR");
1819 let strategy_id = StrategyId::from("STATE-STRATEGY-001");
1820 let actor_load = state("actor-loaded", b"actor-load-value");
1821 let strategy_load = state("strategy-loaded", b"strategy-load-value");
1822 let actor_save = state("actor-saved", b"actor-save-value");
1823 let strategy_save = state("strategy-saved", b"strategy-save-value");
1824 let (database, control) = TestCacheDatabaseControl::create();
1825 control.set_actor_state(actor_id, &actor_load);
1826 control.set_strategy_state(strategy_id, &strategy_load);
1827
1828 let event_store_control = control.clone();
1829 let mut kernel = NautilusKernelBuilder::default()
1830 .with_cache_database(Box::new(database))
1831 .with_event_store(move |_instance_id, _clock| {
1832 Ok(Box::new(RecordingEventStore {
1833 control: event_store_control,
1834 opened: false,
1835 }))
1836 })
1837 .build()
1838 .unwrap();
1839
1840 let actor = StateActor::new(actor_id, control.clone(), actor_save.clone());
1841 let strategy = StateStrategy::new(strategy_id, control.clone(), strategy_save.clone());
1842 add_state_components(&kernel, &control, actor, strategy);
1843
1844 kernel.start();
1845 kernel.start_trader().unwrap();
1846
1847 let actor_state = get_actor_unchecked::<StateActor>(&actor_id.inner())
1848 .state_load()
1849 .cloned();
1850 let strategy_state = get_actor_unchecked::<StateStrategy>(&strategy_id.inner())
1851 .state_load()
1852 .cloned();
1853 assert_eq!(actor_state, Some(actor_load));
1854 assert_eq!(strategy_state, Some(strategy_load));
1855
1856 kernel.stop_trader();
1857 kernel.stop_trader();
1858 finalize(&mut kernel).unwrap();
1859 finalize(&mut kernel).unwrap();
1860 kernel.dispose();
1861
1862 assert_eq!(
1863 control.events(),
1864 vec![
1865 "components.registered",
1866 "event_store.restore",
1867 "event_store.open",
1868 "actor.load:STATE-ACTOR",
1869 "actor.on_load",
1870 "strategy.load:STATE-STRATEGY-001",
1871 "strategy.on_load",
1872 "actor.on_start",
1873 "strategy.on_start",
1874 "actor.on_stop",
1875 "strategy.on_stop",
1876 "actor.on_save",
1877 "actor.update:STATE-ACTOR",
1878 "strategy.on_save",
1879 "strategy.update:STATE-STRATEGY-001",
1880 "event_store.seal",
1881 "database.close",
1882 ]
1883 );
1884 assert_eq!(control.actor_state(&actor_id), Some(actor_save));
1885 assert_eq!(control.strategy_state(&strategy_id), Some(strategy_save));
1886 }
1887
1888 #[rstest]
1889 fn test_state_persistence_skips_callbacks_without_cache_backing() {
1890 let actor_id = ActorId::from("NO-BACKING-ACTOR");
1891 let strategy_id = StrategyId::from("NO-BACKING-STRATEGY-001");
1892 let control = TestCacheDatabaseControl::default();
1893 let mut kernel = NautilusKernelBuilder::default().build().unwrap();
1894 let actor = StateActor::new(actor_id, control.clone(), state("actor", b"save"));
1895 let strategy = StateStrategy::new(strategy_id, control.clone(), state("strategy", b"save"));
1896 add_state_components(&kernel, &control, actor, strategy);
1897
1898 kernel.start();
1899 kernel.start_trader().unwrap();
1900 kernel.stop_trader();
1901 finalize(&mut kernel).unwrap();
1902 kernel.dispose();
1903
1904 assert_eq!(
1905 control.events(),
1906 vec![
1907 "components.registered",
1908 "actor.on_start",
1909 "strategy.on_start",
1910 "actor.on_stop",
1911 "strategy.on_stop",
1912 ]
1913 );
1914 }
1915
1916 #[rstest]
1917 fn test_state_persistence_skips_empty_load_and_persists_empty_save() {
1918 let actor_id = ActorId::from("EMPTY-STATE-ACTOR");
1919 let strategy_id = StrategyId::from("EMPTY-STATE-STRATEGY-001");
1920 let (database, control) = TestCacheDatabaseControl::create();
1921 let mut kernel = NautilusKernelBuilder::default()
1922 .with_cache_database(Box::new(database))
1923 .build()
1924 .unwrap();
1925 let actor = StateActor::new(actor_id, control.clone(), IndexMap::new());
1926 let strategy = StateStrategy::new(strategy_id, control.clone(), IndexMap::new());
1927 add_state_components(&kernel, &control, actor, strategy);
1928
1929 kernel.start();
1930 kernel.start_trader().unwrap();
1931 kernel.stop_trader();
1932 finalize(&mut kernel).unwrap();
1933
1934 assert_eq!(
1935 control.events(),
1936 vec![
1937 "components.registered",
1938 "actor.load:EMPTY-STATE-ACTOR",
1939 "strategy.load:EMPTY-STATE-STRATEGY-001",
1940 "actor.on_start",
1941 "strategy.on_start",
1942 "actor.on_stop",
1943 "strategy.on_stop",
1944 "actor.on_save",
1945 "actor.update:EMPTY-STATE-ACTOR",
1946 "strategy.on_save",
1947 "strategy.update:EMPTY-STATE-STRATEGY-001",
1948 ]
1949 );
1950 assert_eq!(control.actor_state(&actor_id), Some(IndexMap::new()));
1951 assert_eq!(control.strategy_state(&strategy_id), Some(IndexMap::new()));
1952 kernel.dispose();
1953 }
1954
1955 #[rstest]
1956 fn test_state_save_reports_all_callback_errors_and_continues_shutdown() {
1957 let actor_id = ActorId::from("FAIL-SAVE-ACTOR");
1958 let strategy_id = StrategyId::from("FAIL-SAVE-STRATEGY-001");
1959 let (database, control) = TestCacheDatabaseControl::create();
1960 let event_store_control = control.clone();
1961 let mut kernel = NautilusKernelBuilder::default()
1962 .with_cache_database(Box::new(database))
1963 .with_event_store(move |_instance_id, _clock| {
1964 Ok(Box::new(RecordingEventStore {
1965 control: event_store_control,
1966 opened: false,
1967 }))
1968 })
1969 .build()
1970 .unwrap();
1971 let actor = StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_save();
1972 let strategy =
1973 StateStrategy::new(strategy_id, control.clone(), IndexMap::new()).with_fail_save();
1974 add_state_components(&kernel, &control, actor, strategy);
1975
1976 kernel.start();
1977 kernel.start_trader().unwrap();
1978 kernel.stop_trader();
1979 let expected_shutdown = kernel.clock.borrow().timestamp_ns();
1980 let error = finalize(&mut kernel).unwrap_err();
1981 kernel.dispose();
1982
1983 assert_eq!(
1984 error.to_string(),
1985 "Failed to save component state: actor FAIL-SAVE-ACTOR callback: test actor on_save \
1986 failure; strategy FAIL-SAVE-STRATEGY-001 callback: test strategy on_save failure"
1987 );
1988 assert_eq!(kernel.ts_shutdown, Some(expected_shutdown));
1989 assert_eq!(
1990 control.events(),
1991 vec![
1992 "components.registered",
1993 "event_store.restore",
1994 "event_store.open",
1995 "actor.load:FAIL-SAVE-ACTOR",
1996 "strategy.load:FAIL-SAVE-STRATEGY-001",
1997 "actor.on_start",
1998 "strategy.on_start",
1999 "actor.on_stop",
2000 "strategy.on_stop",
2001 "actor.on_save",
2002 "strategy.on_save",
2003 "event_store.seal",
2004 "database.close",
2005 ]
2006 );
2007 }
2008
2009 #[rstest]
2010 fn test_state_load_callback_failure_prevents_start_and_save() {
2011 let actor_id = ActorId::from("FAIL-LOAD-ACTOR");
2012 let strategy_id = StrategyId::from("FAIL-LOAD-STRATEGY-001");
2013 let (database, control) = TestCacheDatabaseControl::create();
2014 control.set_actor_state(actor_id, &state("actor", b"load"));
2015 control.set_strategy_state(strategy_id, &state("strategy", b"load"));
2016 let mut kernel = NautilusKernelBuilder::default()
2017 .with_cache_database(Box::new(database))
2018 .build()
2019 .unwrap();
2020 let actor = StateActor::new(actor_id, control.clone(), IndexMap::new()).with_fail_load();
2021 let strategy = StateStrategy::new(strategy_id, control.clone(), IndexMap::new());
2022 add_state_components(&kernel, &control, actor, strategy);
2023
2024 kernel.start();
2025 let error = kernel.start_trader().unwrap_err();
2026 kernel.dispose();
2027
2028 assert_eq!(
2029 error.to_string(),
2030 "Failed to load actor and strategy state: Failed to restore actor FAIL-LOAD-ACTOR \
2031 state: test actor on_load failure"
2032 );
2033 assert_eq!(
2034 control.events(),
2035 vec![
2036 "components.registered",
2037 "actor.load:FAIL-LOAD-ACTOR",
2038 "actor.on_load",
2039 "database.close",
2040 ]
2041 );
2042 }
2043
2044 #[rstest]
2045 fn test_state_save_persistence_errors_do_not_fail_finalization() {
2046 let actor_id = ActorId::from("FAIL-UPDATE-ACTOR");
2047 let strategy_id = StrategyId::from("FAIL-UPDATE-STRATEGY-001");
2048 let (database, control) = TestCacheDatabaseControl::create();
2049 control.set_fail_update_actor(true);
2050 control.set_fail_update_strategy(true);
2051 let mut kernel = NautilusKernelBuilder::default()
2052 .with_cache_database(Box::new(database))
2053 .build()
2054 .unwrap();
2055 let actor = StateActor::new(actor_id, control.clone(), state("actor", b"save"));
2056 let strategy = StateStrategy::new(strategy_id, control.clone(), state("strategy", b"save"));
2057 add_state_components(&kernel, &control, actor, strategy);
2058
2059 kernel.start();
2060 kernel.start_trader().unwrap();
2061 kernel.stop_trader();
2062 finalize(&mut kernel).unwrap();
2063 kernel.dispose();
2064
2065 assert_eq!(
2066 control.events(),
2067 vec![
2068 "components.registered",
2069 "actor.load:FAIL-UPDATE-ACTOR",
2070 "strategy.load:FAIL-UPDATE-STRATEGY-001",
2071 "actor.on_start",
2072 "strategy.on_start",
2073 "actor.on_stop",
2074 "strategy.on_stop",
2075 "actor.on_save",
2076 "actor.update:FAIL-UPDATE-ACTOR",
2077 "strategy.on_save",
2078 "strategy.update:FAIL-UPDATE-STRATEGY-001",
2079 "database.close",
2080 ]
2081 );
2082 }
2083
2084 #[rstest]
2085 fn test_partial_startup_stops_and_saves_once() {
2086 let actor_id = ActorId::from("PARTIAL-ACTOR");
2087 let strategy_id = StrategyId::from("PARTIAL-STRATEGY-001");
2088 let (database, control) = TestCacheDatabaseControl::create();
2089 let mut kernel = NautilusKernelBuilder::default()
2090 .with_cache_database(Box::new(database))
2091 .build()
2092 .unwrap();
2093 let actor = StateActor::new(actor_id, control.clone(), state("actor", b"partial"));
2094 let strategy =
2095 StateStrategy::new(strategy_id, control.clone(), state("strategy", b"partial"))
2096 .with_fail_start();
2097 add_state_components(&kernel, &control, actor, strategy);
2098
2099 kernel.start();
2100 let error = kernel.start_trader().unwrap_err();
2101 kernel.dispose();
2102
2103 assert_eq!(
2104 error.to_string(),
2105 "Failed to start trader: test strategy on_start failure"
2106 );
2107 assert_eq!(
2108 control.events(),
2109 vec![
2110 "components.registered",
2111 "actor.load:PARTIAL-ACTOR",
2112 "strategy.load:PARTIAL-STRATEGY-001",
2113 "actor.on_start",
2114 "strategy.on_start",
2115 "actor.on_stop",
2116 "strategy.on_stop",
2117 "actor.on_save",
2118 "actor.update:PARTIAL-ACTOR",
2119 "strategy.on_save",
2120 "strategy.update:PARTIAL-STRATEGY-001",
2121 "database.close",
2122 ]
2123 );
2124 assert_eq!(
2125 control.actor_state(&actor_id),
2126 Some(state("actor", b"partial"))
2127 );
2128 assert_eq!(
2129 control.strategy_state(&strategy_id),
2130 Some(state("strategy", b"partial"))
2131 );
2132 }
2133
2134 #[rstest]
2135 fn test_forced_dispose_stops_and_saves_once() {
2136 let actor_id = ActorId::from("FORCED-ACTOR");
2137 let strategy_id = StrategyId::from("FORCED-STRATEGY-001");
2138 let (database, control) = TestCacheDatabaseControl::create();
2139 let mut kernel = NautilusKernelBuilder::default()
2140 .with_cache_database(Box::new(database))
2141 .build()
2142 .unwrap();
2143 let actor = StateActor::new(actor_id, control.clone(), state("actor", b"forced"));
2144 let strategy =
2145 StateStrategy::new(strategy_id, control.clone(), state("strategy", b"forced"));
2146 add_state_components(&kernel, &control, actor, strategy);
2147
2148 kernel.start();
2149 kernel.start_trader().unwrap();
2150 kernel.dispose();
2151
2152 assert_eq!(
2153 control.events(),
2154 vec![
2155 "components.registered",
2156 "actor.load:FORCED-ACTOR",
2157 "strategy.load:FORCED-STRATEGY-001",
2158 "actor.on_start",
2159 "strategy.on_start",
2160 "actor.on_stop",
2161 "strategy.on_stop",
2162 "actor.on_save",
2163 "actor.update:FORCED-ACTOR",
2164 "strategy.on_save",
2165 "strategy.update:FORCED-STRATEGY-001",
2166 "database.close",
2167 ]
2168 );
2169 }
2170
2171 #[rstest]
2172 fn test_start_trader_starts_order_emulator_for_cached_emulated_orders() {
2173 let mut kernel = NautilusKernelBuilder::default().build().unwrap();
2174 let data_commands = register_data_command_handler("DataEngine.queue_execute.kernel_start");
2175 let instrument = crypto_perpetual_ethusdt();
2176 let instrument_id = instrument.id();
2177 let first_order = create_stop_market_order(&instrument, "O-KERNEL-001");
2178 let second_order = create_stop_market_order(&instrument, "O-KERNEL-002");
2179 let first_client_order_id = first_order.client_order_id();
2180 let second_client_order_id = second_order.client_order_id();
2181 kernel
2182 .cache
2183 .borrow_mut()
2184 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
2185 .unwrap();
2186 kernel
2187 .cache
2188 .borrow_mut()
2189 .add_order(first_order, None, None, false)
2190 .unwrap();
2191 kernel
2192 .cache
2193 .borrow_mut()
2194 .add_order(second_order, None, None, false)
2195 .unwrap();
2196
2197 kernel.start();
2198 assert!(
2199 kernel
2200 .order_emulator
2201 .get_emulator()
2202 .get_matching_core(&instrument_id)
2203 .is_none()
2204 );
2205 kernel.start_trader().unwrap();
2206
2207 let commands = data_commands.get_messages();
2208 let cache = kernel.cache.borrow();
2209 let first_status = cache.order(&first_client_order_id).unwrap().status();
2210 let second_status = cache.order(&second_client_order_id).unwrap().status();
2211 drop(cache);
2212 let emulator = kernel.order_emulator.get_emulator();
2213 assert!(emulator.get_matching_core(&instrument_id).is_some());
2214 assert_eq!(emulator.subscribed_quotes(), vec![instrument_id]);
2215 assert_eq!(first_status, OrderStatus::Emulated);
2216 assert_eq!(second_status, OrderStatus::Emulated);
2217 assert!(commands.iter().any(|command| matches!(
2218 command,
2219 DataCommand::Subscribe(SubscribeCommand::Quotes(command))
2220 if command.instrument_ids == [instrument_id]
2221 )));
2222
2223 data_commands.clear();
2224 drop(emulator);
2225 kernel.stop_trader();
2226 kernel.dispose();
2227
2228 let commands = data_commands.get_messages();
2229 let emulator = kernel.order_emulator.get_emulator();
2230 assert!(emulator.subscribed_quotes().is_empty());
2231 assert!(emulator.get_matching_core(&instrument_id).is_none());
2232 assert!(commands.iter().any(|command| matches!(
2233 command,
2234 DataCommand::Unsubscribe(UnsubscribeCommand::Quotes(command))
2235 if command.instrument_ids == [instrument_id]
2236 )));
2237 }
2238
2239 #[rstest]
2240 fn test_reset_resets_order_emulator_state() {
2241 let mut kernel = NautilusKernelBuilder::default().build().unwrap();
2242 let data_commands = register_data_command_handler("DataEngine.queue_execute.kernel_reset");
2243 let instrument = crypto_perpetual_ethusdt();
2244 let instrument_id = instrument.id();
2245 let order = create_stop_market_order(&instrument, "O-KERNEL-RESET-001");
2246 kernel
2247 .cache
2248 .borrow_mut()
2249 .add_instrument(InstrumentAny::CryptoPerpetual(instrument))
2250 .unwrap();
2251 kernel
2252 .cache
2253 .borrow_mut()
2254 .add_order(order, None, None, false)
2255 .unwrap();
2256
2257 kernel.start();
2258 kernel.start_trader().unwrap();
2259 assert!(
2260 kernel
2261 .order_emulator
2262 .get_emulator()
2263 .get_matching_core(&instrument_id)
2264 .is_some()
2265 );
2266 kernel.stop_trader();
2267 data_commands.clear();
2268
2269 kernel.reset();
2270
2271 let commands = data_commands.get_messages();
2272 let emulator = kernel.order_emulator.get_emulator();
2273 assert!(emulator.subscribed_quotes().is_empty());
2274 assert!(emulator.get_matching_core(&instrument_id).is_none());
2275 assert!(commands.iter().any(|command| matches!(
2276 command,
2277 DataCommand::Unsubscribe(UnsubscribeCommand::Quotes(command))
2278 if command.instrument_ids == [instrument_id]
2279 )));
2280
2281 drop(emulator);
2282 kernel.dispose();
2283 }
2284
2285 #[rstest]
2286 fn test_start_after_reset_skips_trader_initialize() {
2287 let mut kernel = NautilusKernelBuilder::default().build().unwrap();
2288 kernel.start();
2289 kernel.start_trader().unwrap();
2290 kernel.stop_trader();
2291 kernel.reset();
2292 let restart_ns = UnixNanos::from(1_700_000_000_000_000_000);
2293 kernel
2294 .clock
2295 .borrow_mut()
2296 .as_any_mut()
2297 .downcast_mut::<VirtualClock>()
2298 .unwrap()
2299 .set_time(restart_ns);
2300
2301 kernel.start();
2302 let ts_started = kernel.ts_started();
2303 kernel.start_trader().unwrap();
2304 let trader_state = kernel.trader.borrow().state();
2305 kernel.stop_trader();
2306 kernel.dispose();
2307
2308 assert_eq!(ts_started, Some(restart_ns));
2309 assert_eq!(trader_state, ComponentState::Running);
2310 }
2311}