1use std::{
27 any::Any,
28 cell::RefCell,
29 fmt::Debug,
30 path::{Path, PathBuf},
31 rc::Rc,
32 sync::{
33 Arc,
34 atomic::{AtomicBool, AtomicU64, Ordering},
35 },
36 thread,
37 time::{Duration, Instant},
38};
39
40use bytes::Bytes;
41use nautilus_common::{
42 cache::{Cache, CacheSnapshotRef},
43 clock::Clock,
44 enums::Environment,
45 msgbus::{self, BusTap, Endpoint, MStr, MessagingSwitchboard},
46};
47#[cfg(feature = "live")]
48use nautilus_core::time::get_atomic_clock_realtime;
49use nautilus_core::{
50 UUID4, UnixNanos,
51 time::{AtomicTime, get_atomic_clock_static},
52};
53use nautilus_execution::engine::SnapshotAnchorer;
54use nautilus_system::{
55 KernelEventStore as KernelEventStoreTrait, RegisteredComponents,
56 event_store::{DataMarkerClass, DataMarkerConfig, EventStoreConfig, RetentionMode},
57};
58use parking_lot::Mutex;
59use ustr::Ustr;
60
61use crate::{
62 BusCaptureAdapter, CacheReplayError, CacheReplayReport, CaptureError, EncoderRegistry,
63 EntryDraft, EventStore, EventStoreError, EventStoreWriter, HaltCallback, HaltReason, Headers,
64 RedbBackend, RunId, RunManifest, RunStatus, ScanDirection, Topic, WriterConfig,
65 compute_snapshot_content_hash, default_registry,
66 markers::{
67 DataClass, DataMarkerCapture, DataMarkerExtractorRegistry, MarkerBackend, MarkerManifest,
68 MarkerWriter, MarkerWriterConfig, RedbMarkerBackend,
69 },
70 restore_cache_from_sealed_run, validate_event_store_replay_source,
71};
72
73const RUN_STARTED_TOPIC: &str = "run.lifecycle.RunStarted";
74const RUN_STARTED_PAYLOAD_TYPE: &str = "RunStarted";
75const RUN_ENDED_TOPIC: &str = "run.lifecycle.RunEnded";
76const RUN_ENDED_PAYLOAD_TYPE: &str = "RunEnded";
77
78#[derive(Clone, Debug, PartialEq, Eq)]
80pub struct RecoveredRun {
81 pub run_id: RunId,
83 pub status: RunStatus,
86}
87
88#[derive(Debug, Default)]
90pub struct RecoveryOutcome {
91 pub recovered: Vec<RecoveredRun>,
93 pub parent_run_id: Option<RunId>,
97}
98
99type RegistryFactory = dyn Fn() -> EncoderRegistry + Send + Sync + 'static;
100type BackendOpenResult = Result<Box<dyn EventStore + Send>, EventStoreError>;
101type BackendOpener =
102 dyn Fn(&EventStoreConfig, &RunManifest) -> BackendOpenResult + Send + Sync + 'static;
103type MarkerRegistryFactory =
104 dyn Fn(&[DataClass]) -> DataMarkerExtractorRegistry + Send + Sync + 'static;
105type SharedMarkerCapture = Rc<RefCell<Option<DataMarkerCapture>>>;
106
107#[derive(Clone)]
113pub struct EventStoreLifecycleOptions {
114 registry_factory: Arc<RegistryFactory>,
115 backend_opener: Arc<BackendOpener>,
116 marker_registry_factory: Arc<MarkerRegistryFactory>,
117}
118
119impl Debug for EventStoreLifecycleOptions {
120 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
121 f.debug_struct(stringify!(EventStoreLifecycleOptions))
122 .finish_non_exhaustive()
123 }
124}
125
126impl Default for EventStoreLifecycleOptions {
127 fn default() -> Self {
128 Self {
129 registry_factory: Arc::new(default_registry),
130 backend_opener: Arc::new(default_backend_opener),
131 marker_registry_factory: Arc::new(DataMarkerExtractorRegistry::default_registry),
132 }
133 }
134}
135
136impl EventStoreLifecycleOptions {
137 #[must_use]
139 pub fn new() -> Self {
140 Self::default()
141 }
142
143 #[must_use]
145 pub fn with_registry_factory<F>(mut self, factory: F) -> Self
146 where
147 F: Fn() -> EncoderRegistry + Send + Sync + 'static,
148 {
149 self.registry_factory = Arc::new(factory);
150 self
151 }
152
153 #[must_use]
155 pub fn with_encoder_registry(self, registry: EncoderRegistry) -> Self {
156 self.with_registry_factory(move || registry.clone())
157 }
158
159 #[must_use]
161 pub fn with_backend_opener<F>(mut self, opener: F) -> Self
162 where
163 F: Fn(&EventStoreConfig, &RunManifest) -> BackendOpenResult + Send + Sync + 'static,
164 {
165 self.backend_opener = Arc::new(opener);
166 self
167 }
168
169 #[must_use]
171 pub fn with_marker_registry_factory<F>(mut self, factory: F) -> Self
172 where
173 F: Fn(&[DataClass]) -> DataMarkerExtractorRegistry + Send + Sync + 'static,
174 {
175 self.marker_registry_factory = Arc::new(factory);
176 self
177 }
178
179 fn build_registry(&self) -> EncoderRegistry {
180 (self.registry_factory)()
181 }
182
183 fn open_backend(&self, config: &EventStoreConfig, manifest: &RunManifest) -> BackendOpenResult {
184 (self.backend_opener)(config, manifest)
185 }
186
187 fn build_marker_registry(&self, classes: &[DataClass]) -> DataMarkerExtractorRegistry {
188 (self.marker_registry_factory)(classes)
189 }
190}
191
192fn default_backend_opener(config: &EventStoreConfig, manifest: &RunManifest) -> BackendOpenResult {
193 let mut backend = RedbBackend::new(config.base_dir.clone());
194 backend.open_run(manifest.clone())?;
195 Ok(Box::new(backend))
196}
197
198#[derive(Debug, thiserror::Error)]
200pub enum BootError {
201 #[error(transparent)]
204 EventStore(#[from] EventStoreError),
205 #[error("RunStarted submit failed: {0}")]
207 RunStartedSubmit(String),
208 #[error("RunStarted did not durably commit within {timeout:?}")]
211 RunStartedTimeout {
212 timeout: Duration,
215 },
216 #[error("event store halted during boot: {0:?}")]
219 HaltedDuringBoot(HaltReason),
220}
221
222#[derive(Clone, Debug)]
229pub struct HaltSignal {
230 halted: Arc<AtomicBool>,
231 reason: Arc<Mutex<Option<HaltReason>>>,
232}
233
234impl Default for HaltSignal {
235 fn default() -> Self {
236 Self::new()
237 }
238}
239
240impl HaltSignal {
241 #[must_use]
243 pub fn new() -> Self {
244 Self {
245 halted: Arc::new(AtomicBool::new(false)),
246 reason: Arc::new(Mutex::new(None)),
247 }
248 }
249
250 #[must_use]
258 pub fn callback(&self) -> HaltCallback {
259 let halted = Arc::clone(&self.halted);
260 let reason = Arc::clone(&self.reason);
261 Arc::new(move |r| {
262 let mut slot = reason.lock();
263 if slot.is_none() {
264 *slot = Some(r);
265 }
266 drop(slot);
267 halted.store(true, Ordering::Release);
268 })
269 }
270
271 #[must_use]
273 pub fn is_halted(&self) -> bool {
274 self.halted.load(Ordering::Acquire)
275 }
276
277 #[must_use]
282 pub fn reason(&self) -> Option<HaltReason> {
283 self.reason.lock().clone()
284 }
285}
286
287pub struct EventStoreSession {
289 writer: Option<Arc<EventStoreWriter>>,
290 adapter: Option<Arc<BusCaptureAdapter>>,
291 marker_capture: Option<SharedMarkerCapture>,
292 manifest: RunManifest,
293 halt_signal: HaltSignal,
294}
295
296impl Debug for EventStoreSession {
297 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
298 f.debug_struct(stringify!(EventStoreSession))
299 .field("run_id", &self.manifest.run_id)
300 .field("parent_run_id", &self.manifest.parent_run_id)
301 .field("instance_id", &self.manifest.instance_id)
302 .field("halted", &self.halt_signal.is_halted())
303 .field("writer_attached", &self.writer.is_some())
304 .field("marker_capture_attached", &self.marker_capture.is_some())
305 .finish_non_exhaustive()
306 }
307}
308
309impl EventStoreSession {
310 #[must_use]
315 pub const fn manifest(&self) -> &RunManifest {
316 &self.manifest
317 }
318
319 #[must_use]
321 pub fn run_id(&self) -> &str {
322 self.manifest.run_id.as_str()
323 }
324
325 #[must_use]
327 pub fn parent_run_id(&self) -> Option<&str> {
328 self.manifest.parent_run_id.as_deref()
329 }
330
331 #[must_use]
333 pub fn is_halted(&self) -> bool {
334 self.halt_signal.is_halted()
335 }
336
337 #[must_use]
341 pub fn high_watermark(&self) -> u64 {
342 self.writer.as_ref().map_or(0, |w| w.high_watermark())
343 }
344
345 #[must_use]
354 pub fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
355 let writer = Arc::clone(self.writer.as_ref()?);
356
357 Some(Rc::new(move |snapshot_ref: CacheSnapshotRef| {
358 let content_hash = compute_snapshot_content_hash(snapshot_ref.blob.as_ref());
359 writer
360 .record_snapshot_anchor(snapshot_ref.blob_ref, content_hash)
361 .map(|_| ())
362 .map_err(|e| anyhow::anyhow!("record snapshot anchor: {e}"))
363 }))
364 }
365
366 #[must_use]
370 pub fn adapter(&self) -> Option<&Arc<BusCaptureAdapter>> {
371 self.adapter.as_ref()
372 }
373
374 pub fn close(&mut self, ts_init: UnixNanos) -> Result<(), EventStoreError> {
385 self.adapter = None;
389 let marker_capture = self.marker_capture.take();
390
391 let Some(writer_arc) = self.writer.take() else {
392 close_marker_capture(marker_capture);
393 return Ok(());
394 };
395 let Ok(writer) = Arc::try_unwrap(writer_arc) else {
396 close_marker_capture(marker_capture);
397 return Err(EventStoreError::Backend(
398 "event store writer has multiple owners; clear the bus tap before close"
399 .to_string(),
400 ));
401 };
402
403 let run_ended = run_ended_draft(ts_init);
404 let result = writer.close(run_ended);
405 close_marker_capture(marker_capture);
406 result?;
407 Ok(())
408 }
409}
410
411impl Drop for EventStoreSession {
412 fn drop(&mut self) {
413 self.adapter.take();
416 self.marker_capture.take();
417 self.writer.take();
418 }
419}
420
421fn close_marker_capture(marker_capture: Option<SharedMarkerCapture>) {
422 if let Some(marker_capture) = marker_capture
423 && let Some(capture) = marker_capture.borrow_mut().take()
424 {
425 capture.close();
426 }
427}
428
429#[derive(Debug, thiserror::Error)]
434pub enum KernelError {
435 #[error("event store boot failed: {0}")]
437 EventStoreBoot(#[from] BootError),
438 #[error("event store cache replay failed: {0}")]
440 CacheReplay(#[from] CacheReplayError),
441 #[error("event store halted: {0:?}")]
443 EventStoreHalted(HaltReason),
444}
445
446#[derive(Debug)]
456pub struct EventStoreLifecycle {
457 config: Option<EventStoreConfig>,
458 options: EventStoreLifecycleOptions,
459 recovered: Vec<RecoveredRun>,
460 parent_run_id: Option<String>,
461 session: Option<EventStoreSession>,
462 halt: HaltSignal,
463 clock: Rc<RefCell<dyn Clock>>,
467}
468
469impl EventStoreLifecycle {
470 pub fn boot(
480 config: Option<EventStoreConfig>,
481 instance_id: UUID4,
482 clock: Rc<RefCell<dyn Clock>>,
483 ) -> anyhow::Result<Self> {
484 Self::boot_with_options(
485 config,
486 instance_id,
487 clock,
488 EventStoreLifecycleOptions::default(),
489 )
490 }
491
492 pub fn boot_with_options(
502 config: Option<EventStoreConfig>,
503 instance_id: UUID4,
504 clock: Rc<RefCell<dyn Clock>>,
505 options: EventStoreLifecycleOptions,
506 ) -> anyhow::Result<Self> {
507 let (recovered, parent_run_id) = if let Some(cfg) = config.as_ref() {
508 let outcome = recover_predecessors(&cfg.base_dir, &instance_id.to_string())?;
509 if !outcome.recovered.is_empty() {
510 log::info!(
511 "Sealed {} crashed event-store predecessor(s); parent_run_id={:?}",
512 outcome.recovered.len(),
513 outcome.parent_run_id,
514 );
515 }
516 (outcome.recovered, outcome.parent_run_id)
517 } else {
518 (Vec::new(), None)
519 };
520 Ok(Self {
521 config,
522 options,
523 recovered,
524 parent_run_id,
525 session: None,
526 halt: HaltSignal::new(),
527 clock,
528 })
529 }
530
531 pub fn open(
546 &mut self,
547 instance_id: UUID4,
548 components: &RegisteredComponents,
549 environment: Environment,
550 ) -> Result<(), KernelError> {
551 let Some(config) = self.config.clone() else {
552 return Ok(());
553 };
554
555 if self.session.is_some() {
556 let ts = self.clock.borrow().timestamp_ns();
559 self.seal(ts);
560 }
561
562 self.halt = HaltSignal::new();
566
567 let clock = Self::clock_for(environment);
568 let start_ts_init = self.clock.borrow().timestamp_ns();
569 let run_id = build_run_id(start_ts_init);
570 let parent_run_id = if let Some(replay_run_id) = config.replay_from_run_id.as_deref() {
571 validate_event_store_replay_source(
572 config.base_dir.clone(),
573 &instance_id.to_string(),
574 replay_run_id,
575 )?;
576 Some(replay_run_id.to_string())
577 } else {
578 self.parent_run_id.clone()
579 };
580 let session = open_run_with_options(
581 &config,
582 &instance_id.to_string(),
583 run_id,
584 parent_run_id,
585 start_ts_init,
586 components,
587 self.halt.clone(),
588 clock,
589 &self.options,
590 )?;
591 log::info!(
592 "Opened event-store run {} (parent_run_id={:?})",
593 session.run_id(),
594 session.parent_run_id(),
595 );
596
597 if let Some(adapter) = session.adapter() {
598 install_bus_tap(Arc::clone(adapter), session.marker_capture.clone(), clock);
599 }
600 self.session = Some(session);
601 Ok(())
602 }
603
604 pub fn restore_parent_cache(
614 &self,
615 instance_id: UUID4,
616 cache: &mut Cache,
617 ) -> Result<Option<CacheReplayReport>, KernelError> {
618 let Some(config) = self.config.as_ref() else {
619 return Ok(None);
620 };
621 let replay_run_id = config
622 .replay_from_run_id
623 .as_deref()
624 .or(self.parent_run_id.as_deref());
625 let Some(replay_run_id) = replay_run_id else {
626 return Ok(None);
627 };
628 let source = if config.replay_from_run_id.is_some() {
629 "configured replay run"
630 } else {
631 "parent run"
632 };
633
634 let report = restore_cache_from_sealed_run(
635 cache,
636 config.base_dir.clone(),
637 &instance_id.to_string(),
638 replay_run_id,
639 )?;
640
641 log::info!(
642 "Restored cache from event-store {source} {replay_run_id}: from_seq={}, to_seq={}, applied={}, ignored={}",
643 report.cache.plan.from_seq,
644 report.cache.plan.to_seq,
645 report.cache.applied_entries,
646 report.cache.ignored_entries,
647 );
648
649 Ok(Some(report.cache))
650 }
651
652 pub fn seal(&mut self, ts_init: UnixNanos) {
656 let Some(mut session) = self.session.take() else {
657 return;
658 };
659
660 msgbus::clear_bus_tap();
663
664 if session.is_halted() {
665 log::warn!(
666 "Event-store writer fail-stopped before close; run {} sealed by recovery sweep on next boot",
667 session.run_id(),
668 );
669 return;
670 }
671 let run_id = session.run_id().to_string();
672 if let Err(e) = session.close(ts_init) {
673 log::error!(
674 "Failed to seal event-store run {run_id} on graceful stop: {e}; run will be sealed as CrashedRecovered on next boot",
675 );
676 } else {
677 log::info!("Sealed event-store run {run_id}");
678 }
679 }
680
681 #[must_use]
683 pub fn recovered(&self) -> &[RecoveredRun] {
684 &self.recovered
685 }
686
687 #[must_use]
689 pub fn parent_run_id(&self) -> Option<&str> {
690 self.config
691 .as_ref()
692 .and_then(|config| config.replay_from_run_id.as_deref())
693 .or(self.parent_run_id.as_deref())
694 }
695
696 #[must_use]
698 pub fn is_event_store_replay_configured(&self) -> bool {
699 self.config
700 .as_ref()
701 .is_some_and(|config| config.replay_from_run_id.is_some())
702 }
703
704 #[must_use]
706 pub fn run_id(&self) -> Option<&str> {
707 self.session.as_ref().map(EventStoreSession::run_id)
708 }
709
710 #[must_use]
712 pub fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
713 self.session
714 .as_ref()
715 .and_then(EventStoreSession::snapshot_anchorer)
716 }
717
718 #[must_use]
720 pub fn is_halted(&self) -> bool {
721 self.halt.is_halted()
722 }
723
724 #[must_use]
726 pub fn halt_reason(&self) -> Option<HaltReason> {
727 self.halt.reason()
728 }
729
730 #[must_use]
733 pub fn check_halt(&self) -> Option<KernelError> {
734 self.halt_reason().map(KernelError::EventStoreHalted)
735 }
736
737 #[cfg(feature = "live")]
738 fn clock_for(environment: Environment) -> &'static AtomicTime {
739 match environment {
740 Environment::Backtest => get_atomic_clock_static(),
741 Environment::Live | Environment::Sandbox => get_atomic_clock_realtime(),
742 }
743 }
744
745 #[cfg(not(feature = "live"))]
746 fn clock_for(_environment: Environment) -> &'static AtomicTime {
747 get_atomic_clock_static()
748 }
749}
750
751impl Drop for EventStoreLifecycle {
752 fn drop(&mut self) {
753 if self.session.is_none() {
755 return;
756 }
757 let ts = self
758 .clock
759 .try_borrow()
760 .map(|c| c.timestamp_ns())
761 .unwrap_or_default();
762 self.seal(ts);
763 }
764}
765
766pub fn recover_predecessors(
788 base_dir: &Path,
789 instance_id: &str,
790) -> Result<RecoveryOutcome, EventStoreError> {
791 let manifests = RedbBackend::list_runs(base_dir, instance_id)?;
792 let crashed: Vec<RunManifest> = manifests
793 .into_iter()
794 .filter(|m| matches!(m.status, RunStatus::Running))
795 .collect();
796
797 let mut outcome = RecoveryOutcome::default();
798
799 for predecessor in crashed {
800 let run_id = predecessor.run_id.clone();
801 let mut backend = RedbBackend::new(base_dir.to_path_buf());
802
803 match backend.open_run(predecessor) {
804 Err(EventStoreError::CrashedPredecessor) => {}
805 Ok(()) => {
806 return Err(EventStoreError::Backend(format!(
807 "expected CrashedPredecessor reopening {run_id}, was Ok",
808 )));
809 }
810 Err(other) => {
811 log::error!("Skipping recovery of run {run_id}, reopen failed: {other}");
812 continue;
813 }
814 }
815
816 let high_watermark = backend.high_watermark()?;
817 let final_status = if high_watermark == 0 {
818 RunStatus::CrashedRecovered
819 } else {
820 match backend.scan_range(1, high_watermark, ScanDirection::Forward) {
821 Ok(entries) => {
822 let tail_is_run_ended = entries.last().is_some_and(|e| {
830 e.topic.as_ref() == RUN_ENDED_TOPIC
831 && e.payload_type.as_str() == RUN_ENDED_PAYLOAD_TYPE
832 });
833
834 if tail_is_run_ended {
835 RunStatus::Ended
836 } else {
837 RunStatus::CrashedRecovered
838 }
839 }
840 Err(
841 EventStoreError::HashMismatch { .. }
842 | EventStoreError::SeqMismatch { .. }
843 | EventStoreError::Corrupted(_)
844 | EventStoreError::Gap { .. },
845 ) => RunStatus::Quarantined,
846 Err(other) => {
847 log::error!("Skipping recovery of run {run_id}, scan failed: {other}");
848 continue;
849 }
850 }
851 };
852
853 if let Err(e) = backend.seal(final_status) {
854 log::error!("Skipping recovery of run {run_id}, seal as {final_status:?} failed: {e}");
855 continue;
856 }
857 outcome.recovered.push(RecoveredRun {
858 run_id: run_id.clone(),
859 status: final_status,
860 });
861
862 if matches!(final_status, RunStatus::CrashedRecovered) {
863 outcome.parent_run_id = Some(run_id);
864 }
865 }
866
867 Ok(outcome)
868}
869
870#[must_use]
877pub fn build_run_id(start_ts_init: UnixNanos) -> RunId {
878 let suffix: String = UUID4::new().to_string().chars().take(8).collect();
879 format!("{}-{suffix}", u64::from(start_ts_init))
880}
881
882#[expect(
899 clippy::too_many_arguments,
900 reason = "run opening requires the boot context and event-store handles"
901)]
902pub fn open_run(
903 config: &EventStoreConfig,
904 instance_id: &str,
905 run_id: RunId,
906 parent_run_id: Option<RunId>,
907 start_ts_init: UnixNanos,
908 components: &RegisteredComponents,
909 halt_signal: HaltSignal,
910 clock: &'static AtomicTime,
911) -> Result<EventStoreSession, BootError> {
912 open_run_with_options(
913 config,
914 instance_id,
915 run_id,
916 parent_run_id,
917 start_ts_init,
918 components,
919 halt_signal,
920 clock,
921 &EventStoreLifecycleOptions::default(),
922 )
923}
924
925#[expect(
936 clippy::too_many_arguments,
937 reason = "run opening with options requires the boot context and lifecycle handles"
938)]
939pub fn open_run_with_options(
940 config: &EventStoreConfig,
941 instance_id: &str,
942 run_id: RunId,
943 parent_run_id: Option<RunId>,
944 start_ts_init: UnixNanos,
945 components: &RegisteredComponents,
946 halt_signal: HaltSignal,
947 clock: &'static AtomicTime,
948 options: &EventStoreLifecycleOptions,
949) -> Result<EventStoreSession, BootError> {
950 let manifest = build_manifest(
951 config,
952 instance_id,
953 run_id,
954 parent_run_id,
955 start_ts_init,
956 components.clone(),
957 );
958
959 let backend = options.open_backend(config, &manifest)?;
960
961 let writer = Arc::new(EventStoreWriter::spawn(
962 backend,
963 clock,
964 halt_signal.callback(),
965 writer_config_from(config),
966 )?);
967
968 submit_run_started_blocking(
969 &writer,
970 components,
971 start_ts_init,
972 &halt_signal,
973 config.run_started_timeout,
974 )?;
975
976 let (marker_capture, submit_counter) =
977 build_marker_capture(config, &manifest, writer.high_watermark(), clock, options);
978 let mut adapter = BusCaptureAdapter::new(
979 Arc::clone(&writer),
980 Arc::new(options.build_registry()),
981 halt_signal.callback(),
982 );
983
984 if let Some(submit_counter) = submit_counter {
985 adapter = adapter.with_submit_counter(submit_counter);
986 }
987 let adapter = Arc::new(adapter);
988
989 Ok(EventStoreSession {
990 writer: Some(writer),
991 adapter: Some(adapter),
992 marker_capture,
993 manifest,
994 halt_signal,
995 })
996}
997
998fn build_marker_capture(
999 config: &EventStoreConfig,
1000 manifest: &RunManifest,
1001 initial_submit_counter: u64,
1002 clock: &'static AtomicTime,
1003 options: &EventStoreLifecycleOptions,
1004) -> (Option<SharedMarkerCapture>, Option<Arc<AtomicU64>>) {
1005 let Some(marker_config) = config.data_markers.as_ref() else {
1006 return (None, None);
1007 };
1008
1009 match open_marker_capture(
1010 config,
1011 manifest,
1012 marker_config,
1013 initial_submit_counter,
1014 clock,
1015 options,
1016 ) {
1017 Ok((capture, submit_counter)) => (
1018 Some(Rc::new(RefCell::new(Some(capture)))),
1019 Some(submit_counter),
1020 ),
1021 Err(e) => {
1022 log::warn!(
1023 "Data marker sidecar disabled for run {} after marker setup failed: {e}",
1024 manifest.run_id,
1025 );
1026 (None, None)
1027 }
1028 }
1029}
1030
1031fn open_marker_capture(
1032 config: &EventStoreConfig,
1033 manifest: &RunManifest,
1034 marker_config: &DataMarkerConfig,
1035 initial_submit_counter: u64,
1036 clock: &'static AtomicTime,
1037 options: &EventStoreLifecycleOptions,
1038) -> Result<(DataMarkerCapture, Arc<AtomicU64>), EventStoreError> {
1039 let classes = marker_config
1040 .classes
1041 .iter()
1042 .copied()
1043 .map(data_marker_class_to_data_class)
1044 .collect::<Vec<_>>();
1045 let marker_manifest = marker_manifest_for(manifest, classes.clone(), marker_config);
1046 let marker_path = marker_file_path(config, &manifest.instance_id, &manifest.run_id);
1047 let mut marker_backend = RedbMarkerBackend::new(marker_path);
1048 marker_backend.open_run(marker_manifest)?;
1049 let writer = MarkerWriter::spawn(
1050 Box::new(marker_backend),
1051 clock,
1052 MarkerWriterConfig {
1053 channel_capacity: marker_config.channel_capacity,
1054 ..MarkerWriterConfig::default()
1055 },
1056 )?;
1057 let submit_counter = Arc::new(AtomicU64::new(initial_submit_counter));
1058 let registry = options.build_marker_registry(&classes);
1059 let capture =
1060 DataMarkerCapture::new(registry, writer, Arc::clone(&submit_counter), marker_config);
1061
1062 Ok((capture, submit_counter))
1063}
1064
1065fn marker_file_path(config: &EventStoreConfig, instance_id: &str, run_id: &str) -> PathBuf {
1066 config
1067 .base_dir
1068 .join(instance_id)
1069 .join(format!("{run_id}.markers.redb"))
1070}
1071
1072fn marker_manifest_for(
1073 manifest: &RunManifest,
1074 enabled_classes: Vec<DataClass>,
1075 config: &DataMarkerConfig,
1076) -> MarkerManifest {
1077 MarkerManifest {
1078 run_id: manifest.run_id.clone(),
1079 enabled_classes,
1080 high_fidelity: !config.high_fidelity.is_empty(),
1081 snapshot_count: 0,
1082 hifi_count: 0,
1083 gap_count: 0,
1084 dict_count: 0,
1085 status: RunStatus::Running,
1086 }
1087}
1088
1089const fn data_marker_class_to_data_class(class: DataMarkerClass) -> DataClass {
1090 match class {
1091 DataMarkerClass::BookDeltas => DataClass::BookDeltas,
1092 DataMarkerClass::BookDepth => DataClass::BookDepth,
1093 DataMarkerClass::Quote => DataClass::Quote,
1094 DataMarkerClass::Trade => DataClass::Trade,
1095 DataMarkerClass::Bar => DataClass::Bar,
1096 }
1097}
1098
1099fn build_manifest(
1100 config: &EventStoreConfig,
1101 instance_id: &str,
1102 run_id: RunId,
1103 parent_run_id: Option<RunId>,
1104 start_ts_init: UnixNanos,
1105 components: RegisteredComponents,
1106) -> RunManifest {
1107 let mut feature_flags = config.identity.feature_flags.clone();
1108 feature_flags.push(format!("retention={}", retention_tag(config.retention)));
1109
1110 RunManifest {
1111 run_id,
1112 parent_run_id,
1113 instance_id: instance_id.to_string(),
1114 binary_hash: config.identity.binary_hash.clone(),
1115 schema_version: config.identity.schema_version,
1116 crate_versions: config.identity.crate_versions.clone(),
1117 feature_flags,
1118 adapter_versions: config.identity.adapter_versions.clone(),
1119 config_hash: config.identity.config_hash.clone(),
1120 registered_components: components,
1121 seed: config.identity.seed,
1122 start_ts_init,
1123 end_ts_init: None,
1124 high_watermark: 0,
1125 status: RunStatus::Running,
1126 }
1127}
1128
1129const fn retention_tag(mode: RetentionMode) -> &'static str {
1130 match mode {
1131 RetentionMode::Full => "full",
1132 RetentionMode::Bounded { .. } => "bounded",
1133 RetentionMode::SnapshotAnchored => "snapshot",
1134 }
1135}
1136
1137fn writer_config_from(config: &EventStoreConfig) -> WriterConfig {
1138 WriterConfig {
1139 channel_capacity: config.channel_capacity,
1140 max_batch_entries: config.max_batch_entries,
1141 max_batch_latency: config.max_batch_latency,
1142 halt_threshold: config.halt_threshold,
1143 }
1144}
1145
1146pub(crate) fn submit_run_started_blocking(
1157 writer: &EventStoreWriter,
1158 components: &RegisteredComponents,
1159 ts_init: UnixNanos,
1160 halt_signal: &HaltSignal,
1161 timeout: Duration,
1162) -> Result<(), BootError> {
1163 let payload = encode_run_started(components);
1164 let draft = EntryDraft::without_indices(
1165 Headers::empty(),
1166 Topic::from(RUN_STARTED_TOPIC),
1167 Ustr::from(RUN_STARTED_PAYLOAD_TYPE),
1168 payload,
1169 ts_init,
1170 );
1171
1172 writer
1173 .submit(draft)
1174 .map_err(|e| BootError::RunStartedSubmit(e.to_string()))?;
1175
1176 let start = Instant::now(); while writer.high_watermark() == 0 {
1181 if halt_signal.is_halted() {
1182 return Err(BootError::HaltedDuringBoot(
1183 halt_signal.reason().unwrap_or_else(|| {
1184 HaltReason::BackendError("event store halted during boot".to_string())
1185 }),
1186 ));
1187 }
1188
1189 let elapsed = start.elapsed();
1190
1191 if elapsed >= timeout {
1192 return Err(BootError::RunStartedTimeout { timeout });
1193 }
1194 thread::sleep(Duration::from_millis(1));
1195 }
1196
1197 Ok(())
1198}
1199
1200fn encode_run_started(components: &RegisteredComponents) -> Bytes {
1201 let bytes = crate::codec::encode_to_vec(components).expect(
1203 "RegisteredComponents serializes via serde, must not fail under the positional codec",
1204 );
1205 Bytes::from(bytes)
1206}
1207
1208fn run_ended_draft(ts_init: UnixNanos) -> EntryDraft {
1209 EntryDraft::without_indices(
1210 Headers::empty(),
1211 Topic::from(RUN_ENDED_TOPIC),
1212 Ustr::from(RUN_ENDED_PAYLOAD_TYPE),
1213 Bytes::new(),
1214 ts_init,
1215 )
1216}
1217
1218struct EventStoreBusTap {
1225 adapter: Arc<BusCaptureAdapter>,
1226 marker_capture: Option<SharedMarkerCapture>,
1227 clock: &'static AtomicTime,
1228 halted_logged: AtomicBool,
1231}
1232
1233impl Debug for EventStoreBusTap {
1234 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1235 f.debug_struct(stringify!(EventStoreBusTap))
1236 .field("halted", &self.adapter.is_halted())
1237 .field("marker_capture_attached", &self.marker_capture.is_some())
1238 .finish_non_exhaustive()
1239 }
1240}
1241
1242impl BusTap for EventStoreBusTap {
1243 fn on_publish(&self, topic: Topic, message: &dyn Any) {
1244 let ts_init = self.clock.get_time_ns();
1245 self.capture(topic, message, ts_init);
1246 }
1247
1248 fn on_send(&self, endpoint: MStr<Endpoint>, message: &dyn Any) {
1249 let ts_init = self.clock.get_time_ns();
1250 let topic = Topic::from(*endpoint);
1253 self.capture(topic, message, ts_init);
1254 }
1255
1256 fn on_response(&self, _correlation_id: &UUID4, message: &dyn Any) {
1257 let ts_init = self.clock.get_time_ns();
1258 let topic = MessagingSwitchboard::data_response_topic();
1259 self.capture(topic, message, ts_init);
1260 }
1261}
1262
1263impl EventStoreBusTap {
1264 fn capture(&self, topic: Topic, message: &dyn Any, ts_init: UnixNanos) {
1265 let headers = self
1270 .adapter
1271 .registry()
1272 .headers_for_any(message)
1273 .unwrap_or_else(Headers::empty);
1274 match self.adapter.capture_any(topic, message, headers, ts_init) {
1277 Ok(captured) => {
1278 self.capture_marker(topic, message, ts_init, captured);
1279 }
1280 Err(CaptureError::Halted) => {
1281 if !self.halted_logged.swap(true, Ordering::AcqRel) {
1282 log::error!(
1283 "Event store capture is halted; state-affecting messages are no longer recorded for this run"
1284 );
1285 }
1286 }
1287 Err(CaptureError::Submit(e)) => {
1288 log::error!("Event store capture submit failed on {topic}: {e}");
1289 }
1290 Err(CaptureError::Encode(e)) => {
1291 log::warn!("Event store encoder rejected message on {topic}: {e}");
1292 }
1293 }
1294 }
1295
1296 fn capture_marker(&self, topic: Topic, message: &dyn Any, ts_init: UnixNanos, captured: bool) {
1297 let Some(marker_capture) = self.marker_capture.as_ref() else {
1298 return;
1299 };
1300 let mut marker_capture = marker_capture.borrow_mut();
1301 let Some(capture) = marker_capture.as_mut() else {
1302 return;
1303 };
1304
1305 if captured {
1306 capture.on_entry_submitted(ts_init);
1307 } else {
1308 capture.observe_publish(topic, message, ts_init);
1309 }
1310 capture.maybe_safety_flush(ts_init);
1311 }
1312}
1313
1314fn install_bus_tap(
1315 adapter: Arc<BusCaptureAdapter>,
1316 marker_capture: Option<SharedMarkerCapture>,
1317 clock: &'static AtomicTime,
1318) {
1319 let tap: Rc<dyn BusTap> = Rc::new(EventStoreBusTap {
1320 adapter,
1321 marker_capture,
1322 clock,
1323 halted_logged: AtomicBool::new(false),
1324 });
1325 msgbus::set_bus_tap(tap);
1326}
1327
1328#[expect(
1331 clippy::use_self,
1332 reason = "Self would dispatch back into this trait impl"
1333)]
1334impl KernelEventStoreTrait for EventStoreLifecycle {
1335 fn restore_parent_cache(
1336 &mut self,
1337 instance_id: UUID4,
1338 cache: &mut Cache,
1339 ) -> anyhow::Result<()> {
1340 EventStoreLifecycle::restore_parent_cache(self, instance_id, cache)
1341 .map(|_| ())
1342 .map_err(Into::into)
1343 }
1344
1345 fn open(
1346 &mut self,
1347 instance_id: UUID4,
1348 components: &RegisteredComponents,
1349 environment: Environment,
1350 ) -> anyhow::Result<()> {
1351 EventStoreLifecycle::open(self, instance_id, components, environment).map_err(Into::into)
1352 }
1353
1354 fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
1355 EventStoreLifecycle::snapshot_anchorer(self)
1356 }
1357
1358 fn seal(&mut self, ts_init: UnixNanos) {
1359 EventStoreLifecycle::seal(self, ts_init);
1360 }
1361
1362 fn run_id(&self) -> Option<&str> {
1363 EventStoreLifecycle::run_id(self)
1364 }
1365
1366 fn parent_run_id(&self) -> Option<&str> {
1367 EventStoreLifecycle::parent_run_id(self)
1368 }
1369
1370 fn is_event_store_replay_configured(&self) -> bool {
1371 EventStoreLifecycle::is_event_store_replay_configured(self)
1372 }
1373
1374 fn is_halted(&self) -> bool {
1375 EventStoreLifecycle::is_halted(self)
1376 }
1377}
1378
1379#[cfg(test)]
1380mod tests {
1381 #[cfg(madsim)]
1382 use std::path::Path;
1383 use std::path::PathBuf;
1384
1385 use indexmap::IndexMap;
1386 use nautilus_common::{
1387 clock::VirtualClock,
1388 messages::{
1389 data::{
1390 DataCommand, DataResponse, DataResponseEnvelope, QuotesResponse, RequestCommand,
1391 RequestQuotes, SubscribeCommand, SubscribeQuotes,
1392 },
1393 execution::{SubmitOrder, TradingCommand},
1394 },
1395 msgbus::TypedHandler,
1396 timer::{TimeEvent, TimeEventCallback, TimeEventHandler},
1397 };
1398 use nautilus_core::time::get_atomic_clock_static;
1399 use nautilus_model::{
1400 data::stubs::{quote_ethusdt_binance, stub_deltas},
1401 enums::TimeInForce,
1402 events::{
1403 OrderEventAny, OrderFilled,
1404 order::spec::{OrderFilledSpec, OrderInitializedSpec},
1405 },
1406 identifiers::{
1407 AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, TradeId, TraderId, Venue,
1408 VenueOrderId,
1409 },
1410 types::{Currency, Money, Price, Quantity},
1411 };
1412 use nautilus_system::event_store::{DataMarkerClass, DataMarkerConfig, RunIdentity};
1413 use rstest::rstest;
1414 use tempfile::TempDir;
1415
1416 use super::*;
1417 use crate::{
1418 AppendEntry, DataClass, EncodedPayload, EventStoreEntry, IndexKind, MarkerBackend,
1419 MemoryBackend, RedbMarkerBackend, SnapshotAnchor,
1420 capture::builtins::PAYLOAD_TYPE_TIME_EVENT, compute_entry_hash,
1421 };
1422
1423 const INSTANCE_ID: &str = "trader-001";
1424
1425 fn make_config(base_dir: PathBuf) -> EventStoreConfig {
1426 EventStoreConfig {
1427 base_dir,
1428 identity: RunIdentity {
1429 binary_hash: "deadbeef".to_string(),
1430 schema_version: 1,
1431 crate_versions: "feedface".to_string(),
1432 feature_flags: Vec::new(),
1433 adapter_versions: IndexMap::new(),
1434 config_hash: "cafebabe".to_string(),
1435 seed: None,
1436 },
1437 retention: RetentionMode::Full,
1438 replay_from_run_id: None,
1439 data_markers: None,
1440 channel_capacity: 64,
1441 max_batch_entries: 1,
1442 max_batch_latency: Duration::from_millis(2),
1443 halt_threshold: Duration::from_secs(2),
1444 run_started_timeout: Duration::from_secs(2),
1445 }
1446 }
1447
1448 #[derive(Clone, Copy, Debug)]
1449 enum CrashPoint {
1450 BeforeEnqueue,
1451 AfterEnqueueBeforeCommit,
1452 AfterCommitBeforeSnapshot,
1453 AfterSnapshot,
1454 }
1455
1456 fn append_entry(seq: u64, topic: &str, payload_type: &str, payload: Bytes) -> AppendEntry {
1457 let ts = UnixNanos::from(seq);
1458 let headers = Headers::empty();
1459 let hash = compute_entry_hash(seq, ts, ts, topic, payload_type, &payload, &headers);
1460 let entry = EventStoreEntry::new(
1461 hash,
1462 seq,
1463 headers,
1464 Topic::from(topic),
1465 Ustr::from(payload_type),
1466 payload,
1467 ts,
1468 ts,
1469 );
1470 AppendEntry::without_indices(entry)
1471 }
1472
1473 fn make_submit_order(client_order_id: ClientOrderId) -> SubmitOrder {
1474 let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE");
1475 let order_init = OrderInitializedSpec::builder()
1476 .instrument_id(instrument_id)
1477 .client_order_id(client_order_id)
1478 .quantity(Quantity::from("1"))
1479 .time_in_force(TimeInForce::Gtc)
1480 .ts_event(UnixNanos::from(1))
1481 .ts_init(UnixNanos::from(2))
1482 .build();
1483 SubmitOrder::new(
1484 TraderId::from("TRADER-001"),
1485 Some(ClientId::from("BINANCE")),
1486 StrategyId::from("S-001"),
1487 instrument_id,
1488 client_order_id,
1489 order_init,
1490 None,
1491 None,
1492 None,
1493 UUID4::new(),
1494 UnixNanos::from(3),
1495 None, )
1497 }
1498
1499 fn append_run_started(seq: u64) -> AppendEntry {
1500 append_entry(
1501 seq,
1502 RUN_STARTED_TOPIC,
1503 RUN_STARTED_PAYLOAD_TYPE,
1504 encode_run_started(&RegisteredComponents::default()),
1505 )
1506 }
1507
1508 #[derive(Debug)]
1509 struct TestAuditMessage {
1510 value: u8,
1511 }
1512
1513 fn test_registry() -> EncoderRegistry {
1514 let mut registry = EncoderRegistry::new();
1515 registry.register::<TestAuditMessage, _>(Ustr::from("TestAuditMessage"), |message| {
1516 Ok(EncodedPayload::without_indices(Bytes::copy_from_slice(&[
1517 message.value,
1518 ])))
1519 });
1520 registry
1521 }
1522
1523 #[derive(Debug, Clone)]
1524 struct SharedMemoryBackend(Arc<Mutex<MemoryBackend>>);
1525
1526 impl EventStore for SharedMemoryBackend {
1527 fn open_run(&mut self, manifest: RunManifest) -> Result<(), EventStoreError> {
1528 self.0.lock().open_run(manifest)
1529 }
1530
1531 fn append_batch(&mut self, entries: &[AppendEntry]) -> Result<u64, EventStoreError> {
1532 self.0.lock().append_batch(entries)
1533 }
1534
1535 fn scan_range(
1536 &self,
1537 from: u64,
1538 to: u64,
1539 direction: ScanDirection,
1540 ) -> Result<Vec<EventStoreEntry>, EventStoreError> {
1541 self.0.lock().scan_range(from, to, direction)
1542 }
1543
1544 fn scan_seq(&self, seq: u64) -> Result<Option<EventStoreEntry>, EventStoreError> {
1545 self.0.lock().scan_seq(seq)
1546 }
1547
1548 fn lookup(&self, kind: IndexKind, key: &str) -> Result<Option<u64>, EventStoreError> {
1549 self.0.lock().lookup(kind, key)
1550 }
1551
1552 fn iter_index_keys(&self, kind: IndexKind) -> Result<Vec<(String, u64)>, EventStoreError> {
1553 self.0.lock().iter_index_keys(kind)
1554 }
1555
1556 fn record_snapshot_anchor(
1557 &mut self,
1558 anchor: SnapshotAnchor,
1559 ) -> Result<(), EventStoreError> {
1560 self.0.lock().record_snapshot_anchor(anchor)
1561 }
1562
1563 fn latest_snapshot_anchor(&self) -> Result<Option<SnapshotAnchor>, EventStoreError> {
1564 self.0.lock().latest_snapshot_anchor()
1565 }
1566
1567 fn seal(&mut self, status: RunStatus) -> Result<(), EventStoreError> {
1568 self.0.lock().seal(status)
1569 }
1570
1571 fn manifest(&self) -> Result<RunManifest, EventStoreError> {
1572 self.0.lock().manifest()
1573 }
1574
1575 fn high_watermark(&self) -> Result<u64, EventStoreError> {
1576 self.0.lock().high_watermark()
1577 }
1578 }
1579
1580 fn seed_crashed_predecessor(config: &EventStoreConfig, run_id: &str, crash_point: CrashPoint) {
1581 let mut backend = RedbBackend::new(config.base_dir.clone());
1582 backend
1583 .open_run(build_manifest(
1584 config,
1585 INSTANCE_ID,
1586 run_id.to_string(),
1587 None,
1588 UnixNanos::from(1_000),
1589 RegisteredComponents::default(),
1590 ))
1591 .expect("open predecessor");
1592
1593 match crash_point {
1594 CrashPoint::BeforeEnqueue | CrashPoint::AfterEnqueueBeforeCommit => {}
1598 CrashPoint::AfterCommitBeforeSnapshot => {
1599 backend
1600 .append_batch(&[append_run_started(1)])
1601 .expect("append committed entry");
1602 }
1603 CrashPoint::AfterSnapshot => {
1604 backend
1605 .append_batch(&[append_run_started(1)])
1606 .expect("append committed entry");
1607 backend
1608 .record_snapshot_anchor(SnapshotAnchor::new(
1609 1,
1610 "cache://snapshot/run-crash/1",
1611 "blake3:abc",
1612 ))
1613 .expect("record snapshot anchor");
1614 }
1615 }
1616 }
1617
1618 #[rstest]
1619 fn halt_signal_callback_records_first_reason() {
1620 let signal = HaltSignal::new();
1621 let cb = signal.callback();
1622 cb(HaltReason::BackendDisk("ENOSPC".to_string()));
1623 cb(HaltReason::BackendError("second".to_string()));
1624
1625 assert!(signal.is_halted());
1626 match signal.reason() {
1627 Some(HaltReason::BackendDisk(msg)) => assert!(msg.contains("ENOSPC")),
1628 other => panic!("expected first reason BackendDisk, was {other:?}"),
1629 }
1630 }
1631
1632 #[rstest]
1633 fn recover_predecessors_returns_empty_for_missing_directory() {
1634 let tmp = TempDir::new().expect("tempdir");
1635 let outcome =
1636 recover_predecessors(tmp.path(), INSTANCE_ID).expect("recover empty directory");
1637 assert!(outcome.recovered.is_empty());
1638 assert!(outcome.parent_run_id.is_none());
1639 }
1640
1641 #[rstest]
1642 fn restore_cache_snapshot_blob_rejects_hash_mismatch() {
1643 let mut cache = Cache::default();
1644 let blob = Bytes::from_static(b"snapshot");
1645 let anchor =
1646 crate::SnapshotAnchor::new(0, "cache://position-snapshots/P-1/0", "blake3:bad");
1647
1648 cache
1649 .add(&anchor.blob_ref, blob)
1650 .expect("seed snapshot blob");
1651 let err =
1652 crate::restore_cache_snapshot_blob(&mut cache, Some(&anchor)).expect_err("hash error");
1653
1654 assert!(
1655 err.to_string().contains("content_hash mismatch"),
1656 "err was: {err}",
1657 );
1658 }
1659
1660 #[rstest]
1661 fn open_run_writes_run_started_and_advances_watermark() {
1662 let tmp = TempDir::new().expect("tempdir");
1663 let config = make_config(tmp.path().to_path_buf());
1664 let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover empty");
1665 assert!(outcome.parent_run_id.is_none());
1666
1667 let halt = HaltSignal::new();
1668 let session = open_run(
1669 &config,
1670 INSTANCE_ID,
1671 build_run_id(UnixNanos::from(1_000)),
1672 outcome.parent_run_id,
1673 UnixNanos::from(1_000),
1674 &RegisteredComponents::default(),
1675 halt,
1676 get_atomic_clock_static(),
1677 )
1678 .expect("open run");
1679
1680 assert_eq!(session.high_watermark(), 1);
1682 assert_eq!(session.parent_run_id(), None);
1683
1684 let manifest = session.manifest();
1688 assert_eq!(manifest.instance_id, INSTANCE_ID);
1689 assert_eq!(manifest.status, RunStatus::Running);
1690 assert_eq!(manifest.binary_hash, "deadbeef");
1691 assert_eq!(manifest.schema_version, 1);
1692 assert_eq!(manifest.crate_versions, "feedface");
1693 assert_eq!(manifest.config_hash, "cafebabe");
1694 assert_eq!(manifest.start_ts_init, UnixNanos::from(1_000));
1695 assert_eq!(manifest.end_ts_init, None);
1696 assert!(
1697 manifest
1698 .feature_flags
1699 .contains(&"retention=full".to_string()),
1700 "feature_flags must record the retention mode, was {:?}",
1701 manifest.feature_flags,
1702 );
1703 }
1704
1705 #[rstest]
1706 fn lifecycle_options_default_registry_keeps_builtin_encoders() {
1707 let registry = EventStoreLifecycleOptions::default().build_registry();
1708
1709 assert!(registry.contains::<SubmitOrder>());
1710 assert!(registry.contains::<TradingCommand>());
1711 assert!(!registry.contains::<TestAuditMessage>());
1712 }
1713
1714 #[rstest]
1715 fn lifecycle_options_custom_registry_captures_registered_message() {
1716 let tmp = TempDir::new().expect("tempdir");
1717 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1718 let instance_id = UUID4::new();
1719 let options = EventStoreLifecycleOptions::new().with_encoder_registry(test_registry());
1720
1721 let mut store = EventStoreLifecycle::boot_with_options(
1722 Some(make_config(tmp.path().to_path_buf())),
1723 instance_id,
1724 clock_rc,
1725 options,
1726 )
1727 .expect("boot store");
1728 store
1729 .open(
1730 instance_id,
1731 &RegisteredComponents::default(),
1732 Environment::Backtest,
1733 )
1734 .expect("open run");
1735 let run_id = store.run_id().expect("run open").to_string();
1736
1737 let topic: MStr<msgbus::Topic> = MStr::from("events.test.audit");
1738 msgbus::publish_any(topic, &TestAuditMessage { value: 42 });
1739 store.seal(UnixNanos::from(0));
1740
1741 let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
1742 .expect("open sealed");
1743 let captured = sealed
1744 .scan_seq(2)
1745 .expect("scan")
1746 .expect("captured entry present");
1747
1748 assert_eq!(captured.payload_type.as_str(), "TestAuditMessage");
1749 assert_eq!(captured.topic.as_ref(), topic.as_str());
1750 assert_eq!(captured.payload.as_ref(), &[42]);
1751 }
1752
1753 #[rstest]
1754 fn lifecycle_options_memory_backend_opener_captures_and_seals() {
1755 let tmp = TempDir::new().expect("tempdir");
1756 let memory = Arc::new(Mutex::new(MemoryBackend::new()));
1757 let opener_memory = Arc::clone(&memory);
1758 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1759 let instance_id = UUID4::new();
1760 let options = EventStoreLifecycleOptions::new()
1761 .with_encoder_registry(test_registry())
1762 .with_backend_opener(move |_, manifest| {
1763 opener_memory.lock().open_run(manifest.clone())?;
1764 Ok(Box::new(SharedMemoryBackend(Arc::clone(&opener_memory))))
1765 });
1766
1767 let mut store = EventStoreLifecycle::boot_with_options(
1768 Some(make_config(tmp.path().to_path_buf())),
1769 instance_id,
1770 clock_rc,
1771 options,
1772 )
1773 .expect("boot store");
1774 store
1775 .open(
1776 instance_id,
1777 &RegisteredComponents::default(),
1778 Environment::Backtest,
1779 )
1780 .expect("open run");
1781
1782 let topic: MStr<msgbus::Topic> = MStr::from("events.test.memory");
1783 msgbus::publish_any(topic, &TestAuditMessage { value: 7 });
1784 store.seal(UnixNanos::from(1_000));
1785
1786 let backend = memory.lock();
1787 let manifest = backend.manifest().expect("manifest");
1788 let captured = backend
1789 .scan_seq(2)
1790 .expect("scan")
1791 .expect("captured entry present");
1792
1793 assert_eq!(manifest.instance_id, instance_id.to_string());
1794 assert_eq!(manifest.status, RunStatus::Ended);
1795 assert_eq!(manifest.high_watermark, 3);
1796 assert_eq!(captured.payload_type.as_str(), "TestAuditMessage");
1797 assert_eq!(captured.topic.as_ref(), topic.as_str());
1798 assert_eq!(captured.payload.as_ref(), &[7]);
1799 }
1800
1801 #[cfg(madsim)]
1802 #[rstest]
1803 fn lifecycle_options_memory_backend_opener_captures_deterministic_seq_order_under_madsim() {
1804 let first = capture_madsim_memory_lifecycle_summary(42);
1805 let second = capture_madsim_memory_lifecycle_summary(42);
1806 let expected = expected_madsim_memory_entries();
1807
1808 assert_eq!(first.entries, second.entries);
1809 assert_eq!(first.entries, expected);
1810 assert_eq!(
1811 first
1812 .entries
1813 .iter()
1814 .map(|entry| entry.seq)
1815 .collect::<Vec<_>>(),
1816 vec![1, 2, 3, 4],
1817 );
1818 assert!(
1819 first.redb_files.is_empty(),
1820 "memory opener must not create redb files, was {:?}",
1821 first.redb_files,
1822 );
1823 assert!(
1824 second.redb_files.is_empty(),
1825 "memory opener must not create redb files, was {:?}",
1826 second.redb_files,
1827 );
1828 }
1829
1830 #[cfg(madsim)]
1831 fn expected_madsim_memory_entries() -> Vec<CapturedEntrySummary> {
1832 vec![
1833 CapturedEntrySummary {
1834 seq: 1,
1835 topic: RUN_STARTED_TOPIC.to_string(),
1836 payload_type: RUN_STARTED_PAYLOAD_TYPE.to_string(),
1837 payload: encode_run_started(&RegisteredComponents::default()).to_vec(),
1838 ts_init: UnixNanos::from(0),
1839 ts_publish: UnixNanos::from(10_000),
1840 },
1841 CapturedEntrySummary {
1842 seq: 2,
1843 topic: "events.test.madsim".to_string(),
1844 payload_type: "TestAuditMessage".to_string(),
1845 payload: vec![1],
1846 ts_init: UnixNanos::from(20_000),
1847 ts_publish: UnixNanos::from(20_000),
1848 },
1849 CapturedEntrySummary {
1850 seq: 3,
1851 topic: "events.test.madsim".to_string(),
1852 payload_type: "TestAuditMessage".to_string(),
1853 payload: vec![2],
1854 ts_init: UnixNanos::from(30_000),
1855 ts_publish: UnixNanos::from(30_000),
1856 },
1857 CapturedEntrySummary {
1858 seq: 4,
1859 topic: RUN_ENDED_TOPIC.to_string(),
1860 payload_type: RUN_ENDED_PAYLOAD_TYPE.to_string(),
1861 payload: Vec::new(),
1862 ts_init: UnixNanos::from(40_000),
1863 ts_publish: UnixNanos::from(40_000),
1864 },
1865 ]
1866 }
1867
1868 #[rstest]
1869 fn open_run_with_options_surfaces_backend_opener_error() {
1870 let tmp = TempDir::new().expect("tempdir");
1871 let config = make_config(tmp.path().to_path_buf());
1872 let options = EventStoreLifecycleOptions::new().with_backend_opener(|_, _| {
1873 Err(EventStoreError::Backend(
1874 "test backend open failed".to_string(),
1875 ))
1876 });
1877
1878 let err = open_run_with_options(
1879 &config,
1880 INSTANCE_ID,
1881 "run-open-error".to_string(),
1882 None,
1883 UnixNanos::from(5_000),
1884 &RegisteredComponents::default(),
1885 HaltSignal::new(),
1886 get_atomic_clock_static(),
1887 &options,
1888 )
1889 .expect_err("backend opener error must stop run open");
1890
1891 match err {
1892 BootError::EventStore(EventStoreError::Backend(msg)) => {
1893 assert!(msg.contains("test backend open failed"));
1894 }
1895 other => panic!("expected backend open failure, was {other:?}"),
1896 }
1897 }
1898
1899 #[cfg(madsim)]
1900 #[derive(Debug, PartialEq, Eq)]
1901 struct MadsimMemoryLifecycleCapture {
1902 entries: Vec<CapturedEntrySummary>,
1903 redb_files: Vec<PathBuf>,
1904 }
1905
1906 #[cfg(madsim)]
1907 #[derive(Debug, PartialEq, Eq)]
1908 struct CapturedEntrySummary {
1909 seq: u64,
1910 topic: String,
1911 payload_type: String,
1912 payload: Vec<u8>,
1913 ts_init: UnixNanos,
1914 ts_publish: UnixNanos,
1915 }
1916
1917 #[cfg(madsim)]
1918 fn capture_madsim_memory_lifecycle_summary(seed: u64) -> MadsimMemoryLifecycleCapture {
1919 get_atomic_clock_static().set_time(UnixNanos::from(10_000));
1920
1921 let tmp = TempDir::new().expect("tempdir");
1922 let memory = Arc::new(Mutex::new(MemoryBackend::new()));
1923 let opener_memory = Arc::clone(&memory);
1924 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1925 let instance_id = UUID4::new();
1926 let mut config = make_config(tmp.path().to_path_buf());
1927 config.identity.seed = Some(seed);
1928 let options = EventStoreLifecycleOptions::new()
1929 .with_encoder_registry(test_registry())
1930 .with_backend_opener(move |_, manifest| {
1931 opener_memory.lock().open_run(manifest.clone())?;
1932 Ok(Box::new(SharedMemoryBackend(Arc::clone(&opener_memory))))
1933 });
1934
1935 let mut store =
1936 EventStoreLifecycle::boot_with_options(Some(config), instance_id, clock_rc, options)
1937 .expect("boot store");
1938 store
1939 .open(
1940 instance_id,
1941 &RegisteredComponents::default(),
1942 Environment::Backtest,
1943 )
1944 .expect("open run");
1945
1946 let topic: MStr<msgbus::Topic> = MStr::from("events.test.madsim");
1947 get_atomic_clock_static().set_time(UnixNanos::from(20_000));
1948 msgbus::publish_any(topic, &TestAuditMessage { value: 1 });
1949 get_atomic_clock_static().set_time(UnixNanos::from(30_000));
1950 msgbus::publish_any(topic, &TestAuditMessage { value: 2 });
1951 assert_eq!(
1952 store
1953 .session
1954 .as_ref()
1955 .expect("open session")
1956 .high_watermark(),
1957 3
1958 );
1959
1960 get_atomic_clock_static().set_time(UnixNanos::from(40_000));
1961 store.seal(UnixNanos::from(40_000));
1962
1963 let backend = memory.lock();
1964 let manifest = backend.manifest().expect("manifest");
1965 assert_eq!(manifest.seed, Some(seed));
1966 assert_eq!(manifest.status, RunStatus::Ended);
1967 assert_eq!(manifest.high_watermark, 4);
1968 let entries = backend
1969 .scan_range(1, manifest.high_watermark, ScanDirection::Forward)
1970 .expect("scan entries")
1971 .into_iter()
1972 .map(|entry| CapturedEntrySummary {
1973 seq: entry.seq,
1974 topic: entry.topic.as_ref().to_string(),
1975 payload_type: entry.payload_type.as_str().to_string(),
1976 payload: entry.payload.to_vec(),
1977 ts_init: entry.ts_init,
1978 ts_publish: entry.ts_publish,
1979 })
1980 .collect();
1981 drop(backend);
1982
1983 MadsimMemoryLifecycleCapture {
1984 entries,
1985 redb_files: redb_files_under(tmp.path()),
1986 }
1987 }
1988
1989 #[cfg(madsim)]
1990 fn redb_files_under(dir: &Path) -> Vec<PathBuf> {
1991 let mut paths = Vec::new();
1992 collect_redb_files(dir, &mut paths);
1993 paths.sort();
1994 paths
1995 }
1996
1997 #[cfg(madsim)]
1998 fn collect_redb_files(dir: &Path, paths: &mut Vec<PathBuf>) {
1999 for entry in std::fs::read_dir(dir).expect("read dir") {
2000 let path = entry.expect("dir entry").path();
2001 if path.is_dir() {
2002 collect_redb_files(&path, paths);
2003 } else if path
2004 .extension()
2005 .is_some_and(|extension| extension == "redb")
2006 {
2007 paths.push(path);
2008 }
2009 }
2010 }
2011
2012 #[rstest]
2013 fn close_seals_manifest_and_records_run_ended() {
2014 let tmp = TempDir::new().expect("tempdir");
2015 let config = make_config(tmp.path().to_path_buf());
2016
2017 let halt = HaltSignal::new();
2018 let mut session = open_run(
2019 &config,
2020 INSTANCE_ID,
2021 build_run_id(UnixNanos::from(2_000)),
2022 None,
2023 UnixNanos::from(2_000),
2024 &RegisteredComponents::default(),
2025 halt,
2026 get_atomic_clock_static(),
2027 )
2028 .expect("open run");
2029
2030 let run_id = session.run_id().to_string();
2031 session.close(UnixNanos::from(3_000)).expect("close");
2032
2033 let manifests = RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2034 let manifest = manifests
2035 .into_iter()
2036 .find(|m| m.run_id == run_id)
2037 .expect("manifest present");
2038 assert_eq!(manifest.status, RunStatus::Ended);
2039 assert!(manifest.high_watermark >= 2);
2040 }
2041
2042 #[rstest]
2043 fn snapshot_anchorer_persists_anchor_for_open_session() {
2044 let tmp = TempDir::new().expect("tempdir");
2045 let config = make_config(tmp.path().to_path_buf());
2046
2047 let halt = HaltSignal::new();
2048 let mut session = open_run(
2049 &config,
2050 INSTANCE_ID,
2051 build_run_id(UnixNanos::from(4_000)),
2052 None,
2053 UnixNanos::from(4_000),
2054 &RegisteredComponents::default(),
2055 halt,
2056 get_atomic_clock_static(),
2057 )
2058 .expect("open run");
2059
2060 let run_id = session.run_id().to_string();
2061
2062 {
2063 let anchorer = session.snapshot_anchorer().expect("snapshot anchorer");
2064 anchorer(CacheSnapshotRef::new(
2065 "cache://position-snapshots/P-1/0",
2066 Bytes::from_static(b"snapshot"),
2067 ))
2068 .expect("record snapshot anchor");
2069 }
2070
2071 session.close(UnixNanos::from(4_500)).expect("close");
2072
2073 let reader =
2074 RedbBackend::open_sealed(&config.base_dir, INSTANCE_ID, &run_id).expect("open sealed");
2075 let anchor = reader
2076 .latest_snapshot_anchor()
2077 .expect("latest snapshot anchor")
2078 .expect("anchor present");
2079
2080 assert_eq!(anchor.high_watermark, 1);
2081 assert_eq!(anchor.blob_ref, "cache://position-snapshots/P-1/0");
2082 assert_eq!(
2083 anchor.content_hash,
2084 compute_snapshot_content_hash(b"snapshot"),
2085 );
2086 }
2087
2088 #[rstest]
2089 fn recovery_seals_tail_ending_in_run_ended_as_ended_not_crashed() {
2090 let tmp = TempDir::new().expect("tempdir");
2099 let config = make_config(tmp.path().to_path_buf());
2100
2101 let halt = HaltSignal::new();
2102 let run_id = build_run_id(UnixNanos::from(7_000));
2103 let session = open_run(
2104 &config,
2105 INSTANCE_ID,
2106 run_id.clone(),
2107 None,
2108 UnixNanos::from(7_000),
2109 &RegisteredComponents::default(),
2110 halt,
2111 get_atomic_clock_static(),
2112 )
2113 .expect("open run");
2114
2115 let writer = session.writer.as_ref().expect("writer attached");
2116 writer
2117 .submit(run_ended_draft(UnixNanos::from(7_500)))
2118 .expect("submit RunEnded as tail entry");
2119 let deadline = Instant::now() + Duration::from_secs(2);
2123
2124 while session.high_watermark() < 2 {
2125 assert!(
2126 Instant::now() < deadline,
2127 "writer high_watermark stuck at {} before deadline",
2128 session.high_watermark(),
2129 );
2130 thread::sleep(Duration::from_millis(2));
2131 }
2132 drop(session);
2133
2134 let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2135 assert_eq!(outcome.recovered.len(), 1);
2136 assert_eq!(outcome.recovered[0].run_id, run_id);
2137 assert_eq!(
2138 outcome.recovered[0].status,
2139 RunStatus::Ended,
2140 "tail ending in RunEnded must seal as Ended",
2141 );
2142 assert!(
2143 outcome.parent_run_id.is_none(),
2144 "Ended runs must not become parents",
2145 );
2146
2147 let manifests = RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2148 let manifest = manifests
2149 .into_iter()
2150 .find(|m| m.run_id == run_id)
2151 .expect("manifest present");
2152 assert_eq!(manifest.status, RunStatus::Ended);
2153 }
2154
2155 #[rstest]
2156 fn recovery_quarantines_run_with_rows_swapped_between_keys() {
2157 let tmp = TempDir::new().expect("tempdir");
2160 let config = make_config(tmp.path().to_path_buf());
2161 let run_id = "1700000000-swapped1";
2162
2163 let path = {
2164 let mut backend = RedbBackend::new(config.base_dir.clone());
2165 backend.open_run(manifest_for(run_id)).expect("open run");
2166 backend
2167 .append_batch(&[
2168 append_entry(
2169 1,
2170 "events.order.1",
2171 "OrderAccepted",
2172 Bytes::from_static(b"\x01"),
2173 ),
2174 append_entry(
2175 2,
2176 "events.order.2",
2177 "OrderFilled",
2178 Bytes::from_static(b"\x02"),
2179 ),
2180 ])
2181 .expect("append");
2182 config
2183 .base_dir
2184 .join(INSTANCE_ID)
2185 .join(format!("{run_id}.redb"))
2186 };
2187
2188 {
2189 let entries: redb::TableDefinition<u64, &[u8]> = redb::TableDefinition::new("entries");
2190 let db = redb::Database::create(&path).expect("open redb");
2191 let txn = db.begin_write().expect("begin write");
2192 {
2193 let mut table = txn.open_table(entries).expect("open table");
2194 let bytes_1 = table
2195 .remove(1_u64)
2196 .expect("remove seq 1")
2197 .expect("seq 1 present")
2198 .value()
2199 .to_vec();
2200 let bytes_2 = table
2201 .remove(2_u64)
2202 .expect("remove seq 2")
2203 .expect("seq 2 present")
2204 .value()
2205 .to_vec();
2206 table
2207 .insert(1_u64, bytes_2.as_slice())
2208 .expect("insert under key 1");
2209 table
2210 .insert(2_u64, bytes_1.as_slice())
2211 .expect("insert under key 2");
2212 }
2213 txn.commit().expect("commit swap");
2214 }
2215
2216 let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2217
2218 assert_eq!(outcome.recovered.len(), 1);
2219 assert_eq!(outcome.recovered[0].run_id, run_id);
2220 assert_eq!(outcome.recovered[0].status, RunStatus::Quarantined);
2221 assert!(
2222 outcome.parent_run_id.is_none(),
2223 "quarantined runs must not become parents",
2224 );
2225
2226 let manifests = RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2227 assert_eq!(manifests[0].status, RunStatus::Quarantined);
2228 }
2229
2230 #[rstest]
2231 fn drop_without_close_leaves_run_for_next_boot_to_recover() {
2232 let tmp = TempDir::new().expect("tempdir");
2233 let config = make_config(tmp.path().to_path_buf());
2234
2235 let halt = HaltSignal::new();
2236 let session = open_run(
2237 &config,
2238 INSTANCE_ID,
2239 build_run_id(UnixNanos::from(4_000)),
2240 None,
2241 UnixNanos::from(4_000),
2242 &RegisteredComponents::default(),
2243 halt,
2244 get_atomic_clock_static(),
2245 )
2246 .expect("open run");
2247 let run_id = session.run_id().to_string();
2248 drop(session);
2249
2250 let outcome =
2251 recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover after crash");
2252 assert_eq!(outcome.recovered.len(), 1);
2253 assert_eq!(outcome.recovered[0].run_id, run_id);
2254 assert_eq!(outcome.recovered[0].status, RunStatus::CrashedRecovered);
2255 assert_eq!(outcome.parent_run_id.as_deref(), Some(run_id.as_str()));
2256
2257 let manifests = RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2258 let sealed = manifests
2259 .into_iter()
2260 .find(|m| m.run_id == run_id)
2261 .expect("present");
2262 assert_eq!(sealed.status, RunStatus::CrashedRecovered);
2263 }
2264
2265 #[rstest]
2266 fn build_run_id_uses_expected_format() {
2267 let id = build_run_id(UnixNanos::from(123_456));
2271 let (prefix, suffix) = id.split_once('-').expect("run id must contain a hyphen");
2272 assert_eq!(prefix, "123456");
2273 assert_eq!(suffix.len(), 8, "suffix was {suffix:?}");
2274 assert!(
2275 suffix.chars().all(|c| c.is_ascii_hexdigit()),
2276 "suffix must be hex, was {suffix:?}",
2277 );
2278 }
2279
2280 #[rstest]
2281 fn crash_recovery_seals_predecessor_and_links_parent_run_id() {
2282 let tmp = TempDir::new().expect("tempdir");
2286 let config = make_config(tmp.path().to_path_buf());
2287
2288 let halt_first = HaltSignal::new();
2290 let first = open_run(
2291 &config,
2292 INSTANCE_ID,
2293 build_run_id(UnixNanos::from(10_000)),
2294 None,
2295 UnixNanos::from(10_000),
2296 &RegisteredComponents::default(),
2297 halt_first,
2298 get_atomic_clock_static(),
2299 )
2300 .expect("open first run");
2301 let crashed_run_id = first.run_id().to_string();
2302 drop(first);
2303
2304 let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2306 assert_eq!(outcome.recovered.len(), 1);
2307 assert_eq!(outcome.recovered[0].run_id, crashed_run_id);
2308 assert_eq!(outcome.recovered[0].status, RunStatus::CrashedRecovered);
2309 assert_eq!(
2310 outcome.parent_run_id.as_deref(),
2311 Some(crashed_run_id.as_str())
2312 );
2313
2314 let manifests_after_seal =
2316 RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2317 let predecessor = manifests_after_seal
2318 .iter()
2319 .find(|m| m.run_id == crashed_run_id)
2320 .expect("predecessor present");
2321 assert_eq!(predecessor.status, RunStatus::CrashedRecovered);
2322
2323 let halt_second = HaltSignal::new();
2325 let new_run_id = build_run_id(UnixNanos::from(20_000));
2326 let next = open_run(
2327 &config,
2328 INSTANCE_ID,
2329 new_run_id.clone(),
2330 outcome.parent_run_id,
2331 UnixNanos::from(20_000),
2332 &RegisteredComponents::default(),
2333 halt_second,
2334 get_atomic_clock_static(),
2335 )
2336 .expect("open second run");
2337 assert_eq!(next.parent_run_id(), Some(crashed_run_id.as_str()));
2338 assert_eq!(
2339 next.manifest().parent_run_id.as_deref(),
2340 Some(crashed_run_id.as_str()),
2341 );
2342 assert_eq!(next.high_watermark(), 1, "RunStarted is the first entry");
2343
2344 drop(next);
2347 let outcome_after = recover_predecessors(&config.base_dir, INSTANCE_ID)
2348 .expect("recover after second crash");
2349 assert_eq!(outcome_after.recovered.len(), 1);
2351 assert_eq!(outcome_after.recovered[0].run_id, new_run_id);
2352 assert_eq!(
2353 outcome_after.recovered[0].status,
2354 RunStatus::CrashedRecovered,
2355 );
2356
2357 let sealed = RedbBackend::open_sealed(&config.base_dir, INSTANCE_ID, &new_run_id)
2359 .expect("open sealed");
2360 let first_entry = sealed
2361 .scan_seq(1)
2362 .expect("scan")
2363 .expect("RunStarted present");
2364 assert_eq!(first_entry.payload_type.as_str(), "RunStarted");
2365 assert_eq!(first_entry.topic.as_ref(), "run.lifecycle.RunStarted");
2366 }
2367
2368 #[rstest]
2369 #[case::before_enqueue(CrashPoint::BeforeEnqueue, 0, false)]
2370 #[case::after_enqueue_before_commit(CrashPoint::AfterEnqueueBeforeCommit, 0, false)]
2371 #[case::after_commit_before_snapshot(CrashPoint::AfterCommitBeforeSnapshot, 1, false)]
2372 #[case::after_snapshot(CrashPoint::AfterSnapshot, 1, true)]
2373 fn crash_recovery_matrix_seals_predecessor_and_links_parent_run_id(
2374 #[case] crash_point: CrashPoint,
2375 #[case] expected_hwm: u64,
2376 #[case] expect_snapshot_anchor: bool,
2377 ) {
2378 let tmp = TempDir::new().expect("tempdir");
2379 let config = make_config(tmp.path().to_path_buf());
2380 let predecessor_run_id = format!("3000-{crash_point:?}");
2381 seed_crashed_predecessor(&config, &predecessor_run_id, crash_point);
2382
2383 let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2384 assert_eq!(outcome.recovered.len(), 1);
2385 assert_eq!(outcome.recovered[0].run_id, predecessor_run_id);
2386 assert_eq!(outcome.recovered[0].status, RunStatus::CrashedRecovered);
2387 assert_eq!(
2388 outcome.parent_run_id.as_deref(),
2389 Some(predecessor_run_id.as_str()),
2390 );
2391
2392 let predecessor =
2393 RedbBackend::open_sealed(&config.base_dir, INSTANCE_ID, &predecessor_run_id)
2394 .expect("open sealed predecessor");
2395 let manifest = predecessor.manifest().expect("manifest");
2396 let snapshot_anchor = predecessor.latest_snapshot_anchor().expect("anchor read");
2397
2398 assert_eq!(manifest.status, RunStatus::CrashedRecovered);
2399 assert_eq!(manifest.high_watermark, expected_hwm);
2400 assert_eq!(
2401 snapshot_anchor.is_some(),
2402 expect_snapshot_anchor,
2403 "snapshot anchor presence must match crash point",
2404 );
2405
2406 let next = open_run(
2407 &config,
2408 INSTANCE_ID,
2409 "4000-next".to_string(),
2410 outcome.parent_run_id,
2411 UnixNanos::from(4_000),
2412 &RegisteredComponents::default(),
2413 HaltSignal::new(),
2414 get_atomic_clock_static(),
2415 )
2416 .expect("open next run");
2417 assert_eq!(next.parent_run_id(), Some(predecessor_run_id.as_str()));
2418 assert_eq!(
2419 next.manifest().parent_run_id.as_deref(),
2420 Some(predecessor_run_id.as_str()),
2421 );
2422 }
2423
2424 #[rstest]
2425 fn kernel_event_store_open_seals_leftover_session_before_reopen() {
2426 let tmp = TempDir::new().expect("tempdir");
2431 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2432 let instance_id = UUID4::new();
2433
2434 let mut store = EventStoreLifecycle::boot(
2435 Some(make_config(tmp.path().to_path_buf())),
2436 instance_id,
2437 clock_rc,
2438 )
2439 .expect("boot store");
2440
2441 store
2442 .open(
2443 instance_id,
2444 &RegisteredComponents::default(),
2445 Environment::Backtest,
2446 )
2447 .expect("open first run");
2448 let run_one = store.run_id().expect("run one open").to_string();
2449
2450 store
2451 .open(
2452 instance_id,
2453 &RegisteredComponents::default(),
2454 Environment::Backtest,
2455 )
2456 .expect("open second run");
2457 let run_two = store.run_id().expect("run two open").to_string();
2458
2459 assert_ne!(run_one, run_two, "second open must produce a fresh run id");
2460
2461 drop(store);
2464 let manifests =
2465 RedbBackend::list_runs(tmp.path(), &instance_id.to_string()).expect("list runs");
2466 let m1 = manifests
2467 .iter()
2468 .find(|m| m.run_id == run_one)
2469 .expect("first run present");
2470 let m2 = manifests
2471 .iter()
2472 .find(|m| m.run_id == run_two)
2473 .expect("second run present");
2474 assert_eq!(m1.status, RunStatus::Ended);
2475 assert_eq!(m2.status, RunStatus::Ended);
2476 }
2477
2478 #[rstest]
2479 fn open_after_halt_re_arms_signal_and_next_run_seals_ended() {
2480 let tmp = TempDir::new().expect("tempdir");
2484 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2485 let instance_id = UUID4::new();
2486
2487 let mut store = EventStoreLifecycle::boot(
2488 Some(make_config(tmp.path().to_path_buf())),
2489 instance_id,
2490 clock_rc,
2491 )
2492 .expect("boot store");
2493
2494 store
2495 .open(
2496 instance_id,
2497 &RegisteredComponents::default(),
2498 Environment::Backtest,
2499 )
2500 .expect("open first run");
2501 let run_one = store.run_id().expect("run one open").to_string();
2502
2503 store.halt.callback()(HaltReason::BackendDisk("ENOSPC".to_string()));
2504 assert!(store.halt.is_halted());
2505
2506 store
2507 .open(
2508 instance_id,
2509 &RegisteredComponents::default(),
2510 Environment::Backtest,
2511 )
2512 .expect("open second run after halt");
2513 let run_two = store.run_id().expect("run two open").to_string();
2514
2515 assert_ne!(run_one, run_two);
2516 assert!(!store.halt.is_halted(), "open must re-arm the halt signal");
2517 assert!(store.halt.reason().is_none());
2518
2519 drop(store);
2520 let manifests =
2521 RedbBackend::list_runs(tmp.path(), &instance_id.to_string()).expect("list runs");
2522 let m1 = manifests
2523 .iter()
2524 .find(|m| m.run_id == run_one)
2525 .expect("first run present");
2526 let m2 = manifests
2527 .iter()
2528 .find(|m| m.run_id == run_two)
2529 .expect("second run present");
2530 assert_eq!(m1.status, RunStatus::Running);
2533 assert_eq!(m2.status, RunStatus::Ended);
2534 }
2535
2536 #[rstest]
2537 fn recover_picks_most_recent_crashed_recovered_as_parent() {
2538 let tmp = TempDir::new().expect("tempdir");
2541 let config = make_config(tmp.path().to_path_buf());
2542
2543 for ts in [1_000_u64, 2_000_u64, 3_000_u64] {
2544 let session = open_run(
2545 &config,
2546 INSTANCE_ID,
2547 build_run_id(UnixNanos::from(ts)),
2548 None,
2549 UnixNanos::from(ts),
2550 &RegisteredComponents::default(),
2551 HaltSignal::new(),
2552 get_atomic_clock_static(),
2553 )
2554 .expect("open");
2555 drop(session);
2556 }
2557
2558 let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2559 assert_eq!(outcome.recovered.len(), 3);
2560 assert!(
2561 outcome
2562 .recovered
2563 .iter()
2564 .all(|r| r.status == RunStatus::CrashedRecovered),
2565 "every predecessor must seal CrashedRecovered, was {:?}",
2566 outcome.recovered,
2567 );
2568 let parent = outcome.parent_run_id.as_deref().expect("parent set");
2570 assert!(
2571 parent.starts_with("3000-"),
2572 "parent must be the run with the highest start_ts_init, was {parent}",
2573 );
2574 }
2575
2576 #[cfg(not(madsim))]
2577 #[rstest]
2578 fn submit_run_started_returns_timeout_when_writer_stalls() {
2579 let stub = StallBackend::default();
2583 let manifest = manifest_for("run-timeout");
2584 let mut backend: Box<dyn EventStore + Send> = Box::new(stub.clone());
2585 backend.open_run(manifest).expect("open stub");
2586
2587 let halt = HaltSignal::new();
2588
2589 let writer = EventStoreWriter::spawn(
2590 backend,
2591 get_atomic_clock_static(),
2592 halt.callback(),
2593 WriterConfig::default(),
2594 )
2595 .expect("spawn writer");
2596
2597 let err = submit_run_started_blocking(
2598 &writer,
2599 &RegisteredComponents::default(),
2600 UnixNanos::from(100),
2601 &halt,
2602 Duration::from_millis(20),
2603 )
2604 .expect_err("must time out");
2605
2606 match err {
2607 BootError::RunStartedTimeout { timeout } => {
2608 assert_eq!(timeout, Duration::from_millis(20));
2609 }
2610 other => panic!("expected RunStartedTimeout, was {other:?}"),
2611 }
2612
2613 stub.release();
2615 }
2616
2617 #[cfg(not(madsim))]
2618 #[rstest]
2619 fn submit_run_started_returns_halted_when_writer_halts_during_wait() {
2620 let stub = StallBackend::default();
2623 let manifest = manifest_for("run-halt");
2624 let mut backend: Box<dyn EventStore + Send> = Box::new(stub.clone());
2625 backend.open_run(manifest).expect("open stub");
2626
2627 let halt = HaltSignal::new();
2628
2629 let writer = EventStoreWriter::spawn(
2630 backend,
2631 get_atomic_clock_static(),
2632 halt.callback(),
2633 WriterConfig::default(),
2634 )
2635 .expect("spawn writer");
2636
2637 let halt_for_thread = halt.clone();
2639
2640 let firer = thread::spawn(move || {
2641 thread::sleep(Duration::from_millis(10));
2642 halt_for_thread.callback()(HaltReason::BackendDisk("test stall".to_string()));
2643 });
2644
2645 let err = submit_run_started_blocking(
2646 &writer,
2647 &RegisteredComponents::default(),
2648 UnixNanos::from(200),
2649 &halt,
2650 Duration::from_secs(2),
2651 )
2652 .expect_err("must observe halt");
2653
2654 match err {
2655 BootError::HaltedDuringBoot(HaltReason::BackendDisk(msg)) => {
2656 assert!(msg.contains("test stall"), "reason msg was {msg}");
2657 }
2658 other => panic!("expected HaltedDuringBoot(BackendDisk), was {other:?}"),
2659 }
2660 firer.join().expect("halt firer joined");
2661 stub.release();
2662 }
2663
2664 fn manifest_for(run_id: &str) -> RunManifest {
2665 RunManifest {
2666 run_id: run_id.to_string(),
2667 parent_run_id: None,
2668 instance_id: INSTANCE_ID.to_string(),
2669 binary_hash: String::new(),
2670 schema_version: 1,
2671 crate_versions: String::new(),
2672 feature_flags: Vec::new(),
2673 adapter_versions: IndexMap::new(),
2674 config_hash: String::new(),
2675 registered_components: RegisteredComponents::default(),
2676 seed: None,
2677 start_ts_init: UnixNanos::default(),
2678 end_ts_init: None,
2679 high_watermark: 0,
2680 status: RunStatus::Running,
2681 }
2682 }
2683
2684 #[cfg(not(madsim))]
2688 #[derive(Debug, Default, Clone)]
2689 struct StallBackend {
2690 inner: Arc<Mutex<StallInner>>,
2691 gate: Arc<(Mutex<bool>, parking_lot::Condvar)>,
2692 }
2693
2694 #[cfg(not(madsim))]
2695 #[derive(Debug, Default)]
2696 struct StallInner {
2697 manifest: Option<RunManifest>,
2698 }
2699
2700 #[cfg(not(madsim))]
2701 impl StallBackend {
2702 fn release(&self) {
2703 let (lock, cvar) = &*self.gate;
2704 *lock.lock() = true;
2705 cvar.notify_all();
2706 }
2707 }
2708
2709 #[cfg(not(madsim))]
2710 impl crate::EventStore for StallBackend {
2711 fn open_run(&mut self, manifest: RunManifest) -> Result<(), EventStoreError> {
2712 self.inner.lock().manifest = Some(manifest);
2713 Ok(())
2714 }
2715
2716 fn append_batch(&mut self, _: &[crate::AppendEntry]) -> Result<u64, EventStoreError> {
2717 let (lock, cvar) = &*self.gate;
2718 let mut released = lock.lock();
2719
2720 while !*released {
2721 cvar.wait(&mut released);
2722 }
2723 Ok(0)
2724 }
2725
2726 fn scan_range(
2727 &self,
2728 _: u64,
2729 _: u64,
2730 _: crate::ScanDirection,
2731 ) -> Result<Vec<crate::EventStoreEntry>, EventStoreError> {
2732 Ok(Vec::new())
2733 }
2734
2735 fn scan_seq(&self, _: u64) -> Result<Option<crate::EventStoreEntry>, EventStoreError> {
2736 Ok(None)
2737 }
2738
2739 fn lookup(&self, _: crate::IndexKind, _: &str) -> Result<Option<u64>, EventStoreError> {
2740 Ok(None)
2741 }
2742
2743 fn iter_index_keys(
2744 &self,
2745 _: crate::IndexKind,
2746 ) -> Result<Vec<(String, u64)>, EventStoreError> {
2747 Ok(Vec::new())
2748 }
2749
2750 fn seal(&mut self, _: RunStatus) -> Result<(), EventStoreError> {
2751 Ok(())
2752 }
2753
2754 fn manifest(&self) -> Result<RunManifest, EventStoreError> {
2755 self.inner
2756 .lock()
2757 .manifest
2758 .clone()
2759 .ok_or_else(|| EventStoreError::Backend("no manifest".to_string()))
2760 }
2761
2762 fn high_watermark(&self) -> Result<u64, EventStoreError> {
2763 Ok(0)
2764 }
2765 }
2766
2767 #[rstest]
2772 fn bus_tap_captures_submit_order_sent_through_msgbus() {
2773 let tmp = TempDir::new().expect("tempdir");
2774 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2775 let instance_id = UUID4::new();
2776
2777 let mut store = EventStoreLifecycle::boot(
2778 Some(make_config(tmp.path().to_path_buf())),
2779 instance_id,
2780 clock_rc,
2781 )
2782 .expect("boot store");
2783 store
2784 .open(
2785 instance_id,
2786 &RegisteredComponents::default(),
2787 Environment::Backtest,
2788 )
2789 .expect("open run");
2790 let run_id = store.run_id().expect("run open").to_string();
2791
2792 let trader_id = TraderId::from("TRADER-001");
2793 let strategy_id = StrategyId::from("S-001");
2794 let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE");
2795 let client_order_id = ClientOrderId::from("O-20260510-000001");
2796 let order_init = OrderInitializedSpec::builder()
2797 .instrument_id(instrument_id)
2798 .client_order_id(client_order_id)
2799 .quantity(Quantity::from("1"))
2800 .time_in_force(TimeInForce::Gtc)
2801 .build();
2802 let submit_order = SubmitOrder::new(
2803 trader_id,
2804 Some(ClientId::from("BINANCE")),
2805 strategy_id,
2806 instrument_id,
2807 client_order_id,
2808 order_init,
2809 None,
2810 None,
2811 None,
2812 UUID4::new(),
2813 UnixNanos::from(3),
2814 None, );
2816
2817 let endpoint = MStr::<Endpoint>::from("test.exec.engine.process");
2818 msgbus::send_any_value(endpoint, &submit_order);
2819
2820 store.seal(UnixNanos::from(0));
2821
2822 let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
2823 .expect("open sealed");
2824 let captured = sealed
2825 .scan_seq(2)
2826 .expect("scan")
2827 .expect("captured entry present");
2828 assert_eq!(captured.payload_type.as_str(), "SubmitOrder");
2829 assert_eq!(captured.topic.as_ref(), endpoint.as_str());
2830
2831 let by_client = sealed
2834 .lookup(IndexKind::ClientOrderId, client_order_id.as_str())
2835 .expect("lookup")
2836 .expect("indexed");
2837 assert_eq!(by_client, 2);
2838 }
2839
2840 #[rstest]
2841 fn kernel_with_markers_captures_snapshots_over_synthetic_bus() {
2842 let tmp = TempDir::new().expect("tempdir");
2843 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2844 let instance_id = UUID4::new();
2845 let mut config = make_config(tmp.path().to_path_buf());
2846 config.data_markers = Some(DataMarkerConfig {
2847 classes: vec![DataMarkerClass::BookDeltas, DataMarkerClass::Quote],
2848 safety_flush_interval: Duration::from_secs(1),
2849 channel_capacity: 128,
2850 high_fidelity: Vec::new(),
2851 });
2852
2853 let mut store =
2854 EventStoreLifecycle::boot(Some(config), instance_id, clock_rc).expect("boot store");
2855 store
2856 .open(
2857 instance_id,
2858 &RegisteredComponents::default(),
2859 Environment::Backtest,
2860 )
2861 .expect("open run");
2862 let run_id = store.run_id().expect("run open").to_string();
2863
2864 let first = make_submit_order(ClientOrderId::from("O-marker-1"));
2865 msgbus::send_any_value(MStr::<Endpoint>::from("test.exec.process"), &first);
2866
2867 let quote = quote_ethusdt_binance();
2868 msgbus::publish_quote(MStr::from("data.quotes.BINANCE.ETHUSDT-PERP"), "e);
2869 let deltas = stub_deltas();
2870 msgbus::publish_deltas(MStr::from("data.deltas.XNAS.AAPL"), &deltas);
2871
2872 let second = make_submit_order(ClientOrderId::from("O-marker-2"));
2873 msgbus::send_any_value(MStr::<Endpoint>::from("test.exec.process"), &second);
2874 store.seal(UnixNanos::from(500));
2875
2876 let marker_path = tmp
2877 .path()
2878 .join(instance_id.to_string())
2879 .join(format!("{run_id}.markers.redb"));
2880 let marker = RedbMarkerBackend::open_read_only_file(marker_path).expect("open markers");
2881 let snapshots = marker.scan_snapshots().expect("scan snapshots");
2882 let dict = marker.scan_dict().expect("scan dict");
2883
2884 assert_eq!(snapshots.len(), 1);
2885 assert_eq!(snapshots[0].event_seq_before, 3);
2886 assert_eq!(snapshots[0].advanced.len(), 2);
2887 assert_eq!(
2888 snapshots[0]
2889 .advanced
2890 .iter()
2891 .map(|cursor| cursor.count)
2892 .collect::<Vec<_>>(),
2893 vec![1, 1]
2894 );
2895 assert_eq!(
2896 dict.iter()
2897 .map(|entry| (entry.data_cls, entry.identifier.as_str()))
2898 .collect::<Vec<_>>(),
2899 vec![
2900 (DataClass::Quote, "ETHUSDT-PERP.BINANCE"),
2901 (DataClass::BookDeltas, "AAPL.XNAS"),
2902 ],
2903 );
2904 }
2905
2906 #[rstest]
2907 fn boot_recovery_ignores_marker_sidecar_files() {
2908 let tmp = TempDir::new().expect("tempdir");
2909 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2910 let instance_id = UUID4::new();
2911 let mut config = make_config(tmp.path().to_path_buf());
2912 config.data_markers = Some(DataMarkerConfig {
2913 classes: vec![DataMarkerClass::Quote],
2914 safety_flush_interval: Duration::from_secs(1),
2915 channel_capacity: 128,
2916 high_fidelity: Vec::new(),
2917 });
2918
2919 let mut store =
2920 EventStoreLifecycle::boot(Some(config.clone()), instance_id, Rc::clone(&clock_rc))
2921 .expect("boot store");
2922 store
2923 .open(
2924 instance_id,
2925 &RegisteredComponents::default(),
2926 Environment::Backtest,
2927 )
2928 .expect("open run");
2929 let run_id = store.run_id().expect("run open").to_string();
2930 store.seal(UnixNanos::from(500));
2931
2932 let marker_path = tmp
2933 .path()
2934 .join(instance_id.to_string())
2935 .join(format!("{run_id}.markers.redb"));
2936 assert!(marker_path.exists());
2937
2938 let rebooted = EventStoreLifecycle::boot(Some(config), instance_id, clock_rc)
2939 .expect("boot after marker sidecar");
2940
2941 assert!(rebooted.recovered().is_empty());
2942 }
2943
2944 #[rstest]
2945 fn marker_setup_failure_disables_markers_without_blocking_open() {
2946 let tmp = TempDir::new().expect("tempdir");
2947 let bad_base = tmp.path().join("not-a-directory");
2948 std::fs::write(&bad_base, b"not a directory").expect("write bad base");
2949 let memory = Arc::new(Mutex::new(MemoryBackend::new()));
2950 let opener_memory = Arc::clone(&memory);
2951 let mut config = make_config(bad_base);
2952 config.data_markers = Some(DataMarkerConfig {
2953 classes: vec![DataMarkerClass::Quote],
2954 safety_flush_interval: Duration::from_secs(1),
2955 channel_capacity: 128,
2956 high_fidelity: Vec::new(),
2957 });
2958 let options = EventStoreLifecycleOptions::new()
2959 .with_encoder_registry(test_registry())
2960 .with_backend_opener(move |_, manifest| {
2961 opener_memory.lock().open_run(manifest.clone())?;
2962 Ok(Box::new(SharedMemoryBackend(Arc::clone(&opener_memory))))
2963 });
2964
2965 let mut session = open_run_with_options(
2966 &config,
2967 INSTANCE_ID,
2968 "run-marker-setup-fails".to_string(),
2969 None,
2970 UnixNanos::from(5_000),
2971 &RegisteredComponents::default(),
2972 HaltSignal::new(),
2973 get_atomic_clock_static(),
2974 &options,
2975 )
2976 .expect("open run despite marker failure");
2977
2978 assert!(session.marker_capture.is_none());
2979
2980 let topic: MStr<msgbus::Topic> = MStr::from("events.test.marker-fallback");
2981 session
2982 .adapter()
2983 .expect("adapter")
2984 .capture::<TestAuditMessage>(
2985 topic,
2986 &TestAuditMessage { value: 11 },
2987 Headers::empty(),
2988 UnixNanos::from(5_001),
2989 )
2990 .expect("capture");
2991 session
2992 .close(UnixNanos::from(6_000))
2993 .expect("close session");
2994
2995 let backend = memory.lock();
2996 let captured = backend
2997 .scan_seq(2)
2998 .expect("scan")
2999 .expect("captured entry present");
3000
3001 assert_eq!(captured.payload_type.as_str(), "TestAuditMessage");
3002 assert_eq!(captured.topic.as_ref(), topic.as_str());
3003 assert_eq!(captured.payload.as_ref(), &[11]);
3004 }
3005
3006 #[rstest]
3007 fn marker_registry_factory_receives_enabled_classes() {
3008 let tmp = TempDir::new().expect("tempdir");
3009 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3010 let instance_id = UUID4::new();
3011 let seen_classes: Arc<Mutex<Vec<Vec<DataClass>>>> = Arc::new(Mutex::new(Vec::new()));
3012 let seen_for_factory = Arc::clone(&seen_classes);
3013 let mut config = make_config(tmp.path().to_path_buf());
3014 config.data_markers = Some(DataMarkerConfig {
3015 classes: vec![DataMarkerClass::Trade, DataMarkerClass::Quote],
3016 safety_flush_interval: Duration::from_secs(1),
3017 channel_capacity: 128,
3018 high_fidelity: Vec::new(),
3019 });
3020 let options =
3021 EventStoreLifecycleOptions::new().with_marker_registry_factory(move |classes| {
3022 seen_for_factory.lock().push(classes.to_vec());
3023 DataMarkerExtractorRegistry::default_registry(classes)
3024 });
3025
3026 let mut store =
3027 EventStoreLifecycle::boot_with_options(Some(config), instance_id, clock_rc, options)
3028 .expect("boot store");
3029 store
3030 .open(
3031 instance_id,
3032 &RegisteredComponents::default(),
3033 Environment::Backtest,
3034 )
3035 .expect("open run");
3036 store.seal(UnixNanos::from(1_000));
3037
3038 let seen = seen_classes.lock();
3039 assert_eq!(seen.as_slice(), &[vec![DataClass::Trade, DataClass::Quote]]);
3040 }
3041
3042 #[rstest]
3043 fn markers_disabled_installs_no_file_and_no_cost() {
3044 let tmp = TempDir::new().expect("tempdir");
3045 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3046 let instance_id = UUID4::new();
3047
3048 let mut store = EventStoreLifecycle::boot(
3049 Some(make_config(tmp.path().to_path_buf())),
3050 instance_id,
3051 clock_rc,
3052 )
3053 .expect("boot store");
3054 store
3055 .open(
3056 instance_id,
3057 &RegisteredComponents::default(),
3058 Environment::Backtest,
3059 )
3060 .expect("open run");
3061 let run_id = store.run_id().expect("run open").to_string();
3062
3063 assert!(
3064 store
3065 .session
3066 .as_ref()
3067 .expect("session")
3068 .marker_capture
3069 .is_none()
3070 );
3071
3072 let quote = quote_ethusdt_binance();
3073 msgbus::publish_quote(MStr::from("data.quotes.BINANCE.ETHUSDT-PERP"), "e);
3074 store.seal(UnixNanos::from(500));
3075
3076 let marker_path = tmp
3077 .path()
3078 .join(instance_id.to_string())
3079 .join(format!("{run_id}.markers.redb"));
3080 assert!(!marker_path.exists());
3081 }
3082
3083 #[rstest]
3087 fn bus_tap_captures_time_event_handler_run() {
3088 let tmp = TempDir::new().expect("tempdir");
3089 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3090 let instance_id = UUID4::new();
3091
3092 let mut store = EventStoreLifecycle::boot(
3093 Some(make_config(tmp.path().to_path_buf())),
3094 instance_id,
3095 clock_rc,
3096 )
3097 .expect("boot store");
3098 store
3099 .open(
3100 instance_id,
3101 &RegisteredComponents::default(),
3102 Environment::Backtest,
3103 )
3104 .expect("open run");
3105 let run_id = store.run_id().expect("run open").to_string();
3106
3107 let event = TimeEvent::new(
3108 Ustr::from("strategy.heartbeat"),
3109 UUID4::new(),
3110 UnixNanos::from(100),
3111 UnixNanos::from(99),
3112 );
3113 let callback = TimeEventCallback::from(|_: TimeEvent| {});
3114 TimeEventHandler::new(event, callback).run();
3115
3116 store.seal(UnixNanos::from(0));
3117
3118 let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3119 .expect("open sealed");
3120 let captured = sealed
3121 .scan_seq(2)
3122 .expect("scan")
3123 .expect("captured entry present");
3124
3125 assert_eq!(captured.payload_type.as_str(), PAYLOAD_TYPE_TIME_EVENT);
3126 assert_eq!(captured.topic, MessagingSwitchboard::time_event_topic());
3127 }
3128
3129 #[rstest]
3134 fn seal_clears_bus_tap_so_post_seal_dispatches_do_not_capture() {
3135 let tmp = TempDir::new().expect("tempdir");
3136 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3137 let instance_id = UUID4::new();
3138
3139 let mut store = EventStoreLifecycle::boot(
3140 Some(make_config(tmp.path().to_path_buf())),
3141 instance_id,
3142 clock_rc,
3143 )
3144 .expect("boot store");
3145 store
3146 .open(
3147 instance_id,
3148 &RegisteredComponents::default(),
3149 Environment::Backtest,
3150 )
3151 .expect("open run");
3152 let run_id = store.run_id().expect("run open").to_string();
3153
3154 store.seal(UnixNanos::from(0));
3155
3156 let endpoint = MStr::<Endpoint>::from("test.post.seal.endpoint");
3161 let payload: u32 = 99;
3162 msgbus::send_any_value(endpoint, &payload);
3163
3164 let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3165 .expect("open sealed");
3166 assert!(
3168 sealed.scan_seq(3).expect("scan").is_none(),
3169 "no entry must land after seal",
3170 );
3171 }
3172
3173 #[rstest]
3179 fn bus_tap_captures_trading_command_envelope_with_inner_payload_type() {
3180 let tmp = TempDir::new().expect("tempdir");
3181 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3182 let instance_id = UUID4::new();
3183
3184 let mut store = EventStoreLifecycle::boot(
3185 Some(make_config(tmp.path().to_path_buf())),
3186 instance_id,
3187 clock_rc,
3188 )
3189 .expect("boot store");
3190 store
3191 .open(
3192 instance_id,
3193 &RegisteredComponents::default(),
3194 Environment::Backtest,
3195 )
3196 .expect("open run");
3197 let run_id = store.run_id().expect("run open").to_string();
3198
3199 let trader_id = TraderId::from("TRADER-001");
3200 let strategy_id = StrategyId::from("S-001");
3201 let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE");
3202 let client_order_id = ClientOrderId::from("O-20260510-000002");
3203 let order_init = OrderInitializedSpec::builder()
3204 .instrument_id(instrument_id)
3205 .client_order_id(client_order_id)
3206 .quantity(Quantity::from("1"))
3207 .time_in_force(TimeInForce::Gtc)
3208 .build();
3209 let submit_order = SubmitOrder::new(
3210 trader_id,
3211 Some(ClientId::from("BINANCE")),
3212 strategy_id,
3213 instrument_id,
3214 client_order_id,
3215 order_init,
3216 None,
3217 None,
3218 None,
3219 UUID4::new(),
3220 UnixNanos::from(3),
3221 None, );
3223 let command = TradingCommand::SubmitOrder(submit_order.clone());
3224
3225 let endpoint = MStr::<Endpoint>::from("test.exec.engine.envelope");
3226 msgbus::send_trading_command(endpoint, command);
3227
3228 store.seal(UnixNanos::from(0));
3229
3230 let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3231 .expect("open sealed");
3232 let captured = sealed
3233 .scan_seq(2)
3234 .expect("scan")
3235 .expect("captured entry present");
3236 assert_eq!(
3237 captured.payload_type.as_str(),
3238 "SubmitOrder",
3239 "wrapper envelope must stamp the inner payload_type",
3240 );
3241 assert_eq!(captured.topic.as_ref(), endpoint.as_str());
3242
3243 let by_client = sealed
3244 .lookup(IndexKind::ClientOrderId, client_order_id.as_str())
3245 .expect("lookup")
3246 .expect("indexed");
3247 assert_eq!(by_client, 2);
3248
3249 let decoded: SubmitOrder =
3253 rmp_serde::from_slice(&captured.payload).expect("decode captured SubmitOrder");
3254 assert_eq!(decoded, submit_order);
3255 }
3256
3257 #[rstest]
3261 fn bus_tap_captures_order_event_any_envelope_with_inner_payload_type() {
3262 let tmp = TempDir::new().expect("tempdir");
3263 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3264 let instance_id = UUID4::new();
3265
3266 let mut store = EventStoreLifecycle::boot(
3267 Some(make_config(tmp.path().to_path_buf())),
3268 instance_id,
3269 clock_rc,
3270 )
3271 .expect("boot store");
3272 store
3273 .open(
3274 instance_id,
3275 &RegisteredComponents::default(),
3276 Environment::Backtest,
3277 )
3278 .expect("open run");
3279 let run_id = store.run_id().expect("run open").to_string();
3280
3281 let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE");
3282 let client_order_id = ClientOrderId::from("O-20260510-000003");
3283 let venue_order_id = VenueOrderId::from("V-99");
3284 let filled = OrderFilledSpec::builder()
3285 .instrument_id(instrument_id)
3286 .client_order_id(client_order_id)
3287 .venue_order_id(venue_order_id)
3288 .account_id(AccountId::from("BINANCE-001"))
3289 .trade_id(TradeId::from("T-1"))
3290 .last_qty(Quantity::from("1"))
3291 .last_px(Price::from("100.00"))
3292 .currency(Currency::USDT())
3293 .ts_event(UnixNanos::from(10))
3294 .ts_init(UnixNanos::from(11))
3295 .commission(Money::new(0.10, Currency::USDT()))
3296 .build();
3297 let event = OrderEventAny::Filled(filled.clone());
3298
3299 let topic: MStr<msgbus::Topic> = MStr::from("events.order.ETHUSDT-PERP.BINANCE");
3300 msgbus::publish_order_event(topic, &event);
3301
3302 store.seal(UnixNanos::from(0));
3303
3304 let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3305 .expect("open sealed");
3306 let captured = sealed
3307 .scan_seq(2)
3308 .expect("scan")
3309 .expect("captured entry present");
3310 assert_eq!(
3311 captured.payload_type.as_str(),
3312 "OrderFilled",
3313 "wrapper envelope must stamp the inner payload_type",
3314 );
3315 assert_eq!(captured.topic.as_ref(), topic.as_str());
3316
3317 let by_client = sealed
3318 .lookup(IndexKind::ClientOrderId, client_order_id.as_str())
3319 .expect("lookup")
3320 .expect("indexed");
3321 let by_venue = sealed
3322 .lookup(IndexKind::VenueOrderId, venue_order_id.as_str())
3323 .expect("lookup")
3324 .expect("indexed");
3325 assert_eq!(by_client, 2);
3326 assert_eq!(by_venue, 2);
3327
3328 let decoded: OrderFilled =
3332 rmp_serde::from_slice(&captured.payload).expect("decode captured OrderFilled");
3333 assert_eq!(decoded, filled);
3334 }
3335
3336 #[rstest]
3341 fn bus_tap_captures_data_command_envelopes_with_category_payload_types() {
3342 let tmp = TempDir::new().expect("tempdir");
3343 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3344 let instance_id = UUID4::new();
3345
3346 let mut store = EventStoreLifecycle::boot(
3347 Some(make_config(tmp.path().to_path_buf())),
3348 instance_id,
3349 clock_rc,
3350 )
3351 .expect("boot store");
3352 store
3353 .open(
3354 instance_id,
3355 &RegisteredComponents::default(),
3356 Environment::Backtest,
3357 )
3358 .expect("open run");
3359 let run_id = store.run_id().expect("run open").to_string();
3360
3361 let request = RequestCommand::Quotes(RequestQuotes::new(
3362 InstrumentId::from("ETHUSDT-PERP.BINANCE"),
3363 None,
3364 None,
3365 None,
3366 Some(ClientId::from("BINANCE")),
3367 UUID4::new(),
3368 UnixNanos::from(20),
3369 None,
3370 ));
3371 let subscribe = SubscribeCommand::Quotes(SubscribeQuotes::new(
3372 InstrumentId::from("ETHUSDT-PERP.BINANCE"),
3373 Some(ClientId::from("BINANCE")),
3374 Some(Venue::from("BINANCE")),
3375 UUID4::new(),
3376 UnixNanos::from(21),
3377 Some(UUID4::new()),
3378 None,
3379 ));
3380
3381 let request_endpoint = MStr::<Endpoint>::from("test.data.engine.request");
3382 msgbus::send_data_command(request_endpoint, DataCommand::Request(request.clone()));
3383
3384 let subscribe_endpoint = MStr::<Endpoint>::from("test.data.engine.subscribe");
3385 msgbus::send_data_command(
3386 subscribe_endpoint,
3387 DataCommand::Subscribe(subscribe.clone()),
3388 );
3389
3390 store.seal(UnixNanos::from(0));
3391
3392 let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3393 .expect("open sealed");
3394 let captured_request = sealed
3395 .scan_seq(2)
3396 .expect("scan request")
3397 .expect("captured request present");
3398 assert_eq!(captured_request.payload_type.as_str(), "RequestCommand");
3399 assert_eq!(captured_request.topic.as_ref(), request_endpoint.as_str());
3400
3401 let decoded_request: RequestCommand =
3402 rmp_serde::from_slice(&captured_request.payload).expect("decode RequestCommand");
3403 match (decoded_request, request) {
3404 (RequestCommand::Quotes(decoded), RequestCommand::Quotes(expected)) => {
3405 assert_eq!(decoded.request_id, expected.request_id);
3406 assert_eq!(decoded.instrument_ids, expected.instrument_ids);
3407 assert_eq!(decoded.client_id, expected.client_id);
3408 assert_eq!(decoded.ts_init, expected.ts_init);
3409 }
3410 other => panic!("expected RequestCommand::Quotes round trip, was {other:?}"),
3411 }
3412
3413 let captured_subscribe = sealed
3414 .scan_seq(3)
3415 .expect("scan subscribe")
3416 .expect("captured subscribe present");
3417 assert_eq!(captured_subscribe.payload_type.as_str(), "SubscribeCommand");
3418 assert_eq!(
3419 captured_subscribe.topic.as_ref(),
3420 subscribe_endpoint.as_str()
3421 );
3422
3423 let decoded_subscribe: SubscribeCommand =
3424 rmp_serde::from_slice(&captured_subscribe.payload).expect("decode SubscribeCommand");
3425 match (decoded_subscribe, subscribe) {
3426 (SubscribeCommand::Quotes(decoded), SubscribeCommand::Quotes(expected)) => {
3427 assert_eq!(decoded.command_id, expected.command_id);
3428 assert_eq!(decoded.instrument_ids, expected.instrument_ids);
3429 assert_eq!(decoded.client_id, expected.client_id);
3430 assert_eq!(decoded.venue, expected.venue);
3431 assert_eq!(decoded.ts_init, expected.ts_init);
3432 assert_eq!(decoded.correlation_id, expected.correlation_id);
3433 }
3434 other => panic!("expected SubscribeCommand::Quotes round trip, was {other:?}"),
3435 }
3436 }
3437
3438 #[rstest]
3441 fn bus_tap_captures_data_response_sent_through_correlation_handler() {
3442 let tmp = TempDir::new().expect("tempdir");
3443 let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3444 let instance_id = UUID4::new();
3445
3446 let mut store = EventStoreLifecycle::boot(
3447 Some(make_config(tmp.path().to_path_buf())),
3448 instance_id,
3449 clock_rc,
3450 )
3451 .expect("boot store");
3452 store
3453 .open(
3454 instance_id,
3455 &RegisteredComponents::default(),
3456 Environment::Backtest,
3457 )
3458 .expect("open run");
3459 let run_id = store.run_id().expect("run open").to_string();
3460
3461 let correlation_id = UUID4::new();
3462 let handler_called = Rc::new(RefCell::new(false));
3463 let handler_called_clone = Rc::clone(&handler_called);
3464 msgbus::register_response_handler(
3465 &correlation_id,
3466 TypedHandler::from(move |_resp: &DataResponseEnvelope| {
3467 *handler_called_clone.borrow_mut() = true;
3468 }),
3469 );
3470
3471 let response = QuotesResponse::new(
3472 correlation_id,
3473 ClientId::from("BINANCE"),
3474 InstrumentId::from("ETHUSDT-PERP.BINANCE"),
3475 vec![],
3476 None,
3477 None,
3478 UnixNanos::from(30),
3479 None,
3480 );
3481 msgbus::send_response(&correlation_id, &DataResponse::Quotes(response.clone()));
3482
3483 assert!(*handler_called.borrow());
3484 store.seal(UnixNanos::from(0));
3485
3486 let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3487 .expect("open sealed");
3488 let captured = sealed
3489 .scan_seq(2)
3490 .expect("scan")
3491 .expect("captured response present");
3492 assert_eq!(captured.payload_type.as_str(), "DataResponseEnvelope");
3493 assert_eq!(captured.topic, MessagingSwitchboard::data_response_topic());
3494
3495 let envelope: crate::capture::builtins::ResponseEnvelopeCapture =
3496 rmp_serde::from_slice(&captured.payload).expect("decode response envelope");
3497 assert_eq!(envelope.correlation_id, correlation_id);
3498 assert!(envelope.completed);
3499 assert_eq!(envelope.sub_correlation_id, None);
3500 assert_eq!(envelope.failure_reason, None);
3501 assert_eq!(
3502 envelope.payload_type.as_ref().map(Ustr::as_str),
3503 Some("QuotesResponse")
3504 );
3505 let decoded: QuotesResponse = rmp_serde::from_slice(envelope.payload.as_ref().unwrap())
3506 .expect("decode quotes payload");
3507 assert_eq!(decoded.correlation_id, response.correlation_id);
3508 assert_eq!(decoded.client_id, response.client_id);
3509 assert_eq!(decoded.instrument_ids, response.instrument_ids);
3510 assert_eq!(decoded.ts_init, response.ts_init);
3511 assert!(decoded.data.is_empty());
3512 }
3513}