Skip to main content

nautilus_event_store/
kernel.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Run lifecycle and kernel boot integration for the event store.
17//!
18//! This module owns the kernel side of the SPEC's run lifecycle: it scans the on-disk
19//! instance directory for crashed predecessors before a fresh run opens, seals each
20//! survivor, opens the new run, blocks `start()` until the writer acknowledges the
21//! `RunStarted` entry, and seals the manifest with a final `RunEnded` entry on graceful
22//! stop. The writer's halt callback is wrapped in a typed [`HaltSignal`] that records
23//! the first fail-stop reason for supervision; no runtime component polls it to stop
24//! the trader.
25
26use std::{
27    any::Any,
28    cell::RefCell,
29    fmt::Debug,
30    path::{Path, PathBuf},
31    rc::Rc,
32    sync::{
33        Arc,
34        atomic::{AtomicBool, AtomicU64, Ordering},
35    },
36    thread,
37    time::{Duration, Instant},
38};
39
40use bytes::Bytes;
41use nautilus_common::{
42    cache::{Cache, CacheSnapshotRef},
43    clock::Clock,
44    enums::Environment,
45    msgbus::{self, BusTap, Endpoint, MStr, MessagingSwitchboard},
46};
47#[cfg(feature = "live")]
48use nautilus_core::time::get_atomic_clock_realtime;
49use nautilus_core::{
50    UUID4, UnixNanos,
51    time::{AtomicTime, get_atomic_clock_static},
52};
53use nautilus_execution::engine::SnapshotAnchorer;
54use nautilus_system::{
55    KernelEventStore as KernelEventStoreTrait, RegisteredComponents,
56    event_store::{DataMarkerClass, DataMarkerConfig, EventStoreConfig, RetentionMode},
57};
58use parking_lot::Mutex;
59use ustr::Ustr;
60
61use crate::{
62    BusCaptureAdapter, CacheReplayError, CacheReplayReport, CaptureError, EncoderRegistry,
63    EntryDraft, EventStore, EventStoreError, EventStoreWriter, HaltCallback, HaltReason, Headers,
64    RedbBackend, RunId, RunManifest, RunStatus, ScanDirection, Topic, WriterConfig,
65    compute_snapshot_content_hash, default_registry,
66    markers::{
67        DataClass, DataMarkerCapture, DataMarkerExtractorRegistry, MarkerBackend, MarkerManifest,
68        MarkerWriter, MarkerWriterConfig, RedbMarkerBackend,
69    },
70    restore_cache_from_sealed_run, validate_event_store_replay_source,
71};
72
73const RUN_STARTED_TOPIC: &str = "run.lifecycle.RunStarted";
74const RUN_STARTED_PAYLOAD_TYPE: &str = "RunStarted";
75const RUN_ENDED_TOPIC: &str = "run.lifecycle.RunEnded";
76const RUN_ENDED_PAYLOAD_TYPE: &str = "RunEnded";
77
78/// The outcome of sealing a single crashed predecessor.
79#[derive(Clone, Debug, PartialEq, Eq)]
80pub struct RecoveredRun {
81    /// The id of the sealed predecessor.
82    pub run_id: RunId,
83    /// The terminal status applied: [`RunStatus::CrashedRecovered`] or
84    /// [`RunStatus::Quarantined`].
85    pub status: RunStatus,
86}
87
88/// Result of the predecessor recovery sweep performed in the kernel constructor.
89#[derive(Debug, Default)]
90pub struct RecoveryOutcome {
91    /// One entry per predecessor that was sealed by the sweep.
92    pub recovered: Vec<RecoveredRun>,
93    /// The id of the most-recently-crashed predecessor sealed as
94    /// [`RunStatus::CrashedRecovered`], or `None` when no recoverable predecessor
95    /// existed (or every predecessor was quarantined).
96    pub parent_run_id: Option<RunId>,
97}
98
99type RegistryFactory = dyn Fn() -> EncoderRegistry + Send + Sync + 'static;
100type BackendOpenResult = Result<Box<dyn EventStore + Send>, EventStoreError>;
101type BackendOpener =
102    dyn Fn(&EventStoreConfig, &RunManifest) -> BackendOpenResult + Send + Sync + 'static;
103type MarkerRegistryFactory =
104    dyn Fn(&[DataClass]) -> DataMarkerExtractorRegistry + Send + Sync + 'static;
105type SharedMarkerCapture = Rc<RefCell<Option<DataMarkerCapture>>>;
106
107/// Non-serialized lifecycle policy for advanced event-store callers.
108///
109/// [`EventStoreConfig`] remains the serializable run policy. This type carries process-local
110/// construction choices, such as the encoder registry and backend opener used when a kernel
111/// opens a run.
112#[derive(Clone)]
113pub struct EventStoreLifecycleOptions {
114    registry_factory: Arc<RegistryFactory>,
115    backend_opener: Arc<BackendOpener>,
116    marker_registry_factory: Arc<MarkerRegistryFactory>,
117}
118
119impl Debug for EventStoreLifecycleOptions {
120    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
121        f.debug_struct(stringify!(EventStoreLifecycleOptions))
122            .finish_non_exhaustive()
123    }
124}
125
126impl Default for EventStoreLifecycleOptions {
127    fn default() -> Self {
128        Self {
129            registry_factory: Arc::new(default_registry),
130            backend_opener: Arc::new(default_backend_opener),
131            marker_registry_factory: Arc::new(DataMarkerExtractorRegistry::default_registry),
132        }
133    }
134}
135
136impl EventStoreLifecycleOptions {
137    /// Creates options that use [`default_registry`] and [`RedbBackend`].
138    #[must_use]
139    pub fn new() -> Self {
140        Self::default()
141    }
142
143    /// Uses a caller-supplied encoder registry factory for each opened run.
144    #[must_use]
145    pub fn with_registry_factory<F>(mut self, factory: F) -> Self
146    where
147        F: Fn() -> EncoderRegistry + Send + Sync + 'static,
148    {
149        self.registry_factory = Arc::new(factory);
150        self
151    }
152
153    /// Uses a caller-supplied encoder registry for each opened run.
154    #[must_use]
155    pub fn with_encoder_registry(self, registry: EncoderRegistry) -> Self {
156        self.with_registry_factory(move || registry.clone())
157    }
158
159    /// Uses a caller-supplied backend opener for each opened run.
160    #[must_use]
161    pub fn with_backend_opener<F>(mut self, opener: F) -> Self
162    where
163        F: Fn(&EventStoreConfig, &RunManifest) -> BackendOpenResult + Send + Sync + 'static,
164    {
165        self.backend_opener = Arc::new(opener);
166        self
167    }
168
169    /// Uses a caller-supplied data-marker extractor registry factory for each opened run.
170    #[must_use]
171    pub fn with_marker_registry_factory<F>(mut self, factory: F) -> Self
172    where
173        F: Fn(&[DataClass]) -> DataMarkerExtractorRegistry + Send + Sync + 'static,
174    {
175        self.marker_registry_factory = Arc::new(factory);
176        self
177    }
178
179    fn build_registry(&self) -> EncoderRegistry {
180        (self.registry_factory)()
181    }
182
183    fn open_backend(&self, config: &EventStoreConfig, manifest: &RunManifest) -> BackendOpenResult {
184        (self.backend_opener)(config, manifest)
185    }
186
187    fn build_marker_registry(&self, classes: &[DataClass]) -> DataMarkerExtractorRegistry {
188        (self.marker_registry_factory)(classes)
189    }
190}
191
192fn default_backend_opener(config: &EventStoreConfig, manifest: &RunManifest) -> BackendOpenResult {
193    let mut backend = RedbBackend::new(config.base_dir.clone());
194    backend.open_run(manifest.clone())?;
195    Ok(Box::new(backend))
196}
197
198/// Errors surfaced by the boot path.
199#[derive(Debug, thiserror::Error)]
200pub enum BootError {
201    /// The event store backend rejected an open, scan, or seal during recovery or
202    /// new-run creation.
203    #[error(transparent)]
204    EventStore(#[from] EventStoreError),
205    /// The writer rejected the `RunStarted` submit.
206    #[error("RunStarted submit failed: {0}")]
207    RunStartedSubmit(String),
208    /// The writer accepted `RunStarted` but did not durably commit it inside the
209    /// configured timeout.
210    #[error("RunStarted did not durably commit within {timeout:?}")]
211    RunStartedTimeout {
212        /// The configured ceiling that elapsed before the writer's high-watermark
213        /// advanced.
214        timeout: Duration,
215    },
216    /// The writer signaled fail-stop while the boot path was waiting for the
217    /// `RunStarted` entry to commit.
218    #[error("event store halted during boot: {0:?}")]
219    HaltedDuringBoot(HaltReason),
220}
221
222/// A thread-safe halt signal the kernel registers with the writer.
223///
224/// Each fail-stop source fires the shared callback at most once: the submitting
225/// thread on a backpressure stall, the writer thread on a backend failure, and the
226/// capture adapter on a rejected submit. The signal records the first reason for
227/// supervision; no runtime component polls it to stop the trader.
228#[derive(Clone, Debug)]
229pub struct HaltSignal {
230    halted: Arc<AtomicBool>,
231    reason: Arc<Mutex<Option<HaltReason>>>,
232}
233
234impl Default for HaltSignal {
235    fn default() -> Self {
236        Self::new()
237    }
238}
239
240impl HaltSignal {
241    /// Constructs a fresh, un-fired halt signal.
242    #[must_use]
243    pub fn new() -> Self {
244        Self {
245            halted: Arc::new(AtomicBool::new(false)),
246            reason: Arc::new(Mutex::new(None)),
247        }
248    }
249
250    /// Returns the [`HaltCallback`] the writer fires when an unrecoverable condition
251    /// occurs.
252    ///
253    /// The callback records the [`HaltReason`] (preserving only the first one when
254    /// the writer and the capture adapter both signal) and then flips the halted
255    /// flag, so a poller that observes `is_halted()` never reads back an empty
256    /// reason.
257    #[must_use]
258    pub fn callback(&self) -> HaltCallback {
259        let halted = Arc::clone(&self.halted);
260        let reason = Arc::clone(&self.reason);
261        Arc::new(move |r| {
262            let mut slot = reason.lock();
263            if slot.is_none() {
264                *slot = Some(r);
265            }
266            drop(slot);
267            halted.store(true, Ordering::Release);
268        })
269    }
270
271    /// Returns whether the writer has signaled fail-stop.
272    #[must_use]
273    pub fn is_halted(&self) -> bool {
274        self.halted.load(Ordering::Acquire)
275    }
276
277    /// Returns the [`HaltReason`] recorded on the first fail-stop, if any.
278    ///
279    /// Calling this does not clear the signal; the kernel's halted flag remains set so
280    /// subsequent submits surface as fail-stopped.
281    #[must_use]
282    pub fn reason(&self) -> Option<HaltReason> {
283        self.reason.lock().clone()
284    }
285}
286
287/// Live event-store session owned by the kernel between `start()` and `finalize_stop()`.
288pub struct EventStoreSession {
289    writer: Option<Arc<EventStoreWriter>>,
290    adapter: Option<Arc<BusCaptureAdapter>>,
291    marker_capture: Option<SharedMarkerCapture>,
292    manifest: RunManifest,
293    halt_signal: HaltSignal,
294}
295
296impl Debug for EventStoreSession {
297    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
298        f.debug_struct(stringify!(EventStoreSession))
299            .field("run_id", &self.manifest.run_id)
300            .field("parent_run_id", &self.manifest.parent_run_id)
301            .field("instance_id", &self.manifest.instance_id)
302            .field("halted", &self.halt_signal.is_halted())
303            .field("writer_attached", &self.writer.is_some())
304            .field("marker_capture_attached", &self.marker_capture.is_some())
305            .finish_non_exhaustive()
306    }
307}
308
309impl EventStoreSession {
310    /// Returns the captured manifest as it was written to disk at run start.
311    ///
312    /// The high-watermark and `end_ts_init` advance after seal; the snapshot here is
313    /// frozen at boot time.
314    #[must_use]
315    pub const fn manifest(&self) -> &RunManifest {
316        &self.manifest
317    }
318
319    /// Returns the run id of the currently open run.
320    #[must_use]
321    pub fn run_id(&self) -> &str {
322        self.manifest.run_id.as_str()
323    }
324
325    /// Returns the parent run id for the current run.
326    #[must_use]
327    pub fn parent_run_id(&self) -> Option<&str> {
328        self.manifest.parent_run_id.as_deref()
329    }
330
331    /// Returns whether the writer has fail-stopped.
332    #[must_use]
333    pub fn is_halted(&self) -> bool {
334        self.halt_signal.is_halted()
335    }
336
337    /// Returns the writer's current durable high-watermark.
338    ///
339    /// Returns `0` when the writer has been consumed by a prior `close`.
340    #[must_use]
341    pub fn high_watermark(&self) -> u64 {
342        self.writer.as_ref().map_or(0, |w| w.high_watermark())
343    }
344
345    /// Returns a snapshot anchorer bound to the open writer.
346    ///
347    /// The execution engine installs this callback while the run is open. The callback
348    /// records the cache-owned snapshot reference against the writer's durable
349    /// high-watermark after flushing earlier captured entries.
350    ///
351    /// This records the anchor, not its snapshot blob. Recovery also requires the referenced blob
352    /// to be available; a successful cache write does not confirm durable blob storage.
353    #[must_use]
354    pub fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
355        let writer = Arc::clone(self.writer.as_ref()?);
356
357        Some(Rc::new(move |snapshot_ref: CacheSnapshotRef| {
358            let content_hash = compute_snapshot_content_hash(snapshot_ref.blob.as_ref());
359            writer
360                .record_snapshot_anchor(snapshot_ref.blob_ref, content_hash)
361                .map(|_| ())
362                .map_err(|e| anyhow::anyhow!("record snapshot anchor: {e}"))
363        }))
364    }
365
366    /// Returns the live bus capture adapter, when one was wired into this run.
367    ///
368    /// `None` after [`Self::close`] consumes the writer.
369    #[must_use]
370    pub fn adapter(&self) -> Option<&Arc<BusCaptureAdapter>> {
371        self.adapter.as_ref()
372    }
373
374    /// Submits the terminal `RunEnded` entry, drains pending entries, and seals the
375    /// manifest as [`RunStatus::Ended`].
376    ///
377    /// Consumes the inner writer; subsequent calls return without effect.
378    ///
379    /// # Errors
380    ///
381    /// Returns an error if the writer fails to commit the final batch, the seal step fails, or the
382    /// writer Arc has outstanding clones (the bus tap must be cleared before close to release the
383    /// adapter's writer reference).
384    pub fn close(&mut self, ts_init: UnixNanos) -> Result<(), EventStoreError> {
385        // Drop the adapter first so the writer Arc has no other strong owners on
386        // try_unwrap. The kernel clears the bus tap before this site, so dropping the
387        // session-side adapter clone here is the last release before close.
388        self.adapter = None;
389        let marker_capture = self.marker_capture.take();
390
391        let Some(writer_arc) = self.writer.take() else {
392            close_marker_capture(marker_capture);
393            return Ok(());
394        };
395        let Ok(writer) = Arc::try_unwrap(writer_arc) else {
396            close_marker_capture(marker_capture);
397            return Err(EventStoreError::Backend(
398                "event store writer has multiple owners; clear the bus tap before close"
399                    .to_string(),
400            ));
401        };
402
403        let run_ended = run_ended_draft(ts_init);
404        let result = writer.close(run_ended);
405        close_marker_capture(marker_capture);
406        result?;
407        Ok(())
408    }
409}
410
411impl Drop for EventStoreSession {
412    fn drop(&mut self) {
413        // Drop without close: release adapter then writer so the writer thread exits
414        // unsealed; the next boot recovers.
415        self.adapter.take();
416        self.marker_capture.take();
417        self.writer.take();
418    }
419}
420
421fn close_marker_capture(marker_capture: Option<SharedMarkerCapture>) {
422    if let Some(marker_capture) = marker_capture
423        && let Some(capture) = marker_capture.borrow_mut().take()
424    {
425        capture.close();
426    }
427}
428
429/// Typed error surfaced when the event store fails the run lifecycle.
430///
431/// Wraps the boot-time and shutdown-time failure modes so a kernel caller can react to a
432/// fail-stop without inspecting individual writer/backend errors.
433#[derive(Debug, thiserror::Error)]
434pub enum KernelError {
435    /// The event-store boot path failed.
436    #[error("event store boot failed: {0}")]
437    EventStoreBoot(#[from] BootError),
438    /// Cache state reconstruction from a recovered event-store run failed.
439    #[error("event store cache replay failed: {0}")]
440    CacheReplay(#[from] CacheReplayError),
441    /// The writer signaled fail-stop after the kernel was already started.
442    #[error("event store halted: {0:?}")]
443    EventStoreHalted(HaltReason),
444}
445
446/// Kernel-facing wrapper that bundles every event-store concern: predecessor recovery,
447/// the open run, the halt signal, and the seal-on-drop fail-safe.
448///
449/// One instance is typically owned by [`nautilus_system::NautilusKernel`] via the
450/// [`KernelEventStoreTrait`] seam: the kernel calls [`EventStoreLifecycle::open`] from
451/// `start()`, [`EventStoreLifecycle::seal`] from `finalize_stop()` / `dispose()`, and
452/// the wrapper's [`Drop`] runs as the last-chance seal site for callers that skip both
453/// teardown paths (e.g. imperative `engine.run(...)` followed by drop in
454/// `BacktestEngine`).
455#[derive(Debug)]
456pub struct EventStoreLifecycle {
457    config: Option<EventStoreConfig>,
458    options: EventStoreLifecycleOptions,
459    recovered: Vec<RecoveredRun>,
460    parent_run_id: Option<String>,
461    session: Option<EventStoreSession>,
462    halt: HaltSignal,
463    // Held so `Drop` can stamp the seal even when the kernel never called seal()
464    // explicitly. Cloning the kernel's clock Rc keeps the wrapper independent of
465    // its owner.
466    clock: Rc<RefCell<dyn Clock>>,
467}
468
469impl EventStoreLifecycle {
470    /// Boots the wrapper at kernel construction time.
471    ///
472    /// Runs the predecessor recovery sweep against `<base_dir>/<instance_id>/`. When
473    /// `config` is `None` the wrapper is inert: every method becomes a no-op.
474    ///
475    /// # Errors
476    ///
477    /// Returns an error if the recovery sweep fails for a reason other than the expected handshake
478    /// with a crashed predecessor.
479    pub fn boot(
480        config: Option<EventStoreConfig>,
481        instance_id: UUID4,
482        clock: Rc<RefCell<dyn Clock>>,
483    ) -> anyhow::Result<Self> {
484        Self::boot_with_options(
485            config,
486            instance_id,
487            clock,
488            EventStoreLifecycleOptions::default(),
489        )
490    }
491
492    /// Boots the wrapper at kernel construction time with process-local lifecycle options.
493    ///
494    /// `EventStoreConfig` remains serializable. `options` carries runtime-only construction
495    /// policy for the encoder registry and backend opener.
496    ///
497    /// # Errors
498    ///
499    /// Returns an error if the recovery sweep fails for a reason other than the expected handshake
500    /// with a crashed predecessor.
501    pub fn boot_with_options(
502        config: Option<EventStoreConfig>,
503        instance_id: UUID4,
504        clock: Rc<RefCell<dyn Clock>>,
505        options: EventStoreLifecycleOptions,
506    ) -> anyhow::Result<Self> {
507        let (recovered, parent_run_id) = if let Some(cfg) = config.as_ref() {
508            let outcome = recover_predecessors(&cfg.base_dir, &instance_id.to_string())?;
509            if !outcome.recovered.is_empty() {
510                log::info!(
511                    "Sealed {} crashed event-store predecessor(s); parent_run_id={:?}",
512                    outcome.recovered.len(),
513                    outcome.parent_run_id,
514                );
515            }
516            (outcome.recovered, outcome.parent_run_id)
517        } else {
518            (Vec::new(), None)
519        };
520        Ok(Self {
521            config,
522            options,
523            recovered,
524            parent_run_id,
525            session: None,
526            halt: HaltSignal::new(),
527            clock,
528        })
529    }
530
531    /// Opens a fresh run on kernel `start()`. Idempotent against reset/rerun: a
532    /// leftover session from a prior `start()` is sealed before a new one opens, so
533    /// `RunStarted` remains the first entry of every run.
534    ///
535    /// `components` is the manifest captured into the `RunStarted` payload. `environment`
536    /// selects the static (backtest) or realtime (live) clock used to stamp `ts_publish`
537    /// inside the writer.
538    ///
539    /// Returns without effect when no event-store config was supplied.
540    ///
541    /// # Errors
542    ///
543    /// Returns an error if opening the new run, spawning the writer, or blocking on the
544    /// `RunStarted` ack fails.
545    pub fn open(
546        &mut self,
547        instance_id: UUID4,
548        components: &RegisteredComponents,
549        environment: Environment,
550    ) -> Result<(), KernelError> {
551        let Some(config) = self.config.clone() else {
552            return Ok(());
553        };
554
555        if self.session.is_some() {
556            // Reset/rerun (BacktestEngine::run -> reset -> run) reuses the kernel
557            // across runs. Seal the leftover session before opening a fresh one.
558            let ts = self.clock.borrow().timestamp_ns();
559            self.seal(ts);
560        }
561
562        // Re-arm the fail-stop signal: a halt is terminal for the run that fired it,
563        // not for the kernel. A stale signal fails the rerun's boot or opens it
564        // permanently halted, downgrading its graceful stop to CrashedRecovered.
565        self.halt = HaltSignal::new();
566
567        let clock = Self::clock_for(environment);
568        let start_ts_init = self.clock.borrow().timestamp_ns();
569        let run_id = build_run_id(start_ts_init);
570        let parent_run_id = if let Some(replay_run_id) = config.replay_from_run_id.as_deref() {
571            validate_event_store_replay_source(
572                config.base_dir.clone(),
573                &instance_id.to_string(),
574                replay_run_id,
575            )?;
576            Some(replay_run_id.to_string())
577        } else {
578            self.parent_run_id.clone()
579        };
580        let session = open_run_with_options(
581            &config,
582            &instance_id.to_string(),
583            run_id,
584            parent_run_id,
585            start_ts_init,
586            components,
587            self.halt.clone(),
588            clock,
589            &self.options,
590        )?;
591        log::info!(
592            "Opened event-store run {} (parent_run_id={:?})",
593            session.run_id(),
594            session.parent_run_id(),
595        );
596
597        if let Some(adapter) = session.adapter() {
598            install_bus_tap(Arc::clone(adapter), session.marker_capture.clone(), clock);
599        }
600        self.session = Some(session);
601        Ok(())
602    }
603
604    /// Restores cache state from the configured replay run or recovered parent run.
605    ///
606    /// This is a bootstrap-only reconstruction path. It opens the sealed replay source
607    /// for read-only replay, restores the cache-owned snapshot blob, then replays only
608    /// the entries after the snapshot anchor directly into [`Cache`].
609    ///
610    /// # Errors
611    ///
612    /// Returns an error if the source reader, snapshot restore, decode, or cache apply step fails.
613    pub fn restore_parent_cache(
614        &self,
615        instance_id: UUID4,
616        cache: &mut Cache,
617    ) -> Result<Option<CacheReplayReport>, KernelError> {
618        let Some(config) = self.config.as_ref() else {
619            return Ok(None);
620        };
621        let replay_run_id = config
622            .replay_from_run_id
623            .as_deref()
624            .or(self.parent_run_id.as_deref());
625        let Some(replay_run_id) = replay_run_id else {
626            return Ok(None);
627        };
628        let source = if config.replay_from_run_id.is_some() {
629            "configured replay run"
630        } else {
631            "parent run"
632        };
633
634        let report = restore_cache_from_sealed_run(
635            cache,
636            config.base_dir.clone(),
637            &instance_id.to_string(),
638            replay_run_id,
639        )?;
640
641        log::info!(
642            "Restored cache from event-store {source} {replay_run_id}: from_seq={}, to_seq={}, applied={}, ignored={}",
643            report.cache.plan.from_seq,
644            report.cache.plan.to_seq,
645            report.cache.applied_entries,
646            report.cache.ignored_entries,
647        );
648
649        Ok(Some(report.cache))
650    }
651
652    /// Seals the open session by writing `RunEnded` and updating the manifest to
653    /// `Ended`. Idempotent: a closed or absent session makes this a no-op. Halted
654    /// sessions skip the close (the recovery sweep on next boot owns the seal).
655    pub fn seal(&mut self, ts_init: UnixNanos) {
656        let Some(mut session) = self.session.take() else {
657            return;
658        };
659
660        // Drop the bus tap before close so the adapter's writer Arc is released; the
661        // close path then takes sole ownership of the writer and commits RunEnded.
662        msgbus::clear_bus_tap();
663
664        if session.is_halted() {
665            log::warn!(
666                "Event-store writer fail-stopped before close; run {} sealed by recovery sweep on next boot",
667                session.run_id(),
668            );
669            return;
670        }
671        let run_id = session.run_id().to_string();
672        if let Err(e) = session.close(ts_init) {
673            log::error!(
674                "Failed to seal event-store run {run_id} on graceful stop: {e}; run will be sealed as CrashedRecovered on next boot",
675            );
676        } else {
677            log::info!("Sealed event-store run {run_id}");
678        }
679    }
680
681    /// Returns the recovery report from the boot sweep.
682    #[must_use]
683    pub fn recovered(&self) -> &[RecoveredRun] {
684        &self.recovered
685    }
686
687    /// Returns the configured replay source or recovered parent run id, when present.
688    #[must_use]
689    pub fn parent_run_id(&self) -> Option<&str> {
690        self.config
691            .as_ref()
692            .and_then(|config| config.replay_from_run_id.as_deref())
693            .or(self.parent_run_id.as_deref())
694    }
695
696    /// Returns whether this lifecycle is configured for event-store-only replay.
697    #[must_use]
698    pub fn is_event_store_replay_configured(&self) -> bool {
699        self.config
700            .as_ref()
701            .is_some_and(|config| config.replay_from_run_id.is_some())
702    }
703
704    /// Returns the run id of the open session, when capture is active.
705    #[must_use]
706    pub fn run_id(&self) -> Option<&str> {
707        self.session.as_ref().map(EventStoreSession::run_id)
708    }
709
710    /// Returns a snapshot anchorer for the open run, when capture is active.
711    #[must_use]
712    pub fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
713        self.session
714            .as_ref()
715            .and_then(EventStoreSession::snapshot_anchorer)
716    }
717
718    /// Returns whether the writer has signaled fail-stop.
719    #[must_use]
720    pub fn is_halted(&self) -> bool {
721        self.halt.is_halted()
722    }
723
724    /// Returns the [`HaltReason`] recorded on the first fail-stop, if any.
725    #[must_use]
726    pub fn halt_reason(&self) -> Option<HaltReason> {
727        self.halt.reason()
728    }
729
730    /// Surfaces the current halt as a typed [`KernelError`], or `None` when the
731    /// writer has not halted.
732    #[must_use]
733    pub fn check_halt(&self) -> Option<KernelError> {
734        self.halt_reason().map(KernelError::EventStoreHalted)
735    }
736
737    #[cfg(feature = "live")]
738    fn clock_for(environment: Environment) -> &'static AtomicTime {
739        match environment {
740            Environment::Backtest => get_atomic_clock_static(),
741            Environment::Live | Environment::Sandbox => get_atomic_clock_realtime(),
742        }
743    }
744
745    #[cfg(not(feature = "live"))]
746    fn clock_for(_environment: Environment) -> &'static AtomicTime {
747        get_atomic_clock_static()
748    }
749}
750
751impl Drop for EventStoreLifecycle {
752    fn drop(&mut self) {
753        // Last-chance seal: callers may skip both finalize_stop() and dispose().
754        if self.session.is_none() {
755            return;
756        }
757        let ts = self
758            .clock
759            .try_borrow()
760            .map(|c| c.timestamp_ns())
761            .unwrap_or_default();
762        self.seal(ts);
763    }
764}
765
766/// Sweeps `<base_dir>/<instance_id>/` for crashed predecessor runs and seals each one.
767///
768/// A predecessor is a run file whose manifest still reads [`RunStatus::Running`]: the
769/// previous trader exited (cleanly via drop, or crashed) without sealing. The sweep
770/// scans every entry in the run, validating hashes; on success the manifest seals as
771/// [`RunStatus::CrashedRecovered`], otherwise as [`RunStatus::Quarantined`]. The
772/// most-recently-crashed survivor's `run_id` is returned so the new run records it as
773/// `parent_run_id`.
774///
775/// Quarantined runs do not become parents: a future replay must skip the corrupted
776/// tail rather than chain through it.
777///
778/// A predecessor that cannot be reopened, scanned, or sealed is skipped with a logged
779/// error rather than failing the sweep: recovery must never leave the trader unbootable
780/// because one run file is damaged. Skipped runs keep their on-disk status, so the next
781/// boot retries them.
782///
783/// # Errors
784///
785/// Returns an error if directory enumeration fails or a predecessor unexpectedly reopens without
786/// the handshake the backend uses to report an unsealed run.
787pub fn recover_predecessors(
788    base_dir: &Path,
789    instance_id: &str,
790) -> Result<RecoveryOutcome, EventStoreError> {
791    let manifests = RedbBackend::list_runs(base_dir, instance_id)?;
792    let crashed: Vec<RunManifest> = manifests
793        .into_iter()
794        .filter(|m| matches!(m.status, RunStatus::Running))
795        .collect();
796
797    let mut outcome = RecoveryOutcome::default();
798
799    for predecessor in crashed {
800        let run_id = predecessor.run_id.clone();
801        let mut backend = RedbBackend::new(base_dir.to_path_buf());
802
803        match backend.open_run(predecessor) {
804            Err(EventStoreError::CrashedPredecessor) => {}
805            Ok(()) => {
806                return Err(EventStoreError::Backend(format!(
807                    "expected CrashedPredecessor reopening {run_id}, was Ok",
808                )));
809            }
810            Err(other) => {
811                log::error!("Skipping recovery of run {run_id}, reopen failed: {other}");
812                continue;
813            }
814        }
815
816        let high_watermark = backend.high_watermark()?;
817        let final_status = if high_watermark == 0 {
818            RunStatus::CrashedRecovered
819        } else {
820            match backend.scan_range(1, high_watermark, ScanDirection::Forward) {
821                Ok(entries) => {
822                    // The writer commits RunEnded before seal; a crash between those
823                    // two steps leaves a graceful tail without a sealed manifest.
824                    // Honor the tail: if the last entry is the kernel's RunEnded
825                    // marker, the predecessor closed cleanly and is not a crash to
826                    // chain through. Match both topic and payload_type so a future
827                    // capture-registry entry that happens to share the payload tag
828                    // cannot be misclassified as a graceful close.
829                    let tail_is_run_ended = entries.last().is_some_and(|e| {
830                        e.topic.as_ref() == RUN_ENDED_TOPIC
831                            && e.payload_type.as_str() == RUN_ENDED_PAYLOAD_TYPE
832                    });
833
834                    if tail_is_run_ended {
835                        RunStatus::Ended
836                    } else {
837                        RunStatus::CrashedRecovered
838                    }
839                }
840                Err(
841                    EventStoreError::HashMismatch { .. }
842                    | EventStoreError::SeqMismatch { .. }
843                    | EventStoreError::Corrupted(_)
844                    | EventStoreError::Gap { .. },
845                ) => RunStatus::Quarantined,
846                Err(other) => {
847                    log::error!("Skipping recovery of run {run_id}, scan failed: {other}");
848                    continue;
849                }
850            }
851        };
852
853        if let Err(e) = backend.seal(final_status) {
854            log::error!("Skipping recovery of run {run_id}, seal as {final_status:?} failed: {e}");
855            continue;
856        }
857        outcome.recovered.push(RecoveredRun {
858            run_id: run_id.clone(),
859            status: final_status,
860        });
861
862        if matches!(final_status, RunStatus::CrashedRecovered) {
863            outcome.parent_run_id = Some(run_id);
864        }
865    }
866
867    Ok(outcome)
868}
869
870/// Builds the `<start_ts_init>-<short_uuid>` run id used as the manifest key and on-disk
871/// file name.
872///
873/// The id is sortable by start time so directory listings produce chronological order;
874/// the short uuid suffix keeps it unique even when two kernels start at the same
875/// nanosecond on different machines.
876#[must_use]
877pub fn build_run_id(start_ts_init: UnixNanos) -> RunId {
878    let suffix: String = UUID4::new().to_string().chars().take(8).collect();
879    format!("{}-{suffix}", u64::from(start_ts_init))
880}
881
882/// Opens a fresh run, spawns the writer, and submits a blocking `RunStarted` entry.
883///
884/// The kernel calls this from `start()` after components have registered with the
885/// trader so the captured `RunStarted` payload reflects the actual boot configuration.
886/// The function blocks until the writer's high-watermark advances past zero (i.e. the
887/// `RunStarted` entry has durably committed) or until [`EventStoreConfig::run_started_timeout`]
888/// elapses.
889///
890/// `feature_flags` is appended after the configured `feature_flags` so the retention
891/// mode survives in the manifest as `retention=<mode>`.
892///
893/// # Errors
894///
895/// Returns an error if the backend rejects open, if the writer rejects the submit, if the commit
896/// does not happen inside the configured ceiling, or if the writer fail-stops while waiting for the
897/// commit.
898#[expect(
899    clippy::too_many_arguments,
900    reason = "run opening requires the boot context and event-store handles"
901)]
902pub fn open_run(
903    config: &EventStoreConfig,
904    instance_id: &str,
905    run_id: RunId,
906    parent_run_id: Option<RunId>,
907    start_ts_init: UnixNanos,
908    components: &RegisteredComponents,
909    halt_signal: HaltSignal,
910    clock: &'static AtomicTime,
911) -> Result<EventStoreSession, BootError> {
912    open_run_with_options(
913        config,
914        instance_id,
915        run_id,
916        parent_run_id,
917        start_ts_init,
918        components,
919        halt_signal,
920        clock,
921        &EventStoreLifecycleOptions::default(),
922    )
923}
924
925/// Opens a fresh run with process-local lifecycle options.
926///
927/// This follows [`open_run`] but obtains the backend and encoder registry from
928/// `options`.
929///
930/// # Errors
931///
932/// Returns an error if the backend rejects open, if the writer rejects the submit, if the commit
933/// does not happen inside the configured ceiling, or if the writer fail-stops while waiting for the
934/// commit.
935#[expect(
936    clippy::too_many_arguments,
937    reason = "run opening with options requires the boot context and lifecycle handles"
938)]
939pub fn open_run_with_options(
940    config: &EventStoreConfig,
941    instance_id: &str,
942    run_id: RunId,
943    parent_run_id: Option<RunId>,
944    start_ts_init: UnixNanos,
945    components: &RegisteredComponents,
946    halt_signal: HaltSignal,
947    clock: &'static AtomicTime,
948    options: &EventStoreLifecycleOptions,
949) -> Result<EventStoreSession, BootError> {
950    let manifest = build_manifest(
951        config,
952        instance_id,
953        run_id,
954        parent_run_id,
955        start_ts_init,
956        components.clone(),
957    );
958
959    let backend = options.open_backend(config, &manifest)?;
960
961    let writer = Arc::new(EventStoreWriter::spawn(
962        backend,
963        clock,
964        halt_signal.callback(),
965        writer_config_from(config),
966    )?);
967
968    submit_run_started_blocking(
969        &writer,
970        components,
971        start_ts_init,
972        &halt_signal,
973        config.run_started_timeout,
974    )?;
975
976    let (marker_capture, submit_counter) =
977        build_marker_capture(config, &manifest, writer.high_watermark(), clock, options);
978    let mut adapter = BusCaptureAdapter::new(
979        Arc::clone(&writer),
980        Arc::new(options.build_registry()),
981        halt_signal.callback(),
982    );
983
984    if let Some(submit_counter) = submit_counter {
985        adapter = adapter.with_submit_counter(submit_counter);
986    }
987    let adapter = Arc::new(adapter);
988
989    Ok(EventStoreSession {
990        writer: Some(writer),
991        adapter: Some(adapter),
992        marker_capture,
993        manifest,
994        halt_signal,
995    })
996}
997
998fn build_marker_capture(
999    config: &EventStoreConfig,
1000    manifest: &RunManifest,
1001    initial_submit_counter: u64,
1002    clock: &'static AtomicTime,
1003    options: &EventStoreLifecycleOptions,
1004) -> (Option<SharedMarkerCapture>, Option<Arc<AtomicU64>>) {
1005    let Some(marker_config) = config.data_markers.as_ref() else {
1006        return (None, None);
1007    };
1008
1009    match open_marker_capture(
1010        config,
1011        manifest,
1012        marker_config,
1013        initial_submit_counter,
1014        clock,
1015        options,
1016    ) {
1017        Ok((capture, submit_counter)) => (
1018            Some(Rc::new(RefCell::new(Some(capture)))),
1019            Some(submit_counter),
1020        ),
1021        Err(e) => {
1022            log::warn!(
1023                "Data marker sidecar disabled for run {} after marker setup failed: {e}",
1024                manifest.run_id,
1025            );
1026            (None, None)
1027        }
1028    }
1029}
1030
1031fn open_marker_capture(
1032    config: &EventStoreConfig,
1033    manifest: &RunManifest,
1034    marker_config: &DataMarkerConfig,
1035    initial_submit_counter: u64,
1036    clock: &'static AtomicTime,
1037    options: &EventStoreLifecycleOptions,
1038) -> Result<(DataMarkerCapture, Arc<AtomicU64>), EventStoreError> {
1039    let classes = marker_config
1040        .classes
1041        .iter()
1042        .copied()
1043        .map(data_marker_class_to_data_class)
1044        .collect::<Vec<_>>();
1045    let marker_manifest = marker_manifest_for(manifest, classes.clone(), marker_config);
1046    let marker_path = marker_file_path(config, &manifest.instance_id, &manifest.run_id);
1047    let mut marker_backend = RedbMarkerBackend::new(marker_path);
1048    marker_backend.open_run(marker_manifest)?;
1049    let writer = MarkerWriter::spawn(
1050        Box::new(marker_backend),
1051        clock,
1052        MarkerWriterConfig {
1053            channel_capacity: marker_config.channel_capacity,
1054            ..MarkerWriterConfig::default()
1055        },
1056    )?;
1057    let submit_counter = Arc::new(AtomicU64::new(initial_submit_counter));
1058    let registry = options.build_marker_registry(&classes);
1059    let capture =
1060        DataMarkerCapture::new(registry, writer, Arc::clone(&submit_counter), marker_config);
1061
1062    Ok((capture, submit_counter))
1063}
1064
1065fn marker_file_path(config: &EventStoreConfig, instance_id: &str, run_id: &str) -> PathBuf {
1066    config
1067        .base_dir
1068        .join(instance_id)
1069        .join(format!("{run_id}.markers.redb"))
1070}
1071
1072fn marker_manifest_for(
1073    manifest: &RunManifest,
1074    enabled_classes: Vec<DataClass>,
1075    config: &DataMarkerConfig,
1076) -> MarkerManifest {
1077    MarkerManifest {
1078        run_id: manifest.run_id.clone(),
1079        enabled_classes,
1080        high_fidelity: !config.high_fidelity.is_empty(),
1081        snapshot_count: 0,
1082        hifi_count: 0,
1083        gap_count: 0,
1084        dict_count: 0,
1085        status: RunStatus::Running,
1086    }
1087}
1088
1089const fn data_marker_class_to_data_class(class: DataMarkerClass) -> DataClass {
1090    match class {
1091        DataMarkerClass::BookDeltas => DataClass::BookDeltas,
1092        DataMarkerClass::BookDepth => DataClass::BookDepth,
1093        DataMarkerClass::Quote => DataClass::Quote,
1094        DataMarkerClass::Trade => DataClass::Trade,
1095        DataMarkerClass::Bar => DataClass::Bar,
1096    }
1097}
1098
1099fn build_manifest(
1100    config: &EventStoreConfig,
1101    instance_id: &str,
1102    run_id: RunId,
1103    parent_run_id: Option<RunId>,
1104    start_ts_init: UnixNanos,
1105    components: RegisteredComponents,
1106) -> RunManifest {
1107    let mut feature_flags = config.identity.feature_flags.clone();
1108    feature_flags.push(format!("retention={}", retention_tag(config.retention)));
1109
1110    RunManifest {
1111        run_id,
1112        parent_run_id,
1113        instance_id: instance_id.to_string(),
1114        binary_hash: config.identity.binary_hash.clone(),
1115        schema_version: config.identity.schema_version,
1116        crate_versions: config.identity.crate_versions.clone(),
1117        feature_flags,
1118        adapter_versions: config.identity.adapter_versions.clone(),
1119        config_hash: config.identity.config_hash.clone(),
1120        registered_components: components,
1121        seed: config.identity.seed,
1122        start_ts_init,
1123        end_ts_init: None,
1124        high_watermark: 0,
1125        status: RunStatus::Running,
1126    }
1127}
1128
1129const fn retention_tag(mode: RetentionMode) -> &'static str {
1130    match mode {
1131        RetentionMode::Full => "full",
1132        RetentionMode::Bounded { .. } => "bounded",
1133        RetentionMode::SnapshotAnchored => "snapshot",
1134    }
1135}
1136
1137fn writer_config_from(config: &EventStoreConfig) -> WriterConfig {
1138    WriterConfig {
1139        channel_capacity: config.channel_capacity,
1140        max_batch_entries: config.max_batch_entries,
1141        max_batch_latency: config.max_batch_latency,
1142        halt_threshold: config.halt_threshold,
1143    }
1144}
1145
1146/// Submits the `RunStarted` draft and blocks until the writer durably acknowledges it,
1147/// the writer fail-stops, or `timeout` elapses.
1148///
1149/// Exposed at `pub(crate)` so tests can drive it against a stub backend without going
1150/// through [`open_run`].
1151///
1152/// # Errors
1153///
1154/// Returns an error if the writer rejects the submit, if the writer fail-stops during the wait, or
1155/// if the writer does not commit within `timeout`.
1156pub(crate) fn submit_run_started_blocking(
1157    writer: &EventStoreWriter,
1158    components: &RegisteredComponents,
1159    ts_init: UnixNanos,
1160    halt_signal: &HaltSignal,
1161    timeout: Duration,
1162) -> Result<(), BootError> {
1163    let payload = encode_run_started(components);
1164    let draft = EntryDraft::without_indices(
1165        Headers::empty(),
1166        Topic::from(RUN_STARTED_TOPIC),
1167        Ustr::from(RUN_STARTED_PAYLOAD_TYPE),
1168        payload,
1169        ts_init,
1170    );
1171
1172    writer
1173        .submit(draft)
1174        .map_err(|e| BootError::RunStartedSubmit(e.to_string()))?;
1175
1176    // Wall-clock timeout against the writer thread: the writer drives the seam,
1177    // not the kernel state machine, so monotonic Instant timing is correct here.
1178    let start = Instant::now(); // dst-ok
1179
1180    while writer.high_watermark() == 0 {
1181        if halt_signal.is_halted() {
1182            return Err(BootError::HaltedDuringBoot(
1183                halt_signal.reason().unwrap_or_else(|| {
1184                    HaltReason::BackendError("event store halted during boot".to_string())
1185                }),
1186            ));
1187        }
1188
1189        let elapsed = start.elapsed();
1190
1191        if elapsed >= timeout {
1192            return Err(BootError::RunStartedTimeout { timeout });
1193        }
1194        thread::sleep(Duration::from_millis(1));
1195    }
1196
1197    Ok(())
1198}
1199
1200fn encode_run_started(components: &RegisteredComponents) -> Bytes {
1201    // The payload uses the same positional codec as the event-store envelope.
1202    let bytes = crate::codec::encode_to_vec(components).expect(
1203        "RegisteredComponents serializes via serde, must not fail under the positional codec",
1204    );
1205    Bytes::from(bytes)
1206}
1207
1208fn run_ended_draft(ts_init: UnixNanos) -> EntryDraft {
1209    EntryDraft::without_indices(
1210        Headers::empty(),
1211        Topic::from(RUN_ENDED_TOPIC),
1212        Ustr::from(RUN_ENDED_PAYLOAD_TYPE),
1213        Bytes::new(),
1214        ts_init,
1215    )
1216}
1217
1218/// Bus tap that forwards captured publish and send dispatches to the event store.
1219///
1220/// Built and registered by [`EventStoreLifecycle::open`]; cleared by
1221/// [`EventStoreLifecycle::seal`] and the wrapper's [`Drop`]. The tap reads `ts_init` from
1222/// the kernel's `AtomicTime` at capture time so non-Phase-A headers carry a
1223/// writer-receive timestamp.
1224struct EventStoreBusTap {
1225    adapter: Arc<BusCaptureAdapter>,
1226    marker_capture: Option<SharedMarkerCapture>,
1227    clock: &'static AtomicTime,
1228    // Latch for the one-time halted log: the per-message Halted arm stays silent to
1229    // avoid log spam, but the transition into dropping captures must leave a trace.
1230    halted_logged: AtomicBool,
1231}
1232
1233impl Debug for EventStoreBusTap {
1234    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1235        f.debug_struct(stringify!(EventStoreBusTap))
1236            .field("halted", &self.adapter.is_halted())
1237            .field("marker_capture_attached", &self.marker_capture.is_some())
1238            .finish_non_exhaustive()
1239    }
1240}
1241
1242impl BusTap for EventStoreBusTap {
1243    fn on_publish(&self, topic: Topic, message: &dyn Any) {
1244        let ts_init = self.clock.get_time_ns();
1245        self.capture(topic, message, ts_init);
1246    }
1247
1248    fn on_send(&self, endpoint: MStr<Endpoint>, message: &dyn Any) {
1249        let ts_init = self.clock.get_time_ns();
1250        // Reuse the endpoint string as the captured topic. The MStr markers differ but
1251        // the underlying interned string is the same; offline scans match either way.
1252        let topic = Topic::from(*endpoint);
1253        self.capture(topic, message, ts_init);
1254    }
1255
1256    fn on_response(&self, _correlation_id: &UUID4, message: &dyn Any) {
1257        let ts_init = self.clock.get_time_ns();
1258        let topic = MessagingSwitchboard::data_response_topic();
1259        self.capture(topic, message, ts_init);
1260    }
1261}
1262
1263impl EventStoreBusTap {
1264    fn capture(&self, topic: Topic, message: &dyn Any, ts_init: UnixNanos) {
1265        // The registry both gates capture (no encoder -> no entry) and supplies headers
1266        // for entries that do flow through. Looking the headers up here keeps the
1267        // adapter encoder-only and lets header propagation light up per-type as the
1268        // SPEC's workstream A lands fields on commands and events.
1269        let headers = self
1270            .adapter
1271            .registry()
1272            .headers_for_any(message)
1273            .unwrap_or_else(Headers::empty);
1274        // Submit failures fire the adapter halt callback before returning; HaltSignal
1275        // is the observation path. Halted means the signal already fired.
1276        match self.adapter.capture_any(topic, message, headers, ts_init) {
1277            Ok(captured) => {
1278                self.capture_marker(topic, message, ts_init, captured);
1279            }
1280            Err(CaptureError::Halted) => {
1281                if !self.halted_logged.swap(true, Ordering::AcqRel) {
1282                    log::error!(
1283                        "Event store capture is halted; state-affecting messages are no longer recorded for this run"
1284                    );
1285                }
1286            }
1287            Err(CaptureError::Submit(e)) => {
1288                log::error!("Event store capture submit failed on {topic}: {e}");
1289            }
1290            Err(CaptureError::Encode(e)) => {
1291                log::warn!("Event store encoder rejected message on {topic}: {e}");
1292            }
1293        }
1294    }
1295
1296    fn capture_marker(&self, topic: Topic, message: &dyn Any, ts_init: UnixNanos, captured: bool) {
1297        let Some(marker_capture) = self.marker_capture.as_ref() else {
1298            return;
1299        };
1300        let mut marker_capture = marker_capture.borrow_mut();
1301        let Some(capture) = marker_capture.as_mut() else {
1302            return;
1303        };
1304
1305        if captured {
1306            capture.on_entry_submitted(ts_init);
1307        } else {
1308            capture.observe_publish(topic, message, ts_init);
1309        }
1310        capture.maybe_safety_flush(ts_init);
1311    }
1312}
1313
1314fn install_bus_tap(
1315    adapter: Arc<BusCaptureAdapter>,
1316    marker_capture: Option<SharedMarkerCapture>,
1317    clock: &'static AtomicTime,
1318) {
1319    let tap: Rc<dyn BusTap> = Rc::new(EventStoreBusTap {
1320        adapter,
1321        marker_capture,
1322        clock,
1323        halted_logged: AtomicBool::new(false),
1324    });
1325    msgbus::set_bus_tap(tap);
1326}
1327
1328// Use fully qualified `EventStoreLifecycle::` to dispatch to the inherent methods;
1329// `Self::` would resolve back into this trait impl and recurse.
1330#[expect(
1331    clippy::use_self,
1332    reason = "Self would dispatch back into this trait impl"
1333)]
1334impl KernelEventStoreTrait for EventStoreLifecycle {
1335    fn restore_parent_cache(
1336        &mut self,
1337        instance_id: UUID4,
1338        cache: &mut Cache,
1339    ) -> anyhow::Result<()> {
1340        EventStoreLifecycle::restore_parent_cache(self, instance_id, cache)
1341            .map(|_| ())
1342            .map_err(Into::into)
1343    }
1344
1345    fn open(
1346        &mut self,
1347        instance_id: UUID4,
1348        components: &RegisteredComponents,
1349        environment: Environment,
1350    ) -> anyhow::Result<()> {
1351        EventStoreLifecycle::open(self, instance_id, components, environment).map_err(Into::into)
1352    }
1353
1354    fn snapshot_anchorer(&self) -> Option<SnapshotAnchorer> {
1355        EventStoreLifecycle::snapshot_anchorer(self)
1356    }
1357
1358    fn seal(&mut self, ts_init: UnixNanos) {
1359        EventStoreLifecycle::seal(self, ts_init);
1360    }
1361
1362    fn run_id(&self) -> Option<&str> {
1363        EventStoreLifecycle::run_id(self)
1364    }
1365
1366    fn parent_run_id(&self) -> Option<&str> {
1367        EventStoreLifecycle::parent_run_id(self)
1368    }
1369
1370    fn is_event_store_replay_configured(&self) -> bool {
1371        EventStoreLifecycle::is_event_store_replay_configured(self)
1372    }
1373
1374    fn is_halted(&self) -> bool {
1375        EventStoreLifecycle::is_halted(self)
1376    }
1377}
1378
1379#[cfg(test)]
1380mod tests {
1381    #[cfg(madsim)]
1382    use std::path::Path;
1383    use std::path::PathBuf;
1384
1385    use indexmap::IndexMap;
1386    use nautilus_common::{
1387        clock::VirtualClock,
1388        messages::{
1389            data::{
1390                DataCommand, DataResponse, DataResponseEnvelope, QuotesResponse, RequestCommand,
1391                RequestQuotes, SubscribeCommand, SubscribeQuotes,
1392            },
1393            execution::{SubmitOrder, TradingCommand},
1394        },
1395        msgbus::TypedHandler,
1396        timer::{TimeEvent, TimeEventCallback, TimeEventHandler},
1397    };
1398    use nautilus_core::time::get_atomic_clock_static;
1399    use nautilus_model::{
1400        data::stubs::{quote_ethusdt_binance, stub_deltas},
1401        enums::TimeInForce,
1402        events::{
1403            OrderEventAny, OrderFilled,
1404            order::spec::{OrderFilledSpec, OrderInitializedSpec},
1405        },
1406        identifiers::{
1407            AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, TradeId, TraderId, Venue,
1408            VenueOrderId,
1409        },
1410        types::{Currency, Money, Price, Quantity},
1411    };
1412    use nautilus_system::event_store::{DataMarkerClass, DataMarkerConfig, RunIdentity};
1413    use rstest::rstest;
1414    use tempfile::TempDir;
1415
1416    use super::*;
1417    use crate::{
1418        AppendEntry, DataClass, EncodedPayload, EventStoreEntry, IndexKind, MarkerBackend,
1419        MemoryBackend, RedbMarkerBackend, SnapshotAnchor,
1420        capture::builtins::PAYLOAD_TYPE_TIME_EVENT, compute_entry_hash,
1421    };
1422
1423    const INSTANCE_ID: &str = "trader-001";
1424
1425    fn make_config(base_dir: PathBuf) -> EventStoreConfig {
1426        EventStoreConfig {
1427            base_dir,
1428            identity: RunIdentity {
1429                binary_hash: "deadbeef".to_string(),
1430                schema_version: 1,
1431                crate_versions: "feedface".to_string(),
1432                feature_flags: Vec::new(),
1433                adapter_versions: IndexMap::new(),
1434                config_hash: "cafebabe".to_string(),
1435                seed: None,
1436            },
1437            retention: RetentionMode::Full,
1438            replay_from_run_id: None,
1439            data_markers: None,
1440            channel_capacity: 64,
1441            max_batch_entries: 1,
1442            max_batch_latency: Duration::from_millis(2),
1443            halt_threshold: Duration::from_secs(2),
1444            run_started_timeout: Duration::from_secs(2),
1445        }
1446    }
1447
1448    #[derive(Clone, Copy, Debug)]
1449    enum CrashPoint {
1450        BeforeEnqueue,
1451        AfterEnqueueBeforeCommit,
1452        AfterCommitBeforeSnapshot,
1453        AfterSnapshot,
1454    }
1455
1456    fn append_entry(seq: u64, topic: &str, payload_type: &str, payload: Bytes) -> AppendEntry {
1457        let ts = UnixNanos::from(seq);
1458        let headers = Headers::empty();
1459        let hash = compute_entry_hash(seq, ts, ts, topic, payload_type, &payload, &headers);
1460        let entry = EventStoreEntry::new(
1461            hash,
1462            seq,
1463            headers,
1464            Topic::from(topic),
1465            Ustr::from(payload_type),
1466            payload,
1467            ts,
1468            ts,
1469        );
1470        AppendEntry::without_indices(entry)
1471    }
1472
1473    fn make_submit_order(client_order_id: ClientOrderId) -> SubmitOrder {
1474        let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE");
1475        let order_init = OrderInitializedSpec::builder()
1476            .instrument_id(instrument_id)
1477            .client_order_id(client_order_id)
1478            .quantity(Quantity::from("1"))
1479            .time_in_force(TimeInForce::Gtc)
1480            .ts_event(UnixNanos::from(1))
1481            .ts_init(UnixNanos::from(2))
1482            .build();
1483        SubmitOrder::new(
1484            TraderId::from("TRADER-001"),
1485            Some(ClientId::from("BINANCE")),
1486            StrategyId::from("S-001"),
1487            instrument_id,
1488            client_order_id,
1489            order_init,
1490            None,
1491            None,
1492            None,
1493            UUID4::new(),
1494            UnixNanos::from(3),
1495            None, // correlation_id
1496        )
1497    }
1498
1499    fn append_run_started(seq: u64) -> AppendEntry {
1500        append_entry(
1501            seq,
1502            RUN_STARTED_TOPIC,
1503            RUN_STARTED_PAYLOAD_TYPE,
1504            encode_run_started(&RegisteredComponents::default()),
1505        )
1506    }
1507
1508    #[derive(Debug)]
1509    struct TestAuditMessage {
1510        value: u8,
1511    }
1512
1513    fn test_registry() -> EncoderRegistry {
1514        let mut registry = EncoderRegistry::new();
1515        registry.register::<TestAuditMessage, _>(Ustr::from("TestAuditMessage"), |message| {
1516            Ok(EncodedPayload::without_indices(Bytes::copy_from_slice(&[
1517                message.value,
1518            ])))
1519        });
1520        registry
1521    }
1522
1523    #[derive(Debug, Clone)]
1524    struct SharedMemoryBackend(Arc<Mutex<MemoryBackend>>);
1525
1526    impl EventStore for SharedMemoryBackend {
1527        fn open_run(&mut self, manifest: RunManifest) -> Result<(), EventStoreError> {
1528            self.0.lock().open_run(manifest)
1529        }
1530
1531        fn append_batch(&mut self, entries: &[AppendEntry]) -> Result<u64, EventStoreError> {
1532            self.0.lock().append_batch(entries)
1533        }
1534
1535        fn scan_range(
1536            &self,
1537            from: u64,
1538            to: u64,
1539            direction: ScanDirection,
1540        ) -> Result<Vec<EventStoreEntry>, EventStoreError> {
1541            self.0.lock().scan_range(from, to, direction)
1542        }
1543
1544        fn scan_seq(&self, seq: u64) -> Result<Option<EventStoreEntry>, EventStoreError> {
1545            self.0.lock().scan_seq(seq)
1546        }
1547
1548        fn lookup(&self, kind: IndexKind, key: &str) -> Result<Option<u64>, EventStoreError> {
1549            self.0.lock().lookup(kind, key)
1550        }
1551
1552        fn iter_index_keys(&self, kind: IndexKind) -> Result<Vec<(String, u64)>, EventStoreError> {
1553            self.0.lock().iter_index_keys(kind)
1554        }
1555
1556        fn record_snapshot_anchor(
1557            &mut self,
1558            anchor: SnapshotAnchor,
1559        ) -> Result<(), EventStoreError> {
1560            self.0.lock().record_snapshot_anchor(anchor)
1561        }
1562
1563        fn latest_snapshot_anchor(&self) -> Result<Option<SnapshotAnchor>, EventStoreError> {
1564            self.0.lock().latest_snapshot_anchor()
1565        }
1566
1567        fn seal(&mut self, status: RunStatus) -> Result<(), EventStoreError> {
1568            self.0.lock().seal(status)
1569        }
1570
1571        fn manifest(&self) -> Result<RunManifest, EventStoreError> {
1572            self.0.lock().manifest()
1573        }
1574
1575        fn high_watermark(&self) -> Result<u64, EventStoreError> {
1576            self.0.lock().high_watermark()
1577        }
1578    }
1579
1580    fn seed_crashed_predecessor(config: &EventStoreConfig, run_id: &str, crash_point: CrashPoint) {
1581        let mut backend = RedbBackend::new(config.base_dir.clone());
1582        backend
1583            .open_run(build_manifest(
1584                config,
1585                INSTANCE_ID,
1586                run_id.to_string(),
1587                None,
1588                UnixNanos::from(1_000),
1589                RegisteredComponents::default(),
1590            ))
1591            .expect("open predecessor");
1592
1593        match crash_point {
1594            // An entry sitting only in the writer channel leaves no durable redb
1595            // footprint after process death, so these two fault points intentionally
1596            // recover from the same on-disk state.
1597            CrashPoint::BeforeEnqueue | CrashPoint::AfterEnqueueBeforeCommit => {}
1598            CrashPoint::AfterCommitBeforeSnapshot => {
1599                backend
1600                    .append_batch(&[append_run_started(1)])
1601                    .expect("append committed entry");
1602            }
1603            CrashPoint::AfterSnapshot => {
1604                backend
1605                    .append_batch(&[append_run_started(1)])
1606                    .expect("append committed entry");
1607                backend
1608                    .record_snapshot_anchor(SnapshotAnchor::new(
1609                        1,
1610                        "cache://snapshot/run-crash/1",
1611                        "blake3:abc",
1612                    ))
1613                    .expect("record snapshot anchor");
1614            }
1615        }
1616    }
1617
1618    #[rstest]
1619    fn halt_signal_callback_records_first_reason() {
1620        let signal = HaltSignal::new();
1621        let cb = signal.callback();
1622        cb(HaltReason::BackendDisk("ENOSPC".to_string()));
1623        cb(HaltReason::BackendError("second".to_string()));
1624
1625        assert!(signal.is_halted());
1626        match signal.reason() {
1627            Some(HaltReason::BackendDisk(msg)) => assert!(msg.contains("ENOSPC")),
1628            other => panic!("expected first reason BackendDisk, was {other:?}"),
1629        }
1630    }
1631
1632    #[rstest]
1633    fn recover_predecessors_returns_empty_for_missing_directory() {
1634        let tmp = TempDir::new().expect("tempdir");
1635        let outcome =
1636            recover_predecessors(tmp.path(), INSTANCE_ID).expect("recover empty directory");
1637        assert!(outcome.recovered.is_empty());
1638        assert!(outcome.parent_run_id.is_none());
1639    }
1640
1641    #[rstest]
1642    fn restore_cache_snapshot_blob_rejects_hash_mismatch() {
1643        let mut cache = Cache::default();
1644        let blob = Bytes::from_static(b"snapshot");
1645        let anchor =
1646            crate::SnapshotAnchor::new(0, "cache://position-snapshots/P-1/0", "blake3:bad");
1647
1648        cache
1649            .add(&anchor.blob_ref, blob)
1650            .expect("seed snapshot blob");
1651        let err =
1652            crate::restore_cache_snapshot_blob(&mut cache, Some(&anchor)).expect_err("hash error");
1653
1654        assert!(
1655            err.to_string().contains("content_hash mismatch"),
1656            "err was: {err}",
1657        );
1658    }
1659
1660    #[rstest]
1661    fn open_run_writes_run_started_and_advances_watermark() {
1662        let tmp = TempDir::new().expect("tempdir");
1663        let config = make_config(tmp.path().to_path_buf());
1664        let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover empty");
1665        assert!(outcome.parent_run_id.is_none());
1666
1667        let halt = HaltSignal::new();
1668        let session = open_run(
1669            &config,
1670            INSTANCE_ID,
1671            build_run_id(UnixNanos::from(1_000)),
1672            outcome.parent_run_id,
1673            UnixNanos::from(1_000),
1674            &RegisteredComponents::default(),
1675            halt,
1676            get_atomic_clock_static(),
1677        )
1678        .expect("open run");
1679
1680        // Watermark + run-status snapshot.
1681        assert_eq!(session.high_watermark(), 1);
1682        assert_eq!(session.parent_run_id(), None);
1683
1684        // Every identity field must thread from EventStoreConfig into the manifest.
1685        // A field-swap mutation in build_manifest (e.g. assigning binary_hash from
1686        // config.identity.config_hash) would fail one of these assertions.
1687        let manifest = session.manifest();
1688        assert_eq!(manifest.instance_id, INSTANCE_ID);
1689        assert_eq!(manifest.status, RunStatus::Running);
1690        assert_eq!(manifest.binary_hash, "deadbeef");
1691        assert_eq!(manifest.schema_version, 1);
1692        assert_eq!(manifest.crate_versions, "feedface");
1693        assert_eq!(manifest.config_hash, "cafebabe");
1694        assert_eq!(manifest.start_ts_init, UnixNanos::from(1_000));
1695        assert_eq!(manifest.end_ts_init, None);
1696        assert!(
1697            manifest
1698                .feature_flags
1699                .contains(&"retention=full".to_string()),
1700            "feature_flags must record the retention mode, was {:?}",
1701            manifest.feature_flags,
1702        );
1703    }
1704
1705    #[rstest]
1706    fn lifecycle_options_default_registry_keeps_builtin_encoders() {
1707        let registry = EventStoreLifecycleOptions::default().build_registry();
1708
1709        assert!(registry.contains::<SubmitOrder>());
1710        assert!(registry.contains::<TradingCommand>());
1711        assert!(!registry.contains::<TestAuditMessage>());
1712    }
1713
1714    #[rstest]
1715    fn lifecycle_options_custom_registry_captures_registered_message() {
1716        let tmp = TempDir::new().expect("tempdir");
1717        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1718        let instance_id = UUID4::new();
1719        let options = EventStoreLifecycleOptions::new().with_encoder_registry(test_registry());
1720
1721        let mut store = EventStoreLifecycle::boot_with_options(
1722            Some(make_config(tmp.path().to_path_buf())),
1723            instance_id,
1724            clock_rc,
1725            options,
1726        )
1727        .expect("boot store");
1728        store
1729            .open(
1730                instance_id,
1731                &RegisteredComponents::default(),
1732                Environment::Backtest,
1733            )
1734            .expect("open run");
1735        let run_id = store.run_id().expect("run open").to_string();
1736
1737        let topic: MStr<msgbus::Topic> = MStr::from("events.test.audit");
1738        msgbus::publish_any(topic, &TestAuditMessage { value: 42 });
1739        store.seal(UnixNanos::from(0));
1740
1741        let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
1742            .expect("open sealed");
1743        let captured = sealed
1744            .scan_seq(2)
1745            .expect("scan")
1746            .expect("captured entry present");
1747
1748        assert_eq!(captured.payload_type.as_str(), "TestAuditMessage");
1749        assert_eq!(captured.topic.as_ref(), topic.as_str());
1750        assert_eq!(captured.payload.as_ref(), &[42]);
1751    }
1752
1753    #[rstest]
1754    fn lifecycle_options_memory_backend_opener_captures_and_seals() {
1755        let tmp = TempDir::new().expect("tempdir");
1756        let memory = Arc::new(Mutex::new(MemoryBackend::new()));
1757        let opener_memory = Arc::clone(&memory);
1758        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1759        let instance_id = UUID4::new();
1760        let options = EventStoreLifecycleOptions::new()
1761            .with_encoder_registry(test_registry())
1762            .with_backend_opener(move |_, manifest| {
1763                opener_memory.lock().open_run(manifest.clone())?;
1764                Ok(Box::new(SharedMemoryBackend(Arc::clone(&opener_memory))))
1765            });
1766
1767        let mut store = EventStoreLifecycle::boot_with_options(
1768            Some(make_config(tmp.path().to_path_buf())),
1769            instance_id,
1770            clock_rc,
1771            options,
1772        )
1773        .expect("boot store");
1774        store
1775            .open(
1776                instance_id,
1777                &RegisteredComponents::default(),
1778                Environment::Backtest,
1779            )
1780            .expect("open run");
1781
1782        let topic: MStr<msgbus::Topic> = MStr::from("events.test.memory");
1783        msgbus::publish_any(topic, &TestAuditMessage { value: 7 });
1784        store.seal(UnixNanos::from(1_000));
1785
1786        let backend = memory.lock();
1787        let manifest = backend.manifest().expect("manifest");
1788        let captured = backend
1789            .scan_seq(2)
1790            .expect("scan")
1791            .expect("captured entry present");
1792
1793        assert_eq!(manifest.instance_id, instance_id.to_string());
1794        assert_eq!(manifest.status, RunStatus::Ended);
1795        assert_eq!(manifest.high_watermark, 3);
1796        assert_eq!(captured.payload_type.as_str(), "TestAuditMessage");
1797        assert_eq!(captured.topic.as_ref(), topic.as_str());
1798        assert_eq!(captured.payload.as_ref(), &[7]);
1799    }
1800
1801    #[cfg(madsim)]
1802    #[rstest]
1803    fn lifecycle_options_memory_backend_opener_captures_deterministic_seq_order_under_madsim() {
1804        let first = capture_madsim_memory_lifecycle_summary(42);
1805        let second = capture_madsim_memory_lifecycle_summary(42);
1806        let expected = expected_madsim_memory_entries();
1807
1808        assert_eq!(first.entries, second.entries);
1809        assert_eq!(first.entries, expected);
1810        assert_eq!(
1811            first
1812                .entries
1813                .iter()
1814                .map(|entry| entry.seq)
1815                .collect::<Vec<_>>(),
1816            vec![1, 2, 3, 4],
1817        );
1818        assert!(
1819            first.redb_files.is_empty(),
1820            "memory opener must not create redb files, was {:?}",
1821            first.redb_files,
1822        );
1823        assert!(
1824            second.redb_files.is_empty(),
1825            "memory opener must not create redb files, was {:?}",
1826            second.redb_files,
1827        );
1828    }
1829
1830    #[cfg(madsim)]
1831    fn expected_madsim_memory_entries() -> Vec<CapturedEntrySummary> {
1832        vec![
1833            CapturedEntrySummary {
1834                seq: 1,
1835                topic: RUN_STARTED_TOPIC.to_string(),
1836                payload_type: RUN_STARTED_PAYLOAD_TYPE.to_string(),
1837                payload: encode_run_started(&RegisteredComponents::default()).to_vec(),
1838                ts_init: UnixNanos::from(0),
1839                ts_publish: UnixNanos::from(10_000),
1840            },
1841            CapturedEntrySummary {
1842                seq: 2,
1843                topic: "events.test.madsim".to_string(),
1844                payload_type: "TestAuditMessage".to_string(),
1845                payload: vec![1],
1846                ts_init: UnixNanos::from(20_000),
1847                ts_publish: UnixNanos::from(20_000),
1848            },
1849            CapturedEntrySummary {
1850                seq: 3,
1851                topic: "events.test.madsim".to_string(),
1852                payload_type: "TestAuditMessage".to_string(),
1853                payload: vec![2],
1854                ts_init: UnixNanos::from(30_000),
1855                ts_publish: UnixNanos::from(30_000),
1856            },
1857            CapturedEntrySummary {
1858                seq: 4,
1859                topic: RUN_ENDED_TOPIC.to_string(),
1860                payload_type: RUN_ENDED_PAYLOAD_TYPE.to_string(),
1861                payload: Vec::new(),
1862                ts_init: UnixNanos::from(40_000),
1863                ts_publish: UnixNanos::from(40_000),
1864            },
1865        ]
1866    }
1867
1868    #[rstest]
1869    fn open_run_with_options_surfaces_backend_opener_error() {
1870        let tmp = TempDir::new().expect("tempdir");
1871        let config = make_config(tmp.path().to_path_buf());
1872        let options = EventStoreLifecycleOptions::new().with_backend_opener(|_, _| {
1873            Err(EventStoreError::Backend(
1874                "test backend open failed".to_string(),
1875            ))
1876        });
1877
1878        let err = open_run_with_options(
1879            &config,
1880            INSTANCE_ID,
1881            "run-open-error".to_string(),
1882            None,
1883            UnixNanos::from(5_000),
1884            &RegisteredComponents::default(),
1885            HaltSignal::new(),
1886            get_atomic_clock_static(),
1887            &options,
1888        )
1889        .expect_err("backend opener error must stop run open");
1890
1891        match err {
1892            BootError::EventStore(EventStoreError::Backend(msg)) => {
1893                assert!(msg.contains("test backend open failed"));
1894            }
1895            other => panic!("expected backend open failure, was {other:?}"),
1896        }
1897    }
1898
1899    #[cfg(madsim)]
1900    #[derive(Debug, PartialEq, Eq)]
1901    struct MadsimMemoryLifecycleCapture {
1902        entries: Vec<CapturedEntrySummary>,
1903        redb_files: Vec<PathBuf>,
1904    }
1905
1906    #[cfg(madsim)]
1907    #[derive(Debug, PartialEq, Eq)]
1908    struct CapturedEntrySummary {
1909        seq: u64,
1910        topic: String,
1911        payload_type: String,
1912        payload: Vec<u8>,
1913        ts_init: UnixNanos,
1914        ts_publish: UnixNanos,
1915    }
1916
1917    #[cfg(madsim)]
1918    fn capture_madsim_memory_lifecycle_summary(seed: u64) -> MadsimMemoryLifecycleCapture {
1919        get_atomic_clock_static().set_time(UnixNanos::from(10_000));
1920
1921        let tmp = TempDir::new().expect("tempdir");
1922        let memory = Arc::new(Mutex::new(MemoryBackend::new()));
1923        let opener_memory = Arc::clone(&memory);
1924        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
1925        let instance_id = UUID4::new();
1926        let mut config = make_config(tmp.path().to_path_buf());
1927        config.identity.seed = Some(seed);
1928        let options = EventStoreLifecycleOptions::new()
1929            .with_encoder_registry(test_registry())
1930            .with_backend_opener(move |_, manifest| {
1931                opener_memory.lock().open_run(manifest.clone())?;
1932                Ok(Box::new(SharedMemoryBackend(Arc::clone(&opener_memory))))
1933            });
1934
1935        let mut store =
1936            EventStoreLifecycle::boot_with_options(Some(config), instance_id, clock_rc, options)
1937                .expect("boot store");
1938        store
1939            .open(
1940                instance_id,
1941                &RegisteredComponents::default(),
1942                Environment::Backtest,
1943            )
1944            .expect("open run");
1945
1946        let topic: MStr<msgbus::Topic> = MStr::from("events.test.madsim");
1947        get_atomic_clock_static().set_time(UnixNanos::from(20_000));
1948        msgbus::publish_any(topic, &TestAuditMessage { value: 1 });
1949        get_atomic_clock_static().set_time(UnixNanos::from(30_000));
1950        msgbus::publish_any(topic, &TestAuditMessage { value: 2 });
1951        assert_eq!(
1952            store
1953                .session
1954                .as_ref()
1955                .expect("open session")
1956                .high_watermark(),
1957            3
1958        );
1959
1960        get_atomic_clock_static().set_time(UnixNanos::from(40_000));
1961        store.seal(UnixNanos::from(40_000));
1962
1963        let backend = memory.lock();
1964        let manifest = backend.manifest().expect("manifest");
1965        assert_eq!(manifest.seed, Some(seed));
1966        assert_eq!(manifest.status, RunStatus::Ended);
1967        assert_eq!(manifest.high_watermark, 4);
1968        let entries = backend
1969            .scan_range(1, manifest.high_watermark, ScanDirection::Forward)
1970            .expect("scan entries")
1971            .into_iter()
1972            .map(|entry| CapturedEntrySummary {
1973                seq: entry.seq,
1974                topic: entry.topic.as_ref().to_string(),
1975                payload_type: entry.payload_type.as_str().to_string(),
1976                payload: entry.payload.to_vec(),
1977                ts_init: entry.ts_init,
1978                ts_publish: entry.ts_publish,
1979            })
1980            .collect();
1981        drop(backend);
1982
1983        MadsimMemoryLifecycleCapture {
1984            entries,
1985            redb_files: redb_files_under(tmp.path()),
1986        }
1987    }
1988
1989    #[cfg(madsim)]
1990    fn redb_files_under(dir: &Path) -> Vec<PathBuf> {
1991        let mut paths = Vec::new();
1992        collect_redb_files(dir, &mut paths);
1993        paths.sort();
1994        paths
1995    }
1996
1997    #[cfg(madsim)]
1998    fn collect_redb_files(dir: &Path, paths: &mut Vec<PathBuf>) {
1999        for entry in std::fs::read_dir(dir).expect("read dir") {
2000            let path = entry.expect("dir entry").path();
2001            if path.is_dir() {
2002                collect_redb_files(&path, paths);
2003            } else if path
2004                .extension()
2005                .is_some_and(|extension| extension == "redb")
2006            {
2007                paths.push(path);
2008            }
2009        }
2010    }
2011
2012    #[rstest]
2013    fn close_seals_manifest_and_records_run_ended() {
2014        let tmp = TempDir::new().expect("tempdir");
2015        let config = make_config(tmp.path().to_path_buf());
2016
2017        let halt = HaltSignal::new();
2018        let mut session = open_run(
2019            &config,
2020            INSTANCE_ID,
2021            build_run_id(UnixNanos::from(2_000)),
2022            None,
2023            UnixNanos::from(2_000),
2024            &RegisteredComponents::default(),
2025            halt,
2026            get_atomic_clock_static(),
2027        )
2028        .expect("open run");
2029
2030        let run_id = session.run_id().to_string();
2031        session.close(UnixNanos::from(3_000)).expect("close");
2032
2033        let manifests = RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2034        let manifest = manifests
2035            .into_iter()
2036            .find(|m| m.run_id == run_id)
2037            .expect("manifest present");
2038        assert_eq!(manifest.status, RunStatus::Ended);
2039        assert!(manifest.high_watermark >= 2);
2040    }
2041
2042    #[rstest]
2043    fn snapshot_anchorer_persists_anchor_for_open_session() {
2044        let tmp = TempDir::new().expect("tempdir");
2045        let config = make_config(tmp.path().to_path_buf());
2046
2047        let halt = HaltSignal::new();
2048        let mut session = open_run(
2049            &config,
2050            INSTANCE_ID,
2051            build_run_id(UnixNanos::from(4_000)),
2052            None,
2053            UnixNanos::from(4_000),
2054            &RegisteredComponents::default(),
2055            halt,
2056            get_atomic_clock_static(),
2057        )
2058        .expect("open run");
2059
2060        let run_id = session.run_id().to_string();
2061
2062        {
2063            let anchorer = session.snapshot_anchorer().expect("snapshot anchorer");
2064            anchorer(CacheSnapshotRef::new(
2065                "cache://position-snapshots/P-1/0",
2066                Bytes::from_static(b"snapshot"),
2067            ))
2068            .expect("record snapshot anchor");
2069        }
2070
2071        session.close(UnixNanos::from(4_500)).expect("close");
2072
2073        let reader =
2074            RedbBackend::open_sealed(&config.base_dir, INSTANCE_ID, &run_id).expect("open sealed");
2075        let anchor = reader
2076            .latest_snapshot_anchor()
2077            .expect("latest snapshot anchor")
2078            .expect("anchor present");
2079
2080        assert_eq!(anchor.high_watermark, 1);
2081        assert_eq!(anchor.blob_ref, "cache://position-snapshots/P-1/0");
2082        assert_eq!(
2083            anchor.content_hash,
2084            compute_snapshot_content_hash(b"snapshot"),
2085        );
2086    }
2087
2088    #[rstest]
2089    fn recovery_seals_tail_ending_in_run_ended_as_ended_not_crashed() {
2090        // The writer commits RunEnded before sealing the manifest. A crash between
2091        // those two steps leaves the manifest Running while the tail already proves
2092        // graceful close: recovery must seal as Ended (not CrashedRecovered) and
2093        // must not chain the next run to it as a crashed parent.
2094        //
2095        // Reproduce the in-between state by submitting a RunEnded draft through the
2096        // writer's normal append path and then dropping the session without going
2097        // through close() (which is what would have sealed the manifest).
2098        let tmp = TempDir::new().expect("tempdir");
2099        let config = make_config(tmp.path().to_path_buf());
2100
2101        let halt = HaltSignal::new();
2102        let run_id = build_run_id(UnixNanos::from(7_000));
2103        let session = open_run(
2104            &config,
2105            INSTANCE_ID,
2106            run_id.clone(),
2107            None,
2108            UnixNanos::from(7_000),
2109            &RegisteredComponents::default(),
2110            halt,
2111            get_atomic_clock_static(),
2112        )
2113        .expect("open run");
2114
2115        let writer = session.writer.as_ref().expect("writer attached");
2116        writer
2117            .submit(run_ended_draft(UnixNanos::from(7_500)))
2118            .expect("submit RunEnded as tail entry");
2119        // Wait until the writer durably commits the RunEnded entry before dropping;
2120        // otherwise the on-disk tail might not include it and the recovery test
2121        // would fall back to CrashedRecovered for an unrelated reason.
2122        let deadline = Instant::now() + Duration::from_secs(2);
2123
2124        while session.high_watermark() < 2 {
2125            assert!(
2126                Instant::now() < deadline,
2127                "writer high_watermark stuck at {} before deadline",
2128                session.high_watermark(),
2129            );
2130            thread::sleep(Duration::from_millis(2));
2131        }
2132        drop(session);
2133
2134        let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2135        assert_eq!(outcome.recovered.len(), 1);
2136        assert_eq!(outcome.recovered[0].run_id, run_id);
2137        assert_eq!(
2138            outcome.recovered[0].status,
2139            RunStatus::Ended,
2140            "tail ending in RunEnded must seal as Ended",
2141        );
2142        assert!(
2143            outcome.parent_run_id.is_none(),
2144            "Ended runs must not become parents",
2145        );
2146
2147        let manifests = RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2148        let manifest = manifests
2149            .into_iter()
2150            .find(|m| m.run_id == run_id)
2151            .expect("manifest present");
2152        assert_eq!(manifest.status, RunStatus::Ended);
2153    }
2154
2155    #[rstest]
2156    fn recovery_quarantines_run_with_rows_swapped_between_keys() {
2157        // A row moved under a different table key still hashes correctly; the sweep
2158        // must quarantine rather than chain the next run through a tampered parent.
2159        let tmp = TempDir::new().expect("tempdir");
2160        let config = make_config(tmp.path().to_path_buf());
2161        let run_id = "1700000000-swapped1";
2162
2163        let path = {
2164            let mut backend = RedbBackend::new(config.base_dir.clone());
2165            backend.open_run(manifest_for(run_id)).expect("open run");
2166            backend
2167                .append_batch(&[
2168                    append_entry(
2169                        1,
2170                        "events.order.1",
2171                        "OrderAccepted",
2172                        Bytes::from_static(b"\x01"),
2173                    ),
2174                    append_entry(
2175                        2,
2176                        "events.order.2",
2177                        "OrderFilled",
2178                        Bytes::from_static(b"\x02"),
2179                    ),
2180                ])
2181                .expect("append");
2182            config
2183                .base_dir
2184                .join(INSTANCE_ID)
2185                .join(format!("{run_id}.redb"))
2186        };
2187
2188        {
2189            let entries: redb::TableDefinition<u64, &[u8]> = redb::TableDefinition::new("entries");
2190            let db = redb::Database::create(&path).expect("open redb");
2191            let txn = db.begin_write().expect("begin write");
2192            {
2193                let mut table = txn.open_table(entries).expect("open table");
2194                let bytes_1 = table
2195                    .remove(1_u64)
2196                    .expect("remove seq 1")
2197                    .expect("seq 1 present")
2198                    .value()
2199                    .to_vec();
2200                let bytes_2 = table
2201                    .remove(2_u64)
2202                    .expect("remove seq 2")
2203                    .expect("seq 2 present")
2204                    .value()
2205                    .to_vec();
2206                table
2207                    .insert(1_u64, bytes_2.as_slice())
2208                    .expect("insert under key 1");
2209                table
2210                    .insert(2_u64, bytes_1.as_slice())
2211                    .expect("insert under key 2");
2212            }
2213            txn.commit().expect("commit swap");
2214        }
2215
2216        let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2217
2218        assert_eq!(outcome.recovered.len(), 1);
2219        assert_eq!(outcome.recovered[0].run_id, run_id);
2220        assert_eq!(outcome.recovered[0].status, RunStatus::Quarantined);
2221        assert!(
2222            outcome.parent_run_id.is_none(),
2223            "quarantined runs must not become parents",
2224        );
2225
2226        let manifests = RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2227        assert_eq!(manifests[0].status, RunStatus::Quarantined);
2228    }
2229
2230    #[rstest]
2231    fn drop_without_close_leaves_run_for_next_boot_to_recover() {
2232        let tmp = TempDir::new().expect("tempdir");
2233        let config = make_config(tmp.path().to_path_buf());
2234
2235        let halt = HaltSignal::new();
2236        let session = open_run(
2237            &config,
2238            INSTANCE_ID,
2239            build_run_id(UnixNanos::from(4_000)),
2240            None,
2241            UnixNanos::from(4_000),
2242            &RegisteredComponents::default(),
2243            halt,
2244            get_atomic_clock_static(),
2245        )
2246        .expect("open run");
2247        let run_id = session.run_id().to_string();
2248        drop(session);
2249
2250        let outcome =
2251            recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover after crash");
2252        assert_eq!(outcome.recovered.len(), 1);
2253        assert_eq!(outcome.recovered[0].run_id, run_id);
2254        assert_eq!(outcome.recovered[0].status, RunStatus::CrashedRecovered);
2255        assert_eq!(outcome.parent_run_id.as_deref(), Some(run_id.as_str()));
2256
2257        let manifests = RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2258        let sealed = manifests
2259            .into_iter()
2260            .find(|m| m.run_id == run_id)
2261            .expect("present");
2262        assert_eq!(sealed.status, RunStatus::CrashedRecovered);
2263    }
2264
2265    #[rstest]
2266    fn build_run_id_uses_expected_format() {
2267        // Format: "<start_ts_init>-<8 hex chars>". The prefix is sortable by start
2268        // time so directory listings produce chronological order; the suffix
2269        // disambiguates concurrent starts at the same nanosecond.
2270        let id = build_run_id(UnixNanos::from(123_456));
2271        let (prefix, suffix) = id.split_once('-').expect("run id must contain a hyphen");
2272        assert_eq!(prefix, "123456");
2273        assert_eq!(suffix.len(), 8, "suffix was {suffix:?}");
2274        assert!(
2275            suffix.chars().all(|c| c.is_ascii_hexdigit()),
2276            "suffix must be hex, was {suffix:?}",
2277        );
2278    }
2279
2280    #[rstest]
2281    fn crash_recovery_seals_predecessor_and_links_parent_run_id() {
2282        // SPEC Phase 7 acceptance: kill mid-run, restart, assert the predecessor seals
2283        // as CrashedRecovered, the new run's parent_run_id points to it, and the new
2284        // run's first entry is a RunStarted at seq=1.
2285        let tmp = TempDir::new().expect("tempdir");
2286        let config = make_config(tmp.path().to_path_buf());
2287
2288        // Kernel boot 1: open a run and crash (drop the session without close).
2289        let halt_first = HaltSignal::new();
2290        let first = open_run(
2291            &config,
2292            INSTANCE_ID,
2293            build_run_id(UnixNanos::from(10_000)),
2294            None,
2295            UnixNanos::from(10_000),
2296            &RegisteredComponents::default(),
2297            halt_first,
2298            get_atomic_clock_static(),
2299        )
2300        .expect("open first run");
2301        let crashed_run_id = first.run_id().to_string();
2302        drop(first);
2303
2304        // Kernel boot 2: recover predecessors then open the next run.
2305        let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2306        assert_eq!(outcome.recovered.len(), 1);
2307        assert_eq!(outcome.recovered[0].run_id, crashed_run_id);
2308        assert_eq!(outcome.recovered[0].status, RunStatus::CrashedRecovered);
2309        assert_eq!(
2310            outcome.parent_run_id.as_deref(),
2311            Some(crashed_run_id.as_str())
2312        );
2313
2314        // Predecessor's on-disk manifest is sealed CrashedRecovered.
2315        let manifests_after_seal =
2316            RedbBackend::list_runs(&config.base_dir, INSTANCE_ID).expect("list");
2317        let predecessor = manifests_after_seal
2318            .iter()
2319            .find(|m| m.run_id == crashed_run_id)
2320            .expect("predecessor present");
2321        assert_eq!(predecessor.status, RunStatus::CrashedRecovered);
2322
2323        // New run is opened with parent_run_id pointing to the predecessor.
2324        let halt_second = HaltSignal::new();
2325        let new_run_id = build_run_id(UnixNanos::from(20_000));
2326        let next = open_run(
2327            &config,
2328            INSTANCE_ID,
2329            new_run_id.clone(),
2330            outcome.parent_run_id,
2331            UnixNanos::from(20_000),
2332            &RegisteredComponents::default(),
2333            halt_second,
2334            get_atomic_clock_static(),
2335        )
2336        .expect("open second run");
2337        assert_eq!(next.parent_run_id(), Some(crashed_run_id.as_str()));
2338        assert_eq!(
2339            next.manifest().parent_run_id.as_deref(),
2340            Some(crashed_run_id.as_str()),
2341        );
2342        assert_eq!(next.high_watermark(), 1, "RunStarted is the first entry");
2343
2344        // The first entry in the new run is RunStarted at seq=1; close cleanly so we
2345        // can read the on-disk file without contending with the writer's lock.
2346        drop(next);
2347        let outcome_after = recover_predecessors(&config.base_dir, INSTANCE_ID)
2348            .expect("recover after second crash");
2349        // Only the second run shows up because the first is already sealed.
2350        assert_eq!(outcome_after.recovered.len(), 1);
2351        assert_eq!(outcome_after.recovered[0].run_id, new_run_id);
2352        assert_eq!(
2353            outcome_after.recovered[0].status,
2354            RunStatus::CrashedRecovered,
2355        );
2356
2357        // Open the recovered run read-only and verify seq=1 is RunStarted.
2358        let sealed = RedbBackend::open_sealed(&config.base_dir, INSTANCE_ID, &new_run_id)
2359            .expect("open sealed");
2360        let first_entry = sealed
2361            .scan_seq(1)
2362            .expect("scan")
2363            .expect("RunStarted present");
2364        assert_eq!(first_entry.payload_type.as_str(), "RunStarted");
2365        assert_eq!(first_entry.topic.as_ref(), "run.lifecycle.RunStarted");
2366    }
2367
2368    #[rstest]
2369    #[case::before_enqueue(CrashPoint::BeforeEnqueue, 0, false)]
2370    #[case::after_enqueue_before_commit(CrashPoint::AfterEnqueueBeforeCommit, 0, false)]
2371    #[case::after_commit_before_snapshot(CrashPoint::AfterCommitBeforeSnapshot, 1, false)]
2372    #[case::after_snapshot(CrashPoint::AfterSnapshot, 1, true)]
2373    fn crash_recovery_matrix_seals_predecessor_and_links_parent_run_id(
2374        #[case] crash_point: CrashPoint,
2375        #[case] expected_hwm: u64,
2376        #[case] expect_snapshot_anchor: bool,
2377    ) {
2378        let tmp = TempDir::new().expect("tempdir");
2379        let config = make_config(tmp.path().to_path_buf());
2380        let predecessor_run_id = format!("3000-{crash_point:?}");
2381        seed_crashed_predecessor(&config, &predecessor_run_id, crash_point);
2382
2383        let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2384        assert_eq!(outcome.recovered.len(), 1);
2385        assert_eq!(outcome.recovered[0].run_id, predecessor_run_id);
2386        assert_eq!(outcome.recovered[0].status, RunStatus::CrashedRecovered);
2387        assert_eq!(
2388            outcome.parent_run_id.as_deref(),
2389            Some(predecessor_run_id.as_str()),
2390        );
2391
2392        let predecessor =
2393            RedbBackend::open_sealed(&config.base_dir, INSTANCE_ID, &predecessor_run_id)
2394                .expect("open sealed predecessor");
2395        let manifest = predecessor.manifest().expect("manifest");
2396        let snapshot_anchor = predecessor.latest_snapshot_anchor().expect("anchor read");
2397
2398        assert_eq!(manifest.status, RunStatus::CrashedRecovered);
2399        assert_eq!(manifest.high_watermark, expected_hwm);
2400        assert_eq!(
2401            snapshot_anchor.is_some(),
2402            expect_snapshot_anchor,
2403            "snapshot anchor presence must match crash point",
2404        );
2405
2406        let next = open_run(
2407            &config,
2408            INSTANCE_ID,
2409            "4000-next".to_string(),
2410            outcome.parent_run_id,
2411            UnixNanos::from(4_000),
2412            &RegisteredComponents::default(),
2413            HaltSignal::new(),
2414            get_atomic_clock_static(),
2415        )
2416        .expect("open next run");
2417        assert_eq!(next.parent_run_id(), Some(predecessor_run_id.as_str()));
2418        assert_eq!(
2419            next.manifest().parent_run_id.as_deref(),
2420            Some(predecessor_run_id.as_str()),
2421        );
2422    }
2423
2424    #[rstest]
2425    fn kernel_event_store_open_seals_leftover_session_before_reopen() {
2426        // BacktestEngine::run -> reset -> run reuses the kernel. EventStoreLifecycle::open
2427        // must seal any leftover session before opening a fresh one so RunStarted is
2428        // the first entry of every run. The UUID suffix in build_run_id keeps the
2429        // two ids distinct even though VirtualClock holds start_ts_init at zero.
2430        let tmp = TempDir::new().expect("tempdir");
2431        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2432        let instance_id = UUID4::new();
2433
2434        let mut store = EventStoreLifecycle::boot(
2435            Some(make_config(tmp.path().to_path_buf())),
2436            instance_id,
2437            clock_rc,
2438        )
2439        .expect("boot store");
2440
2441        store
2442            .open(
2443                instance_id,
2444                &RegisteredComponents::default(),
2445                Environment::Backtest,
2446            )
2447            .expect("open first run");
2448        let run_one = store.run_id().expect("run one open").to_string();
2449
2450        store
2451            .open(
2452                instance_id,
2453                &RegisteredComponents::default(),
2454                Environment::Backtest,
2455            )
2456            .expect("open second run");
2457        let run_two = store.run_id().expect("run two open").to_string();
2458
2459        assert_ne!(run_one, run_two, "second open must produce a fresh run id");
2460
2461        // Drop the wrapper so any open run seals via Drop, then read both manifests
2462        // off disk and assert each closed cleanly as Ended.
2463        drop(store);
2464        let manifests =
2465            RedbBackend::list_runs(tmp.path(), &instance_id.to_string()).expect("list runs");
2466        let m1 = manifests
2467            .iter()
2468            .find(|m| m.run_id == run_one)
2469            .expect("first run present");
2470        let m2 = manifests
2471            .iter()
2472            .find(|m| m.run_id == run_two)
2473            .expect("second run present");
2474        assert_eq!(m1.status, RunStatus::Ended);
2475        assert_eq!(m2.status, RunStatus::Ended);
2476    }
2477
2478    #[rstest]
2479    fn open_after_halt_re_arms_signal_and_next_run_seals_ended() {
2480        // One halt must be terminal for the run that fired it, not for the kernel: a
2481        // rerun (reset -> run) opens with a fresh signal, reports no stale halt, and
2482        // its graceful stop still seals Ended.
2483        let tmp = TempDir::new().expect("tempdir");
2484        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2485        let instance_id = UUID4::new();
2486
2487        let mut store = EventStoreLifecycle::boot(
2488            Some(make_config(tmp.path().to_path_buf())),
2489            instance_id,
2490            clock_rc,
2491        )
2492        .expect("boot store");
2493
2494        store
2495            .open(
2496                instance_id,
2497                &RegisteredComponents::default(),
2498                Environment::Backtest,
2499            )
2500            .expect("open first run");
2501        let run_one = store.run_id().expect("run one open").to_string();
2502
2503        store.halt.callback()(HaltReason::BackendDisk("ENOSPC".to_string()));
2504        assert!(store.halt.is_halted());
2505
2506        store
2507            .open(
2508                instance_id,
2509                &RegisteredComponents::default(),
2510                Environment::Backtest,
2511            )
2512            .expect("open second run after halt");
2513        let run_two = store.run_id().expect("run two open").to_string();
2514
2515        assert_ne!(run_one, run_two);
2516        assert!(!store.halt.is_halted(), "open must re-arm the halt signal");
2517        assert!(store.halt.reason().is_none());
2518
2519        drop(store);
2520        let manifests =
2521            RedbBackend::list_runs(tmp.path(), &instance_id.to_string()).expect("list runs");
2522        let m1 = manifests
2523            .iter()
2524            .find(|m| m.run_id == run_one)
2525            .expect("first run present");
2526        let m2 = manifests
2527            .iter()
2528            .find(|m| m.run_id == run_two)
2529            .expect("second run present");
2530        // The halted run skips the in-process seal; the recovery sweep on next boot
2531        // owns it. The post-halt rerun must close cleanly.
2532        assert_eq!(m1.status, RunStatus::Running);
2533        assert_eq!(m2.status, RunStatus::Ended);
2534    }
2535
2536    #[rstest]
2537    fn recover_picks_most_recent_crashed_recovered_as_parent() {
2538        // With multiple unsealed predecessors, the sweep must seal every one and the
2539        // new run's parent_run_id must point to the most recently started survivor.
2540        let tmp = TempDir::new().expect("tempdir");
2541        let config = make_config(tmp.path().to_path_buf());
2542
2543        for ts in [1_000_u64, 2_000_u64, 3_000_u64] {
2544            let session = open_run(
2545                &config,
2546                INSTANCE_ID,
2547                build_run_id(UnixNanos::from(ts)),
2548                None,
2549                UnixNanos::from(ts),
2550                &RegisteredComponents::default(),
2551                HaltSignal::new(),
2552                get_atomic_clock_static(),
2553            )
2554            .expect("open");
2555            drop(session);
2556        }
2557
2558        let outcome = recover_predecessors(&config.base_dir, INSTANCE_ID).expect("recover sweep");
2559        assert_eq!(outcome.recovered.len(), 3);
2560        assert!(
2561            outcome
2562                .recovered
2563                .iter()
2564                .all(|r| r.status == RunStatus::CrashedRecovered),
2565            "every predecessor must seal CrashedRecovered, was {:?}",
2566            outcome.recovered,
2567        );
2568        // Most-recent (start_ts_init=3_000) becomes the parent.
2569        let parent = outcome.parent_run_id.as_deref().expect("parent set");
2570        assert!(
2571            parent.starts_with("3000-"),
2572            "parent must be the run with the highest start_ts_init, was {parent}",
2573        );
2574    }
2575
2576    #[cfg(not(madsim))]
2577    #[rstest]
2578    fn submit_run_started_returns_timeout_when_writer_stalls() {
2579        // A backend whose append_batch never returns simulates a stuck writer. The
2580        // wait loop must surface BootError::RunStartedTimeout after the configured
2581        // ceiling elapses, never block indefinitely.
2582        let stub = StallBackend::default();
2583        let manifest = manifest_for("run-timeout");
2584        let mut backend: Box<dyn EventStore + Send> = Box::new(stub.clone());
2585        backend.open_run(manifest).expect("open stub");
2586
2587        let halt = HaltSignal::new();
2588
2589        let writer = EventStoreWriter::spawn(
2590            backend,
2591            get_atomic_clock_static(),
2592            halt.callback(),
2593            WriterConfig::default(),
2594        )
2595        .expect("spawn writer");
2596
2597        let err = submit_run_started_blocking(
2598            &writer,
2599            &RegisteredComponents::default(),
2600            UnixNanos::from(100),
2601            &halt,
2602            Duration::from_millis(20),
2603        )
2604        .expect_err("must time out");
2605
2606        match err {
2607            BootError::RunStartedTimeout { timeout } => {
2608                assert_eq!(timeout, Duration::from_millis(20));
2609            }
2610            other => panic!("expected RunStartedTimeout, was {other:?}"),
2611        }
2612
2613        // Release the gate so the writer thread can exit cleanly before drop.
2614        stub.release();
2615    }
2616
2617    #[cfg(not(madsim))]
2618    #[rstest]
2619    fn submit_run_started_returns_halted_when_writer_halts_during_wait() {
2620        // A halt signal fired before the writer can commit must surface
2621        // BootError::HaltedDuringBoot with the recorded reason.
2622        let stub = StallBackend::default();
2623        let manifest = manifest_for("run-halt");
2624        let mut backend: Box<dyn EventStore + Send> = Box::new(stub.clone());
2625        backend.open_run(manifest).expect("open stub");
2626
2627        let halt = HaltSignal::new();
2628
2629        let writer = EventStoreWriter::spawn(
2630            backend,
2631            get_atomic_clock_static(),
2632            halt.callback(),
2633            WriterConfig::default(),
2634        )
2635        .expect("spawn writer");
2636
2637        // Fire the halt from a peer thread shortly after we begin waiting.
2638        let halt_for_thread = halt.clone();
2639
2640        let firer = thread::spawn(move || {
2641            thread::sleep(Duration::from_millis(10));
2642            halt_for_thread.callback()(HaltReason::BackendDisk("test stall".to_string()));
2643        });
2644
2645        let err = submit_run_started_blocking(
2646            &writer,
2647            &RegisteredComponents::default(),
2648            UnixNanos::from(200),
2649            &halt,
2650            Duration::from_secs(2),
2651        )
2652        .expect_err("must observe halt");
2653
2654        match err {
2655            BootError::HaltedDuringBoot(HaltReason::BackendDisk(msg)) => {
2656                assert!(msg.contains("test stall"), "reason msg was {msg}");
2657            }
2658            other => panic!("expected HaltedDuringBoot(BackendDisk), was {other:?}"),
2659        }
2660        firer.join().expect("halt firer joined");
2661        stub.release();
2662    }
2663
2664    fn manifest_for(run_id: &str) -> RunManifest {
2665        RunManifest {
2666            run_id: run_id.to_string(),
2667            parent_run_id: None,
2668            instance_id: INSTANCE_ID.to_string(),
2669            binary_hash: String::new(),
2670            schema_version: 1,
2671            crate_versions: String::new(),
2672            feature_flags: Vec::new(),
2673            adapter_versions: IndexMap::new(),
2674            config_hash: String::new(),
2675            registered_components: RegisteredComponents::default(),
2676            seed: None,
2677            start_ts_init: UnixNanos::default(),
2678            end_ts_init: None,
2679            high_watermark: 0,
2680            status: RunStatus::Running,
2681        }
2682    }
2683
2684    /// Stub backend whose `append_batch` blocks until `release()` is called. Used to
2685    /// hold the writer's high-watermark at zero so the boot path's wait loop can
2686    /// exercise its timeout and halt branches deterministically.
2687    #[cfg(not(madsim))]
2688    #[derive(Debug, Default, Clone)]
2689    struct StallBackend {
2690        inner: Arc<Mutex<StallInner>>,
2691        gate: Arc<(Mutex<bool>, parking_lot::Condvar)>,
2692    }
2693
2694    #[cfg(not(madsim))]
2695    #[derive(Debug, Default)]
2696    struct StallInner {
2697        manifest: Option<RunManifest>,
2698    }
2699
2700    #[cfg(not(madsim))]
2701    impl StallBackend {
2702        fn release(&self) {
2703            let (lock, cvar) = &*self.gate;
2704            *lock.lock() = true;
2705            cvar.notify_all();
2706        }
2707    }
2708
2709    #[cfg(not(madsim))]
2710    impl crate::EventStore for StallBackend {
2711        fn open_run(&mut self, manifest: RunManifest) -> Result<(), EventStoreError> {
2712            self.inner.lock().manifest = Some(manifest);
2713            Ok(())
2714        }
2715
2716        fn append_batch(&mut self, _: &[crate::AppendEntry]) -> Result<u64, EventStoreError> {
2717            let (lock, cvar) = &*self.gate;
2718            let mut released = lock.lock();
2719
2720            while !*released {
2721                cvar.wait(&mut released);
2722            }
2723            Ok(0)
2724        }
2725
2726        fn scan_range(
2727            &self,
2728            _: u64,
2729            _: u64,
2730            _: crate::ScanDirection,
2731        ) -> Result<Vec<crate::EventStoreEntry>, EventStoreError> {
2732            Ok(Vec::new())
2733        }
2734
2735        fn scan_seq(&self, _: u64) -> Result<Option<crate::EventStoreEntry>, EventStoreError> {
2736            Ok(None)
2737        }
2738
2739        fn lookup(&self, _: crate::IndexKind, _: &str) -> Result<Option<u64>, EventStoreError> {
2740            Ok(None)
2741        }
2742
2743        fn iter_index_keys(
2744            &self,
2745            _: crate::IndexKind,
2746        ) -> Result<Vec<(String, u64)>, EventStoreError> {
2747            Ok(Vec::new())
2748        }
2749
2750        fn seal(&mut self, _: RunStatus) -> Result<(), EventStoreError> {
2751            Ok(())
2752        }
2753
2754        fn manifest(&self) -> Result<RunManifest, EventStoreError> {
2755            self.inner
2756                .lock()
2757                .manifest
2758                .clone()
2759                .ok_or_else(|| EventStoreError::Backend("no manifest".to_string()))
2760        }
2761
2762        fn high_watermark(&self) -> Result<u64, EventStoreError> {
2763            Ok(0)
2764        }
2765    }
2766
2767    /// Integration: the kernel-installed bus tap forwards a `SubmitOrder` dispatched
2768    /// through the typed-send path into the event store before any subscriber observes
2769    /// it. The captured entry carries the dispatching endpoint as the topic and the
2770    /// canonical `SubmitOrder` payload type tag.
2771    #[rstest]
2772    fn bus_tap_captures_submit_order_sent_through_msgbus() {
2773        let tmp = TempDir::new().expect("tempdir");
2774        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2775        let instance_id = UUID4::new();
2776
2777        let mut store = EventStoreLifecycle::boot(
2778            Some(make_config(tmp.path().to_path_buf())),
2779            instance_id,
2780            clock_rc,
2781        )
2782        .expect("boot store");
2783        store
2784            .open(
2785                instance_id,
2786                &RegisteredComponents::default(),
2787                Environment::Backtest,
2788            )
2789            .expect("open run");
2790        let run_id = store.run_id().expect("run open").to_string();
2791
2792        let trader_id = TraderId::from("TRADER-001");
2793        let strategy_id = StrategyId::from("S-001");
2794        let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE");
2795        let client_order_id = ClientOrderId::from("O-20260510-000001");
2796        let order_init = OrderInitializedSpec::builder()
2797            .instrument_id(instrument_id)
2798            .client_order_id(client_order_id)
2799            .quantity(Quantity::from("1"))
2800            .time_in_force(TimeInForce::Gtc)
2801            .build();
2802        let submit_order = SubmitOrder::new(
2803            trader_id,
2804            Some(ClientId::from("BINANCE")),
2805            strategy_id,
2806            instrument_id,
2807            client_order_id,
2808            order_init,
2809            None,
2810            None,
2811            None,
2812            UUID4::new(),
2813            UnixNanos::from(3),
2814            None, // correlation_id
2815        );
2816
2817        let endpoint = MStr::<Endpoint>::from("test.exec.engine.process");
2818        msgbus::send_any_value(endpoint, &submit_order);
2819
2820        store.seal(UnixNanos::from(0));
2821
2822        let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
2823            .expect("open sealed");
2824        let captured = sealed
2825            .scan_seq(2)
2826            .expect("scan")
2827            .expect("captured entry present");
2828        assert_eq!(captured.payload_type.as_str(), "SubmitOrder");
2829        assert_eq!(captured.topic.as_ref(), endpoint.as_str());
2830
2831        // The SubmitOrder encoder commits a ClientOrderId sidecar index; the lookup
2832        // must resolve to the captured seq.
2833        let by_client = sealed
2834            .lookup(IndexKind::ClientOrderId, client_order_id.as_str())
2835            .expect("lookup")
2836            .expect("indexed");
2837        assert_eq!(by_client, 2);
2838    }
2839
2840    #[rstest]
2841    fn kernel_with_markers_captures_snapshots_over_synthetic_bus() {
2842        let tmp = TempDir::new().expect("tempdir");
2843        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2844        let instance_id = UUID4::new();
2845        let mut config = make_config(tmp.path().to_path_buf());
2846        config.data_markers = Some(DataMarkerConfig {
2847            classes: vec![DataMarkerClass::BookDeltas, DataMarkerClass::Quote],
2848            safety_flush_interval: Duration::from_secs(1),
2849            channel_capacity: 128,
2850            high_fidelity: Vec::new(),
2851        });
2852
2853        let mut store =
2854            EventStoreLifecycle::boot(Some(config), instance_id, clock_rc).expect("boot store");
2855        store
2856            .open(
2857                instance_id,
2858                &RegisteredComponents::default(),
2859                Environment::Backtest,
2860            )
2861            .expect("open run");
2862        let run_id = store.run_id().expect("run open").to_string();
2863
2864        let first = make_submit_order(ClientOrderId::from("O-marker-1"));
2865        msgbus::send_any_value(MStr::<Endpoint>::from("test.exec.process"), &first);
2866
2867        let quote = quote_ethusdt_binance();
2868        msgbus::publish_quote(MStr::from("data.quotes.BINANCE.ETHUSDT-PERP"), &quote);
2869        let deltas = stub_deltas();
2870        msgbus::publish_deltas(MStr::from("data.deltas.XNAS.AAPL"), &deltas);
2871
2872        let second = make_submit_order(ClientOrderId::from("O-marker-2"));
2873        msgbus::send_any_value(MStr::<Endpoint>::from("test.exec.process"), &second);
2874        store.seal(UnixNanos::from(500));
2875
2876        let marker_path = tmp
2877            .path()
2878            .join(instance_id.to_string())
2879            .join(format!("{run_id}.markers.redb"));
2880        let marker = RedbMarkerBackend::open_read_only_file(marker_path).expect("open markers");
2881        let snapshots = marker.scan_snapshots().expect("scan snapshots");
2882        let dict = marker.scan_dict().expect("scan dict");
2883
2884        assert_eq!(snapshots.len(), 1);
2885        assert_eq!(snapshots[0].event_seq_before, 3);
2886        assert_eq!(snapshots[0].advanced.len(), 2);
2887        assert_eq!(
2888            snapshots[0]
2889                .advanced
2890                .iter()
2891                .map(|cursor| cursor.count)
2892                .collect::<Vec<_>>(),
2893            vec![1, 1]
2894        );
2895        assert_eq!(
2896            dict.iter()
2897                .map(|entry| (entry.data_cls, entry.identifier.as_str()))
2898                .collect::<Vec<_>>(),
2899            vec![
2900                (DataClass::Quote, "ETHUSDT-PERP.BINANCE"),
2901                (DataClass::BookDeltas, "AAPL.XNAS"),
2902            ],
2903        );
2904    }
2905
2906    #[rstest]
2907    fn boot_recovery_ignores_marker_sidecar_files() {
2908        let tmp = TempDir::new().expect("tempdir");
2909        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
2910        let instance_id = UUID4::new();
2911        let mut config = make_config(tmp.path().to_path_buf());
2912        config.data_markers = Some(DataMarkerConfig {
2913            classes: vec![DataMarkerClass::Quote],
2914            safety_flush_interval: Duration::from_secs(1),
2915            channel_capacity: 128,
2916            high_fidelity: Vec::new(),
2917        });
2918
2919        let mut store =
2920            EventStoreLifecycle::boot(Some(config.clone()), instance_id, Rc::clone(&clock_rc))
2921                .expect("boot store");
2922        store
2923            .open(
2924                instance_id,
2925                &RegisteredComponents::default(),
2926                Environment::Backtest,
2927            )
2928            .expect("open run");
2929        let run_id = store.run_id().expect("run open").to_string();
2930        store.seal(UnixNanos::from(500));
2931
2932        let marker_path = tmp
2933            .path()
2934            .join(instance_id.to_string())
2935            .join(format!("{run_id}.markers.redb"));
2936        assert!(marker_path.exists());
2937
2938        let rebooted = EventStoreLifecycle::boot(Some(config), instance_id, clock_rc)
2939            .expect("boot after marker sidecar");
2940
2941        assert!(rebooted.recovered().is_empty());
2942    }
2943
2944    #[rstest]
2945    fn marker_setup_failure_disables_markers_without_blocking_open() {
2946        let tmp = TempDir::new().expect("tempdir");
2947        let bad_base = tmp.path().join("not-a-directory");
2948        std::fs::write(&bad_base, b"not a directory").expect("write bad base");
2949        let memory = Arc::new(Mutex::new(MemoryBackend::new()));
2950        let opener_memory = Arc::clone(&memory);
2951        let mut config = make_config(bad_base);
2952        config.data_markers = Some(DataMarkerConfig {
2953            classes: vec![DataMarkerClass::Quote],
2954            safety_flush_interval: Duration::from_secs(1),
2955            channel_capacity: 128,
2956            high_fidelity: Vec::new(),
2957        });
2958        let options = EventStoreLifecycleOptions::new()
2959            .with_encoder_registry(test_registry())
2960            .with_backend_opener(move |_, manifest| {
2961                opener_memory.lock().open_run(manifest.clone())?;
2962                Ok(Box::new(SharedMemoryBackend(Arc::clone(&opener_memory))))
2963            });
2964
2965        let mut session = open_run_with_options(
2966            &config,
2967            INSTANCE_ID,
2968            "run-marker-setup-fails".to_string(),
2969            None,
2970            UnixNanos::from(5_000),
2971            &RegisteredComponents::default(),
2972            HaltSignal::new(),
2973            get_atomic_clock_static(),
2974            &options,
2975        )
2976        .expect("open run despite marker failure");
2977
2978        assert!(session.marker_capture.is_none());
2979
2980        let topic: MStr<msgbus::Topic> = MStr::from("events.test.marker-fallback");
2981        session
2982            .adapter()
2983            .expect("adapter")
2984            .capture::<TestAuditMessage>(
2985                topic,
2986                &TestAuditMessage { value: 11 },
2987                Headers::empty(),
2988                UnixNanos::from(5_001),
2989            )
2990            .expect("capture");
2991        session
2992            .close(UnixNanos::from(6_000))
2993            .expect("close session");
2994
2995        let backend = memory.lock();
2996        let captured = backend
2997            .scan_seq(2)
2998            .expect("scan")
2999            .expect("captured entry present");
3000
3001        assert_eq!(captured.payload_type.as_str(), "TestAuditMessage");
3002        assert_eq!(captured.topic.as_ref(), topic.as_str());
3003        assert_eq!(captured.payload.as_ref(), &[11]);
3004    }
3005
3006    #[rstest]
3007    fn marker_registry_factory_receives_enabled_classes() {
3008        let tmp = TempDir::new().expect("tempdir");
3009        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3010        let instance_id = UUID4::new();
3011        let seen_classes: Arc<Mutex<Vec<Vec<DataClass>>>> = Arc::new(Mutex::new(Vec::new()));
3012        let seen_for_factory = Arc::clone(&seen_classes);
3013        let mut config = make_config(tmp.path().to_path_buf());
3014        config.data_markers = Some(DataMarkerConfig {
3015            classes: vec![DataMarkerClass::Trade, DataMarkerClass::Quote],
3016            safety_flush_interval: Duration::from_secs(1),
3017            channel_capacity: 128,
3018            high_fidelity: Vec::new(),
3019        });
3020        let options =
3021            EventStoreLifecycleOptions::new().with_marker_registry_factory(move |classes| {
3022                seen_for_factory.lock().push(classes.to_vec());
3023                DataMarkerExtractorRegistry::default_registry(classes)
3024            });
3025
3026        let mut store =
3027            EventStoreLifecycle::boot_with_options(Some(config), instance_id, clock_rc, options)
3028                .expect("boot store");
3029        store
3030            .open(
3031                instance_id,
3032                &RegisteredComponents::default(),
3033                Environment::Backtest,
3034            )
3035            .expect("open run");
3036        store.seal(UnixNanos::from(1_000));
3037
3038        let seen = seen_classes.lock();
3039        assert_eq!(seen.as_slice(), &[vec![DataClass::Trade, DataClass::Quote]]);
3040    }
3041
3042    #[rstest]
3043    fn markers_disabled_installs_no_file_and_no_cost() {
3044        let tmp = TempDir::new().expect("tempdir");
3045        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3046        let instance_id = UUID4::new();
3047
3048        let mut store = EventStoreLifecycle::boot(
3049            Some(make_config(tmp.path().to_path_buf())),
3050            instance_id,
3051            clock_rc,
3052        )
3053        .expect("boot store");
3054        store
3055            .open(
3056                instance_id,
3057                &RegisteredComponents::default(),
3058                Environment::Backtest,
3059            )
3060            .expect("open run");
3061        let run_id = store.run_id().expect("run open").to_string();
3062
3063        assert!(
3064            store
3065                .session
3066                .as_ref()
3067                .expect("session")
3068                .marker_capture
3069                .is_none()
3070        );
3071
3072        let quote = quote_ethusdt_binance();
3073        msgbus::publish_quote(MStr::from("data.quotes.BINANCE.ETHUSDT-PERP"), &quote);
3074        store.seal(UnixNanos::from(500));
3075
3076        let marker_path = tmp
3077            .path()
3078            .join(instance_id.to_string())
3079            .join(format!("{run_id}.markers.redb"));
3080        assert!(!marker_path.exists());
3081    }
3082
3083    /// Fired clock events do not pass through normal message bus publish/send calls.
3084    /// `TimeEventHandler::run` must still hit the installed tap so timer-driven
3085    /// strategy logic has a durable trigger record.
3086    #[rstest]
3087    fn bus_tap_captures_time_event_handler_run() {
3088        let tmp = TempDir::new().expect("tempdir");
3089        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3090        let instance_id = UUID4::new();
3091
3092        let mut store = EventStoreLifecycle::boot(
3093            Some(make_config(tmp.path().to_path_buf())),
3094            instance_id,
3095            clock_rc,
3096        )
3097        .expect("boot store");
3098        store
3099            .open(
3100                instance_id,
3101                &RegisteredComponents::default(),
3102                Environment::Backtest,
3103            )
3104            .expect("open run");
3105        let run_id = store.run_id().expect("run open").to_string();
3106
3107        let event = TimeEvent::new(
3108            Ustr::from("strategy.heartbeat"),
3109            UUID4::new(),
3110            UnixNanos::from(100),
3111            UnixNanos::from(99),
3112        );
3113        let callback = TimeEventCallback::from(|_: TimeEvent| {});
3114        TimeEventHandler::new(event, callback).run();
3115
3116        store.seal(UnixNanos::from(0));
3117
3118        let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3119            .expect("open sealed");
3120        let captured = sealed
3121            .scan_seq(2)
3122            .expect("scan")
3123            .expect("captured entry present");
3124
3125        assert_eq!(captured.payload_type.as_str(), PAYLOAD_TYPE_TIME_EVENT);
3126        assert_eq!(captured.topic, MessagingSwitchboard::time_event_topic());
3127    }
3128
3129    /// `EventStoreLifecycle::seal` must clear the bus tap so a publish issued after the
3130    /// run closes cannot reach the sealed writer. Without the clear, the dropped
3131    /// adapter would still receive captures and `Arc::try_unwrap` inside close would
3132    /// fail with multiple owners.
3133    #[rstest]
3134    fn seal_clears_bus_tap_so_post_seal_dispatches_do_not_capture() {
3135        let tmp = TempDir::new().expect("tempdir");
3136        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3137        let instance_id = UUID4::new();
3138
3139        let mut store = EventStoreLifecycle::boot(
3140            Some(make_config(tmp.path().to_path_buf())),
3141            instance_id,
3142            clock_rc,
3143        )
3144        .expect("boot store");
3145        store
3146            .open(
3147                instance_id,
3148                &RegisteredComponents::default(),
3149                Environment::Backtest,
3150            )
3151            .expect("open run");
3152        let run_id = store.run_id().expect("run open").to_string();
3153
3154        store.seal(UnixNanos::from(0));
3155
3156        // Post-seal dispatch: any tap that survived would either capture into the
3157        // dropped writer (panic via the channel close path) or hold the adapter Arc
3158        // and fail the close try_unwrap. The session is already gone, so this just
3159        // exercises the cleared-tap path through msgbus dispatch.
3160        let endpoint = MStr::<Endpoint>::from("test.post.seal.endpoint");
3161        let payload: u32 = 99;
3162        msgbus::send_any_value(endpoint, &payload);
3163
3164        let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3165            .expect("open sealed");
3166        // RunStarted at seq=1, RunEnded at seq=2; no captured u32 entry must exist
3167        assert!(
3168            sealed.scan_seq(3).expect("scan").is_none(),
3169            "no entry must land after seal",
3170        );
3171    }
3172
3173    /// Production code reaches the bus tap with [`TradingCommand`] wrapped around the
3174    /// inner command (the wrapper's `TypeId`, not `SubmitOrder`'s). The envelope
3175    /// dispatcher in [`default_registry`] must unwrap the variant, stamp the inner
3176    /// `payload_type` (`SubmitOrder`), and commit the same indices the bare-type encoder
3177    /// would have produced.
3178    #[rstest]
3179    fn bus_tap_captures_trading_command_envelope_with_inner_payload_type() {
3180        let tmp = TempDir::new().expect("tempdir");
3181        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3182        let instance_id = UUID4::new();
3183
3184        let mut store = EventStoreLifecycle::boot(
3185            Some(make_config(tmp.path().to_path_buf())),
3186            instance_id,
3187            clock_rc,
3188        )
3189        .expect("boot store");
3190        store
3191            .open(
3192                instance_id,
3193                &RegisteredComponents::default(),
3194                Environment::Backtest,
3195            )
3196            .expect("open run");
3197        let run_id = store.run_id().expect("run open").to_string();
3198
3199        let trader_id = TraderId::from("TRADER-001");
3200        let strategy_id = StrategyId::from("S-001");
3201        let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE");
3202        let client_order_id = ClientOrderId::from("O-20260510-000002");
3203        let order_init = OrderInitializedSpec::builder()
3204            .instrument_id(instrument_id)
3205            .client_order_id(client_order_id)
3206            .quantity(Quantity::from("1"))
3207            .time_in_force(TimeInForce::Gtc)
3208            .build();
3209        let submit_order = SubmitOrder::new(
3210            trader_id,
3211            Some(ClientId::from("BINANCE")),
3212            strategy_id,
3213            instrument_id,
3214            client_order_id,
3215            order_init,
3216            None,
3217            None,
3218            None,
3219            UUID4::new(),
3220            UnixNanos::from(3),
3221            None, // correlation_id
3222        );
3223        let command = TradingCommand::SubmitOrder(submit_order.clone());
3224
3225        let endpoint = MStr::<Endpoint>::from("test.exec.engine.envelope");
3226        msgbus::send_trading_command(endpoint, command);
3227
3228        store.seal(UnixNanos::from(0));
3229
3230        let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3231            .expect("open sealed");
3232        let captured = sealed
3233            .scan_seq(2)
3234            .expect("scan")
3235            .expect("captured entry present");
3236        assert_eq!(
3237            captured.payload_type.as_str(),
3238            "SubmitOrder",
3239            "wrapper envelope must stamp the inner payload_type",
3240        );
3241        assert_eq!(captured.topic.as_ref(), endpoint.as_str());
3242
3243        let by_client = sealed
3244            .lookup(IndexKind::ClientOrderId, client_order_id.as_str())
3245            .expect("lookup")
3246            .expect("indexed");
3247        assert_eq!(by_client, 2);
3248
3249        // Round-trip the captured payload back through the inner-type decoder so the
3250        // bytes-equal-bare invariant is checked at the integration layer too: a mutation
3251        // that wrote the wrapper-typed bytes instead of the inner would fail here.
3252        let decoded: SubmitOrder =
3253            rmp_serde::from_slice(&captured.payload).expect("decode captured SubmitOrder");
3254        assert_eq!(decoded, submit_order);
3255    }
3256
3257    /// `publish_order_event` reaches the bus tap with `OrderEventAny::Filled(...)`; the
3258    /// envelope dispatcher must unwrap to `OrderFilled`, stamp `OrderFilled` as the
3259    /// `payload_type`, and commit both the `client_order_id` and `venue_order_id` indices.
3260    #[rstest]
3261    fn bus_tap_captures_order_event_any_envelope_with_inner_payload_type() {
3262        let tmp = TempDir::new().expect("tempdir");
3263        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3264        let instance_id = UUID4::new();
3265
3266        let mut store = EventStoreLifecycle::boot(
3267            Some(make_config(tmp.path().to_path_buf())),
3268            instance_id,
3269            clock_rc,
3270        )
3271        .expect("boot store");
3272        store
3273            .open(
3274                instance_id,
3275                &RegisteredComponents::default(),
3276                Environment::Backtest,
3277            )
3278            .expect("open run");
3279        let run_id = store.run_id().expect("run open").to_string();
3280
3281        let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE");
3282        let client_order_id = ClientOrderId::from("O-20260510-000003");
3283        let venue_order_id = VenueOrderId::from("V-99");
3284        let filled = OrderFilledSpec::builder()
3285            .instrument_id(instrument_id)
3286            .client_order_id(client_order_id)
3287            .venue_order_id(venue_order_id)
3288            .account_id(AccountId::from("BINANCE-001"))
3289            .trade_id(TradeId::from("T-1"))
3290            .last_qty(Quantity::from("1"))
3291            .last_px(Price::from("100.00"))
3292            .currency(Currency::USDT())
3293            .ts_event(UnixNanos::from(10))
3294            .ts_init(UnixNanos::from(11))
3295            .commission(Money::new(0.10, Currency::USDT()))
3296            .build();
3297        let event = OrderEventAny::Filled(filled.clone());
3298
3299        let topic: MStr<msgbus::Topic> = MStr::from("events.order.ETHUSDT-PERP.BINANCE");
3300        msgbus::publish_order_event(topic, &event);
3301
3302        store.seal(UnixNanos::from(0));
3303
3304        let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3305            .expect("open sealed");
3306        let captured = sealed
3307            .scan_seq(2)
3308            .expect("scan")
3309            .expect("captured entry present");
3310        assert_eq!(
3311            captured.payload_type.as_str(),
3312            "OrderFilled",
3313            "wrapper envelope must stamp the inner payload_type",
3314        );
3315        assert_eq!(captured.topic.as_ref(), topic.as_str());
3316
3317        let by_client = sealed
3318            .lookup(IndexKind::ClientOrderId, client_order_id.as_str())
3319            .expect("lookup")
3320            .expect("indexed");
3321        let by_venue = sealed
3322            .lookup(IndexKind::VenueOrderId, venue_order_id.as_str())
3323            .expect("lookup")
3324            .expect("indexed");
3325        assert_eq!(by_client, 2);
3326        assert_eq!(by_venue, 2);
3327
3328        // Round-trip the captured payload back through the inner-type decoder so a
3329        // mutation that wrote the wrapper-typed bytes instead of the inner would fail
3330        // here rather than only at the unit-level bytes-equal-bare check.
3331        let decoded: OrderFilled =
3332            rmp_serde::from_slice(&captured.payload).expect("decode captured OrderFilled");
3333        assert_eq!(decoded, filled);
3334    }
3335
3336    /// `send_data_command` reaches the bus tap with the [`DataCommand`] wrapper. The
3337    /// envelope dispatcher must unwrap to the request/subscription category, stamp that
3338    /// category as the `payload_type`, and write bytes that decode as the inner command
3339    /// enum.
3340    #[rstest]
3341    fn bus_tap_captures_data_command_envelopes_with_category_payload_types() {
3342        let tmp = TempDir::new().expect("tempdir");
3343        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3344        let instance_id = UUID4::new();
3345
3346        let mut store = EventStoreLifecycle::boot(
3347            Some(make_config(tmp.path().to_path_buf())),
3348            instance_id,
3349            clock_rc,
3350        )
3351        .expect("boot store");
3352        store
3353            .open(
3354                instance_id,
3355                &RegisteredComponents::default(),
3356                Environment::Backtest,
3357            )
3358            .expect("open run");
3359        let run_id = store.run_id().expect("run open").to_string();
3360
3361        let request = RequestCommand::Quotes(RequestQuotes::new(
3362            InstrumentId::from("ETHUSDT-PERP.BINANCE"),
3363            None,
3364            None,
3365            None,
3366            Some(ClientId::from("BINANCE")),
3367            UUID4::new(),
3368            UnixNanos::from(20),
3369            None,
3370        ));
3371        let subscribe = SubscribeCommand::Quotes(SubscribeQuotes::new(
3372            InstrumentId::from("ETHUSDT-PERP.BINANCE"),
3373            Some(ClientId::from("BINANCE")),
3374            Some(Venue::from("BINANCE")),
3375            UUID4::new(),
3376            UnixNanos::from(21),
3377            Some(UUID4::new()),
3378            None,
3379        ));
3380
3381        let request_endpoint = MStr::<Endpoint>::from("test.data.engine.request");
3382        msgbus::send_data_command(request_endpoint, DataCommand::Request(request.clone()));
3383
3384        let subscribe_endpoint = MStr::<Endpoint>::from("test.data.engine.subscribe");
3385        msgbus::send_data_command(
3386            subscribe_endpoint,
3387            DataCommand::Subscribe(subscribe.clone()),
3388        );
3389
3390        store.seal(UnixNanos::from(0));
3391
3392        let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3393            .expect("open sealed");
3394        let captured_request = sealed
3395            .scan_seq(2)
3396            .expect("scan request")
3397            .expect("captured request present");
3398        assert_eq!(captured_request.payload_type.as_str(), "RequestCommand");
3399        assert_eq!(captured_request.topic.as_ref(), request_endpoint.as_str());
3400
3401        let decoded_request: RequestCommand =
3402            rmp_serde::from_slice(&captured_request.payload).expect("decode RequestCommand");
3403        match (decoded_request, request) {
3404            (RequestCommand::Quotes(decoded), RequestCommand::Quotes(expected)) => {
3405                assert_eq!(decoded.request_id, expected.request_id);
3406                assert_eq!(decoded.instrument_ids, expected.instrument_ids);
3407                assert_eq!(decoded.client_id, expected.client_id);
3408                assert_eq!(decoded.ts_init, expected.ts_init);
3409            }
3410            other => panic!("expected RequestCommand::Quotes round trip, was {other:?}"),
3411        }
3412
3413        let captured_subscribe = sealed
3414            .scan_seq(3)
3415            .expect("scan subscribe")
3416            .expect("captured subscribe present");
3417        assert_eq!(captured_subscribe.payload_type.as_str(), "SubscribeCommand");
3418        assert_eq!(
3419            captured_subscribe.topic.as_ref(),
3420            subscribe_endpoint.as_str()
3421        );
3422
3423        let decoded_subscribe: SubscribeCommand =
3424            rmp_serde::from_slice(&captured_subscribe.payload).expect("decode SubscribeCommand");
3425        match (decoded_subscribe, subscribe) {
3426            (SubscribeCommand::Quotes(decoded), SubscribeCommand::Quotes(expected)) => {
3427                assert_eq!(decoded.command_id, expected.command_id);
3428                assert_eq!(decoded.instrument_ids, expected.instrument_ids);
3429                assert_eq!(decoded.client_id, expected.client_id);
3430                assert_eq!(decoded.venue, expected.venue);
3431                assert_eq!(decoded.ts_init, expected.ts_init);
3432                assert_eq!(decoded.correlation_id, expected.correlation_id);
3433            }
3434            other => panic!("expected SubscribeCommand::Quotes round trip, was {other:?}"),
3435        }
3436    }
3437
3438    // `send_response` dispatches through a correlation handler rather than an endpoint
3439    // or pub/sub topic. Capture retains the completion envelope and its typed payload.
3440    #[rstest]
3441    fn bus_tap_captures_data_response_sent_through_correlation_handler() {
3442        let tmp = TempDir::new().expect("tempdir");
3443        let clock_rc: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::new()));
3444        let instance_id = UUID4::new();
3445
3446        let mut store = EventStoreLifecycle::boot(
3447            Some(make_config(tmp.path().to_path_buf())),
3448            instance_id,
3449            clock_rc,
3450        )
3451        .expect("boot store");
3452        store
3453            .open(
3454                instance_id,
3455                &RegisteredComponents::default(),
3456                Environment::Backtest,
3457            )
3458            .expect("open run");
3459        let run_id = store.run_id().expect("run open").to_string();
3460
3461        let correlation_id = UUID4::new();
3462        let handler_called = Rc::new(RefCell::new(false));
3463        let handler_called_clone = Rc::clone(&handler_called);
3464        msgbus::register_response_handler(
3465            &correlation_id,
3466            TypedHandler::from(move |_resp: &DataResponseEnvelope| {
3467                *handler_called_clone.borrow_mut() = true;
3468            }),
3469        );
3470
3471        let response = QuotesResponse::new(
3472            correlation_id,
3473            ClientId::from("BINANCE"),
3474            InstrumentId::from("ETHUSDT-PERP.BINANCE"),
3475            vec![],
3476            None,
3477            None,
3478            UnixNanos::from(30),
3479            None,
3480        );
3481        msgbus::send_response(&correlation_id, &DataResponse::Quotes(response.clone()));
3482
3483        assert!(*handler_called.borrow());
3484        store.seal(UnixNanos::from(0));
3485
3486        let sealed = RedbBackend::open_sealed(tmp.path(), &instance_id.to_string(), &run_id)
3487            .expect("open sealed");
3488        let captured = sealed
3489            .scan_seq(2)
3490            .expect("scan")
3491            .expect("captured response present");
3492        assert_eq!(captured.payload_type.as_str(), "DataResponseEnvelope");
3493        assert_eq!(captured.topic, MessagingSwitchboard::data_response_topic());
3494
3495        let envelope: crate::capture::builtins::ResponseEnvelopeCapture =
3496            rmp_serde::from_slice(&captured.payload).expect("decode response envelope");
3497        assert_eq!(envelope.correlation_id, correlation_id);
3498        assert!(envelope.completed);
3499        assert_eq!(envelope.sub_correlation_id, None);
3500        assert_eq!(envelope.failure_reason, None);
3501        assert_eq!(
3502            envelope.payload_type.as_ref().map(Ustr::as_str),
3503            Some("QuotesResponse")
3504        );
3505        let decoded: QuotesResponse = rmp_serde::from_slice(envelope.payload.as_ref().unwrap())
3506            .expect("decode quotes payload");
3507        assert_eq!(decoded.correlation_id, response.correlation_id);
3508        assert_eq!(decoded.client_id, response.client_id);
3509        assert_eq!(decoded.instrument_ids, response.instrument_ids);
3510        assert_eq!(decoded.ts_init, response.ts_init);
3511        assert!(decoded.data.is_empty());
3512    }
3513}