1use std::fmt::Debug;
17
18use anyhow::Context;
19use jiff::tz::TimeZone;
20use nautilus_common::{
21 messages::data::{SubscribeBars, SubscribeCommand},
22 msgbus::{self, MStr, Topic, TypedHandler},
23};
24use nautilus_core::{UUID4, datetime::get_timezone};
25use nautilus_model::data::{Bar, BarType, QuoteTick, TradeTick};
26
27use super::{
28 AggregationSource, BAR_AGGREGATOR_PRIORITY, BarAggregation, BarAggregator, BarBarHandler,
29 BarQuoteHandler, BarTradeHandler, DataCommand, DataEngine, DataResponse, Instrument,
30 InstrumentAny, PriceType, Rc, RefCell, RenkoBarAggregator, RequestBarAggregation,
31 RequestCommand, SubscribeQuotes, SubscribeTrades, TickBarAggregator,
32 TickImbalanceBarAggregator, TickRunsBarAggregator, TimeBarAggregator, UnsubscribeBars,
33 UnsubscribeCommand, UnsubscribeQuotes, UnsubscribeTrades, ValueBarAggregator,
34 ValueImbalanceBarAggregator, ValueRunsBarAggregator, VolumeBarAggregator,
35 VolumeImbalanceBarAggregator, VolumeRunsBarAggregator, handlers::WarmupBuffer,
36 log_error_on_cache_insert, process_engine_bar, request_bar_aggregation_from_params,
37 request_params, requests::time_zone_from_params, switchboard,
38};
39
40impl DataEngine {
41 pub(super) fn prepare_request_bar_aggregators_from_state(
42 &mut self,
43 request_id: UUID4,
44 state: &RequestBarAggregation,
45 ) -> anyhow::Result<()> {
46 if !self.can_start_request_bar_aggregators(request_id, state) {
47 anyhow::bail!(
48 "Cannot request aggregated bars: one of the aggregators in `bar_types` is already running"
49 );
50 }
51
52 self.request_bar_aggregations
53 .insert(request_id, state.clone());
54
55 if let Err(e) = self.init_request_bar_aggregators(request_id, state) {
56 self.cleanup_request_bar_aggregators(&request_id);
57 return Err(e);
58 }
59
60 Ok(())
61 }
62
63 pub(super) fn prepare_request_bar_aggregators(
64 &mut self,
65 req: &RequestCommand,
66 ) -> anyhow::Result<()> {
67 let request_id = *req.request_id();
68
69 let Some(state) = request_bar_aggregation_from_params(request_params(req))? else {
70 return Ok(());
71 };
72
73 self.prepare_request_bar_aggregators_from_state(request_id, &state)
74 }
75
76 fn can_start_request_bar_aggregators(
77 &self,
78 request_id: UUID4,
79 state: &RequestBarAggregation,
80 ) -> bool {
81 let aggregator_request_id = state.aggregator_request_id(request_id);
82 state.bar_types.iter().all(|bar_type| {
83 let key = bar_aggregator_key(*bar_type, aggregator_request_id);
84 self.bar_aggregators
85 .get(&key)
86 .is_none_or(|aggregator| !aggregator.borrow().is_running())
87 })
88 }
89
90 fn init_request_bar_aggregators(
91 &mut self,
92 request_id: UUID4,
93 state: &RequestBarAggregation,
94 ) -> anyhow::Result<()> {
95 let aggregator_request_id = state.aggregator_request_id(request_id);
96
97 for bar_type in &state.bar_types {
98 self.create_bar_aggregator_for_key(
99 *bar_type,
100 aggregator_request_id,
101 state.skip_first_non_full_bar,
102 state.time_zone.clone(),
103 )?;
104 self.setup_bar_aggregator(*bar_type, true, aggregator_request_id)?;
105
106 let key = bar_aggregator_key(*bar_type, aggregator_request_id);
107 if let Some(aggregator) = self.bar_aggregators.get(&key) {
108 if state.disable_build_with_no_updates {
109 aggregator.borrow_mut().set_build_with_no_updates(false);
110 }
111
112 aggregator.borrow_mut().set_is_running(true);
113 }
114 }
115
116 self.set_request_bar_aggregator_chain_handlers(request_id, state);
117
118 Ok(())
119 }
120
121 fn set_request_bar_aggregator_chain_handlers(
122 &self,
123 request_id: UUID4,
124 state: &RequestBarAggregation,
125 ) {
126 let aggregator_request_id = state.aggregator_request_id(request_id);
127
128 for bar_type in &state.bar_types {
129 let key = bar_aggregator_key(*bar_type, aggregator_request_id);
130
131 let Some(aggregator) = self.bar_aggregators.get(&key).cloned() else {
132 continue;
133 };
134
135 let downstream: Vec<_> = state
136 .bar_types
137 .iter()
138 .filter(|candidate| {
139 candidate.is_composite()
140 && candidate.composite().standard() == bar_type.standard()
141 })
142 .filter_map(|candidate| {
143 let key = bar_aggregator_key(*candidate, aggregator_request_id);
144 self.bar_aggregators.get(&key).cloned()
145 })
146 .collect();
147
148 let cache = Rc::clone(&self.cache);
149
150 let handler: Box<dyn FnMut(Bar)> = Box::new(move |bar: Bar| {
151 if let Err(e) = cache.as_ref().borrow_mut().add_bar_historical(bar) {
153 log_error_on_cache_insert(&e);
154 }
155
156 for aggregator in &downstream {
157 aggregator.borrow_mut().handle_bar(bar);
158 }
159 });
160
161 aggregator.borrow_mut().set_historical_mode(true, handler);
162 }
163 }
164
165 pub(super) fn cleanup_request_bar_aggregators(&mut self, request_id: &UUID4) -> bool {
166 let Some(state) = self.request_bar_aggregations.remove(request_id) else {
167 return false;
168 };
169
170 let aggregator_request_id = state.aggregator_request_id(*request_id);
171
172 let bar_types = sources_before_dependents(&state.bar_types);
175
176 for bar_type in bar_types {
177 let key = bar_aggregator_key(bar_type, aggregator_request_id);
178 let has_live_handlers =
179 state.update_subscriptions && self.bar_aggregator_handlers.contains_key(&key);
180
181 self.finish_historical_bar_aggregator(&key, has_live_handlers);
182
183 let keep_running = if has_live_handlers {
184 match self.setup_bar_aggregator(bar_type, false, aggregator_request_id) {
185 Ok(()) => true,
186 Err(e) => {
187 log::error!(
188 "Error starting live request bar aggregator for {bar_type}: {e}"
189 );
190 false
191 }
192 }
193 } else {
194 false
195 };
196
197 if let Some(aggregator) = self.bar_aggregators.get(&key) {
198 aggregator.borrow_mut().set_is_running(keep_running);
199 }
200
201 let stop_after_request = self.bar_aggregators_pending_stop.contains(&key)
202 && !self.has_pending_shared_bar_aggregation(bar_type);
203
204 if (!state.update_subscriptions || stop_after_request)
205 && let Err(e) = self.stop_bar_aggregator(bar_type, aggregator_request_id)
206 {
207 log::error!("Error stopping request bar aggregator for {bar_type}: {e}");
208 }
209 }
210
211 true
212 }
213
214 fn finish_historical_bar_aggregator(
215 &mut self,
216 key: &BarAggregatorKey,
217 has_live_handlers: bool,
218 ) {
219 if has_live_handlers
221 && let Some(warmup) = self.bar_aggregator_warmups.remove(key)
222 && let Some(aggregator) = self.bar_aggregators.get(key)
223 {
224 warmup.release(aggregator.borrow_mut().as_mut());
225 }
226
227 if let Some(aggregator) = self.bar_aggregators.get(key) {
228 let mut aggregator = aggregator.borrow_mut();
229
230 if let Some(aggregator) = aggregator.as_any_mut().downcast_mut::<TimeBarAggregator>() {
231 aggregator.flush_pending_historical_bar();
232 }
233 }
234 }
235
236 pub(super) fn process_request_bar_aggregation_response(
237 &mut self,
238 resp: &DataResponse,
239 completed: bool,
240 ) {
241 let correlation_id = *resp.correlation_id();
242
243 let Some(state) = self.request_bar_aggregations.get(&correlation_id).cloned() else {
244 return;
245 };
246
247 match resp {
248 DataResponse::Quotes(r) => {
249 for quote in r.data.iter() {
250 self.update_request_bar_aggregators_from_quote(&state, correlation_id, *quote);
251 }
252 }
253 DataResponse::Trades(r) => {
254 for trade in r.data.iter() {
255 self.update_request_bar_aggregators_from_trade(&state, correlation_id, *trade);
256 }
257 }
258 DataResponse::Bars(r) => {
259 for bar in r.data.iter() {
260 self.update_request_bar_aggregators_from_bar(&state, correlation_id, *bar);
261 }
262 }
263 _ => {}
264 }
265
266 if completed {
267 self.cleanup_request_bar_aggregators(&correlation_id);
268 }
269 }
270
271 pub(super) fn update_request_bar_aggregators_from_quote(
272 &self,
273 state: &RequestBarAggregation,
274 request_id: UUID4,
275 quote: QuoteTick,
276 ) {
277 let aggregator_request_id = state.aggregator_request_id(request_id);
278
279 for bar_type in &state.bar_types {
280 if bar_type.is_composite()
281 || bar_type.instrument_id() != quote.instrument_id
282 || bar_type.spec().price_type == PriceType::Last
283 {
284 continue;
285 }
286
287 self.update_request_bar_aggregator(*bar_type, aggregator_request_id, |aggregator| {
288 aggregator.handle_quote(quote);
289 });
290 }
291 }
292
293 pub(super) fn update_request_bar_aggregators_from_trade(
294 &self,
295 state: &RequestBarAggregation,
296 request_id: UUID4,
297 trade: TradeTick,
298 ) {
299 let aggregator_request_id = state.aggregator_request_id(request_id);
300
301 for bar_type in &state.bar_types {
302 if bar_type.is_composite()
303 || bar_type.instrument_id() != trade.instrument_id
304 || bar_type.spec().price_type != PriceType::Last
305 {
306 continue;
307 }
308
309 self.update_request_bar_aggregator(*bar_type, aggregator_request_id, |aggregator| {
310 aggregator.handle_trade(trade);
311 });
312 }
313 }
314
315 pub(super) fn update_request_bar_aggregators_from_bar(
316 &self,
317 state: &RequestBarAggregation,
318 request_id: UUID4,
319 bar: Bar,
320 ) {
321 let aggregator_request_id = state.aggregator_request_id(request_id);
322
323 for bar_type in &state.bar_types {
324 if !bar_type.is_composite()
325 || bar_type.composite().standard() != bar.bar_type.standard()
326 {
327 continue;
328 }
329
330 self.update_request_bar_aggregator(*bar_type, aggregator_request_id, |aggregator| {
331 aggregator.handle_bar(bar);
332 });
333 }
334 }
335
336 pub(super) fn update_request_bar_aggregator<F>(
337 &self,
338 bar_type: BarType,
339 request_id: Option<UUID4>,
340 update: F,
341 ) where
342 F: FnOnce(&mut dyn BarAggregator),
343 {
344 let key = bar_aggregator_key(bar_type, request_id);
345
346 let Some(aggregator) = self.bar_aggregators.get(&key) else {
347 log::error!("Cannot update request bar aggregator: no aggregator found for {bar_type}");
348 return;
349 };
350
351 update(aggregator.borrow_mut().as_mut());
352 }
353
354 pub(super) fn subscribe_bars(&mut self, cmd: &SubscribeBars) -> anyhow::Result<()> {
355 let bar_type = cmd
356 .single_bar_type()
357 .expect("native engine receives one bar type");
358
359 match bar_type.aggregation_source() {
360 AggregationSource::Internal => self.start_bar_aggregation(cmd)?,
361 AggregationSource::External => {
362 if bar_type.instrument_id().is_synthetic() {
363 anyhow::bail!(
364 "Cannot subscribe for externally aggregated synthetic instrument bar data"
365 );
366 }
367 }
368 }
369
370 Ok(())
371 }
372
373 pub(super) fn unsubscribe_bars(&mut self, cmd: &UnsubscribeBars) {
374 let bar_type = cmd
375 .single_bar_type()
376 .expect("native engine receives one bar type");
377
378 let topic = switchboard::get_bars_topic(bar_type.standard());
380 if msgbus::exact_subscriber_count_bars(topic) > 0 {
381 return;
382 }
383
384 let retained = self
385 .subscriptions_bar_aggregation
386 .get(&bar_type.standard())
387 .map(|subscription| subscription.command.clone());
388
389 let command = retained.map_or_else(
390 || cmd.clone(),
391 |subscribe| {
392 UnsubscribeBars::new(
393 subscribe.bar_types,
394 subscribe.client_id,
395 subscribe.venue,
396 cmd.command_id,
397 cmd.ts_init,
398 Some(subscribe.command_id),
399 subscribe.params,
400 )
401 },
402 );
403
404 let key = bar_aggregator_key(bar_type, None);
405
406 if self.bar_aggregators_pending_stop.contains(&key) {
408 return;
409 }
410
411 if self.bar_aggregators.contains_key(&key) {
412 let result = if self.has_pending_shared_bar_aggregation(bar_type) {
413 self.unsubscribe_bar_aggregator_handlers(&key);
415 self.bar_aggregators_pending_stop.insert(key);
416 Ok(())
417 } else {
418 self.stop_bar_aggregator(bar_type, None)
419 };
420
421 match result {
422 Ok(()) => {
423 self.subscriptions_bar_aggregation
424 .remove(&bar_type.standard());
425 self.unsubscribe_bar_aggregator(&command);
426 }
427 Err(e) => log::error!("Error stopping bar aggregator for {bar_type}: {e}"),
428 }
429 }
430
431 self.release_composite_bar_source(&command);
432 }
433
434 fn release_composite_bar_source(&mut self, command: &UnsubscribeBars) {
435 if command
438 .single_bar_type()
439 .expect("retained aggregation has one bar type")
440 .is_composite()
441 {
442 let source_type = command
443 .single_bar_type()
444 .expect("retained aggregation has one bar type")
445 .composite();
446
447 if self
448 .bar_aggregators
449 .contains_key(&bar_aggregator_key(source_type, None))
450 {
451 self.unsubscribe_bars(&UnsubscribeBars::new(
452 source_type,
453 command.client_id,
454 command.venue,
455 UUID4::new(),
456 command.ts_init,
457 Some(command.command_id),
458 command.params.clone(),
459 ));
460 }
461 }
462 }
463
464 fn has_pending_shared_bar_aggregation(&self, bar_type: BarType) -> bool {
465 self.request_bar_aggregations.values().any(|state| {
466 state.update_subscriptions
467 && state
468 .bar_types
469 .iter()
470 .any(|requested| requested.standard() == bar_type.standard())
471 })
472 }
473
474 fn create_bar_aggregator(
475 &self,
476 instrument: &InstrumentAny,
477 bar_type: BarType,
478 skip_first_non_full_bar: Option<bool>,
479 time_zone: TimeZone,
480 ) -> Box<dyn BarAggregator> {
481 let cache = Rc::clone(&self.cache);
482 let validate_sequence = self.config.validate_data_sequence;
483
484 let handler = move |bar: Bar| {
485 process_engine_bar(&cache, validate_sequence, true, bar);
486 };
487
488 let clock = Rc::clone(&self.clock);
489 let config = self.config.clone();
490
491 let price_precision = instrument.price_precision();
492 let size_precision = instrument.size_precision();
493
494 if bar_type.spec().is_time_aggregated() {
495 let time_bars_origin_offset = config
496 .time_bars_origin_offset
497 .get(&bar_type.spec().aggregation)
498 .map(|duration| jiff::SignedDuration::try_from(*duration).unwrap_or_default());
499
500 Box::new(TimeBarAggregator::new(
501 bar_type,
502 price_precision,
503 size_precision,
504 clock,
505 handler,
506 config.time_bars_build_with_no_updates,
507 config.time_bars_timestamp_on_close,
508 config.time_bars_interval_type,
509 time_bars_origin_offset,
510 time_zone,
511 config.time_bars_build_delay,
512 skip_first_non_full_bar.unwrap_or(config.time_bars_skip_first_non_full_bar),
513 ))
514 } else {
515 match bar_type.spec().aggregation {
516 BarAggregation::Tick => Box::new(TickBarAggregator::new(
517 bar_type,
518 price_precision,
519 size_precision,
520 handler,
521 )) as Box<dyn BarAggregator>,
522 BarAggregation::TickImbalance => Box::new(TickImbalanceBarAggregator::new(
523 bar_type,
524 price_precision,
525 size_precision,
526 handler,
527 )) as Box<dyn BarAggregator>,
528 BarAggregation::TickRuns => Box::new(TickRunsBarAggregator::new(
529 bar_type,
530 price_precision,
531 size_precision,
532 handler,
533 )) as Box<dyn BarAggregator>,
534 BarAggregation::Volume => Box::new(VolumeBarAggregator::new(
535 bar_type,
536 price_precision,
537 size_precision,
538 handler,
539 )) as Box<dyn BarAggregator>,
540 BarAggregation::VolumeImbalance => Box::new(VolumeImbalanceBarAggregator::new(
541 bar_type,
542 price_precision,
543 size_precision,
544 handler,
545 )) as Box<dyn BarAggregator>,
546 BarAggregation::VolumeRuns => Box::new(VolumeRunsBarAggregator::new(
547 bar_type,
548 price_precision,
549 size_precision,
550 handler,
551 )) as Box<dyn BarAggregator>,
552 BarAggregation::Value => Box::new(ValueBarAggregator::new(
553 bar_type,
554 price_precision,
555 size_precision,
556 handler,
557 )) as Box<dyn BarAggregator>,
558 BarAggregation::ValueImbalance => Box::new(ValueImbalanceBarAggregator::new(
559 bar_type,
560 price_precision,
561 size_precision,
562 handler,
563 )) as Box<dyn BarAggregator>,
564 BarAggregation::ValueRuns => Box::new(ValueRunsBarAggregator::new(
565 bar_type,
566 price_precision,
567 size_precision,
568 handler,
569 )) as Box<dyn BarAggregator>,
570 BarAggregation::Renko => Box::new(RenkoBarAggregator::new(
571 bar_type,
572 price_precision,
573 size_precision,
574 instrument.price_increment(),
575 handler,
576 )) as Box<dyn BarAggregator>,
577 other => unreachable!(
578 "Unsupported internal bar aggregation dispatch for {other:?}; update `create_bar_aggregator`"
579 ),
580 }
581 }
582 }
583
584 pub(super) fn create_bar_aggregator_for_key(
585 &mut self,
586 bar_type: BarType,
587 request_id: Option<UUID4>,
588 skip_first_non_full_bar: Option<bool>,
589 time_zone: Option<TimeZone>,
590 ) -> anyhow::Result<()> {
591 let key = bar_aggregator_key(bar_type, request_id);
592
593 let time_zone = match time_zone {
594 Some(zone) => zone,
595 None => get_timezone(self.config.time_bars_time_zone.as_deref().unwrap_or("UTC"))
596 .context("invalid `time_bars_time_zone` configuration")?,
597 };
598
599 if let Some(aggregator) = self.bar_aggregators.get(&key) {
600 let aggregator = aggregator.borrow();
601 if let Some(aggregator) = aggregator.as_any().downcast_ref::<TimeBarAggregator>() {
602 anyhow::ensure!(
603 aggregator.time_zone() == &time_zone,
604 "Cannot reuse bar aggregator {bar_type} with a different time zone"
605 );
606 }
607
608 return Ok(());
609 }
610
611 let instrument = {
612 let cache = self.cache.borrow();
613 cache
614 .instrument(&bar_type.instrument_id())
615 .ok_or_else(|| {
616 anyhow::anyhow!(
617 "Cannot start bar aggregation: no instrument found for {}",
618 bar_type.instrument_id(),
619 )
620 })?
621 .clone()
622 };
623
624 let aggregator =
625 self.create_bar_aggregator(&instrument, bar_type, skip_first_non_full_bar, time_zone);
626 debug_assert_eq!(
627 aggregator.bar_type(),
628 key.0,
629 "aggregator bar type must match its standardized key"
630 );
631 self.bar_aggregators
632 .insert(key, Rc::new(RefCell::new(aggregator)));
633
634 Ok(())
635 }
636
637 fn start_bar_aggregation(&mut self, cmd: &SubscribeBars) -> anyhow::Result<()> {
638 let bar_type = cmd
639 .single_bar_type()
640 .expect("native engine receives one bar type");
641 let time_zone = time_zone_from_params(cmd.params.as_ref())?;
642 let skip_first_non_full_bar = cmd
643 .params
644 .as_ref()
645 .and_then(|params| params.get_bool("skip_first_non_full_bar"));
646 self.create_bar_aggregator_for_key(
647 bar_type,
648 None,
649 skip_first_non_full_bar,
650 time_zone.clone(),
651 )?;
652 let key = bar_aggregator_key(bar_type, None);
653
654 if self
655 .bar_aggregators
656 .get(&key)
657 .is_some_and(|aggregator| aggregator.borrow().is_running())
658 && self.bar_aggregator_handlers.contains_key(&key)
659 {
660 if let Some(source_command) = self
661 .subscriptions_bar_aggregation
662 .get(&bar_type.standard())
663 .and_then(|subscription| subscription.source.clone())
664 {
665 self.execute(DataCommand::Subscribe(source_command));
666 }
667
668 log::debug!("Reusing running bar aggregator for {bar_type}");
669 return Ok(());
670 }
671
672 self.start_bar_aggregator(bar_type, None, skip_first_non_full_bar, time_zone)?;
673 self.bar_aggregators_pending_stop.remove(&key);
674 let source = self.subscribe_bar_aggregator(cmd);
675 self.subscriptions_bar_aggregation.insert(
676 bar_type.standard(),
677 BarAggregationSubscription {
678 command: cmd.clone(),
679 source,
680 },
681 );
682
683 Ok(())
684 }
685
686 fn start_bar_aggregator(
687 &mut self,
688 bar_type: BarType,
689 request_id: Option<UUID4>,
690 skip_first_non_full_bar: Option<bool>,
691 time_zone: Option<TimeZone>,
692 ) -> anyhow::Result<()> {
693 let key = bar_aggregator_key(bar_type, request_id);
694 self.create_bar_aggregator_for_key(
695 bar_type,
696 request_id,
697 skip_first_non_full_bar,
698 time_zone,
699 )?;
700 let aggregator = self
701 .bar_aggregators
702 .get(&key)
703 .map(Rc::clone)
704 .ok_or_else(|| anyhow::anyhow!("Cannot start bar aggregation for {bar_type}"))?;
705 let defer_subscription_activation = request_id.is_none()
708 && aggregator.borrow().is_running()
709 && !self.bar_aggregator_handlers.contains_key(&key);
710
711 if !self.bar_aggregator_handlers.contains_key(&key) {
712 self.subscribe_bar_aggregator_handlers(
713 key,
714 bar_type,
715 &aggregator,
716 defer_subscription_activation,
717 );
718 }
719
720 if defer_subscription_activation {
721 return Ok(());
722 }
723
724 self.setup_bar_aggregator(bar_type, false, request_id)?;
725
726 aggregator.borrow_mut().set_is_running(true);
727
728 Ok(())
729 }
730
731 fn subscribe_bar_aggregator_handlers(
732 &mut self,
733 key: BarAggregatorKey,
734 bar_type: BarType,
735 aggregator: &Rc<RefCell<Box<dyn BarAggregator>>>,
736 defer: bool,
737 ) {
738 let mut subscriptions = Vec::new();
740
741 let bar_type_std = bar_type.standard();
742 let warmup;
743
744 if bar_type.is_composite() {
745 let topic = switchboard::get_bars_topic(bar_type.composite());
746 let buffer = defer.then(WarmupBuffer::new);
747
748 let handler = TypedHandler::new(
749 BarBarHandler::new(aggregator, bar_type_std).with_warmup(buffer.clone()),
750 );
751
752 warmup = buffer.map(BarAggregatorWarmup::Bar);
753 msgbus::subscribe_bars(topic.into(), handler.clone(), None);
754 subscriptions.push(BarAggregatorSubscription::Bar { topic, handler });
755 } else if bar_type.spec().price_type == PriceType::Last {
756 let topic = switchboard::get_trades_topic(bar_type.instrument_id());
757 let buffer = defer.then(WarmupBuffer::new);
758
759 let handler = TypedHandler::new(
760 BarTradeHandler::new(aggregator, bar_type_std).with_warmup(buffer.clone()),
761 );
762
763 warmup = buffer.map(BarAggregatorWarmup::Trade);
764 msgbus::subscribe_trades(topic.into(), handler.clone(), Some(BAR_AGGREGATOR_PRIORITY));
765 subscriptions.push(BarAggregatorSubscription::Trade { topic, handler });
766 } else {
767 if matches!(
769 bar_type.spec().aggregation,
770 BarAggregation::TickImbalance
771 | BarAggregation::VolumeImbalance
772 | BarAggregation::ValueImbalance
773 | BarAggregation::TickRuns
774 | BarAggregation::VolumeRuns
775 | BarAggregation::ValueRuns
776 ) {
777 log::warn!(
778 "Bar type {bar_type} uses imbalance/runs aggregation which requires trade \
779 data with `aggressor_side`, but `price_type` is not LAST so it will receive \
780 quote data: bars will not emit correctly",
781 );
782 }
783
784 let topic = switchboard::get_quotes_topic(bar_type.instrument_id());
785 let buffer = defer.then(WarmupBuffer::new);
786
787 let handler = TypedHandler::new(
788 BarQuoteHandler::new(aggregator, bar_type_std).with_warmup(buffer.clone()),
789 );
790
791 warmup = buffer.map(BarAggregatorWarmup::Quote);
792 msgbus::subscribe_quotes(topic.into(), handler.clone(), Some(BAR_AGGREGATOR_PRIORITY));
793 subscriptions.push(BarAggregatorSubscription::Quote { topic, handler });
794 }
795
796 self.bar_aggregator_handlers.insert(key, subscriptions);
797 if let Some(warmup) = warmup {
798 self.bar_aggregator_warmups.insert(key, warmup);
799 }
800 }
801
802 fn subscribe_bar_aggregator(&mut self, cmd: &SubscribeBars) -> Option<SubscribeCommand> {
803 let subscribe = self.bar_aggregator_source_command(cmd)?;
804 self.execute(DataCommand::Subscribe(subscribe.clone()));
805 Some(subscribe)
806 }
807
808 fn bar_aggregator_source_command(&self, cmd: &SubscribeBars) -> Option<SubscribeCommand> {
809 let bar_type = cmd
810 .single_bar_type()
811 .expect("native engine receives one bar type");
812 let key = bar_aggregator_key(bar_type, None);
813 if !self.bar_aggregators.contains_key(&key) {
814 log::error!("Cannot subscribe bar aggregator: no aggregator found for {bar_type}");
815 return None;
816 }
817
818 if bar_type.is_composite() {
819 let composite_bar_type = bar_type.composite();
820 if composite_bar_type.is_externally_aggregated() {
821 let subscribe = SubscribeBars::new(
822 composite_bar_type,
823 cmd.client_id,
824 cmd.venue,
825 UUID4::new(),
826 cmd.ts_init,
827 Some(cmd.command_id),
828 cmd.params.clone(),
829 );
830 return Some(SubscribeCommand::Bars(subscribe));
831 }
832 } else if bar_type.spec().price_type == PriceType::Last {
833 let subscribe = SubscribeTrades::new(
834 bar_type.instrument_id(),
835 cmd.client_id,
836 cmd.venue,
837 UUID4::new(),
838 cmd.ts_init,
839 Some(cmd.command_id),
840 cmd.params.clone(),
841 );
842 return Some(SubscribeCommand::Trades(subscribe));
843 } else {
844 let subscribe = SubscribeQuotes::new(
845 bar_type.instrument_id(),
846 cmd.client_id,
847 cmd.venue,
848 UUID4::new(),
849 cmd.ts_init,
850 Some(cmd.command_id),
851 cmd.params.clone(),
852 );
853 return Some(SubscribeCommand::Quotes(subscribe));
854 }
855
856 None
857 }
858
859 pub(super) fn setup_bar_aggregator(
863 &self,
864 bar_type: BarType,
865 historical: bool,
866 request_id: Option<UUID4>,
867 ) -> anyhow::Result<()> {
868 let key = bar_aggregator_key(bar_type, request_id);
869
870 let aggregator = self.bar_aggregators.get(&key).ok_or_else(|| {
871 anyhow::anyhow!("Cannot setup bar aggregator: no aggregator found for {bar_type}")
872 })?;
873
874 let cache = Rc::clone(&self.cache);
876 let validate_sequence = self.config.validate_data_sequence;
877 let publish = !historical;
878
879 let handler: Box<dyn FnMut(Bar)> = Box::new(move |bar: Bar| {
880 process_engine_bar(&cache, validate_sequence, publish, bar);
881 });
882
883 aggregator
884 .borrow_mut()
885 .set_historical_mode(historical, handler);
886
887 if bar_type.spec().is_time_aggregated() {
889 use nautilus_common::clock::VirtualClock;
890
891 if historical {
892 let test_clock = Rc::new(RefCell::new(VirtualClock::new()));
894 aggregator.borrow_mut().set_clock(test_clock);
895 let aggregator_weak = Rc::downgrade(aggregator);
898 aggregator.borrow_mut().set_aggregator_weak(aggregator_weak);
899 } else {
900 aggregator.borrow_mut().set_clock(Rc::clone(&self.clock));
901 aggregator
902 .borrow_mut()
903 .start_timer(Some(Rc::clone(aggregator)));
904 }
905 }
906
907 if !historical && request_id.is_none() {
908 self.connect_time_bar_sources(bar_type);
909 }
910
911 Ok(())
912 }
913
914 fn connect_time_bar_sources(&self, bar_type: BarType) {
915 let bar_types = self
916 .subscriptions_bar_aggregation
917 .values()
918 .map(|subscription| {
919 subscription
920 .command
921 .single_bar_type()
922 .expect("retained aggregation has one bar type")
923 })
924 .chain(std::iter::once(bar_type));
925
926 for bar_type in bar_types {
927 if !bar_type.is_composite()
928 || !bar_type.spec().is_time_aggregated()
929 || !bar_type.composite().is_internally_aggregated()
930 || !bar_type.composite().spec().is_time_aggregated()
931 {
932 continue;
933 }
934
935 let Some(source) = self
936 .bar_aggregators
937 .get(&bar_aggregator_key(bar_type.composite(), None))
938 else {
939 continue;
940 };
941
942 let Some(aggregator) = self
943 .bar_aggregators
944 .get(&bar_aggregator_key(bar_type, None))
945 else {
946 continue;
947 };
948
949 if let Some(aggregator) = aggregator
950 .borrow_mut()
951 .as_any_mut()
952 .downcast_mut::<TimeBarAggregator>()
953 {
954 aggregator.set_timer_source(Rc::downgrade(source));
955 }
956 }
957 }
958
959 fn unsubscribe_bar_aggregator(&mut self, cmd: &UnsubscribeBars) {
960 let bar_type = cmd
961 .single_bar_type()
962 .expect("native engine receives one bar type");
963
964 if bar_type.is_composite() {
965 let composite_bar_type = bar_type.composite();
966 if composite_bar_type.is_externally_aggregated() {
967 let unsubscribe = UnsubscribeBars::new(
968 composite_bar_type,
969 cmd.client_id,
970 cmd.venue,
971 UUID4::new(),
972 cmd.ts_init,
973 Some(cmd.command_id),
974 cmd.params.clone(),
975 );
976 self.execute(DataCommand::Unsubscribe(UnsubscribeCommand::Bars(
977 unsubscribe,
978 )));
979 }
980 } else if bar_type.spec().price_type == PriceType::Last {
981 let unsubscribe = UnsubscribeTrades::new(
982 bar_type.instrument_id(),
983 cmd.client_id,
984 cmd.venue,
985 UUID4::new(),
986 cmd.ts_init,
987 Some(cmd.command_id),
988 cmd.params.clone(),
989 );
990 self.execute(DataCommand::Unsubscribe(UnsubscribeCommand::Trades(
991 unsubscribe,
992 )));
993 } else {
994 let unsubscribe = UnsubscribeQuotes::new(
995 bar_type.instrument_id(),
996 cmd.client_id,
997 cmd.venue,
998 UUID4::new(),
999 cmd.ts_init,
1000 Some(cmd.command_id),
1001 cmd.params.clone(),
1002 );
1003 self.execute(DataCommand::Unsubscribe(UnsubscribeCommand::Quotes(
1004 unsubscribe,
1005 )));
1006 }
1007 }
1008
1009 pub(super) fn clear_bar_aggregators(&mut self) {
1010 let keys: Vec<BarAggregatorKey> = self.bar_aggregators.keys().copied().collect();
1011 for (bar_type, request_id) in keys {
1012 if let Err(e) = self.stop_bar_aggregator(bar_type, request_id) {
1013 log::error!("Error stopping bar aggregator during reset for {bar_type}: {e}");
1014 }
1015 }
1016
1017 self.subscriptions_bar_aggregation.clear();
1018 }
1019
1020 pub(super) fn stop_bar_aggregator(
1021 &mut self,
1022 bar_type: BarType,
1023 request_id: Option<UUID4>,
1024 ) -> anyhow::Result<()> {
1025 let key = bar_aggregator_key(bar_type, request_id);
1026
1027 let aggregator = self.bar_aggregators.shift_remove(&key).ok_or_else(|| {
1028 anyhow::anyhow!("Cannot stop bar aggregator: no aggregator to stop for {bar_type}")
1029 })?;
1030
1031 aggregator.borrow_mut().stop();
1032 self.bar_aggregators_pending_stop.remove(&key);
1033 self.unsubscribe_bar_aggregator_handlers(&key);
1034
1035 Ok(())
1036 }
1037
1038 fn unsubscribe_bar_aggregator_handlers(&mut self, key: &BarAggregatorKey) {
1039 self.bar_aggregator_warmups.remove(key);
1040
1041 if let Some(subs) = self.bar_aggregator_handlers.remove(key) {
1043 for sub in subs {
1044 match sub {
1045 BarAggregatorSubscription::Bar { topic, handler } => {
1046 msgbus::unsubscribe_bars(topic.into(), &handler);
1047 }
1048 BarAggregatorSubscription::Trade { topic, handler } => {
1049 msgbus::unsubscribe_trades(topic.into(), &handler);
1050 }
1051 BarAggregatorSubscription::Quote { topic, handler } => {
1052 msgbus::unsubscribe_quotes(topic.into(), &handler);
1053 }
1054 }
1055 }
1056 }
1057 }
1058}
1059
1060#[derive(Clone, Debug)]
1061pub(super) struct BarAggregationSubscription {
1062 pub(super) command: SubscribeBars,
1063 pub(super) source: Option<SubscribeCommand>,
1064}
1065
1066pub(crate) type BarAggregatorKey = (BarType, Option<UUID4>);
1072
1073#[inline]
1074pub(crate) fn bar_aggregator_key(bar_type: BarType, request_id: Option<UUID4>) -> BarAggregatorKey {
1075 (bar_type.standard(), request_id)
1076}
1077
1078fn sources_before_dependents(bar_types: &[BarType]) -> Vec<BarType> {
1082 let depth = |bar_type: &BarType| {
1084 let mut depth = 0;
1085 let mut current = *bar_type;
1086
1087 while current.is_composite() && depth < bar_types.len() {
1088 let Some(source) = bar_types
1089 .iter()
1090 .find(|candidate| candidate.standard() == current.composite().standard())
1091 else {
1092 break;
1093 };
1094 current = *source;
1095 depth += 1;
1096 }
1097
1098 depth
1099 };
1100
1101 let mut ordered = bar_types.to_vec();
1102 ordered.sort_by_cached_key(depth);
1103 ordered
1104}
1105
1106#[derive(Debug)]
1108pub(super) enum BarAggregatorWarmup {
1109 Bar(WarmupBuffer<Bar>),
1110 Trade(WarmupBuffer<TradeTick>),
1111 Quote(WarmupBuffer<QuoteTick>),
1112}
1113
1114impl BarAggregatorWarmup {
1115 pub(super) fn release(&self, aggregator: &mut dyn BarAggregator) {
1117 match self {
1118 Self::Bar(buffer) => buffer
1119 .release()
1120 .into_iter()
1121 .for_each(|bar| aggregator.handle_bar(bar)),
1122 Self::Trade(buffer) => buffer
1123 .release()
1124 .into_iter()
1125 .for_each(|trade| aggregator.handle_trade(trade)),
1126 Self::Quote(buffer) => buffer
1127 .release()
1128 .into_iter()
1129 .for_each(|quote| aggregator.handle_quote(quote)),
1130 }
1131 }
1132}
1133
1134#[derive(Clone)]
1139pub enum BarAggregatorSubscription {
1140 Bar {
1141 topic: MStr<Topic>,
1142 handler: TypedHandler<Bar>,
1143 },
1144 Trade {
1145 topic: MStr<Topic>,
1146 handler: TypedHandler<TradeTick>,
1147 },
1148 Quote {
1149 topic: MStr<Topic>,
1150 handler: TypedHandler<QuoteTick>,
1151 },
1152}
1153
1154impl Debug for BarAggregatorSubscription {
1155 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1156 match self {
1157 Self::Bar { topic, handler } => f
1158 .debug_struct(stringify!(Bar))
1159 .field("topic", topic)
1160 .field("handler_id", &handler.id())
1161 .finish(),
1162 Self::Trade { topic, handler } => f
1163 .debug_struct(stringify!(Trade))
1164 .field("topic", topic)
1165 .field("handler_id", &handler.id())
1166 .finish(),
1167 Self::Quote { topic, handler } => f
1168 .debug_struct(stringify!(Quote))
1169 .field("topic", topic)
1170 .field("handler_id", &handler.id())
1171 .finish(),
1172 }
1173 }
1174}
1175
1176#[cfg(test)]
1177mod tests {
1178 use std::{cell::RefCell, rc::Rc};
1179
1180 use indexmap::IndexMap;
1181 use jiff::Timestamp;
1182 use nautilus_common::{
1183 cache::Cache,
1184 clock::{Clock, VirtualClock},
1185 messages::data::SubscribeBars,
1186 msgbus::MessageBus,
1187 };
1188 use nautilus_core::{UUID4, UnixNanos, datetime::get_timezone};
1189 use nautilus_model::{
1190 data::{Bar, BarType},
1191 identifiers::TraderId,
1192 instruments::{InstrumentAny, stubs::audusd_sim},
1193 types::{Price, Quantity},
1194 };
1195 use rstest::rstest;
1196
1197 use super::{
1198 bar_aggregator_key, request_bar_aggregation_from_params, sources_before_dependents,
1199 time_zone_from_params,
1200 };
1201 use crate::engine::{DataEngine, config::DataEngineConfig};
1202
1203 #[rstest]
1204 #[case::utc_default(None, None, "2026-11-02T00:00:00Z")]
1205 #[case::config(Some("America/New_York"), None, "2026-11-02T05:00:00Z")]
1206 #[case::utc_override(Some("America/New_York"), Some("UTC"), "2026-11-02T00:00:00Z")]
1207 fn test_calendar_zone_precedence_and_reuse(
1208 #[case] config_zone: Option<&str>,
1209 #[case] override_zone: Option<&str>,
1210 #[case] expected: &str,
1211 ) {
1212 let _msgbus = MessageBus::new(TraderId::new("TESTER-001"), UUID4::new(), None, None)
1213 .register_message_bus();
1214 let clock = Rc::new(RefCell::new(VirtualClock::new()));
1215 clock.borrow_mut().set_time(UnixNanos::from(
1216 "2026-11-01T12:00:00Z".parse::<Timestamp>().unwrap(),
1217 ));
1218 let cache = Rc::new(RefCell::new(Cache::default()));
1219 cache
1220 .borrow_mut()
1221 .add_instrument(InstrumentAny::CurrencyPair(audusd_sim()))
1222 .unwrap();
1223 let config = DataEngineConfig::builder()
1224 .maybe_time_bars_time_zone(config_zone.map(str::to_owned))
1225 .time_bars_skip_first_non_full_bar(true)
1226 .build();
1227 let mut engine = DataEngine::new(
1228 Rc::clone(&clock) as Rc<RefCell<dyn Clock>>,
1229 Rc::clone(&cache),
1230 Some(config),
1231 );
1232 let bar_type = BarType::from("AUD/USD.SIM-1-DAY-LAST-INTERNAL");
1233 let mut params = IndexMap::from([(
1234 "skip_first_non_full_bar".to_owned(),
1235 serde_json::json!(false),
1236 )]);
1237
1238 if let Some(zone) = override_zone {
1239 params.insert("time_zone".to_owned(), serde_json::json!(zone));
1240 }
1241
1242 let cmd = SubscribeBars::new(
1243 bar_type,
1244 None,
1245 Some(bar_type.instrument_id().venue),
1246 UUID4::new(),
1247 UnixNanos::default(),
1248 None,
1249 Some(nautilus_core::Params::new(params)),
1250 );
1251 engine.subscribe_bars(&cmd).unwrap();
1252 engine.subscribe_bars(&cmd).unwrap();
1253 let mut params = IndexMap::clone(cmd.params.as_ref().unwrap());
1254 params.insert("time_zone".to_owned(), serde_json::json!("Asia/Tokyo"));
1255
1256 let cmd = SubscribeBars {
1257 params: Some(nautilus_core::Params::new(params)),
1258 ..cmd
1259 };
1260
1261 let e = engine.subscribe_bars(&cmd).unwrap_err();
1262
1263 engine
1265 .create_bar_aggregator_for_key(
1266 bar_type,
1267 Some(UUID4::new()),
1268 None,
1269 Some(get_timezone("Asia/Tokyo").unwrap()),
1270 )
1271 .unwrap();
1272 assert_eq!(
1273 e.to_string(),
1274 format!("Cannot reuse bar aggregator {bar_type} with a different time zone")
1275 );
1276 assert_eq!(
1277 clock.borrow().next_time_ns(&format!("TIME_BAR_{bar_type}")),
1278 Some(UnixNanos::from(expected.parse::<Timestamp>().unwrap()))
1279 );
1280 let now = clock.borrow().timestamp_ns();
1281 engine
1282 .bar_aggregators
1283 .get(&bar_aggregator_key(bar_type, None))
1284 .unwrap()
1285 .borrow_mut()
1286 .update(Price::from("0.65000"), Quantity::from(2), now);
1287 let close = UnixNanos::from(expected.parse::<Timestamp>().unwrap());
1288 let events = clock.borrow_mut().advance_time(close, true);
1289 let handlers = clock.borrow().match_handlers(events);
1290 for handler in handlers {
1291 handler.callback.call(handler.event);
1292 }
1293
1294 assert_eq!(
1295 cache.borrow().bar(&bar_type).copied(),
1296 Some(Bar::new(
1297 bar_type,
1298 Price::from("0.65000"),
1299 Price::from("0.65000"),
1300 Price::from("0.65000"),
1301 Price::from("0.65000"),
1302 Quantity::from(2),
1303 close,
1304 close
1305 ))
1306 );
1307 }
1308
1309 #[rstest]
1310 #[case::wrong_type(serde_json::json!(17), "`time_zone` parameter must be a string")]
1311 #[case::unknown_zone(serde_json::json!("Not/A_Zone"), "invalid `time_zone` parameter \"Not/A_Zone\"")]
1312 fn test_calendar_zone_parameter_rejects_invalid_value(
1313 #[case] value: serde_json::Value,
1314 #[case] expected: &str,
1315 ) {
1316 let params = nautilus_core::Params::new([("time_zone".to_owned(), value)].into());
1317 assert_eq!(
1318 time_zone_from_params(Some(¶ms))
1319 .unwrap_err()
1320 .to_string(),
1321 expected
1322 );
1323 }
1324
1325 #[rstest]
1326 fn test_calendar_request_zone_checks_shared_instance() {
1327 let clock = Rc::new(RefCell::new(VirtualClock::new()));
1328 let cache = Rc::new(RefCell::new(Cache::default()));
1329 cache
1330 .borrow_mut()
1331 .add_instrument(InstrumentAny::CurrencyPair(audusd_sim()))
1332 .unwrap();
1333 let mut engine = DataEngine::new(clock, cache, None);
1334 let bar_type = BarType::from("AUD/USD.SIM-1-MONTH-LAST-INTERNAL");
1335 engine
1336 .create_bar_aggregator_for_key(bar_type, None, None, None)
1337 .unwrap();
1338 let params: nautilus_core::Params = serde_json::from_value(serde_json::json!({
1339 "bar_types": [bar_type.to_string()], "update_subscriptions": true,
1340 "time_zone": "America/New_York"
1341 }))
1342 .unwrap();
1343 let state = request_bar_aggregation_from_params(Some(¶ms))
1344 .unwrap()
1345 .unwrap();
1346 let id = UUID4::new();
1347 let error = engine
1348 .prepare_request_bar_aggregators_from_state(id, &state)
1349 .unwrap_err();
1350 let mut params = IndexMap::clone(¶ms);
1351 params.insert("update_subscriptions".to_owned(), serde_json::json!(false));
1352 let params = nautilus_core::Params::new(params);
1353 let state = request_bar_aggregation_from_params(Some(¶ms))
1354 .unwrap()
1355 .unwrap();
1356 engine
1357 .prepare_request_bar_aggregators_from_state(id, &state)
1358 .unwrap();
1359 let instance = engine
1360 .bar_aggregators
1361 .get(&bar_aggregator_key(bar_type, Some(id)))
1362 .unwrap()
1363 .borrow();
1364 let request_zone = instance
1365 .as_any()
1366 .downcast_ref::<crate::aggregation::TimeBarAggregator>()
1367 .unwrap()
1368 .time_zone()
1369 .clone();
1370 let live = engine
1371 .bar_aggregators
1372 .get(&bar_aggregator_key(bar_type, None))
1373 .unwrap()
1374 .borrow();
1375 let live_zone = live
1376 .as_any()
1377 .downcast_ref::<crate::aggregation::TimeBarAggregator>()
1378 .unwrap()
1379 .time_zone()
1380 .clone();
1381 assert_eq!(
1382 error.to_string(),
1383 format!("Cannot reuse bar aggregator {bar_type} with a different time zone")
1384 );
1385 assert_eq!(request_zone, get_timezone("America/New_York").unwrap());
1386 assert_eq!(live_zone, jiff::tz::TimeZone::UTC);
1387 assert_eq!(engine.bar_aggregators.len(), 2);
1388 }
1389
1390 #[rstest]
1391 #[case::source_first(&["1-MINUTE-MID-INTERNAL", "5-MINUTE-MID-INTERNAL@1-MINUTE-INTERNAL"])]
1392 #[case::composite_first(&["5-MINUTE-MID-INTERNAL@1-MINUTE-INTERNAL", "1-MINUTE-MID-INTERNAL"])]
1393 fn test_sources_before_dependents_orders_two_level_chain(#[case] raw: &[&str]) {
1394 let bar_types = to_bar_types(raw);
1395
1396 let ordered = sources_before_dependents(&bar_types);
1397
1398 assert_eq!(
1399 ordered,
1400 to_bar_types(&[
1401 "1-MINUTE-MID-INTERNAL",
1402 "5-MINUTE-MID-INTERNAL@1-MINUTE-INTERNAL",
1403 ])
1404 );
1405 }
1406
1407 #[rstest]
1408 fn test_sources_before_dependents_orders_three_level_chain_and_keeps_unrelated_order() {
1409 let bar_types = to_bar_types(&[
1410 "15-MINUTE-MID-INTERNAL@5-MINUTE-INTERNAL",
1411 "100-TICK-LAST-INTERNAL",
1412 "5-MINUTE-MID-INTERNAL@1-MINUTE-INTERNAL",
1413 "1-MINUTE-MID-INTERNAL",
1414 "50-TICK-LAST-INTERNAL",
1415 ]);
1416
1417 let ordered = sources_before_dependents(&bar_types);
1418
1419 assert_eq!(
1420 ordered,
1421 to_bar_types(&[
1422 "100-TICK-LAST-INTERNAL",
1423 "1-MINUTE-MID-INTERNAL",
1424 "50-TICK-LAST-INTERNAL",
1425 "5-MINUTE-MID-INTERNAL@1-MINUTE-INTERNAL",
1426 "15-MINUTE-MID-INTERNAL@5-MINUTE-INTERNAL",
1427 ])
1428 );
1429 }
1430
1431 fn to_bar_types(raw: &[&str]) -> Vec<BarType> {
1432 raw.iter()
1433 .map(|bar_type| BarType::from(format!("AUD/USD.SIM-{bar_type}").as_str()))
1434 .collect()
1435 }
1436}