1use std::{any::Any, cell::Cell};
19
20use anyhow::Context;
21use nautilus_model::{
22 data::{CustomData, Data, deserialize_custom_from_json},
23 instruments::Instrument,
24 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
25};
26use serde::de::DeserializeOwned;
27use ustr::Ustr;
28
29pub(crate) mod codec;
30
31use self::codec::{PayloadCodecError, deserialize_payload};
32use super::{
33 BusMessage, BusMessageSource, BusPayloadType, HAS_EXTERNAL_DATA_INGRESS, HAS_EXTERNAL_EGRESS,
34 MessagingSwitchboard, SUPPRESS_EXTERNAL_DEPTH, SuppressExternalGuard,
35 api::{
36 publish_any, publish_bar, publish_deltas, publish_depth, publish_funding_rate,
37 publish_index_price, publish_instrument, publish_mark_price, publish_option_greeks,
38 publish_order_event, publish_portfolio_snapshot, publish_position_event, publish_quote,
39 publish_trade, send_account_state, send_data,
40 },
41 get_message_bus,
42 mstr::{MStr, Topic},
43 switchboard::{
44 get_bars_topic, get_book_deltas_topic, get_book_depth_topic, get_funding_rate_topic,
45 get_index_price_topic, get_instrument_topic, get_mark_price_topic, get_option_greeks_topic,
46 get_quotes_topic, get_trades_topic,
47 },
48};
49use crate::{
50 enums::SerializationEncoding,
51 messages::{
52 data::{
53 DataResponse, DataResponseEnvelope, RequestCommand, SubscribeCommand,
54 UnsubscribeCommand,
55 },
56 execution::{GenerateExecutionMassStatus, TradingCommand},
57 },
58};
59
60#[inline(always)]
61pub(super) fn forward_to_external_egress<T>(
62 topic: MStr<Topic>,
63 payload_type: BusPayloadType,
64 message: &T,
65) where
66 T: serde::Serialize + Any,
67{
68 if !HAS_EXTERNAL_EGRESS.with(Cell::get) {
69 return;
70 }
71
72 forward_external_message(topic, payload_type, message);
73}
74
75#[inline]
76pub(super) fn forward_any_to_external_egress(topic: MStr<Topic>, message: &dyn Any) {
77 if !HAS_EXTERNAL_EGRESS.with(Cell::get) {
78 return;
79 }
80
81 if let Some(custom) = message.downcast_ref::<CustomData>() {
82 forward_external_message(
83 topic,
84 BusPayloadType::Custom(Ustr::from(custom.data.type_name())),
85 custom,
86 );
87 return;
88 }
89
90 if forward_downcast::<SubscribeCommand>(topic, BusPayloadType::SubscribeCommand, message)
91 || forward_downcast::<UnsubscribeCommand>(
92 topic,
93 BusPayloadType::UnsubscribeCommand,
94 message,
95 )
96 || forward_downcast::<DataResponse>(topic, BusPayloadType::DataResponse, message)
97 || forward_downcast::<DataResponseEnvelope>(
98 topic,
99 BusPayloadType::DataResponseEnvelope,
100 message,
101 )
102 || forward_downcast::<RequestCommand>(topic, BusPayloadType::RequestCommand, message)
103 || forward_downcast::<TradingCommand>(topic, BusPayloadType::TradingCommand, message)
104 || forward_downcast::<GenerateExecutionMassStatus>(
105 topic,
106 BusPayloadType::GenerateExecutionMassStatus,
107 message,
108 )
109 || forward_downcast::<OrderStatusReport>(topic, BusPayloadType::OrderStatusReport, message)
110 || forward_downcast::<FillReport>(topic, BusPayloadType::FillReport, message)
111 || forward_downcast::<PositionStatusReport>(
112 topic,
113 BusPayloadType::PositionStatusReport,
114 message,
115 )
116 {
117 return;
118 }
119
120 forward_downcast::<ExecutionMassStatus>(topic, BusPayloadType::ExecutionMassStatus, message);
121}
122
123fn forward_downcast<T>(topic: MStr<Topic>, payload_type: BusPayloadType, message: &dyn Any) -> bool
124where
125 T: serde::Serialize + Any,
126{
127 let Some(message) = message.downcast_ref::<T>() else {
128 return false;
129 };
130
131 forward_external_message(topic, payload_type, message);
132 true
133}
134
135#[cold]
136#[inline(never)]
137fn forward_external_message<T>(topic: MStr<Topic>, payload_type: BusPayloadType, message: &T)
138where
139 T: serde::Serialize + Any,
140{
141 if SUPPRESS_EXTERNAL_DEPTH.with(Cell::get) > 0 {
142 return;
143 }
144
145 let (external_egress, encoding, timestamps_as_iso8601) = {
146 let bus_rc = get_message_bus();
147 let bus = bus_rc.borrow();
148
149 let Some(external_egress) = bus.external_egress() else {
150 return;
151 };
152
153 if payload_type.is_typed_message() && !bus.has_external_streams() {
154 return;
155 }
156
157 if bus.types_filter().contains(&payload_type) {
158 return;
159 }
160
161 (
162 external_egress,
163 bus.encoding_for(payload_type),
164 bus.timestamps_as_iso8601(),
165 )
166 };
167
168 if external_egress.borrow().is_closed() {
169 return;
170 }
171
172 let payload =
173 match codec::serialize_payload(encoding, payload_type, message, timestamps_as_iso8601) {
174 Ok(payload) => payload,
175 Err(PayloadCodecError::Dropped(e)) => {
176 log::debug!("{e}");
177 return;
178 }
179 Err(PayloadCodecError::Failed(e)) => {
180 log::error!("{e}");
181 return;
182 }
183 };
184
185 let external_egress = external_egress.borrow();
186 if external_egress.is_closed() {
187 return;
188 }
189
190 external_egress.publish(BusMessage::new(*topic, payload_type, payload, encoding));
192}
193
194thread_local! {
195 static EXTERNAL_SOURCE: Cell<Option<BusMessageSource>> = const { Cell::new(None) };
196}
197
198#[must_use]
200pub fn current_external_source() -> Option<BusMessageSource> {
201 EXTERNAL_SOURCE.get()
202}
203
204struct ExternalSourceGuard(Option<BusMessageSource>);
205
206impl ExternalSourceGuard {
207 fn new(source: Option<BusMessageSource>) -> Self {
208 Self(EXTERNAL_SOURCE.replace(source))
209 }
210}
211
212impl Drop for ExternalSourceGuard {
213 fn drop(&mut self) {
214 EXTERNAL_SOURCE.set(self.0);
215 }
216}
217
218pub fn republish_pending_external_response(message: &BusMessage) -> anyhow::Result<bool> {
231 if !matches!(
232 message.payload_type,
233 BusPayloadType::DataResponse | BusPayloadType::DataResponseEnvelope
234 ) || !is_registered_streaming_type(message)
235 {
236 return Ok(false);
237 }
238
239 match message.payload_type {
240 BusPayloadType::DataResponseEnvelope => republish_pending_response::<DataResponseEnvelope>(
241 message,
242 DataResponseEnvelope::correlation_id,
243 ),
244 _ => republish_pending_response::<DataResponse>(message, DataResponse::correlation_id),
245 }
246}
247
248fn republish_pending_response<T: serde::de::DeserializeOwned + 'static>(
249 message: &BusMessage,
250 correlation_id: fn(&T) -> &nautilus_core::UUID4,
251) -> anyhow::Result<bool> {
252 let Some(response) =
253 codec::deserialize_payload::<T>(message.payload_type, message.encoding, &message.payload)?
254 else {
255 return Ok(false);
256 };
257
258 if get_message_bus()
259 .borrow()
260 .get_response_handler(correlation_id(&response))
261 .is_none()
262 {
263 return Ok(false);
264 }
265
266 let topic =
267 MStr::<Topic>::topic_from_ustr(message.topic).context("invalid external message topic")?;
268 let _guard = SuppressExternalGuard::new();
269 let _source = ExternalSourceGuard::new(message.source);
270 publish_any(topic, &response);
271 Ok(true)
272}
273
274pub fn republish_external_message(message: &BusMessage) -> anyhow::Result<()> {
296 let topic =
297 MStr::<Topic>::topic_from_ustr(message.topic).context("invalid external message topic")?;
298
299 if !is_registered_streaming_type(message) {
300 return Ok(());
301 }
302
303 let _guard = SuppressExternalGuard::new();
304 let _source = ExternalSourceGuard::new(message.source);
305
306 match message.payload_type {
307 BusPayloadType::Custom(_) => {
308 handle_custom_data(
309 topic,
310 message.payload_type,
311 message.encoding,
312 &message.payload,
313 )?;
314 }
315 BusPayloadType::Instrument => {
316 handle_market_data(
317 topic,
318 message.encoding,
319 &message.payload,
320 |encoding, payload| {
321 deserialize_payload(BusPayloadType::Instrument, encoding, payload)
322 },
323 publish_instrument,
324 |instrument| get_instrument_topic(instrument.id()),
325 )?;
326 }
327 BusPayloadType::OrderBookDeltas => handle_market_data(
328 topic,
329 message.encoding,
330 &message.payload,
331 codec::deserialize_order_book_deltas,
332 publish_deltas,
333 |deltas| get_book_deltas_topic(deltas.instrument_id),
334 )?,
335 BusPayloadType::OrderBookDepth => handle_market_data(
336 topic,
337 message.encoding,
338 &message.payload,
339 codec::deserialize_order_book_depth,
340 publish_depth,
341 |depth| get_book_depth_topic(depth.instrument_id),
342 )?,
343 BusPayloadType::QuoteTick => handle_market_data(
344 topic,
345 message.encoding,
346 &message.payload,
347 codec::deserialize_quote,
348 publish_quote,
349 |quote| get_quotes_topic(quote.instrument_id),
350 )?,
351 BusPayloadType::TradeTick => handle_market_data(
352 topic,
353 message.encoding,
354 &message.payload,
355 codec::deserialize_trade,
356 publish_trade,
357 |trade| get_trades_topic(trade.instrument_id),
358 )?,
359 BusPayloadType::Bar => handle_market_data(
360 topic,
361 message.encoding,
362 &message.payload,
363 codec::deserialize_bar,
364 publish_bar,
365 |bar| get_bars_topic(bar.bar_type),
366 )?,
367 BusPayloadType::MarkPriceUpdate => handle_market_data(
368 topic,
369 message.encoding,
370 &message.payload,
371 codec::deserialize_mark_price,
372 publish_mark_price,
373 |mark_price| get_mark_price_topic(mark_price.instrument_id),
374 )?,
375 BusPayloadType::IndexPriceUpdate => handle_market_data(
376 topic,
377 message.encoding,
378 &message.payload,
379 codec::deserialize_index_price,
380 publish_index_price,
381 |index_price| get_index_price_topic(index_price.instrument_id),
382 )?,
383 BusPayloadType::FundingRateUpdate => handle_market_data(
384 topic,
385 message.encoding,
386 &message.payload,
387 codec::deserialize_funding_rate,
388 publish_funding_rate,
389 |funding_rate| get_funding_rate_topic(funding_rate.instrument_id),
390 )?,
391 BusPayloadType::OptionGreeks => {
392 handle_market_data(
393 topic,
394 message.encoding,
395 &message.payload,
396 codec::deserialize_option_greeks,
397 publish_option_greeks,
398 |greeks| get_option_greeks_topic(greeks.instrument_id),
399 )?;
400 }
401 BusPayloadType::AccountState => {
402 handle_payload(
403 topic,
404 message.payload_type,
405 message.encoding,
406 &message.payload,
407 |_, state| {
408 send_account_state(MessagingSwitchboard::portfolio_update_account(), state);
409 },
410 )?;
411 }
412 BusPayloadType::OrderEvent => {
413 handle_payload(
414 topic,
415 message.payload_type,
416 message.encoding,
417 &message.payload,
418 publish_order_event,
419 )?;
420 }
421 BusPayloadType::PositionEvent => {
422 handle_payload(
423 topic,
424 message.payload_type,
425 message.encoding,
426 &message.payload,
427 publish_position_event,
428 )?;
429 }
430 BusPayloadType::PortfolioSnapshot => {
431 handle_payload(
432 topic,
433 message.payload_type,
434 message.encoding,
435 &message.payload,
436 publish_portfolio_snapshot,
437 )?;
438 }
439 BusPayloadType::SubscribeCommand => {
440 handle_payload_any::<SubscribeCommand>(topic, message)?;
441 }
442 BusPayloadType::UnsubscribeCommand => {
443 handle_payload_any::<UnsubscribeCommand>(topic, message)?;
444 }
445 BusPayloadType::DataResponse => {
446 handle_payload_any::<DataResponse>(topic, message)?;
447 }
448 BusPayloadType::DataResponseEnvelope => {
449 handle_payload_any::<DataResponseEnvelope>(topic, message)?;
450 }
451 BusPayloadType::RequestCommand => {
452 handle_payload_any::<RequestCommand>(topic, message)?;
453 }
454 BusPayloadType::TradingCommand => {
455 handle_payload_any::<TradingCommand>(topic, message)?;
456 }
457 BusPayloadType::GenerateExecutionMassStatus => {
458 handle_payload_any::<GenerateExecutionMassStatus>(topic, message)?;
459 }
460 BusPayloadType::OrderStatusReport => {
461 handle_payload_any::<OrderStatusReport>(topic, message)?;
462 }
463 BusPayloadType::FillReport => {
464 handle_payload_any::<FillReport>(topic, message)?;
465 }
466 BusPayloadType::PositionStatusReport => {
467 handle_payload_any::<PositionStatusReport>(topic, message)?;
468 }
469 BusPayloadType::ExecutionMassStatus => {
470 handle_payload_any::<ExecutionMassStatus>(topic, message)?;
471 }
472 #[cfg(feature = "defi")]
473 BusPayloadType::Block
474 | BusPayloadType::Pool
475 | BusPayloadType::PoolLiquidityUpdate
476 | BusPayloadType::PoolFeeCollect
477 | BusPayloadType::PoolFlash => {
478 crate::defi::msgbus::republish_external_message(
479 topic,
480 message.payload_type,
481 message.encoding,
482 &message.payload,
483 )?;
484 }
485 }
486
487 Ok(())
488}
489
490pub fn process_external_typed_message(
512 message: &BusMessage,
513 processor: &mut dyn FnMut(&dyn Any, &serde_json::Value) -> anyhow::Result<()>,
514) -> anyhow::Result<()> {
515 let topic =
516 MStr::<Topic>::topic_from_ustr(message.topic).context("invalid external message topic")?;
517 let _guard = SuppressExternalGuard::new();
518 let _source = ExternalSourceGuard::new(message.source);
519
520 #[rustfmt::skip]
521 let result = match message.payload_type {
522 BusPayloadType::SubscribeCommand => process_typed_payload::<SubscribeCommand>(topic, message, processor),
523 BusPayloadType::UnsubscribeCommand => process_typed_payload::<UnsubscribeCommand>(topic, message, processor),
524 BusPayloadType::DataResponse => process_typed_payload::<DataResponse>(topic, message, processor),
525 BusPayloadType::DataResponseEnvelope => process_typed_payload::<DataResponseEnvelope>(topic, message, processor),
526 BusPayloadType::RequestCommand => process_typed_payload::<RequestCommand>(topic, message, processor),
527 BusPayloadType::TradingCommand => process_typed_payload::<TradingCommand>(topic, message, processor),
528 BusPayloadType::GenerateExecutionMassStatus => process_typed_payload::<GenerateExecutionMassStatus>(topic, message, processor),
529 BusPayloadType::OrderStatusReport => process_typed_payload::<OrderStatusReport>(topic, message, processor),
530 BusPayloadType::FillReport => process_typed_payload::<FillReport>(topic, message, processor),
531 BusPayloadType::PositionStatusReport => process_typed_payload::<PositionStatusReport>(topic, message, processor),
532 BusPayloadType::ExecutionMassStatus => process_typed_payload::<ExecutionMassStatus>(topic, message, processor),
533 _ => republish_external_message(message),
534 };
535 result
536}
537
538fn handle_payload_any<T>(topic: MStr<Topic>, message: &BusMessage) -> anyhow::Result<()>
539where
540 T: DeserializeOwned + Any,
541{
542 handle_payload(
543 topic,
544 message.payload_type,
545 message.encoding,
546 &message.payload,
547 |topic, value: &T| publish_any(topic, value),
548 )
549}
550
551fn process_typed_payload<T>(
552 topic: MStr<Topic>,
553 message: &BusMessage,
554 processor: &mut dyn FnMut(&dyn Any, &serde_json::Value) -> anyhow::Result<()>,
555) -> anyhow::Result<()>
556where
557 T: DeserializeOwned + serde::Serialize + Any,
558{
559 let Some(value) =
560 codec::deserialize_payload::<T>(message.payload_type, message.encoding, &message.payload)?
561 else {
562 return Ok(());
563 };
564
565 let mut mapping = serde_json::to_value(&value).with_context(|| {
566 format!(
567 "failed to map decoded {} stream payload",
568 message.payload_type
569 )
570 })?;
571
572 let mapping_object = mapping.as_object_mut().with_context(|| {
573 format!(
574 "decoded {} stream payload did not map to an object",
575 message.payload_type
576 )
577 })?;
578
579 anyhow::ensure!(
580 !mapping_object.contains_key("payload_type"),
581 "decoded {} stream payload contains reserved payload_type field",
582 message.payload_type
583 );
584 mapping_object.insert(
585 "payload_type".to_string(),
586 serde_json::Value::String(message.payload_type.as_str().to_string()),
587 );
588
589 processor(&value, &mapping)?;
590 if is_registered_streaming_type(message) {
591 publish_any(topic, &value);
592 }
593
594 Ok(())
595}
596
597fn is_registered_streaming_type(message: &BusMessage) -> bool {
598 if get_message_bus()
599 .borrow()
600 .is_streaming_type(message.payload_type)
601 {
602 return true;
603 }
604
605 let type_name = message.payload_type.as_str();
606 if type_name.is_empty() {
607 log::debug!(
608 "Skipping external message on topic '{}' with no payload type for inbound republishing",
609 message.topic
610 );
611 } else {
612 log::debug!(
613 "Skipping external {type_name} message on topic '{}' because the type is not registered for streaming",
614 message.topic
615 );
616 }
617
618 false
619}
620
621pub(crate) fn handle_payload<T>(
622 topic: MStr<Topic>,
623 payload_type: BusPayloadType,
624 encoding: SerializationEncoding,
625 payload: &[u8],
626 publish: impl FnOnce(MStr<Topic>, &T),
627) -> anyhow::Result<()>
628where
629 T: DeserializeOwned + 'static,
630{
631 let Some(value) = codec::deserialize_payload(payload_type, encoding, payload)? else {
632 return Ok(());
633 };
634
635 publish(topic, &value);
636 Ok(())
637}
638
639fn handle_market_data<T>(
640 topic: MStr<Topic>,
641 encoding: SerializationEncoding,
642 payload: &[u8],
643 deserialize: fn(SerializationEncoding, &[u8]) -> anyhow::Result<Option<T>>,
644 publish: impl FnOnce(MStr<Topic>, &T),
645 canonical_topic: fn(&T) -> MStr<Topic>,
646) -> anyhow::Result<()>
647where
648 T: Into<Data>,
649{
650 let Some(value) = deserialize(encoding, payload)? else {
651 return Ok(());
652 };
653
654 if !HAS_EXTERNAL_DATA_INGRESS.with(Cell::get) {
655 publish(topic, &value);
656 return Ok(());
657 }
658
659 let endpoint = MessagingSwitchboard::data_engine_process_data();
660 let process = topic == canonical_topic(&value)
661 && get_message_bus()
662 .borrow()
663 .endpoints_data
664 .is_registered(endpoint);
665
666 if process {
667 let data = value.into();
668 send_data(endpoint, data);
669 } else {
670 publish(topic, &value);
671 }
672
673 Ok(())
674}
675
676fn handle_custom_data(
677 topic: MStr<Topic>,
678 payload_type: BusPayloadType,
679 encoding: SerializationEncoding,
680 payload: &[u8],
681) -> anyhow::Result<()> {
682 let Some(custom) = decode_custom_data_payload(payload_type, encoding, payload)? else {
683 return Ok(());
684 };
685
686 publish_any(topic, &custom);
687 Ok(())
688}
689
690fn decode_custom_data_payload(
691 payload_type: BusPayloadType,
692 encoding: SerializationEncoding,
693 payload: &[u8],
694) -> anyhow::Result<Option<CustomData>> {
695 let BusPayloadType::Custom(custom_type_name) = payload_type else {
696 unreachable!("custom data payload decoding requires a custom payload type");
697 };
698
699 if custom_type_name.is_empty() {
700 log::warn!("External payload has no type for inbound republishing");
701 return Ok(None);
702 } else if !payload_type.supports(encoding) {
703 codec::warn_unsupported_inbound(payload_type, encoding);
704 return Ok(None);
705 }
706
707 match encoding {
708 SerializationEncoding::Json => {
709 let value =
710 codec::deserialize_json_payload::<serde_json::Value>(payload, "CustomData")?;
711 decode_custom_data_value(custom_type_name, &value)
712 .context("failed to decode JSON CustomData")
713 }
714 SerializationEncoding::MsgPack => {
715 let value =
716 codec::deserialize_msgpack_payload::<serde_json::Value>(payload, "CustomData")?;
717 decode_custom_data_value(custom_type_name, &value)
718 .context("failed to decode MsgPack CustomData")
719 }
720 SerializationEncoding::Sbe => {
721 codec::warn_unsupported_inbound(payload_type, encoding);
722 Ok(None)
723 }
724 }
725}
726
727fn decode_custom_data_value(
728 custom_type_name: Ustr,
729 value: &serde_json::Value,
730) -> anyhow::Result<Option<CustomData>> {
731 let Some(data) = deserialize_custom_from_json(custom_type_name.as_str(), value)? else {
732 log::warn!(
733 "External custom payload type '{custom_type_name}' is not registered for inbound republishing"
734 );
735 return Ok(None);
736 };
737
738 let envelope_type_name = value
739 .get("type")
740 .and_then(serde_json::Value::as_str)
741 .context("CustomData JSON missing 'type' field")?;
742 anyhow::ensure!(
743 envelope_type_name == custom_type_name.as_str(),
744 "CustomData envelope type '{envelope_type_name}' does not match message type '{custom_type_name}'"
745 );
746
747 let Data::Custom(custom) = data else {
748 anyhow::bail!("CustomData registry returned non-custom data");
749 };
750
751 Ok(Some(custom))
752}
753
754#[cfg(test)]
755mod tests {
756 use std::{cell::RefCell, rc::Rc};
757
758 use bytes::Bytes;
759 use rstest::rstest;
760 use serde::{Serialize, Serializer};
761
762 use super::*;
763 use crate::msgbus::{
764 MessageBus, TypedHandler, backing::MessageBusExternalEgress, set_message_bus,
765 };
766
767 #[rstest]
768 fn test_external_data_ingress_follows_active_bus_and_disposal() {
769 let first = Rc::new(RefCell::new(MessageBus::default()));
770 let second = Rc::new(RefCell::new(MessageBus::default()));
771 set_message_bus(Rc::clone(&first));
772 assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
773
774 first.borrow_mut().set_external_data_ingress(true);
775 assert!(HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
776
777 set_message_bus(Rc::clone(&second));
778 assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
779
780 first.borrow_mut().set_external_data_ingress(true);
781 assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
782
783 set_message_bus(Rc::clone(&first));
784 assert!(HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
785
786 second.borrow_mut().dispose();
787 assert!(HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
788
789 first.borrow_mut().close().unwrap();
790 assert!(HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
791
792 first.borrow_mut().dispose();
793 assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
794
795 set_message_bus(first);
796 assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
797 }
798
799 struct RecordingEgress {
800 messages: Rc<RefCell<Vec<BusMessage>>>,
801 closed: bool,
802 }
803
804 impl MessageBusExternalEgress for RecordingEgress {
805 fn is_closed(&self) -> bool {
806 self.closed
807 }
808
809 fn publish(&self, message: BusMessage) {
810 self.messages.borrow_mut().push(message);
811 }
812
813 fn close(&mut self) {
814 self.closed = true;
815 }
816 }
817
818 struct ReentrantPayload<F>(F);
819
820 impl<F: Fn()> Serialize for ReentrantPayload<F> {
821 fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
822 (self.0)();
823 serializer.serialize_u64(37)
824 }
825 }
826
827 #[rstest]
828 fn serialization_can_publish_before_original_egress_delivery() {
829 let messages = install_egress();
830
831 let payload = ReentrantPayload(|| {
832 forward_external_message("nested".into(), BusPayloadType::QuoteTick, &19_u64);
833 });
834
835 forward_external_message("outer".into(), BusPayloadType::QuoteTick, &payload);
836
837 let messages = messages.borrow();
838 assert_eq!(messages.len(), 2);
839 assert_message(&messages[0], "nested", b"19");
840 assert_message(&messages[1], "outer", b"37");
841 }
842
843 #[rstest]
844 #[case::close(false)]
845 #[case::dispose(true)]
846 fn serialization_can_close_original_egress(#[case] dispose: bool) {
847 let messages = install_egress();
848
849 let payload = ReentrantPayload(move || {
850 let bus = get_message_bus();
851 let mut bus = bus.borrow_mut();
852 if dispose {
853 bus.dispose();
854 } else {
855 bus.close().unwrap();
856 }
857 });
858
859 forward_external_message("outer".into(), BusPayloadType::QuoteTick, &payload);
860
861 assert!(messages.borrow().is_empty());
862 assert!(!get_message_bus().borrow().has_external_egress());
863 }
864
865 #[rstest]
866 fn serialization_keeps_original_destination_when_egress_is_replaced() {
867 let original = install_egress();
868 let replacement = Rc::new(RefCell::new(Vec::new()));
869 let replacement_clone = replacement.clone();
870
871 let payload = ReentrantPayload(move || {
872 get_message_bus().borrow_mut().set_external_egress(
873 Box::new(RecordingEgress {
874 messages: replacement_clone.clone(),
875 closed: false,
876 }),
877 SerializationEncoding::Json,
878 );
879 });
880
881 forward_external_message("outer".into(), BusPayloadType::QuoteTick, &payload);
882 forward_external_message("later".into(), BusPayloadType::QuoteTick, &23_u64);
883
884 let original = original.borrow();
885 let replacement = replacement.borrow();
886 assert_eq!(original.len(), 1);
887 assert_message(&original[0], "outer", b"37");
888 assert_eq!(replacement.len(), 1);
889 assert_message(&replacement[0], "later", b"23");
890 }
891
892 #[rstest]
893 #[case(SerializationEncoding::Json)]
894 #[case(SerializationEncoding::MsgPack)]
895 fn test_native_requests_reach_processors_without_stream_registration(
896 #[case] encoding: SerializationEncoding,
897 ) {
898 use nautilus_core::UUID4;
899 use nautilus_model::{data::DataType, identifiers::ClientId};
900
901 use crate::messages::data::RequestCustomData;
902
903 let publications = install_egress();
904 get_message_bus()
905 .borrow_mut()
906 .set_external_egress_config(
907 Box::new(RecordingEgress {
908 messages: publications.clone(),
909 closed: false,
910 }),
911 &crate::msgbus::MessageBusConfig {
912 encoding,
913 external_streams: Some(vec!["external.test".into()]),
914 ..Default::default()
915 },
916 )
917 .unwrap();
918
919 let request = RequestCommand::Data(RequestCustomData::new(
920 ClientId::from("FEED"),
921 DataType::new("DistinctRequest", None, None),
922 None,
923 None,
924 None,
925 UUID4::new(),
926 71.into(),
927 None,
928 ));
929
930 let payload_type = BusPayloadType::RequestCommand;
931 publish_any("commands.data.FEED".into(), &request);
932
933 let message = publications.borrow()[0].clone();
934 let mut received = None;
935 process_external_typed_message(&message, &mut |value, mapping| {
936 let decoded =
937 serde_json::to_value(value.downcast_ref::<RequestCommand>().unwrap()).unwrap();
938
939 received = Some((decoded, mapping.clone()));
940 Ok(())
941 })
942 .unwrap();
943
944 let expected_value = serde_json::to_value(&request).unwrap();
945
946 let mut expected_mapping = expected_value.clone();
947 expected_mapping
948 .as_object_mut()
949 .unwrap()
950 .insert("payload_type".into(), payload_type.as_str().into());
951 assert_eq!(publications.borrow().len(), 1);
952 assert_eq!(message.topic.as_str(), "commands.data.FEED");
953 assert_eq!(message.payload_type, payload_type);
954 assert_eq!(message.encoding, encoding);
955 assert_eq!(received, Some((expected_value, expected_mapping)));
956 }
957
958 #[rstest]
959 #[case(BusPayloadType::RequestCommand, "data request")]
960 #[case(BusPayloadType::DataResponse, "data response")]
961 fn test_native_messages_reject_unsupported_encoding_without_dispatch(
962 #[case] payload_type: BusPayloadType,
963 #[case] name: &str,
964 #[values(SerializationEncoding::Sbe)] encoding: SerializationEncoding,
965 ) {
966 set_message_bus(Rc::new(RefCell::new(MessageBus::default())));
967 get_message_bus()
968 .borrow_mut()
969 .add_streaming_type(payload_type);
970 let message = BusMessage::with_str_topic(
971 "external.test",
972 payload_type,
973 Bytes::from_static(b"unsupported"),
974 encoding,
975 );
976 let expected = format!("{name} encoding is not supported: {encoding}");
977 let mut calls = 0;
978 let error = process_external_typed_message(&message, &mut |_, _| {
979 calls += 1;
980 Ok(())
981 })
982 .unwrap_err();
983
984 assert_eq!(error.to_string(), expected);
985 assert_eq!(
986 republish_external_message(&message)
987 .unwrap_err()
988 .to_string(),
989 expected
990 );
991 assert_eq!(calls, 0);
992 }
993
994 #[rstest]
995 #[case(false)]
996 #[case(true)]
997 fn test_external_source_restored_after_nested_dispatch_and_processor_failure(
998 #[case] panic: bool,
999 ) {
1000 use std::panic::{AssertUnwindSafe, catch_unwind};
1001
1002 use nautilus_core::UUID4;
1003 use nautilus_model::{data::DataType, identifiers::ClientId};
1004
1005 use crate::{
1006 messages::data::RequestCustomData,
1007 msgbus::{ShareableMessageHandler, subscribe_any},
1008 };
1009
1010 set_message_bus(Rc::new(RefCell::new(MessageBus::default())));
1011 get_message_bus()
1012 .borrow_mut()
1013 .add_streaming_type(BusPayloadType::RequestCommand);
1014 let request = RequestCommand::Data(RequestCustomData::new(
1015 ClientId::from("FEED"),
1016 DataType::new("DistinctRequest", None, None),
1017 None,
1018 None,
1019 None,
1020 UUID4::new(),
1021 71.into(),
1022 None,
1023 ));
1024 let payload = codec::serialize_payload(
1025 SerializationEncoding::Json,
1026 BusPayloadType::RequestCommand,
1027 &request,
1028 false,
1029 )
1030 .unwrap();
1031 let mut message = BusMessage::with_str_topic(
1032 "external.outer",
1033 BusPayloadType::RequestCommand,
1034 payload,
1035 SerializationEncoding::Json,
1036 );
1037
1038 let source = BusMessageSource {
1039 publisher_id: "PUBLISHER-017".into(),
1040 instance_id: UUID4::new(),
1041 stream: "snapshot".into(),
1042 sequence: 37,
1043 };
1044
1045 message.source = Some(source);
1046 let mut nested = message.clone();
1047 nested.topic = "external.nested".into();
1048 nested.source = None;
1049 let received = Rc::new(RefCell::new(Vec::new()));
1050 let shared = received.clone();
1051 subscribe_any(
1052 nested.topic.as_str().into(),
1053 ShareableMessageHandler::from_typed::<RequestCommand, _>(move |_| {
1054 shared.borrow_mut().push(current_external_source());
1055 }),
1056 None,
1057 );
1058
1059 let result = catch_unwind(AssertUnwindSafe(|| {
1060 process_external_typed_message(&message, &mut |_, _| {
1061 assert_eq!(current_external_source(), Some(source));
1062 republish_external_message(&nested)?;
1063 assert_eq!(current_external_source(), Some(source));
1064 assert_eq!(*received.borrow(), vec![None]);
1065
1066 assert!(!panic, "processor panicked");
1067
1068 anyhow::bail!("processor rejected payload")
1069 })
1070 }));
1071
1072 if panic {
1073 assert_eq!(
1074 result.unwrap_err().downcast_ref::<&str>(),
1075 Some(&"processor panicked")
1076 );
1077 } else {
1078 assert_eq!(
1079 result.unwrap().unwrap_err().to_string(),
1080 "processor rejected payload"
1081 );
1082 }
1083
1084 assert_eq!(current_external_source(), None);
1085 }
1086
1087 #[rstest]
1088 #[case(false, false)]
1089 #[case(true, false)]
1090 #[case(false, true)]
1091 #[case(true, true)]
1092 fn test_native_response_processors_precede_streaming_filters(
1093 #[case] streaming: bool,
1094 #[case] rejected: bool,
1095 ) {
1096 use nautilus_core::UUID4;
1097 use nautilus_model::{data::DataType, identifiers::ClientId};
1098
1099 use crate::{
1100 messages::data::CustomDataResponse,
1101 msgbus::{self, ShareableMessageHandler},
1102 };
1103
1104 set_message_bus(Rc::new(RefCell::new(MessageBus::default())));
1105
1106 if streaming {
1107 get_message_bus()
1108 .borrow_mut()
1109 .add_streaming_type(BusPayloadType::DataResponse);
1110 }
1111
1112 let response = CustomDataResponse::new(
1113 UUID4::new(),
1114 ClientId::from("FEED"),
1115 None,
1116 DataType::new("DistinctReply", None, None),
1117 Bytes::from_static(b"native-bytes"),
1118 Some(67.into()),
1119 Some(71.into()),
1120 73.into(),
1121 None,
1122 );
1123 let publications = Rc::new(RefCell::new(Vec::new()));
1124 get_message_bus()
1125 .borrow_mut()
1126 .set_external_egress_config(
1127 Box::new(RecordingEgress {
1128 messages: publications.clone(),
1129 closed: false,
1130 }),
1131 &crate::msgbus::MessageBusConfig {
1132 encoding: SerializationEncoding::MsgPack,
1133 external_streams: Some(vec!["external.test".into()]),
1134 ..Default::default()
1135 },
1136 )
1137 .unwrap();
1138
1139 publish_any(
1140 "responses.data.FEED".into(),
1141 &DataResponse::Data(response.clone()),
1142 );
1143 let message = publications.borrow()[0].clone();
1144 let published = Rc::new(RefCell::new(0));
1145 let shared = published.clone();
1146 msgbus::subscribe_any(
1147 message.topic.as_str().into(),
1148 ShareableMessageHandler::from_typed::<DataResponse, _>(move |_| {
1149 *shared.borrow_mut() += 1;
1150 }),
1151 None,
1152 );
1153
1154 let mut calls = 0;
1155 let result = process_external_typed_message(&message, &mut |value, mapping| {
1156 calls += 1;
1157
1158 let DataResponse::Data(actual) = value.downcast_ref::<DataResponse>().unwrap() else {
1159 panic!("Expected native custom response");
1160 };
1161
1162 assert_eq!(actual.correlation_id, response.correlation_id);
1163 assert_eq!(
1164 actual.data.downcast_ref::<Bytes>(),
1165 response.data.downcast_ref::<Bytes>()
1166 );
1167 assert_eq!(
1168 mapping,
1169 &serde_json::json!({
1170 "payload_type": "DataResponse",
1171 "Data": {
1172 "correlation_id": response.correlation_id, "client_id": "FEED",
1173 "start": 67, "end": 71, "ts_init": 73, "params": null,
1174 "venue": null, "data_type": response.data_type,
1175 "data": {"Buffer": b"native-bytes".to_vec()},
1176 }
1177 })
1178 );
1179
1180 if rejected {
1181 anyhow::bail!("processor rejected reply");
1182 }
1183
1184 Ok(())
1185 });
1186
1187 assert_eq!(calls, 1);
1188 assert_eq!(publications.borrow().len(), 1);
1189 assert_eq!(message.topic.as_str(), "responses.data.FEED");
1190 assert_eq!(message.payload_type, BusPayloadType::DataResponse);
1191 assert_eq!(message.encoding, SerializationEncoding::MsgPack);
1192 assert_eq!(
1193 result.err().map(|e| e.to_string()),
1194 rejected.then(|| "processor rejected reply".to_string())
1195 );
1196 assert_eq!(*published.borrow(), usize::from(streaming && !rejected));
1197 }
1198
1199 #[rstest]
1200 fn test_startup_dispatches_only_registered_pending_data_responses() {
1201 use nautilus_core::UUID4;
1202 use nautilus_model::{data::DataType, identifiers::ClientId};
1203
1204 use crate::{
1205 messages::data::CustomDataResponse,
1206 msgbus::{self, ShareableMessageHandler},
1207 };
1208
1209 let bus = Rc::new(RefCell::new(MessageBus::default()));
1210 set_message_bus(bus.clone());
1211 bus.borrow_mut()
1212 .add_streaming_type(BusPayloadType::DataResponse);
1213 let id = UUID4::new();
1214 let response = DataResponse::Data(CustomDataResponse::new(
1215 id,
1216 ClientId::from("FEED"),
1217 None,
1218 DataType::new("DistinctReply", None, None),
1219 Bytes::from_static(b"native-bytes"),
1220 None,
1221 None,
1222 73.into(),
1223 None,
1224 ));
1225 let message = BusMessage::with_str_topic(
1226 "responses.data.FEED",
1227 BusPayloadType::DataResponse,
1228 codec::serialize_payload(
1229 SerializationEncoding::MsgPack,
1230 BusPayloadType::DataResponse,
1231 &response,
1232 false,
1233 )
1234 .unwrap(),
1235 SerializationEncoding::MsgPack,
1236 );
1237
1238 let topic_handler = ShareableMessageHandler::from_typed::<DataResponse, _>(|reply| {
1239 msgbus::send_response(reply.correlation_id(), reply);
1240 });
1241
1242 msgbus::subscribe_any(message.topic.as_str().into(), topic_handler, None);
1243 let received = Rc::new(RefCell::new(Vec::new()));
1244 let shared = received.clone();
1245 assert!(!republish_pending_external_response(&message).unwrap());
1246 msgbus::register_response_handler(
1247 &id,
1248 TypedHandler::from(move |envelope: &DataResponseEnvelope| {
1249 let Some(DataResponse::Data(reply)) = envelope.response() else {
1250 panic!("expected custom reply");
1251 };
1252
1253 shared.borrow_mut().push((
1254 reply.correlation_id,
1255 reply.data.downcast_ref::<Bytes>().unwrap().clone(),
1256 reply.ts_init,
1257 ));
1258 }),
1259 );
1260
1261 assert!(republish_pending_external_response(&message).unwrap());
1262 assert!(!republish_pending_external_response(&message).unwrap());
1263 assert_eq!(
1264 *received.borrow(),
1265 vec![(id, Bytes::from_static(b"native-bytes"), 73.into())]
1266 );
1267 }
1268
1269 #[rstest]
1270 #[case(SerializationEncoding::Json)]
1271 #[case(SerializationEncoding::MsgPack)]
1272 fn test_startup_dispatches_payload_free_completion(#[case] encoding: SerializationEncoding) {
1273 use crate::msgbus::{self, ShareableMessageHandler};
1274
1275 let bus = Rc::new(RefCell::new(MessageBus::default()));
1276 set_message_bus(bus.clone());
1277 bus.borrow_mut()
1278 .add_streaming_type(BusPayloadType::DataResponseEnvelope);
1279 let id = nautilus_core::UUID4::new();
1280 let envelope = DataResponseEnvelope::terminal(id, Some("Source unavailable".to_owned()));
1281 let payload = codec::serialize_payload(
1282 encoding,
1283 BusPayloadType::DataResponseEnvelope,
1284 &envelope,
1285 false,
1286 )
1287 .unwrap();
1288 let message = BusMessage::with_str_topic(
1289 "responses.data.FEED",
1290 BusPayloadType::DataResponseEnvelope,
1291 payload,
1292 encoding,
1293 );
1294 let received = Rc::new(RefCell::new(Vec::new()));
1295 let shared = received.clone();
1296 msgbus::subscribe_any(
1297 message.topic.as_str().into(),
1298 ShareableMessageHandler::from_typed::<DataResponseEnvelope, _>(
1299 msgbus::send_response_envelope,
1300 ),
1301 None,
1302 );
1303 msgbus::register_response_handler(
1304 &id,
1305 TypedHandler::from(move |response: &DataResponseEnvelope| {
1306 shared.borrow_mut().push(response.clone());
1307 }),
1308 );
1309
1310 assert!(republish_pending_external_response(&message).unwrap());
1311
1312 let received = received.borrow();
1313 assert_eq!(received.len(), 1);
1314 assert_eq!(received[0].correlation_id(), &id);
1315 assert!(received[0].completed());
1316 assert!(received[0].response().is_none());
1317 assert_eq!(received[0].sub_correlation_id(), None);
1318 assert_eq!(received[0].failure_reason(), Some("Source unavailable"));
1319 assert!(bus.borrow().get_response_handler(&id).is_none());
1320 assert!(!republish_pending_external_response(&message).unwrap());
1321 }
1322
1323 fn install_egress() -> Rc<RefCell<Vec<BusMessage>>> {
1324 let messages = Rc::new(RefCell::new(Vec::new()));
1325 let mut bus = MessageBus::default();
1326 bus.set_external_egress(
1327 Box::new(RecordingEgress {
1328 messages: messages.clone(),
1329 closed: false,
1330 }),
1331 SerializationEncoding::Json,
1332 );
1333
1334 set_message_bus(Rc::new(RefCell::new(bus)));
1335 messages
1336 }
1337
1338 fn assert_message(message: &BusMessage, topic: &str, payload: &[u8]) {
1339 assert_eq!(message.topic.as_str(), topic);
1340 assert_eq!(message.payload_type, BusPayloadType::QuoteTick);
1341 assert_eq!(message.encoding, SerializationEncoding::Json);
1342 assert_eq!(message.payload.as_ref(), payload);
1343 }
1344}