Skip to main content

nautilus_common/msgbus/external/
mod.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! External message bus stream encoding and republishing.
17
18use std::{any::Any, cell::Cell};
19
20use anyhow::Context;
21use nautilus_model::{
22    data::{CustomData, Data, deserialize_custom_from_json},
23    instruments::Instrument,
24    reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
25};
26use serde::de::DeserializeOwned;
27use ustr::Ustr;
28
29pub(crate) mod codec;
30
31use self::codec::{PayloadCodecError, deserialize_payload};
32use super::{
33    BusMessage, BusMessageSource, BusPayloadType, HAS_EXTERNAL_DATA_INGRESS, HAS_EXTERNAL_EGRESS,
34    MessagingSwitchboard, SUPPRESS_EXTERNAL_DEPTH, SuppressExternalGuard,
35    api::{
36        publish_any, publish_bar, publish_deltas, publish_depth, publish_funding_rate,
37        publish_index_price, publish_instrument, publish_mark_price, publish_option_greeks,
38        publish_order_event, publish_portfolio_snapshot, publish_position_event, publish_quote,
39        publish_trade, send_account_state, send_data,
40    },
41    get_message_bus,
42    mstr::{MStr, Topic},
43    switchboard::{
44        get_bars_topic, get_book_deltas_topic, get_book_depth_topic, get_funding_rate_topic,
45        get_index_price_topic, get_instrument_topic, get_mark_price_topic, get_option_greeks_topic,
46        get_quotes_topic, get_trades_topic,
47    },
48};
49use crate::{
50    enums::SerializationEncoding,
51    messages::{
52        data::{
53            DataResponse, DataResponseEnvelope, RequestCommand, SubscribeCommand,
54            UnsubscribeCommand,
55        },
56        execution::{GenerateExecutionMassStatus, TradingCommand},
57    },
58};
59
60#[inline(always)]
61pub(super) fn forward_to_external_egress<T>(
62    topic: MStr<Topic>,
63    payload_type: BusPayloadType,
64    message: &T,
65) where
66    T: serde::Serialize + Any,
67{
68    if !HAS_EXTERNAL_EGRESS.with(Cell::get) {
69        return;
70    }
71
72    forward_external_message(topic, payload_type, message);
73}
74
75#[inline]
76pub(super) fn forward_any_to_external_egress(topic: MStr<Topic>, message: &dyn Any) {
77    if !HAS_EXTERNAL_EGRESS.with(Cell::get) {
78        return;
79    }
80
81    if let Some(custom) = message.downcast_ref::<CustomData>() {
82        forward_external_message(
83            topic,
84            BusPayloadType::Custom(Ustr::from(custom.data.type_name())),
85            custom,
86        );
87        return;
88    }
89
90    if forward_downcast::<SubscribeCommand>(topic, BusPayloadType::SubscribeCommand, message)
91        || forward_downcast::<UnsubscribeCommand>(
92            topic,
93            BusPayloadType::UnsubscribeCommand,
94            message,
95        )
96        || forward_downcast::<DataResponse>(topic, BusPayloadType::DataResponse, message)
97        || forward_downcast::<DataResponseEnvelope>(
98            topic,
99            BusPayloadType::DataResponseEnvelope,
100            message,
101        )
102        || forward_downcast::<RequestCommand>(topic, BusPayloadType::RequestCommand, message)
103        || forward_downcast::<TradingCommand>(topic, BusPayloadType::TradingCommand, message)
104        || forward_downcast::<GenerateExecutionMassStatus>(
105            topic,
106            BusPayloadType::GenerateExecutionMassStatus,
107            message,
108        )
109        || forward_downcast::<OrderStatusReport>(topic, BusPayloadType::OrderStatusReport, message)
110        || forward_downcast::<FillReport>(topic, BusPayloadType::FillReport, message)
111        || forward_downcast::<PositionStatusReport>(
112            topic,
113            BusPayloadType::PositionStatusReport,
114            message,
115        )
116    {
117        return;
118    }
119
120    forward_downcast::<ExecutionMassStatus>(topic, BusPayloadType::ExecutionMassStatus, message);
121}
122
123fn forward_downcast<T>(topic: MStr<Topic>, payload_type: BusPayloadType, message: &dyn Any) -> bool
124where
125    T: serde::Serialize + Any,
126{
127    let Some(message) = message.downcast_ref::<T>() else {
128        return false;
129    };
130
131    forward_external_message(topic, payload_type, message);
132    true
133}
134
135#[cold]
136#[inline(never)]
137fn forward_external_message<T>(topic: MStr<Topic>, payload_type: BusPayloadType, message: &T)
138where
139    T: serde::Serialize + Any,
140{
141    if SUPPRESS_EXTERNAL_DEPTH.with(Cell::get) > 0 {
142        return;
143    }
144
145    let (external_egress, encoding, timestamps_as_iso8601) = {
146        let bus_rc = get_message_bus();
147        let bus = bus_rc.borrow();
148
149        let Some(external_egress) = bus.external_egress() else {
150            return;
151        };
152
153        if payload_type.is_typed_message() && !bus.has_external_streams() {
154            return;
155        }
156
157        if bus.types_filter().contains(&payload_type) {
158            return;
159        }
160
161        (
162            external_egress,
163            bus.encoding_for(payload_type),
164            bus.timestamps_as_iso8601(),
165        )
166    };
167
168    if external_egress.borrow().is_closed() {
169        return;
170    }
171
172    let payload =
173        match codec::serialize_payload(encoding, payload_type, message, timestamps_as_iso8601) {
174            Ok(payload) => payload,
175            Err(PayloadCodecError::Dropped(e)) => {
176                log::debug!("{e}");
177                return;
178            }
179            Err(PayloadCodecError::Failed(e)) => {
180                log::error!("{e}");
181                return;
182            }
183        };
184
185    let external_egress = external_egress.borrow();
186    if external_egress.is_closed() {
187        return;
188    }
189
190    // Serialization can close the original destination or replace it on the bus
191    external_egress.publish(BusMessage::new(*topic, payload_type, payload, encoding));
192}
193
194thread_local! {
195    static EXTERNAL_SOURCE: Cell<Option<BusMessageSource>> = const { Cell::new(None) };
196}
197
198/// Returns the source of the external message currently being dispatched on this thread.
199#[must_use]
200pub fn current_external_source() -> Option<BusMessageSource> {
201    EXTERNAL_SOURCE.get()
202}
203
204struct ExternalSourceGuard(Option<BusMessageSource>);
205
206impl ExternalSourceGuard {
207    fn new(source: Option<BusMessageSource>) -> Self {
208        Self(EXTERNAL_SOURCE.replace(source))
209    }
210}
211
212impl Drop for ExternalSourceGuard {
213    fn drop(&mut self) {
214        EXTERNAL_SOURCE.set(self.0);
215    }
216}
217
218/// Republishes a native data reply only when its correlation handler is pending.
219///
220/// Returns `false` without dispatching when the message is not a streaming-registered
221/// `DataResponse` or `DataResponseEnvelope`, or no handler is pending for its correlation ID.
222/// `LiveNode` uses this during data-client connection, before actors and strategies subscribe.
223/// The reply dispatches through application topic handlers, with external provenance and egress
224/// suppression active during dispatch. Startup replies bypass stream processors and can overtake
225/// buffered messages from the same ingress.
226///
227/// # Errors
228///
229/// Returns an error if a native data reply cannot be decoded or its topic is invalid.
230pub fn republish_pending_external_response(message: &BusMessage) -> anyhow::Result<bool> {
231    if !matches!(
232        message.payload_type,
233        BusPayloadType::DataResponse | BusPayloadType::DataResponseEnvelope
234    ) || !is_registered_streaming_type(message)
235    {
236        return Ok(false);
237    }
238
239    match message.payload_type {
240        BusPayloadType::DataResponseEnvelope => republish_pending_response::<DataResponseEnvelope>(
241            message,
242            DataResponseEnvelope::correlation_id,
243        ),
244        _ => republish_pending_response::<DataResponse>(message, DataResponse::correlation_id),
245    }
246}
247
248fn republish_pending_response<T: serde::de::DeserializeOwned + 'static>(
249    message: &BusMessage,
250    correlation_id: fn(&T) -> &nautilus_core::UUID4,
251) -> anyhow::Result<bool> {
252    let Some(response) =
253        codec::deserialize_payload::<T>(message.payload_type, message.encoding, &message.payload)?
254    else {
255        return Ok(false);
256    };
257
258    if get_message_bus()
259        .borrow()
260        .get_response_handler(correlation_id(&response))
261        .is_none()
262    {
263        return Ok(false);
264    }
265
266    let topic =
267        MStr::<Topic>::topic_from_ustr(message.topic).context("invalid external message topic")?;
268    let _guard = SuppressExternalGuard::new();
269    let _source = ExternalSourceGuard::new(message.source);
270    publish_any(topic, &response);
271    Ok(true)
272}
273
274/// Decodes an externally-received [`BusMessage`] and republishes it onto the internal bus.
275///
276/// The message `payload_type` header selects the concrete type and the message `encoding` selects
277/// the wire codec, so the message is decoded with the producer's encoding rather than the local
278/// configuration. Republishing runs under a [`SuppressExternalGuard`] so the message is not
279/// forwarded straight back out through external egress, which would create an echo loop on a node
280/// that has both external ingress and egress.
281///
282/// External data-client registrations in the cache enable built-in market-data processing when
283/// the `DataEngine` registers its message bus handlers. Engine processing applies only when the
284/// inbound topic exactly matches the topic the engine publishes for that payload. Custom data,
285/// DeFi data, other topics, and buses without a registered data endpoint retain direct publication.
286///
287/// Account states are sent to Portfolio's account endpoint for application before Portfolio
288/// publishes them on `events.account.{account_id}`. Portfolio must be registered to process them.
289///
290/// # Errors
291///
292/// Returns an error if the topic is invalid or a supported payload cannot be decoded. Unsupported
293/// encodings for native data requests and responses also return an error. Other unsupported
294/// type/encoding pairs are skipped with a warning.
295pub fn republish_external_message(message: &BusMessage) -> anyhow::Result<()> {
296    let topic =
297        MStr::<Topic>::topic_from_ustr(message.topic).context("invalid external message topic")?;
298
299    if !is_registered_streaming_type(message) {
300        return Ok(());
301    }
302
303    let _guard = SuppressExternalGuard::new();
304    let _source = ExternalSourceGuard::new(message.source);
305
306    match message.payload_type {
307        BusPayloadType::Custom(_) => {
308            handle_custom_data(
309                topic,
310                message.payload_type,
311                message.encoding,
312                &message.payload,
313            )?;
314        }
315        BusPayloadType::Instrument => {
316            handle_market_data(
317                topic,
318                message.encoding,
319                &message.payload,
320                |encoding, payload| {
321                    deserialize_payload(BusPayloadType::Instrument, encoding, payload)
322                },
323                publish_instrument,
324                |instrument| get_instrument_topic(instrument.id()),
325            )?;
326        }
327        BusPayloadType::OrderBookDeltas => handle_market_data(
328            topic,
329            message.encoding,
330            &message.payload,
331            codec::deserialize_order_book_deltas,
332            publish_deltas,
333            |deltas| get_book_deltas_topic(deltas.instrument_id),
334        )?,
335        BusPayloadType::OrderBookDepth => handle_market_data(
336            topic,
337            message.encoding,
338            &message.payload,
339            codec::deserialize_order_book_depth,
340            publish_depth,
341            |depth| get_book_depth_topic(depth.instrument_id),
342        )?,
343        BusPayloadType::QuoteTick => handle_market_data(
344            topic,
345            message.encoding,
346            &message.payload,
347            codec::deserialize_quote,
348            publish_quote,
349            |quote| get_quotes_topic(quote.instrument_id),
350        )?,
351        BusPayloadType::TradeTick => handle_market_data(
352            topic,
353            message.encoding,
354            &message.payload,
355            codec::deserialize_trade,
356            publish_trade,
357            |trade| get_trades_topic(trade.instrument_id),
358        )?,
359        BusPayloadType::Bar => handle_market_data(
360            topic,
361            message.encoding,
362            &message.payload,
363            codec::deserialize_bar,
364            publish_bar,
365            |bar| get_bars_topic(bar.bar_type),
366        )?,
367        BusPayloadType::MarkPriceUpdate => handle_market_data(
368            topic,
369            message.encoding,
370            &message.payload,
371            codec::deserialize_mark_price,
372            publish_mark_price,
373            |mark_price| get_mark_price_topic(mark_price.instrument_id),
374        )?,
375        BusPayloadType::IndexPriceUpdate => handle_market_data(
376            topic,
377            message.encoding,
378            &message.payload,
379            codec::deserialize_index_price,
380            publish_index_price,
381            |index_price| get_index_price_topic(index_price.instrument_id),
382        )?,
383        BusPayloadType::FundingRateUpdate => handle_market_data(
384            topic,
385            message.encoding,
386            &message.payload,
387            codec::deserialize_funding_rate,
388            publish_funding_rate,
389            |funding_rate| get_funding_rate_topic(funding_rate.instrument_id),
390        )?,
391        BusPayloadType::OptionGreeks => {
392            handle_market_data(
393                topic,
394                message.encoding,
395                &message.payload,
396                codec::deserialize_option_greeks,
397                publish_option_greeks,
398                |greeks| get_option_greeks_topic(greeks.instrument_id),
399            )?;
400        }
401        BusPayloadType::AccountState => {
402            handle_payload(
403                topic,
404                message.payload_type,
405                message.encoding,
406                &message.payload,
407                |_, state| {
408                    send_account_state(MessagingSwitchboard::portfolio_update_account(), state);
409                },
410            )?;
411        }
412        BusPayloadType::OrderEvent => {
413            handle_payload(
414                topic,
415                message.payload_type,
416                message.encoding,
417                &message.payload,
418                publish_order_event,
419            )?;
420        }
421        BusPayloadType::PositionEvent => {
422            handle_payload(
423                topic,
424                message.payload_type,
425                message.encoding,
426                &message.payload,
427                publish_position_event,
428            )?;
429        }
430        BusPayloadType::PortfolioSnapshot => {
431            handle_payload(
432                topic,
433                message.payload_type,
434                message.encoding,
435                &message.payload,
436                publish_portfolio_snapshot,
437            )?;
438        }
439        BusPayloadType::SubscribeCommand => {
440            handle_payload_any::<SubscribeCommand>(topic, message)?;
441        }
442        BusPayloadType::UnsubscribeCommand => {
443            handle_payload_any::<UnsubscribeCommand>(topic, message)?;
444        }
445        BusPayloadType::DataResponse => {
446            handle_payload_any::<DataResponse>(topic, message)?;
447        }
448        BusPayloadType::DataResponseEnvelope => {
449            handle_payload_any::<DataResponseEnvelope>(topic, message)?;
450        }
451        BusPayloadType::RequestCommand => {
452            handle_payload_any::<RequestCommand>(topic, message)?;
453        }
454        BusPayloadType::TradingCommand => {
455            handle_payload_any::<TradingCommand>(topic, message)?;
456        }
457        BusPayloadType::GenerateExecutionMassStatus => {
458            handle_payload_any::<GenerateExecutionMassStatus>(topic, message)?;
459        }
460        BusPayloadType::OrderStatusReport => {
461            handle_payload_any::<OrderStatusReport>(topic, message)?;
462        }
463        BusPayloadType::FillReport => {
464            handle_payload_any::<FillReport>(topic, message)?;
465        }
466        BusPayloadType::PositionStatusReport => {
467            handle_payload_any::<PositionStatusReport>(topic, message)?;
468        }
469        BusPayloadType::ExecutionMassStatus => {
470            handle_payload_any::<ExecutionMassStatus>(topic, message)?;
471        }
472        #[cfg(feature = "defi")]
473        BusPayloadType::Block
474        | BusPayloadType::Pool
475        | BusPayloadType::PoolLiquidityUpdate
476        | BusPayloadType::PoolFeeCollect
477        | BusPayloadType::PoolFlash => {
478            crate::defi::msgbus::republish_external_message(
479                topic,
480                message.payload_type,
481                message.encoding,
482                &message.payload,
483            )?;
484        }
485    }
486
487    Ok(())
488}
489
490/// Decodes a supported typed payload and passes it to `processor` before applying normal inbound
491/// republishing rules.
492///
493/// Supported typed JSON or MessagePack payloads reach `processor` regardless of internal streaming
494/// registration. Internal republishing still requires registration. Native requests and replies
495/// use the shared JSON/MessagePack codecs; unsupported encodings return an error.
496/// Other unsupported type/encoding pairs are skipped with a warning. The processor mapping
497/// includes a `payload_type` field containing the [`BusPayloadType`] name. External egress is
498/// suppressed for the duration of processing, including processor callbacks. Other payloads bypass
499/// the processor and follow normal inbound republishing. A processor error stops processing and
500/// skips internal republishing.
501///
502/// # Errors
503///
504/// Returns an error if:
505/// - The topic is invalid.
506/// - A native data request or response uses an unsupported encoding.
507/// - The typed payload cannot be decoded or mapped to an object.
508/// - The mapping already contains the reserved `payload_type` field.
509/// - The processor fails.
510/// - Normal inbound republishing fails.
511pub fn process_external_typed_message(
512    message: &BusMessage,
513    processor: &mut dyn FnMut(&dyn Any, &serde_json::Value) -> anyhow::Result<()>,
514) -> anyhow::Result<()> {
515    let topic =
516        MStr::<Topic>::topic_from_ustr(message.topic).context("invalid external message topic")?;
517    let _guard = SuppressExternalGuard::new();
518    let _source = ExternalSourceGuard::new(message.source);
519
520    #[rustfmt::skip]
521    let result = match message.payload_type {
522        BusPayloadType::SubscribeCommand => process_typed_payload::<SubscribeCommand>(topic, message, processor),
523        BusPayloadType::UnsubscribeCommand => process_typed_payload::<UnsubscribeCommand>(topic, message, processor),
524        BusPayloadType::DataResponse => process_typed_payload::<DataResponse>(topic, message, processor),
525        BusPayloadType::DataResponseEnvelope => process_typed_payload::<DataResponseEnvelope>(topic, message, processor),
526        BusPayloadType::RequestCommand => process_typed_payload::<RequestCommand>(topic, message, processor),
527        BusPayloadType::TradingCommand => process_typed_payload::<TradingCommand>(topic, message, processor),
528        BusPayloadType::GenerateExecutionMassStatus => process_typed_payload::<GenerateExecutionMassStatus>(topic, message, processor),
529        BusPayloadType::OrderStatusReport => process_typed_payload::<OrderStatusReport>(topic, message, processor),
530        BusPayloadType::FillReport => process_typed_payload::<FillReport>(topic, message, processor),
531        BusPayloadType::PositionStatusReport => process_typed_payload::<PositionStatusReport>(topic, message, processor),
532        BusPayloadType::ExecutionMassStatus => process_typed_payload::<ExecutionMassStatus>(topic, message, processor),
533        _ => republish_external_message(message),
534    };
535    result
536}
537
538fn handle_payload_any<T>(topic: MStr<Topic>, message: &BusMessage) -> anyhow::Result<()>
539where
540    T: DeserializeOwned + Any,
541{
542    handle_payload(
543        topic,
544        message.payload_type,
545        message.encoding,
546        &message.payload,
547        |topic, value: &T| publish_any(topic, value),
548    )
549}
550
551fn process_typed_payload<T>(
552    topic: MStr<Topic>,
553    message: &BusMessage,
554    processor: &mut dyn FnMut(&dyn Any, &serde_json::Value) -> anyhow::Result<()>,
555) -> anyhow::Result<()>
556where
557    T: DeserializeOwned + serde::Serialize + Any,
558{
559    let Some(value) =
560        codec::deserialize_payload::<T>(message.payload_type, message.encoding, &message.payload)?
561    else {
562        return Ok(());
563    };
564
565    let mut mapping = serde_json::to_value(&value).with_context(|| {
566        format!(
567            "failed to map decoded {} stream payload",
568            message.payload_type
569        )
570    })?;
571
572    let mapping_object = mapping.as_object_mut().with_context(|| {
573        format!(
574            "decoded {} stream payload did not map to an object",
575            message.payload_type
576        )
577    })?;
578
579    anyhow::ensure!(
580        !mapping_object.contains_key("payload_type"),
581        "decoded {} stream payload contains reserved payload_type field",
582        message.payload_type
583    );
584    mapping_object.insert(
585        "payload_type".to_string(),
586        serde_json::Value::String(message.payload_type.as_str().to_string()),
587    );
588
589    processor(&value, &mapping)?;
590    if is_registered_streaming_type(message) {
591        publish_any(topic, &value);
592    }
593
594    Ok(())
595}
596
597fn is_registered_streaming_type(message: &BusMessage) -> bool {
598    if get_message_bus()
599        .borrow()
600        .is_streaming_type(message.payload_type)
601    {
602        return true;
603    }
604
605    let type_name = message.payload_type.as_str();
606    if type_name.is_empty() {
607        log::debug!(
608            "Skipping external message on topic '{}' with no payload type for inbound republishing",
609            message.topic
610        );
611    } else {
612        log::debug!(
613            "Skipping external {type_name} message on topic '{}' because the type is not registered for streaming",
614            message.topic
615        );
616    }
617
618    false
619}
620
621pub(crate) fn handle_payload<T>(
622    topic: MStr<Topic>,
623    payload_type: BusPayloadType,
624    encoding: SerializationEncoding,
625    payload: &[u8],
626    publish: impl FnOnce(MStr<Topic>, &T),
627) -> anyhow::Result<()>
628where
629    T: DeserializeOwned + 'static,
630{
631    let Some(value) = codec::deserialize_payload(payload_type, encoding, payload)? else {
632        return Ok(());
633    };
634
635    publish(topic, &value);
636    Ok(())
637}
638
639fn handle_market_data<T>(
640    topic: MStr<Topic>,
641    encoding: SerializationEncoding,
642    payload: &[u8],
643    deserialize: fn(SerializationEncoding, &[u8]) -> anyhow::Result<Option<T>>,
644    publish: impl FnOnce(MStr<Topic>, &T),
645    canonical_topic: fn(&T) -> MStr<Topic>,
646) -> anyhow::Result<()>
647where
648    T: Into<Data>,
649{
650    let Some(value) = deserialize(encoding, payload)? else {
651        return Ok(());
652    };
653
654    if !HAS_EXTERNAL_DATA_INGRESS.with(Cell::get) {
655        publish(topic, &value);
656        return Ok(());
657    }
658
659    let endpoint = MessagingSwitchboard::data_engine_process_data();
660    let process = topic == canonical_topic(&value)
661        && get_message_bus()
662            .borrow()
663            .endpoints_data
664            .is_registered(endpoint);
665
666    if process {
667        let data = value.into();
668        send_data(endpoint, data);
669    } else {
670        publish(topic, &value);
671    }
672
673    Ok(())
674}
675
676fn handle_custom_data(
677    topic: MStr<Topic>,
678    payload_type: BusPayloadType,
679    encoding: SerializationEncoding,
680    payload: &[u8],
681) -> anyhow::Result<()> {
682    let Some(custom) = decode_custom_data_payload(payload_type, encoding, payload)? else {
683        return Ok(());
684    };
685
686    publish_any(topic, &custom);
687    Ok(())
688}
689
690fn decode_custom_data_payload(
691    payload_type: BusPayloadType,
692    encoding: SerializationEncoding,
693    payload: &[u8],
694) -> anyhow::Result<Option<CustomData>> {
695    let BusPayloadType::Custom(custom_type_name) = payload_type else {
696        unreachable!("custom data payload decoding requires a custom payload type");
697    };
698
699    if custom_type_name.is_empty() {
700        log::warn!("External payload has no type for inbound republishing");
701        return Ok(None);
702    } else if !payload_type.supports(encoding) {
703        codec::warn_unsupported_inbound(payload_type, encoding);
704        return Ok(None);
705    }
706
707    match encoding {
708        SerializationEncoding::Json => {
709            let value =
710                codec::deserialize_json_payload::<serde_json::Value>(payload, "CustomData")?;
711            decode_custom_data_value(custom_type_name, &value)
712                .context("failed to decode JSON CustomData")
713        }
714        SerializationEncoding::MsgPack => {
715            let value =
716                codec::deserialize_msgpack_payload::<serde_json::Value>(payload, "CustomData")?;
717            decode_custom_data_value(custom_type_name, &value)
718                .context("failed to decode MsgPack CustomData")
719        }
720        SerializationEncoding::Sbe => {
721            codec::warn_unsupported_inbound(payload_type, encoding);
722            Ok(None)
723        }
724    }
725}
726
727fn decode_custom_data_value(
728    custom_type_name: Ustr,
729    value: &serde_json::Value,
730) -> anyhow::Result<Option<CustomData>> {
731    let Some(data) = deserialize_custom_from_json(custom_type_name.as_str(), value)? else {
732        log::warn!(
733            "External custom payload type '{custom_type_name}' is not registered for inbound republishing"
734        );
735        return Ok(None);
736    };
737
738    let envelope_type_name = value
739        .get("type")
740        .and_then(serde_json::Value::as_str)
741        .context("CustomData JSON missing 'type' field")?;
742    anyhow::ensure!(
743        envelope_type_name == custom_type_name.as_str(),
744        "CustomData envelope type '{envelope_type_name}' does not match message type '{custom_type_name}'"
745    );
746
747    let Data::Custom(custom) = data else {
748        anyhow::bail!("CustomData registry returned non-custom data");
749    };
750
751    Ok(Some(custom))
752}
753
754#[cfg(test)]
755mod tests {
756    use std::{cell::RefCell, rc::Rc};
757
758    use bytes::Bytes;
759    use rstest::rstest;
760    use serde::{Serialize, Serializer};
761
762    use super::*;
763    use crate::msgbus::{
764        MessageBus, TypedHandler, backing::MessageBusExternalEgress, set_message_bus,
765    };
766
767    #[rstest]
768    fn test_external_data_ingress_follows_active_bus_and_disposal() {
769        let first = Rc::new(RefCell::new(MessageBus::default()));
770        let second = Rc::new(RefCell::new(MessageBus::default()));
771        set_message_bus(Rc::clone(&first));
772        assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
773
774        first.borrow_mut().set_external_data_ingress(true);
775        assert!(HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
776
777        set_message_bus(Rc::clone(&second));
778        assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
779
780        first.borrow_mut().set_external_data_ingress(true);
781        assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
782
783        set_message_bus(Rc::clone(&first));
784        assert!(HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
785
786        second.borrow_mut().dispose();
787        assert!(HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
788
789        first.borrow_mut().close().unwrap();
790        assert!(HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
791
792        first.borrow_mut().dispose();
793        assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
794
795        set_message_bus(first);
796        assert!(!HAS_EXTERNAL_DATA_INGRESS.with(Cell::get));
797    }
798
799    struct RecordingEgress {
800        messages: Rc<RefCell<Vec<BusMessage>>>,
801        closed: bool,
802    }
803
804    impl MessageBusExternalEgress for RecordingEgress {
805        fn is_closed(&self) -> bool {
806            self.closed
807        }
808
809        fn publish(&self, message: BusMessage) {
810            self.messages.borrow_mut().push(message);
811        }
812
813        fn close(&mut self) {
814            self.closed = true;
815        }
816    }
817
818    struct ReentrantPayload<F>(F);
819
820    impl<F: Fn()> Serialize for ReentrantPayload<F> {
821        fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
822            (self.0)();
823            serializer.serialize_u64(37)
824        }
825    }
826
827    #[rstest]
828    fn serialization_can_publish_before_original_egress_delivery() {
829        let messages = install_egress();
830
831        let payload = ReentrantPayload(|| {
832            forward_external_message("nested".into(), BusPayloadType::QuoteTick, &19_u64);
833        });
834
835        forward_external_message("outer".into(), BusPayloadType::QuoteTick, &payload);
836
837        let messages = messages.borrow();
838        assert_eq!(messages.len(), 2);
839        assert_message(&messages[0], "nested", b"19");
840        assert_message(&messages[1], "outer", b"37");
841    }
842
843    #[rstest]
844    #[case::close(false)]
845    #[case::dispose(true)]
846    fn serialization_can_close_original_egress(#[case] dispose: bool) {
847        let messages = install_egress();
848
849        let payload = ReentrantPayload(move || {
850            let bus = get_message_bus();
851            let mut bus = bus.borrow_mut();
852            if dispose {
853                bus.dispose();
854            } else {
855                bus.close().unwrap();
856            }
857        });
858
859        forward_external_message("outer".into(), BusPayloadType::QuoteTick, &payload);
860
861        assert!(messages.borrow().is_empty());
862        assert!(!get_message_bus().borrow().has_external_egress());
863    }
864
865    #[rstest]
866    fn serialization_keeps_original_destination_when_egress_is_replaced() {
867        let original = install_egress();
868        let replacement = Rc::new(RefCell::new(Vec::new()));
869        let replacement_clone = replacement.clone();
870
871        let payload = ReentrantPayload(move || {
872            get_message_bus().borrow_mut().set_external_egress(
873                Box::new(RecordingEgress {
874                    messages: replacement_clone.clone(),
875                    closed: false,
876                }),
877                SerializationEncoding::Json,
878            );
879        });
880
881        forward_external_message("outer".into(), BusPayloadType::QuoteTick, &payload);
882        forward_external_message("later".into(), BusPayloadType::QuoteTick, &23_u64);
883
884        let original = original.borrow();
885        let replacement = replacement.borrow();
886        assert_eq!(original.len(), 1);
887        assert_message(&original[0], "outer", b"37");
888        assert_eq!(replacement.len(), 1);
889        assert_message(&replacement[0], "later", b"23");
890    }
891
892    #[rstest]
893    #[case(SerializationEncoding::Json)]
894    #[case(SerializationEncoding::MsgPack)]
895    fn test_native_requests_reach_processors_without_stream_registration(
896        #[case] encoding: SerializationEncoding,
897    ) {
898        use nautilus_core::UUID4;
899        use nautilus_model::{data::DataType, identifiers::ClientId};
900
901        use crate::messages::data::RequestCustomData;
902
903        let publications = install_egress();
904        get_message_bus()
905            .borrow_mut()
906            .set_external_egress_config(
907                Box::new(RecordingEgress {
908                    messages: publications.clone(),
909                    closed: false,
910                }),
911                &crate::msgbus::MessageBusConfig {
912                    encoding,
913                    external_streams: Some(vec!["external.test".into()]),
914                    ..Default::default()
915                },
916            )
917            .unwrap();
918
919        let request = RequestCommand::Data(RequestCustomData::new(
920            ClientId::from("FEED"),
921            DataType::new("DistinctRequest", None, None),
922            None,
923            None,
924            None,
925            UUID4::new(),
926            71.into(),
927            None,
928        ));
929
930        let payload_type = BusPayloadType::RequestCommand;
931        publish_any("commands.data.FEED".into(), &request);
932
933        let message = publications.borrow()[0].clone();
934        let mut received = None;
935        process_external_typed_message(&message, &mut |value, mapping| {
936            let decoded =
937                serde_json::to_value(value.downcast_ref::<RequestCommand>().unwrap()).unwrap();
938
939            received = Some((decoded, mapping.clone()));
940            Ok(())
941        })
942        .unwrap();
943
944        let expected_value = serde_json::to_value(&request).unwrap();
945
946        let mut expected_mapping = expected_value.clone();
947        expected_mapping
948            .as_object_mut()
949            .unwrap()
950            .insert("payload_type".into(), payload_type.as_str().into());
951        assert_eq!(publications.borrow().len(), 1);
952        assert_eq!(message.topic.as_str(), "commands.data.FEED");
953        assert_eq!(message.payload_type, payload_type);
954        assert_eq!(message.encoding, encoding);
955        assert_eq!(received, Some((expected_value, expected_mapping)));
956    }
957
958    #[rstest]
959    #[case(BusPayloadType::RequestCommand, "data request")]
960    #[case(BusPayloadType::DataResponse, "data response")]
961    fn test_native_messages_reject_unsupported_encoding_without_dispatch(
962        #[case] payload_type: BusPayloadType,
963        #[case] name: &str,
964        #[values(SerializationEncoding::Sbe)] encoding: SerializationEncoding,
965    ) {
966        set_message_bus(Rc::new(RefCell::new(MessageBus::default())));
967        get_message_bus()
968            .borrow_mut()
969            .add_streaming_type(payload_type);
970        let message = BusMessage::with_str_topic(
971            "external.test",
972            payload_type,
973            Bytes::from_static(b"unsupported"),
974            encoding,
975        );
976        let expected = format!("{name} encoding is not supported: {encoding}");
977        let mut calls = 0;
978        let error = process_external_typed_message(&message, &mut |_, _| {
979            calls += 1;
980            Ok(())
981        })
982        .unwrap_err();
983
984        assert_eq!(error.to_string(), expected);
985        assert_eq!(
986            republish_external_message(&message)
987                .unwrap_err()
988                .to_string(),
989            expected
990        );
991        assert_eq!(calls, 0);
992    }
993
994    #[rstest]
995    #[case(false)]
996    #[case(true)]
997    fn test_external_source_restored_after_nested_dispatch_and_processor_failure(
998        #[case] panic: bool,
999    ) {
1000        use std::panic::{AssertUnwindSafe, catch_unwind};
1001
1002        use nautilus_core::UUID4;
1003        use nautilus_model::{data::DataType, identifiers::ClientId};
1004
1005        use crate::{
1006            messages::data::RequestCustomData,
1007            msgbus::{ShareableMessageHandler, subscribe_any},
1008        };
1009
1010        set_message_bus(Rc::new(RefCell::new(MessageBus::default())));
1011        get_message_bus()
1012            .borrow_mut()
1013            .add_streaming_type(BusPayloadType::RequestCommand);
1014        let request = RequestCommand::Data(RequestCustomData::new(
1015            ClientId::from("FEED"),
1016            DataType::new("DistinctRequest", None, None),
1017            None,
1018            None,
1019            None,
1020            UUID4::new(),
1021            71.into(),
1022            None,
1023        ));
1024        let payload = codec::serialize_payload(
1025            SerializationEncoding::Json,
1026            BusPayloadType::RequestCommand,
1027            &request,
1028            false,
1029        )
1030        .unwrap();
1031        let mut message = BusMessage::with_str_topic(
1032            "external.outer",
1033            BusPayloadType::RequestCommand,
1034            payload,
1035            SerializationEncoding::Json,
1036        );
1037
1038        let source = BusMessageSource {
1039            publisher_id: "PUBLISHER-017".into(),
1040            instance_id: UUID4::new(),
1041            stream: "snapshot".into(),
1042            sequence: 37,
1043        };
1044
1045        message.source = Some(source);
1046        let mut nested = message.clone();
1047        nested.topic = "external.nested".into();
1048        nested.source = None;
1049        let received = Rc::new(RefCell::new(Vec::new()));
1050        let shared = received.clone();
1051        subscribe_any(
1052            nested.topic.as_str().into(),
1053            ShareableMessageHandler::from_typed::<RequestCommand, _>(move |_| {
1054                shared.borrow_mut().push(current_external_source());
1055            }),
1056            None,
1057        );
1058
1059        let result = catch_unwind(AssertUnwindSafe(|| {
1060            process_external_typed_message(&message, &mut |_, _| {
1061                assert_eq!(current_external_source(), Some(source));
1062                republish_external_message(&nested)?;
1063                assert_eq!(current_external_source(), Some(source));
1064                assert_eq!(*received.borrow(), vec![None]);
1065
1066                assert!(!panic, "processor panicked");
1067
1068                anyhow::bail!("processor rejected payload")
1069            })
1070        }));
1071
1072        if panic {
1073            assert_eq!(
1074                result.unwrap_err().downcast_ref::<&str>(),
1075                Some(&"processor panicked")
1076            );
1077        } else {
1078            assert_eq!(
1079                result.unwrap().unwrap_err().to_string(),
1080                "processor rejected payload"
1081            );
1082        }
1083
1084        assert_eq!(current_external_source(), None);
1085    }
1086
1087    #[rstest]
1088    #[case(false, false)]
1089    #[case(true, false)]
1090    #[case(false, true)]
1091    #[case(true, true)]
1092    fn test_native_response_processors_precede_streaming_filters(
1093        #[case] streaming: bool,
1094        #[case] rejected: bool,
1095    ) {
1096        use nautilus_core::UUID4;
1097        use nautilus_model::{data::DataType, identifiers::ClientId};
1098
1099        use crate::{
1100            messages::data::CustomDataResponse,
1101            msgbus::{self, ShareableMessageHandler},
1102        };
1103
1104        set_message_bus(Rc::new(RefCell::new(MessageBus::default())));
1105
1106        if streaming {
1107            get_message_bus()
1108                .borrow_mut()
1109                .add_streaming_type(BusPayloadType::DataResponse);
1110        }
1111
1112        let response = CustomDataResponse::new(
1113            UUID4::new(),
1114            ClientId::from("FEED"),
1115            None,
1116            DataType::new("DistinctReply", None, None),
1117            Bytes::from_static(b"native-bytes"),
1118            Some(67.into()),
1119            Some(71.into()),
1120            73.into(),
1121            None,
1122        );
1123        let publications = Rc::new(RefCell::new(Vec::new()));
1124        get_message_bus()
1125            .borrow_mut()
1126            .set_external_egress_config(
1127                Box::new(RecordingEgress {
1128                    messages: publications.clone(),
1129                    closed: false,
1130                }),
1131                &crate::msgbus::MessageBusConfig {
1132                    encoding: SerializationEncoding::MsgPack,
1133                    external_streams: Some(vec!["external.test".into()]),
1134                    ..Default::default()
1135                },
1136            )
1137            .unwrap();
1138
1139        publish_any(
1140            "responses.data.FEED".into(),
1141            &DataResponse::Data(response.clone()),
1142        );
1143        let message = publications.borrow()[0].clone();
1144        let published = Rc::new(RefCell::new(0));
1145        let shared = published.clone();
1146        msgbus::subscribe_any(
1147            message.topic.as_str().into(),
1148            ShareableMessageHandler::from_typed::<DataResponse, _>(move |_| {
1149                *shared.borrow_mut() += 1;
1150            }),
1151            None,
1152        );
1153
1154        let mut calls = 0;
1155        let result = process_external_typed_message(&message, &mut |value, mapping| {
1156            calls += 1;
1157
1158            let DataResponse::Data(actual) = value.downcast_ref::<DataResponse>().unwrap() else {
1159                panic!("Expected native custom response");
1160            };
1161
1162            assert_eq!(actual.correlation_id, response.correlation_id);
1163            assert_eq!(
1164                actual.data.downcast_ref::<Bytes>(),
1165                response.data.downcast_ref::<Bytes>()
1166            );
1167            assert_eq!(
1168                mapping,
1169                &serde_json::json!({
1170                    "payload_type": "DataResponse",
1171                    "Data": {
1172                        "correlation_id": response.correlation_id, "client_id": "FEED",
1173                        "start": 67, "end": 71, "ts_init": 73, "params": null,
1174                        "venue": null, "data_type": response.data_type,
1175                        "data": {"Buffer": b"native-bytes".to_vec()},
1176                    }
1177                })
1178            );
1179
1180            if rejected {
1181                anyhow::bail!("processor rejected reply");
1182            }
1183
1184            Ok(())
1185        });
1186
1187        assert_eq!(calls, 1);
1188        assert_eq!(publications.borrow().len(), 1);
1189        assert_eq!(message.topic.as_str(), "responses.data.FEED");
1190        assert_eq!(message.payload_type, BusPayloadType::DataResponse);
1191        assert_eq!(message.encoding, SerializationEncoding::MsgPack);
1192        assert_eq!(
1193            result.err().map(|e| e.to_string()),
1194            rejected.then(|| "processor rejected reply".to_string())
1195        );
1196        assert_eq!(*published.borrow(), usize::from(streaming && !rejected));
1197    }
1198
1199    #[rstest]
1200    fn test_startup_dispatches_only_registered_pending_data_responses() {
1201        use nautilus_core::UUID4;
1202        use nautilus_model::{data::DataType, identifiers::ClientId};
1203
1204        use crate::{
1205            messages::data::CustomDataResponse,
1206            msgbus::{self, ShareableMessageHandler},
1207        };
1208
1209        let bus = Rc::new(RefCell::new(MessageBus::default()));
1210        set_message_bus(bus.clone());
1211        bus.borrow_mut()
1212            .add_streaming_type(BusPayloadType::DataResponse);
1213        let id = UUID4::new();
1214        let response = DataResponse::Data(CustomDataResponse::new(
1215            id,
1216            ClientId::from("FEED"),
1217            None,
1218            DataType::new("DistinctReply", None, None),
1219            Bytes::from_static(b"native-bytes"),
1220            None,
1221            None,
1222            73.into(),
1223            None,
1224        ));
1225        let message = BusMessage::with_str_topic(
1226            "responses.data.FEED",
1227            BusPayloadType::DataResponse,
1228            codec::serialize_payload(
1229                SerializationEncoding::MsgPack,
1230                BusPayloadType::DataResponse,
1231                &response,
1232                false,
1233            )
1234            .unwrap(),
1235            SerializationEncoding::MsgPack,
1236        );
1237
1238        let topic_handler = ShareableMessageHandler::from_typed::<DataResponse, _>(|reply| {
1239            msgbus::send_response(reply.correlation_id(), reply);
1240        });
1241
1242        msgbus::subscribe_any(message.topic.as_str().into(), topic_handler, None);
1243        let received = Rc::new(RefCell::new(Vec::new()));
1244        let shared = received.clone();
1245        assert!(!republish_pending_external_response(&message).unwrap());
1246        msgbus::register_response_handler(
1247            &id,
1248            TypedHandler::from(move |envelope: &DataResponseEnvelope| {
1249                let Some(DataResponse::Data(reply)) = envelope.response() else {
1250                    panic!("expected custom reply");
1251                };
1252
1253                shared.borrow_mut().push((
1254                    reply.correlation_id,
1255                    reply.data.downcast_ref::<Bytes>().unwrap().clone(),
1256                    reply.ts_init,
1257                ));
1258            }),
1259        );
1260
1261        assert!(republish_pending_external_response(&message).unwrap());
1262        assert!(!republish_pending_external_response(&message).unwrap());
1263        assert_eq!(
1264            *received.borrow(),
1265            vec![(id, Bytes::from_static(b"native-bytes"), 73.into())]
1266        );
1267    }
1268
1269    #[rstest]
1270    #[case(SerializationEncoding::Json)]
1271    #[case(SerializationEncoding::MsgPack)]
1272    fn test_startup_dispatches_payload_free_completion(#[case] encoding: SerializationEncoding) {
1273        use crate::msgbus::{self, ShareableMessageHandler};
1274
1275        let bus = Rc::new(RefCell::new(MessageBus::default()));
1276        set_message_bus(bus.clone());
1277        bus.borrow_mut()
1278            .add_streaming_type(BusPayloadType::DataResponseEnvelope);
1279        let id = nautilus_core::UUID4::new();
1280        let envelope = DataResponseEnvelope::terminal(id, Some("Source unavailable".to_owned()));
1281        let payload = codec::serialize_payload(
1282            encoding,
1283            BusPayloadType::DataResponseEnvelope,
1284            &envelope,
1285            false,
1286        )
1287        .unwrap();
1288        let message = BusMessage::with_str_topic(
1289            "responses.data.FEED",
1290            BusPayloadType::DataResponseEnvelope,
1291            payload,
1292            encoding,
1293        );
1294        let received = Rc::new(RefCell::new(Vec::new()));
1295        let shared = received.clone();
1296        msgbus::subscribe_any(
1297            message.topic.as_str().into(),
1298            ShareableMessageHandler::from_typed::<DataResponseEnvelope, _>(
1299                msgbus::send_response_envelope,
1300            ),
1301            None,
1302        );
1303        msgbus::register_response_handler(
1304            &id,
1305            TypedHandler::from(move |response: &DataResponseEnvelope| {
1306                shared.borrow_mut().push(response.clone());
1307            }),
1308        );
1309
1310        assert!(republish_pending_external_response(&message).unwrap());
1311
1312        let received = received.borrow();
1313        assert_eq!(received.len(), 1);
1314        assert_eq!(received[0].correlation_id(), &id);
1315        assert!(received[0].completed());
1316        assert!(received[0].response().is_none());
1317        assert_eq!(received[0].sub_correlation_id(), None);
1318        assert_eq!(received[0].failure_reason(), Some("Source unavailable"));
1319        assert!(bus.borrow().get_response_handler(&id).is_none());
1320        assert!(!republish_pending_external_response(&message).unwrap());
1321    }
1322
1323    fn install_egress() -> Rc<RefCell<Vec<BusMessage>>> {
1324        let messages = Rc::new(RefCell::new(Vec::new()));
1325        let mut bus = MessageBus::default();
1326        bus.set_external_egress(
1327            Box::new(RecordingEgress {
1328                messages: messages.clone(),
1329                closed: false,
1330            }),
1331            SerializationEncoding::Json,
1332        );
1333
1334        set_message_bus(Rc::new(RefCell::new(bus)));
1335        messages
1336    }
1337
1338    fn assert_message(message: &BusMessage, topic: &str, payload: &[u8]) {
1339        assert_eq!(message.topic.as_str(), topic);
1340        assert_eq!(message.payload_type, BusPayloadType::QuoteTick);
1341        assert_eq!(message.encoding, SerializationEncoding::Json);
1342        assert_eq!(message.payload.as_ref(), payload);
1343    }
1344}