Skip to main content

nautilus_data/engine/
streaming.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use ahash::{AHashMap, AHashSet};
17use indexmap::IndexMap;
18use jiff::Timestamp;
19use nautilus_common::messages::data::{
20    BarsResponse, BookDeltasResponse, BookDepthResponse, CustomDataResponse, DataRequestFailed,
21    DataResponse, FundingRatesResponse, InstrumentResponse, InstrumentsResponse, QuotesResponse,
22    RequestBars, RequestBookDeltas, RequestBookDepth, RequestCommand, RequestCustomData,
23    RequestFundingRates, RequestInstrument, RequestInstruments, RequestQuotes, RequestTrades,
24    SubscribeBars, SubscribeCommand, SubscribeCustomData, SubscribeQuotes, SubscribeTrades,
25    TradesResponse,
26};
27use nautilus_core::{
28    Params, UUID4, UnixNanos,
29    correctness::{FAILED, check_key_not_in_map},
30};
31use nautilus_model::{
32    data::{
33        Bar, BarType, BatchView, CustomData, DataBatch, FromDataBatch, FundingRateUpdate,
34        NautilusDataType, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick,
35    },
36    identifiers::{ClientId, InstrumentId, Venue},
37    instruments::{Instrument, InstrumentAny},
38};
39use nautilus_persistence::catalog::traits::{CatalogInstrumentQuery, CatalogQuery, DataCatalog};
40use serde_json::Value;
41use ustr::Ustr;
42
43use super::{DataEngine, requests::request_params};
44
45const PARAM_SKIP_CATALOG_DATA: &str = "skip_catalog_data";
46pub(super) const PARAM_UPDATE_CATALOG: &str = "update_catalog";
47const PARAM_FORCE_INSTRUMENT_UPDATE: &str = "force_instrument_update";
48const PARAM_SUBSCRIPTION_NAME: &str = "subscription_name";
49const PARAM_FROM_DAY_START: &str = "from_day_start";
50const CATALOG_CLIENT_ID: &str = "CATALOG";
51
52pub(crate) type CatalogMap = AHashMap<Ustr, DataCatalog>;
53
54impl DataEngine {
55    /// Registers the `catalog` with the engine with an optional specific `name`.
56    ///
57    /// # Panics
58    ///
59    /// Panics if a catalog with the same `name` has already been registered.
60    pub fn register_catalog(&mut self, catalog: DataCatalog, name: Option<&str>) {
61        let name = Ustr::from(name.unwrap_or("catalog_0"));
62
63        check_key_not_in_map(&name, &self.catalogs, "name", "catalogs").expect(FAILED);
64
65        self.catalogs.insert(name, catalog);
66        log::info!("Registered catalog <{name}>");
67    }
68
69    pub(super) fn subscribe_command_with_prefilled_start_ns(
70        &mut self,
71        cmd: SubscribeCommand,
72    ) -> anyhow::Result<SubscribeCommand> {
73        match cmd {
74            SubscribeCommand::Quotes(cmd)
75                if cmd.instrument_ids.len() == 1
76                    && Self::is_start_ns_missing(cmd.params.as_ref()) =>
77            {
78                let identifier = cmd.instrument_ids[0].to_string();
79                let params = self.params_with_prefilled_start_ns(
80                    cmd.params.as_ref(),
81                    &NautilusDataType::QuoteTick,
82                    &identifier,
83                )?;
84                Ok(SubscribeCommand::Quotes(SubscribeQuotes { params, ..cmd }))
85            }
86            SubscribeCommand::Trades(cmd)
87                if cmd.instrument_ids.len() == 1
88                    && Self::is_start_ns_missing(cmd.params.as_ref()) =>
89            {
90                let identifier = cmd.instrument_ids[0].to_string();
91                let params = self.params_with_prefilled_start_ns(
92                    cmd.params.as_ref(),
93                    &NautilusDataType::TradeTick,
94                    &identifier,
95                )?;
96                Ok(SubscribeCommand::Trades(SubscribeTrades { params, ..cmd }))
97            }
98            SubscribeCommand::Bars(cmd)
99                if cmd.bar_types.len() == 1
100                    && cmd.single_bar_type()?.is_externally_aggregated()
101                    && Self::is_start_ns_missing(cmd.params.as_ref()) =>
102            {
103                let identifier = cmd.single_bar_type()?.to_string();
104                let params = self.params_with_prefilled_start_ns(
105                    cmd.params.as_ref(),
106                    &NautilusDataType::Bar,
107                    &identifier,
108                )?;
109                Ok(SubscribeCommand::Bars(SubscribeBars { params, ..cmd }))
110            }
111            SubscribeCommand::Data(cmd) if Self::is_start_ns_missing(cmd.params.as_ref()) => {
112                let type_name = cmd.data_type.type_name().to_string();
113                let identifier = cmd.data_type.identifier().map(String::from);
114                let params = self.params_with_custom_data_prefilled_start_ns(
115                    cmd.params.as_ref(),
116                    &type_name,
117                    identifier.as_deref(),
118                )?;
119                Ok(SubscribeCommand::Data(SubscribeCustomData {
120                    params,
121                    ..cmd
122                }))
123            }
124            _ => Ok(cmd),
125        }
126    }
127
128    fn is_start_ns_missing(params: Option<&Params>) -> bool {
129        params.is_none_or(|params| !params.contains_key("start_ns"))
130    }
131
132    fn params_with_prefilled_start_ns(
133        &mut self,
134        params: Option<&Params>,
135        data_type: &NautilusDataType,
136        identifier: &str,
137    ) -> anyhow::Result<Option<Params>> {
138        let last_timestamp = self.catalog_last_timestamp(data_type, identifier)?;
139
140        Ok(Some(Self::params_with_start_ns(params, last_timestamp)))
141    }
142
143    fn params_with_custom_data_prefilled_start_ns(
144        &mut self,
145        params: Option<&Params>,
146        type_name: &str,
147        identifier: Option<&str>,
148    ) -> anyhow::Result<Option<Params>> {
149        let last_timestamp = self.catalog_custom_data_last_timestamp(type_name, identifier)?;
150
151        Ok(Some(Self::params_with_start_ns(params, last_timestamp)))
152    }
153
154    fn params_with_start_ns(params: Option<&Params>, last_timestamp: Option<u64>) -> Params {
155        let start_ns = last_timestamp.map_or(Value::Null, |last_timestamp| {
156            Value::from(last_timestamp.saturating_add(1))
157        });
158
159        let mut params = params.map_or_default(|params| IndexMap::clone(params));
160
161        params.insert("start_ns".to_string(), start_ns);
162
163        Params::new(params)
164    }
165
166    fn catalog_last_timestamp(
167        &mut self,
168        data_type: &NautilusDataType,
169        identifier: &str,
170    ) -> anyhow::Result<Option<u64>> {
171        for catalog in self.catalogs.values_mut() {
172            if let Some(last_timestamp) =
173                catalog.query_last_timestamp(data_type.clone(), Some(identifier))?
174            {
175                return Ok(Some(last_timestamp));
176            }
177        }
178
179        Ok(None)
180    }
181
182    fn catalog_custom_data_last_timestamp(
183        &mut self,
184        type_name: &str,
185        identifier: Option<&str>,
186    ) -> anyhow::Result<Option<u64>> {
187        // `make_path_custom_data` / `get_directory_intervals` are inherent on
188        // `ParquetDataCatalog` and not exposed through `Catalog`. Use the
189        // trait-level `query_last_timestamp` with `NautilusDataType::Custom` so the
190        // path works for any backend that implements the trait.
191        let data_type = NautilusDataType::Custom {
192            type_name: type_name.to_string(),
193        };
194
195        for catalog in self.catalogs.values_mut() {
196            if let Some(last_timestamp) =
197                catalog.query_last_timestamp(data_type.clone(), identifier)?
198            {
199                return Ok(Some(last_timestamp));
200            }
201        }
202
203        Ok(None)
204    }
205
206    /// Rejects a request asking for catalog write-back (`update_catalog`) that cannot get one:
207    /// no catalog is registered, or the request carries a source schema or a record limit.
208    pub(super) fn validate_catalog_update(&self, request: &RequestCommand) -> anyhow::Result<()> {
209        if request_params(request).and_then(|p| p.get_bool(PARAM_UPDATE_CATALOG)) != Some(true) {
210            return Ok(());
211        }
212
213        anyhow::ensure!(
214            !self.catalogs.is_empty(),
215            "Catalog write-back requires a registered catalog"
216        );
217        anyhow::ensure!(
218            !request_has_schema(request),
219            "Catalog write-back is unsupported for requests with a source schema"
220        );
221        anyhow::ensure!(
222            request_limit(request).is_none(),
223            "Catalog write-back is unsupported for requests with a record limit"
224        );
225        Ok(())
226    }
227
228    /// Writes `response` back to a catalog when `request` asks for it (`update_catalog`).
229    /// `validate_catalog_update` has accepted the request.
230    ///
231    /// # Errors
232    ///
233    /// Returns an error if the requested write-back fails, so the response data is not
234    /// persisted: an identifier's coverage is unverified, or a write fails.
235    pub(super) fn update_catalog_from_response(
236        &mut self,
237        response: &DataResponse,
238        request: &RequestCommand,
239    ) -> anyhow::Result<()> {
240        let Some(params) =
241            request_params(request).filter(|p| p.get_bool(PARAM_UPDATE_CATALOG).unwrap_or(false))
242        else {
243            return Ok(());
244        };
245
246        if response
247            .params()
248            .is_some_and(|p| p.get_bool(PARAM_UPDATE_CATALOG) == Some(false))
249        {
250            return Ok(());
251        }
252
253        if let DataResponse::Instrument(value) = response {
254            return self.write_response_instruments(std::slice::from_ref(&value.data), params);
255        }
256
257        if let DataResponse::Instruments(value) = response {
258            return self.write_response_instruments(&value.data, params);
259        }
260
261        let batches = match response_batches(response) {
262            Ok(batches) => batches,
263            Err(e) => {
264                log::error!("Cannot prepare response catalog write: {e}");
265                anyhow::bail!(
266                    "Catalog write-back failed: cannot prepare the response batches: {e}"
267                );
268            }
269        };
270
271        let bounds = response_write_bounds(request, response);
272        let data_type = response.data_kind();
273        let mut failures = Vec::new();
274
275        for (identifier, batch) in batches {
276            if !request_contains_batch(request, identifier.as_deref(), &batch) {
277                log::warn!(
278                    "Skipping catalog write-back because identifier coverage is unverified, request={}, identifier={identifier:?}",
279                    request.request_id()
280                );
281                failures.push(format!(
282                    "Catalog write-back failed: identifier coverage is unverified, identifier={identifier:?}"
283                ));
284                continue;
285            }
286
287            if let Err(e) =
288                self.write_response_batch(&batch, &data_type, identifier.as_deref(), params, bounds)
289            {
290                log::error!("Catalog write-back failed for {identifier:?}: {e}");
291                failures.push(format!("Catalog write-back failed for {identifier:?}: {e}"));
292            }
293        }
294
295        if !failures.is_empty() {
296            anyhow::bail!(failures.join("; "));
297        }
298        Ok(())
299    }
300
301    fn write_response_batch(
302        &mut self,
303        batch: &DataBatch,
304        data_type: &NautilusDataType,
305        identifier: Option<&str>,
306        params: &Params,
307        bounds: (Option<UnixNanos>, Option<UnixNanos>),
308    ) -> anyhow::Result<()> {
309        let name = self.catalog_for_update(data_type, identifier, params)?;
310        let catalog = self
311            .catalogs
312            .get_mut(&name)
313            .expect("selected catalog is registered");
314        let (start, end) = bounds;
315        if batch.is_empty() {
316            if let (Some(start), Some(end)) = (start, end) {
317                catalog.record_empty_coverage(data_type.clone(), identifier, start, end)?;
318            }
319
320            return Ok(());
321        }
322
323        catalog.write_data_batch(batch, start, end, Some(params.clone()))
324    }
325
326    fn write_response_instruments(
327        &mut self,
328        instruments: &[InstrumentAny],
329        params: &Params,
330    ) -> anyhow::Result<()> {
331        let force = params
332            .get_bool(PARAM_FORCE_INSTRUMENT_UPDATE)
333            .unwrap_or(false);
334        let mut failures = Vec::new();
335
336        for instrument in instruments {
337            let identifier = instrument.id().to_string();
338
339            let result = (|| {
340                let name = self.catalog_for_update(
341                    &NautilusDataType::Instrument,
342                    Some(&identifier),
343                    params,
344                )?;
345                let catalog = self
346                    .catalogs
347                    .get_mut(&name)
348                    .expect("selected catalog is registered");
349
350                if !force {
351                    let query = CatalogInstrumentQuery::new()
352                        .with_instrument_ids(Some(vec![identifier.clone()]));
353
354                    if !catalog.instruments(&query)?.is_empty() {
355                        return Ok(());
356                    }
357                }
358
359                catalog.write_instruments(std::slice::from_ref(instrument))
360            })();
361
362            if let Err(e) = result {
363                log::error!("Instrument catalog write-back failed for {identifier}: {e}");
364                failures.push(format!(
365                    "Catalog write-back failed for instrument {identifier}: {e}"
366                ));
367            }
368        }
369
370        if !failures.is_empty() {
371            anyhow::bail!(failures.join("; "));
372        }
373        Ok(())
374    }
375
376    fn catalog_for_update(
377        &mut self,
378        data_type: &NautilusDataType,
379        identifier: Option<&str>,
380        params: &Params,
381    ) -> anyhow::Result<Ustr> {
382        if let Some(name) = params.get_str("catalog_name").map(Ustr::from) {
383            anyhow::ensure!(
384                self.catalogs.contains_key(&name),
385                "Catalog '{name}' is not registered"
386            );
387            return Ok(name);
388        }
389
390        let mut names: Vec<_> = self.catalogs.keys().copied().collect();
391        names.sort_by(|a, b| a.as_str().cmp(b.as_str()));
392        for name in &names {
393            let catalog = self.catalogs.get_mut(name).expect("catalog key exists");
394            if catalog
395                .query_last_timestamp(data_type.clone(), identifier)?
396                .is_some()
397            {
398                return Ok(*name);
399            }
400        }
401
402        names
403            .first()
404            .copied()
405            .ok_or_else(|| anyhow::anyhow!("No catalog registered"))
406    }
407
408    pub(super) fn catalogs_registered(&self) -> bool {
409        !self.catalogs.is_empty()
410    }
411
412    // Bounds the request window, walks the catalogs to find one whose missing
413    // intervals differ from the full requested range, then fans the parent out via
414    // the pipeline with one catalog leg plus one client leg per missing interval.
415    // With no catalog match and no resolvable client, the engine emits an empty
416    // response keyed by the parent request ID.
417    pub(super) fn dispatch_date_range_request(
418        &mut self,
419        req: RequestCommand,
420    ) -> anyhow::Result<()> {
421        if request_has_schema(&req) {
422            return self.dispatch_request_to_client(req).map(|_| ());
423        }
424
425        if matches!(
426            req,
427            RequestCommand::Instrument(_) | RequestCommand::Instruments(_)
428        ) {
429            return self.dispatch_instrument_catalog_request(req);
430        }
431
432        if matches!(&req, RequestCommand::Quotes(r) if r.instrument_ids.len() > 1)
433            || matches!(&req, RequestCommand::Trades(r) if r.instrument_ids.len() > 1)
434        {
435            return self.dispatch_batch_catalog_request(req);
436        }
437
438        let Some(key) = request_identifier(&req) else {
439            return self.dispatch_request_to_client(req).map(|_| ());
440        };
441
442        let now_ns = self.clock.borrow().timestamp_ns();
443        let now_dt = now_ns.to_datetime_utc();
444        let query_past_data = request_params(&req)
445            .and_then(|p| p.get(PARAM_SUBSCRIPTION_NAME))
446            .is_none();
447
448        let (start_dt, end_dt) = bound_request_dates(
449            request_start(&req),
450            request_end(&req),
451            now_dt,
452            query_past_data,
453        );
454        let start_ns = datetime_to_unix_nanos_or_zero(start_dt);
455        let end_ns = datetime_to_unix_nanos_or_zero(end_dt);
456
457        if start_ns > end_ns {
458            anyhow::bail!(
459                "Cannot dispatch request, start {start_ns} was greater than end {end_ns}"
460            );
461        }
462
463        let catalog_start_dt = catalog_request_start(&req, start_dt);
464        let catalog_start_ns = datetime_to_unix_nanos_or_zero(catalog_start_dt);
465        let query_interval = vec![(start_ns.as_u64(), end_ns.as_u64())];
466        let coverage = self.catalog_coverage_for_request(
467            &key,
468            start_ns.as_u64(),
469            end_ns.as_u64(),
470            catalog_start_ns.as_u64(),
471        )?;
472
473        let (winning_catalog, mut missing_intervals) = match coverage {
474            Some(coverage) => (Some(coverage.catalog), coverage.missing),
475            None => (None, query_interval.clone()),
476        };
477
478        let skip_catalog_data = request_params(&req)
479            .and_then(|p| p.get_bool(PARAM_SKIP_CATALOG_DATA))
480            .unwrap_or(false);
481
482        // When `skip_catalog_data` is set the client must serve the full parent window;
483        // dropping the catalog leg without resetting the missing intervals would leave
484        // the catalog-covered range unanswered.
485        if skip_catalog_data {
486            missing_intervals = query_interval;
487        }
488
489        let client_id = req.client_id().copied();
490        let venue = req.venue().copied();
491        let has_client = self
492            .get_client(client_id.as_ref(), venue.as_ref())
493            .is_some();
494
495        if !has_client {
496            missing_intervals.clear();
497        }
498
499        let mut plan = Vec::new();
500
501        if !skip_catalog_data && let Some(catalog) = winning_catalog {
502            let request =
503                with_dates_for_pipeline(&req, Some(catalog_start_dt), Some(end_dt), now_ns);
504            plan.push(RequestLeg::Catalog { request, catalog });
505        }
506
507        for (start, end) in missing_intervals {
508            let start = UnixNanos::from(start).to_datetime_utc();
509            let end = UnixNanos::from(end).to_datetime_utc();
510            let request = with_dates_for_pipeline(&req, Some(start), Some(end), now_ns);
511            plan.push(RequestLeg::Client(request));
512        }
513
514        self.dispatch_request_plan(req, plan)
515    }
516
517    fn dispatch_batch_catalog_request(&mut self, parent: RequestCommand) -> anyhow::Result<()> {
518        let ids = match &parent {
519            RequestCommand::Quotes(r) => r.instrument_ids.clone(),
520            RequestCommand::Trades(r) => r.instrument_ids.clone(),
521            _ => unreachable!("only quote and trade lists use catalog batch planning"),
522        };
523
524        let now_ns = self.clock.borrow().timestamp_ns();
525        let query_past = request_params(&parent)
526            .and_then(|p| p.get(PARAM_SUBSCRIPTION_NAME))
527            .is_none();
528        let (start, end) = bound_request_dates(
529            request_start(&parent),
530            request_end(&parent),
531            now_ns.to_datetime_utc(),
532            query_past,
533        );
534        let start_ns = datetime_to_unix_nanos_or_zero(start);
535        let end_ns = datetime_to_unix_nanos_or_zero(end);
536        anyhow::ensure!(start_ns <= end_ns, "Request start exceeds end");
537        let skip_catalog = request_params(&parent)
538            .and_then(|p| p.get_bool(PARAM_SKIP_CATALOG_DATA))
539            .unwrap_or(false);
540        let full = vec![(start_ns.as_u64(), end_ns.as_u64())];
541        let mut clients: IndexMap<(u64, u64), Vec<InstrumentId>> = IndexMap::new();
542        let mut plan = Vec::new();
543
544        for id in ids {
545            let mut member = with_request_instrument_ids(&parent, vec![id]);
546            let key = request_identifier(&member).expect("single quote or trade identifier");
547            let mut missing = full.clone();
548            if !skip_catalog
549                && let Some(CatalogCoverage {
550                    catalog,
551                    missing: intervals,
552                }) = self.catalog_coverage(&key, start_ns.as_u64(), end_ns.as_u64())?
553            {
554                missing = intervals;
555                member = with_dates_for_pipeline(&member, Some(start), Some(end), now_ns);
556                plan.push(RequestLeg::Catalog {
557                    request: member,
558                    catalog,
559                });
560            }
561
562            for interval in missing {
563                clients.entry(interval).or_default().push(id);
564            }
565        }
566
567        for ((start, end), ids) in clients {
568            let member = with_request_instrument_ids(&parent, ids);
569            let client_id = member.client_id().copied();
570            let venue = member.venue().copied();
571
572            if self
573                .get_client(client_id.as_ref(), venue.as_ref())
574                .is_none()
575            {
576                continue;
577            }
578
579            let start = UnixNanos::from(start).to_datetime_utc();
580            let end = UnixNanos::from(end).to_datetime_utc();
581            let request = with_dates_for_pipeline(&member, Some(start), Some(end), now_ns);
582            plan.push(RequestLeg::Client(request));
583        }
584
585        self.dispatch_request_plan(parent, plan)
586    }
587
588    fn catalog_coverage_for_request(
589        &mut self,
590        key: &RequestCatalogKey,
591        start: u64,
592        end: u64,
593        catalog_start: u64,
594    ) -> anyhow::Result<Option<CatalogCoverage>> {
595        let Some(mut coverage) = self.catalog_coverage(key, catalog_start, end)? else {
596            return Ok(None);
597        };
598
599        // Client gaps exclude the pre-start window used to replay day-start snapshots
600        if catalog_start != start {
601            let catalog = self
602                .catalogs
603                .get_mut(&coverage.catalog)
604                .expect("selected catalog is registered");
605            coverage.missing = catalog_missing_intervals(catalog, start, end, key)?;
606        }
607
608        Ok(Some(coverage))
609    }
610
611    fn catalog_coverage(
612        &mut self,
613        key: &RequestCatalogKey,
614        start: u64,
615        end: u64,
616    ) -> anyhow::Result<Option<CatalogCoverage>> {
617        for (name, catalog) in &mut self.catalogs {
618            let missing = catalog_missing_intervals(catalog, start, end, key)?;
619            if missing != [(start, end)] {
620                return Ok(Some(CatalogCoverage {
621                    catalog: *name,
622                    missing,
623                }));
624            }
625        }
626
627        Ok(None)
628    }
629
630    fn dispatch_request_plan(
631        &mut self,
632        parent: RequestCommand,
633        plan: Vec<RequestLeg>,
634    ) -> anyhow::Result<()> {
635        let parent_id = *parent.request_id();
636
637        if plan.is_empty() {
638            let now = self.clock.borrow().timestamp_ns();
639            let (start, end) = bound_request_dates(
640                request_start(&parent),
641                request_end(&parent),
642                now.to_datetime_utc(),
643                true,
644            );
645            let response = build_empty_response(
646                &parent,
647                datetime_to_unix_nanos_or_zero(start),
648                datetime_to_unix_nanos_or_zero(end),
649                None,
650                now,
651            )?;
652            self.response(response);
653            return Ok(());
654        }
655
656        self.new_request_pipeline(parent, plan.len());
657        for leg in &plan {
658            self.register_request_pipeline_leg(leg.request_id(), parent_id);
659        }
660
661        for leg in plan {
662            let request_id = leg.request_id();
663
664            let result = match leg {
665                RequestLeg::Catalog { request, catalog } => self
666                    .query_catalog_leg(&request, catalog)
667                    .map(|response| self.response(response)),
668                RequestLeg::Client(request) => self.dispatch_request_to_client(request).map(|_| ()),
669                RequestLeg::Response(response) => {
670                    self.response(response);
671                    Ok(())
672                }
673            };
674
675            if let Err(e) = result {
676                self.request_failed(DataRequestFailed {
677                    request_id,
678                    reason: e.to_string(),
679                    response: None,
680                });
681            }
682        }
683
684        Ok(())
685    }
686
687    fn query_catalog_leg(
688        &mut self,
689        leg: &RequestCommand,
690        catalog_name: Ustr,
691    ) -> anyhow::Result<DataResponse> {
692        let start_ns = request_start(leg)
693            .map(datetime_to_unix_nanos_or_zero)
694            .expect("catalog leg has bounded start");
695        let end_ns = request_end(leg)
696            .map(datetime_to_unix_nanos_or_zero)
697            .expect("catalog leg has bounded end");
698        let ts_init = self.clock.borrow().timestamp_ns();
699        let client_id = leg.client_id().copied();
700        let venue = leg.venue().copied();
701        let used_client_id = self
702            .get_client(client_id.as_ref(), venue.as_ref())
703            .map(|value| value.client_id());
704
705        let catalog = self.catalogs.get_mut(&catalog_name).ok_or_else(|| {
706            anyhow::anyhow!("Catalog {catalog_name} disappeared between intervals query and read")
707        })?;
708
709        match leg {
710            RequestCommand::Quotes(cmd) => {
711                let data = quote_ticks_from_query_result(
712                    catalog.query_batch(
713                        &CatalogQuery::new(NautilusDataType::QuoteTick)
714                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
715                            .with_range(Some(start_ns), Some(end_ns)),
716                    )?,
717                )?;
718                Ok(build_quotes_catalog_response(
719                    cmd,
720                    data,
721                    start_ns,
722                    end_ns,
723                    used_client_id,
724                    ts_init,
725                ))
726            }
727            RequestCommand::Trades(cmd) => {
728                let data = trade_ticks_from_query_result(
729                    catalog.query_batch(
730                        &CatalogQuery::new(NautilusDataType::TradeTick)
731                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
732                            .with_range(Some(start_ns), Some(end_ns)),
733                    )?,
734                )?;
735                Ok(build_trades_catalog_response(
736                    cmd,
737                    data,
738                    start_ns,
739                    end_ns,
740                    used_client_id,
741                    ts_init,
742                ))
743            }
744            RequestCommand::FundingRates(cmd) => {
745                let data = funding_rates_from_query_result(
746                    catalog.query_batch(
747                        &CatalogQuery::new(NautilusDataType::FundingRateUpdate)
748                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
749                            .with_range(Some(start_ns), Some(end_ns)),
750                    )?,
751                )?;
752                build_funding_rates_catalog_response(
753                    cmd,
754                    data,
755                    start_ns,
756                    end_ns,
757                    used_client_id,
758                    ts_init,
759                )
760            }
761            RequestCommand::Bars(cmd) => {
762                let data = bars_from_query_result(
763                    catalog.query_batch(
764                        &CatalogQuery::new(NautilusDataType::Bar)
765                            .with_identifiers(Some(vec![cmd.single_bar_type()?.to_string()]))
766                            .with_range(Some(start_ns), Some(end_ns)),
767                    )?,
768                )?;
769                build_bars_catalog_response(cmd, data, start_ns, end_ns, used_client_id, ts_init)
770            }
771            RequestCommand::Data(cmd) => {
772                let identifiers = cmd
773                    .data_type
774                    .identifier()
775                    .map(|identifier| vec![identifier.to_string()]);
776                let where_clause = request_filter_expr(cmd.params.as_ref());
777                let data = catalog.query_batch(
778                    &CatalogQuery::new(NautilusDataType::Custom {
779                        type_name: cmd.data_type.type_name().to_string(),
780                    })
781                    .with_identifiers(identifiers)
782                    .with_range(Some(start_ns), Some(end_ns))
783                    .with_where_clause(where_clause),
784                )?;
785                Ok(build_custom_data_catalog_response(
786                    cmd,
787                    custom_data_from_query_result(data)?,
788                    start_ns,
789                    end_ns,
790                    ts_init,
791                ))
792            }
793            RequestCommand::BookDeltas(cmd) => {
794                let data = order_book_deltas_from_query_result(
795                    catalog.query_batch(
796                        &CatalogQuery::new(NautilusDataType::OrderBookDelta)
797                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
798                            .with_range(Some(start_ns), Some(end_ns)),
799                    )?,
800                )?;
801                build_book_deltas_catalog_response(
802                    cmd,
803                    data,
804                    start_ns,
805                    end_ns,
806                    used_client_id,
807                    ts_init,
808                )
809            }
810            RequestCommand::BookDepth(cmd) => {
811                let data = order_book_depths_from_query_result(
812                    catalog.query_batch(
813                        &CatalogQuery::new(NautilusDataType::OrderBookDepth)
814                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
815                            .with_range(Some(start_ns), Some(end_ns)),
816                    )?,
817                )?;
818                build_book_depth_catalog_response(
819                    cmd,
820                    data,
821                    start_ns,
822                    end_ns,
823                    used_client_id,
824                    ts_init,
825                )
826            }
827            _ => {
828                anyhow::bail!("query_catalog_leg called with non-catalog-eligible variant {leg:?}")
829            }
830        }
831    }
832
833    fn dispatch_instrument_catalog_request(&mut self, req: RequestCommand) -> anyhow::Result<()> {
834        match req {
835            RequestCommand::Instrument(cmd) => self.dispatch_instrument_request(cmd),
836            RequestCommand::Instruments(cmd) => self.dispatch_instruments_request(cmd),
837            _ => self.dispatch_request_to_client(req).map(|_| ()),
838        }
839    }
840
841    fn dispatch_instrument_request(&mut self, cmd: RequestInstrument) -> anyhow::Result<()> {
842        if cmd.instrument_ids.len() > 1 {
843            return self.dispatch_instrument_list(cmd);
844        }
845
846        let force_instrument_update = cmd
847            .params
848            .as_ref()
849            .and_then(|params| params.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
850            .unwrap_or(false);
851
852        if force_instrument_update {
853            return self
854                .dispatch_request_to_client(RequestCommand::Instrument(cmd))
855                .map(|_| ());
856        }
857
858        let identifier = cmd.single_instrument_id()?.to_string();
859        let now_ns = self.clock.borrow().timestamp_ns();
860        let used_client_id = self
861            .get_client(
862                cmd.client_id.as_ref(),
863                Some(&cmd.single_instrument_id()?.venue),
864            )
865            .map(|client| client.client_id());
866        let (start_dt, end_dt) =
867            bound_request_dates(cmd.start, cmd.end, now_ns.to_datetime_utc(), true);
868        let start_ns = datetime_to_unix_nanos_or_zero(start_dt);
869        let end_ns = datetime_to_unix_nanos_or_zero(end_dt);
870        let start = Some(start_ns);
871        let end = cmd.end.map(datetime_to_unix_nanos_or_zero);
872
873        for catalog in self.catalogs.values_mut() {
874            let data = latest_instruments(
875                catalog.instruments(
876                    &CatalogInstrumentQuery::new()
877                        .with_instrument_ids(Some(vec![identifier.clone()]))
878                        .with_range(start, end),
879                )?,
880            );
881
882            if let Some(instrument) = data.into_iter().next() {
883                let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
884                    cmd.request_id,
885                    resolve_response_client_id(cmd.client_id, used_client_id),
886                    cmd.single_instrument_id()?,
887                    instrument,
888                    Some(start_ns),
889                    Some(end_ns),
890                    now_ns,
891                    Some(catalog_response_params(cmd.params.as_ref())),
892                )));
893                self.response(response);
894                return Ok(());
895            }
896        }
897
898        self.dispatch_request_to_client(RequestCommand::Instrument(cmd))
899            .map(|_| ())
900    }
901
902    fn dispatch_instrument_list(&mut self, command: RequestInstrument) -> anyhow::Result<()> {
903        let parent = RequestCommand::Instrument(command.clone());
904        let force = command
905            .params
906            .as_ref()
907            .and_then(|p| p.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
908            .unwrap_or(false);
909
910        if force {
911            return self.dispatch_request_to_client(parent).map(|_| ());
912        }
913
914        let now = self.clock.borrow().timestamp_ns();
915        let (start, end) =
916            bound_request_dates(command.start, command.end, now.to_datetime_utc(), true);
917        let start_ns = datetime_to_unix_nanos_or_zero(start);
918        let end_ns = datetime_to_unix_nanos_or_zero(end);
919        let mut missing = Vec::new();
920        let mut plan = Vec::new();
921
922        for id in &command.instrument_ids {
923            let mut instrument = None;
924
925            for catalog in self.catalogs.values_mut() {
926                let query = CatalogInstrumentQuery::new()
927                    .with_instrument_ids(Some(vec![id.to_string()]))
928                    .with_range(
929                        Some(start_ns),
930                        command.end.map(datetime_to_unix_nanos_or_zero),
931                    );
932                instrument = latest_instruments(catalog.instruments(&query)?)
933                    .into_iter()
934                    .find(|value| value.id() == *id);
935
936                if instrument.is_some() {
937                    break;
938                }
939            }
940
941            if let Some(instrument) = instrument {
942                let response = InstrumentResponse::new(
943                    UUID4::new(),
944                    command
945                        .client_id
946                        .unwrap_or_else(|| ClientId::new(CATALOG_CLIENT_ID)),
947                    *id,
948                    instrument,
949                    Some(start_ns),
950                    Some(end_ns),
951                    now,
952                    Some(catalog_response_params(command.params.as_ref())),
953                );
954                plan.push(RequestLeg::Response(DataResponse::Instrument(Box::new(
955                    response,
956                ))));
957            } else {
958                missing.push(*id);
959            }
960        }
961
962        if !missing.is_empty() {
963            let mut child = command;
964            child.request_id = UUID4::new();
965            child.instrument_ids = missing;
966            plan.push(RequestLeg::Client(RequestCommand::Instrument(child)));
967        }
968
969        self.dispatch_request_plan(parent, plan)
970    }
971
972    fn dispatch_instruments_request(&mut self, cmd: RequestInstruments) -> anyhow::Result<()> {
973        let update_catalog = cmd
974            .params
975            .as_ref()
976            .and_then(|params| params.get_bool(PARAM_UPDATE_CATALOG))
977            .unwrap_or(false);
978        let force_instrument_update = cmd
979            .params
980            .as_ref()
981            .and_then(|params| params.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
982            .unwrap_or(false);
983
984        if update_catalog || force_instrument_update {
985            return self
986                .dispatch_request_to_client(RequestCommand::Instruments(cmd))
987                .map(|_| ());
988        }
989
990        let now_ns = self.clock.borrow().timestamp_ns();
991        let used_client_id = self
992            .get_client(cmd.client_id.as_ref(), cmd.venue.as_ref())
993            .map(|client| client.client_id());
994        let (start_dt, _) = bound_request_dates(cmd.start, cmd.end, now_ns.to_datetime_utc(), true);
995        let start = Some(datetime_to_unix_nanos_or_zero(start_dt));
996        let end = cmd.end.map(datetime_to_unix_nanos_or_zero);
997        let mut data = Vec::new();
998
999        for catalog in self.catalogs.values_mut() {
1000            data.extend(
1001                catalog.instruments(
1002                    &CatalogInstrumentQuery::new()
1003                        .with_range(start, end)
1004                        .with_where_clause(request_filter_expr(cmd.params.as_ref())),
1005                )?,
1006            );
1007        }
1008
1009        if let Some(venue) = cmd.venue {
1010            data.retain(|instrument| instrument.venue() == venue);
1011        }
1012
1013        if instrument_only_last(cmd.params.as_ref()) {
1014            data = latest_instruments(data);
1015        }
1016
1017        let response = DataResponse::Instruments(InstrumentsResponse::new(
1018            cmd.request_id,
1019            resolve_response_client_id(cmd.client_id, used_client_id),
1020            instrument_response_venue(cmd.venue, &data),
1021            data,
1022            // Report the queried window: results sorted by ID cannot be trimmed by `ts_init`
1023            start,
1024            end,
1025            now_ns,
1026            Some(catalog_response_params(cmd.params.as_ref())),
1027        ));
1028        self.response(response);
1029        Ok(())
1030    }
1031}
1032
1033struct RequestCatalogKey {
1034    data_type: NautilusDataType,
1035    identifier: Option<String>,
1036}
1037
1038fn response_write_bounds(
1039    request: &RequestCommand,
1040    response: &DataResponse,
1041) -> (Option<UnixNanos>, Option<UnixNanos>) {
1042    (
1043        request_start(request)
1044            .map(datetime_to_unix_nanos_or_zero)
1045            .or(response.start()),
1046        request_end(request)
1047            .map(datetime_to_unix_nanos_or_zero)
1048            .or(response.end()),
1049    )
1050}
1051
1052fn response_batches(response: &DataResponse) -> anyhow::Result<Vec<(Option<String>, DataBatch)>> {
1053    let batches = match response {
1054        DataResponse::Quotes(r) => response_instrument_batches(
1055            &r.instrument_ids,
1056            &r.data,
1057            |row| row.instrument_id,
1058            DataBatch::Quote,
1059        ),
1060        DataResponse::Trades(r) => response_instrument_batches(
1061            &r.instrument_ids,
1062            &r.data,
1063            |row| row.instrument_id,
1064            DataBatch::Trade,
1065        ),
1066        DataResponse::BookDeltas(r) => vec![(
1067            Some(r.instrument_id.to_string()),
1068            DataBatch::BookDelta(r.data.clone()),
1069        )],
1070        DataResponse::BookDepth(r) => vec![(
1071            Some(r.instrument_id.to_string()),
1072            DataBatch::BookDepth(r.data.clone()),
1073        )],
1074        DataResponse::FundingRates(r) => vec![(
1075            Some(r.instrument_id.to_string()),
1076            DataBatch::FundingRate(r.data.clone()),
1077        )],
1078        DataResponse::Bars(r) => {
1079            vec![(Some(r.bar_type.to_string()), DataBatch::Bar(r.data.clone()))]
1080        }
1081        DataResponse::Data(r) => {
1082            let values = if let Some(data) = r.data.downcast_ref::<Vec<CustomData>>() {
1083                data.clone()
1084            } else if let Some(data) = r.data.downcast_ref::<CustomData>() {
1085                vec![data.clone()]
1086            } else {
1087                anyhow::bail!("Custom response payload is not catalog data");
1088            };
1089
1090            vec![(
1091                r.data_type.identifier().map(String::from),
1092                DataBatch::Custom(values.into()),
1093            )]
1094        }
1095        _ => Vec::new(),
1096    };
1097
1098    Ok(batches)
1099}
1100
1101fn response_instrument_batches<T: Copy>(
1102    requested: &[InstrumentId],
1103    data: &BatchView<T>,
1104    instrument_id: impl Fn(&T) -> InstrumentId,
1105    into_batch: impl Fn(BatchView<T>) -> DataBatch,
1106) -> Vec<(Option<String>, DataBatch)> {
1107    let ids = response_instrument_ids(requested, data.iter().map(&instrument_id));
1108    let single = ids.len() == 1;
1109    ids.into_iter()
1110        .map(|id| {
1111            let data = if single {
1112                data.clone()
1113            } else {
1114                data.iter()
1115                    .filter(|row| instrument_id(row) == id)
1116                    .copied()
1117                    .collect::<Vec<_>>()
1118                    .into()
1119            };
1120
1121            (Some(id.to_string()), into_batch(data))
1122        })
1123        .collect()
1124}
1125
1126struct CatalogCoverage {
1127    catalog: Ustr,
1128    missing: Vec<(u64, u64)>,
1129}
1130
1131enum RequestLeg {
1132    Catalog {
1133        request: RequestCommand,
1134        catalog: Ustr,
1135    },
1136    Client(RequestCommand),
1137    Response(DataResponse),
1138}
1139
1140impl RequestLeg {
1141    fn request_id(&self) -> UUID4 {
1142        match self {
1143            Self::Catalog { request, .. } | Self::Client(request) => *request.request_id(),
1144            Self::Response(response) => *response.correlation_id(),
1145        }
1146    }
1147}
1148
1149fn with_request_instrument_ids(parent: &RequestCommand, ids: Vec<InstrumentId>) -> RequestCommand {
1150    let mut request = parent.clone();
1151    match &mut request {
1152        RequestCommand::Quotes(r) => r.instrument_ids = ids,
1153        RequestCommand::Trades(r) => r.instrument_ids = ids,
1154        _ => unreachable!("catalog batch planning only supports quotes and trades"),
1155    }
1156
1157    request
1158}
1159
1160pub(super) fn is_date_range_variant(req: &RequestCommand) -> bool {
1161    matches!(
1162        req,
1163        RequestCommand::Data(_)
1164            | RequestCommand::Instrument(_)
1165            | RequestCommand::Instruments(_)
1166            | RequestCommand::Quotes(_)
1167            | RequestCommand::Trades(_)
1168            | RequestCommand::FundingRates(_)
1169            | RequestCommand::Bars(_)
1170            | RequestCommand::BookDeltas(_)
1171            | RequestCommand::BookDepth(_)
1172    )
1173}
1174
1175fn request_identifier(req: &RequestCommand) -> Option<RequestCatalogKey> {
1176    match req {
1177        RequestCommand::Data(cmd) => Some(RequestCatalogKey {
1178            data_type: NautilusDataType::Custom {
1179                type_name: cmd.data_type.type_name().to_string(),
1180            },
1181            identifier: cmd.data_type.identifier().map(String::from),
1182        }),
1183        RequestCommand::Quotes(cmd) => Some(RequestCatalogKey::new(
1184            NautilusDataType::QuoteTick,
1185            Some(cmd.single_instrument_id().ok()?.to_string()),
1186        )),
1187        RequestCommand::Trades(cmd) => Some(RequestCatalogKey::new(
1188            NautilusDataType::TradeTick,
1189            Some(cmd.single_instrument_id().ok()?.to_string()),
1190        )),
1191        RequestCommand::FundingRates(cmd) => Some(RequestCatalogKey::new(
1192            NautilusDataType::FundingRateUpdate,
1193            Some(cmd.single_instrument_id().ok()?.to_string()),
1194        )),
1195        RequestCommand::Bars(cmd) => Some(RequestCatalogKey::new(
1196            NautilusDataType::Bar,
1197            Some(cmd.single_bar_type().ok()?.to_string()),
1198        )),
1199        RequestCommand::BookDeltas(cmd) => Some(RequestCatalogKey::new(
1200            NautilusDataType::OrderBookDelta,
1201            Some(cmd.single_instrument_id().ok()?.to_string()),
1202        )),
1203        RequestCommand::BookDepth(cmd) => Some(RequestCatalogKey::new(
1204            NautilusDataType::OrderBookDepth,
1205            Some(cmd.single_instrument_id().ok()?.to_string()),
1206        )),
1207        _ => None,
1208    }
1209}
1210
1211impl RequestCatalogKey {
1212    fn new(data_type: NautilusDataType, identifier: Option<String>) -> Self {
1213        Self {
1214            data_type,
1215            identifier,
1216        }
1217    }
1218
1219    fn catalog_data_type(&self) -> NautilusDataType {
1220        self.data_type.clone()
1221    }
1222}
1223
1224fn catalog_missing_intervals(
1225    catalog: &mut DataCatalog,
1226    start: u64,
1227    end: u64,
1228    key: &RequestCatalogKey,
1229) -> anyhow::Result<Vec<(u64, u64)>> {
1230    catalog.get_missing_intervals_for_request(
1231        UnixNanos::from(start),
1232        UnixNanos::from(end),
1233        key.catalog_data_type(),
1234        key.identifier.as_deref(),
1235    )
1236}
1237
1238fn quote_ticks_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<QuoteTick>> {
1239    match data {
1240        DataBatch::Quote(data) => Ok(data),
1241        _ => anyhow::bail!("expected a Quote batch, found another data batch variant"),
1242    }
1243}
1244
1245fn trade_ticks_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<TradeTick>> {
1246    match data {
1247        DataBatch::Trade(data) => Ok(data),
1248        _ => anyhow::bail!("expected a Trade batch, found another data batch variant"),
1249    }
1250}
1251
1252fn funding_rates_from_query_result(
1253    data: DataBatch,
1254) -> anyhow::Result<BatchView<FundingRateUpdate>> {
1255    match data {
1256        DataBatch::FundingRate(data) => Ok(data),
1257        _ => anyhow::bail!("expected a FundingRate batch, found another data batch variant"),
1258    }
1259}
1260
1261fn bars_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<Bar>> {
1262    match data {
1263        DataBatch::Bar(data) => Ok(data),
1264        _ => anyhow::bail!("expected a Bar batch, found another data batch variant"),
1265    }
1266}
1267
1268fn order_book_deltas_from_query_result(
1269    data: DataBatch,
1270) -> anyhow::Result<BatchView<OrderBookDelta>> {
1271    match data {
1272        DataBatch::BookDelta(data) => Ok(data),
1273        _ => anyhow::bail!("expected a BookDelta batch, found another data batch variant"),
1274    }
1275}
1276
1277fn order_book_depths_from_query_result(
1278    data: DataBatch,
1279) -> anyhow::Result<BatchView<OrderBookDepth>> {
1280    match data {
1281        DataBatch::BookDepth(data) => Ok(data),
1282        _ => anyhow::bail!("expected a BookDepth batch, found another data batch variant"),
1283    }
1284}
1285
1286fn request_limit(request: &RequestCommand) -> Option<std::num::NonZeroUsize> {
1287    match request {
1288        RequestCommand::Quotes(r) => r.limit,
1289        RequestCommand::Trades(r) => r.limit,
1290        RequestCommand::Bars(r) => r.limit,
1291        RequestCommand::FundingRates(r) => r.limit,
1292        RequestCommand::BookDeltas(r) => r.limit,
1293        RequestCommand::BookDepth(r) => r.limit,
1294        RequestCommand::Data(r) => r.limit,
1295        _ => None,
1296    }
1297}
1298
1299fn request_has_schema(request: &RequestCommand) -> bool {
1300    matches!(
1301        request,
1302        RequestCommand::Quotes(_) | RequestCommand::Trades(_)
1303    ) && request_params(request).is_some_and(|params| params.contains_key("schema"))
1304}
1305
1306fn request_contains_batch(
1307    request: &RequestCommand,
1308    identifier: Option<&str>,
1309    batch: &DataBatch,
1310) -> bool {
1311    let contains = |ids: &[InstrumentId]| {
1312        identifier
1313            .and_then(|value| value.parse::<InstrumentId>().ok())
1314            .is_some_and(|id| ids.contains(&id))
1315    };
1316
1317    match (request, batch) {
1318        (RequestCommand::Quotes(r), DataBatch::Quote(_)) => contains(&r.instrument_ids),
1319        (RequestCommand::Trades(r), DataBatch::Trade(_)) => contains(&r.instrument_ids),
1320        (RequestCommand::BookDeltas(r), DataBatch::BookDelta(rows)) => {
1321            contains(&r.instrument_ids)
1322                && rows
1323                    .iter()
1324                    .all(|row| r.instrument_ids.contains(&row.instrument_id))
1325        }
1326        (RequestCommand::BookDepth(r), DataBatch::BookDepth(rows)) => {
1327            contains(&r.instrument_ids)
1328                && rows
1329                    .iter()
1330                    .all(|row| r.instrument_ids.contains(&row.instrument_id))
1331        }
1332        (RequestCommand::FundingRates(r), DataBatch::FundingRate(rows)) => {
1333            contains(&r.instrument_ids)
1334                && rows
1335                    .iter()
1336                    .all(|row| r.instrument_ids.contains(&row.instrument_id))
1337        }
1338        (RequestCommand::Bars(r), DataBatch::Bar(rows)) => {
1339            identifier
1340                .and_then(|value| value.parse::<BarType>().ok())
1341                .is_some_and(|bar_type| r.bar_types.contains(&bar_type))
1342                && rows.iter().all(|row| r.bar_types.contains(&row.bar_type))
1343        }
1344        (RequestCommand::Data(_), DataBatch::Custom(_)) => {
1345            request_identifier(request).is_some_and(|key| key.identifier.as_deref() == identifier)
1346        }
1347        _ => false,
1348    }
1349}
1350
1351fn response_instrument_ids(
1352    requested: &[InstrumentId],
1353    actual: impl Iterator<Item = InstrumentId>,
1354) -> Vec<InstrumentId> {
1355    let actual = actual.collect::<AHashSet<_>>();
1356    if actual.iter().all(|id| requested.contains(id)) {
1357        return requested.to_vec();
1358    }
1359
1360    let mut ids = actual.into_iter().collect::<Vec<_>>();
1361    ids.sort_unstable();
1362    ids
1363}
1364
1365fn request_start(req: &RequestCommand) -> Option<Timestamp> {
1366    match req {
1367        RequestCommand::Data(cmd) => cmd.start,
1368        RequestCommand::Instrument(cmd) => cmd.start,
1369        RequestCommand::Instruments(cmd) => cmd.start,
1370        RequestCommand::Quotes(cmd) => cmd.start,
1371        RequestCommand::Trades(cmd) => cmd.start,
1372        RequestCommand::FundingRates(cmd) => cmd.start,
1373        RequestCommand::Bars(cmd) => cmd.start,
1374        RequestCommand::BookDeltas(cmd) => cmd.start,
1375        RequestCommand::BookDepth(cmd) => cmd.start,
1376        _ => None,
1377    }
1378}
1379
1380fn request_end(req: &RequestCommand) -> Option<Timestamp> {
1381    match req {
1382        RequestCommand::Data(cmd) => cmd.end,
1383        RequestCommand::Instrument(cmd) => cmd.end,
1384        RequestCommand::Instruments(cmd) => cmd.end,
1385        RequestCommand::Quotes(cmd) => cmd.end,
1386        RequestCommand::Trades(cmd) => cmd.end,
1387        RequestCommand::FundingRates(cmd) => cmd.end,
1388        RequestCommand::Bars(cmd) => cmd.end,
1389        RequestCommand::BookDeltas(cmd) => cmd.end,
1390        RequestCommand::BookDepth(cmd) => cmd.end,
1391        _ => None,
1392    }
1393}
1394
1395fn request_filter_expr(params: Option<&Params>) -> Option<String> {
1396    params
1397        .and_then(|params| params.get_str("filter_expr"))
1398        .filter(|filter_expr| !filter_expr.is_empty())
1399        .map(ToString::to_string)
1400}
1401
1402fn bound_request_dates(
1403    start: Option<Timestamp>,
1404    end: Option<Timestamp>,
1405    now: Timestamp,
1406    query_past_data: bool,
1407) -> (Timestamp, Timestamp) {
1408    let zero = Timestamp::UNIX_EPOCH;
1409    let mut start = start.unwrap_or(zero);
1410    let mut end = end.unwrap_or(now);
1411
1412    if query_past_data {
1413        if start > now {
1414            start = now;
1415        }
1416
1417        if end > now {
1418            end = now;
1419        }
1420    }
1421
1422    (start, end)
1423}
1424
1425fn datetime_to_unix_nanos_or_zero(dt: Timestamp) -> UnixNanos {
1426    UnixNanos::from(u64::try_from(dt.as_nanosecond().max(0)).unwrap_or(0))
1427}
1428
1429fn catalog_request_start(request: &RequestCommand, start: Timestamp) -> Timestamp {
1430    // The parent keeps its requested start so fan-in trims the replayed snapshot window
1431    if matches!(request, RequestCommand::BookDeltas(_))
1432        && request_params(request)
1433            .and_then(|p| p.get_bool(PARAM_FROM_DAY_START))
1434            .unwrap_or(true)
1435    {
1436        floor_to_utc_day(start)
1437    } else {
1438        start
1439    }
1440}
1441
1442fn floor_to_utc_day(dt: Timestamp) -> Timestamp {
1443    let midnight = jiff::tz::Offset::UTC.to_datetime(dt).date().at(0, 0, 0, 0);
1444    jiff::tz::Offset::UTC
1445        .to_timestamp(midnight)
1446        .expect("midnight UTC is always valid")
1447}
1448
1449fn with_dates_for_pipeline(
1450    req: &RequestCommand,
1451    start: Option<Timestamp>,
1452    end: Option<Timestamp>,
1453    ts_init: UnixNanos,
1454) -> RequestCommand {
1455    let new_id = UUID4::new();
1456
1457    match req {
1458        RequestCommand::Quotes(cmd) => RequestCommand::Quotes(RequestQuotes {
1459            instrument_ids: cmd.instrument_ids.clone(),
1460            start,
1461            end,
1462            limit: cmd.limit,
1463            client_id: cmd.client_id,
1464            request_id: new_id,
1465            ts_init,
1466            params: cmd.params.clone(),
1467        }),
1468        RequestCommand::Trades(cmd) => RequestCommand::Trades(RequestTrades {
1469            instrument_ids: cmd.instrument_ids.clone(),
1470            start,
1471            end,
1472            limit: cmd.limit,
1473            client_id: cmd.client_id,
1474            request_id: new_id,
1475            ts_init,
1476            params: cmd.params.clone(),
1477        }),
1478        RequestCommand::FundingRates(cmd) => RequestCommand::FundingRates(RequestFundingRates {
1479            instrument_ids: cmd.instrument_ids.clone(),
1480            start,
1481            end,
1482            limit: cmd.limit,
1483            client_id: cmd.client_id,
1484            request_id: new_id,
1485            ts_init,
1486            params: cmd.params.clone(),
1487        }),
1488        RequestCommand::BookDeltas(cmd) => RequestCommand::BookDeltas(RequestBookDeltas {
1489            instrument_ids: cmd.instrument_ids.clone(),
1490            start,
1491            end,
1492            limit: cmd.limit,
1493            client_id: cmd.client_id,
1494            request_id: new_id,
1495            ts_init,
1496            params: cmd.params.clone(),
1497        }),
1498        RequestCommand::BookDepth(cmd) => RequestCommand::BookDepth(RequestBookDepth {
1499            instrument_ids: cmd.instrument_ids.clone(),
1500            start,
1501            end,
1502            limit: cmd.limit,
1503            depth: cmd.depth,
1504            client_id: cmd.client_id,
1505            request_id: new_id,
1506            ts_init,
1507            params: cmd.params.clone(),
1508        }),
1509        RequestCommand::Data(cmd) => RequestCommand::Data(RequestCustomData {
1510            client_id: cmd.client_id,
1511            data_type: cmd.data_type.clone(),
1512            start,
1513            end,
1514            limit: cmd.limit,
1515            request_id: new_id,
1516            ts_init,
1517            params: cmd.params.clone(),
1518        }),
1519        RequestCommand::Bars(cmd) => RequestCommand::Bars(RequestBars {
1520            bar_types: cmd.bar_types.clone(),
1521            start,
1522            end,
1523            limit: cmd.limit,
1524            client_id: cmd.client_id,
1525            request_id: new_id,
1526            ts_init,
1527            params: cmd.params.clone(),
1528        }),
1529        // `Join` and the non-date-range variants should never reach this path; the dispatcher
1530        // gates on `is_date_range_variant` first. Cloning preserves behavior if a caller
1531        // reaches this arm.
1532        _ => req.clone(),
1533    }
1534}
1535
1536fn build_empty_response(
1537    req: &RequestCommand,
1538    start: UnixNanos,
1539    end: UnixNanos,
1540    used_client_id: Option<ClientId>,
1541    ts_init: UnixNanos,
1542) -> anyhow::Result<DataResponse> {
1543    let response = match req {
1544        RequestCommand::Data(cmd) => DataResponse::Data(CustomDataResponse::new(
1545            cmd.request_id,
1546            cmd.client_id,
1547            None,
1548            cmd.data_type.clone(),
1549            Vec::<CustomData>::new(),
1550            Some(start),
1551            Some(end),
1552            ts_init,
1553            cmd.params.clone(),
1554        )),
1555        RequestCommand::Quotes(cmd) => DataResponse::Quotes(QuotesResponse::new(
1556            cmd.request_id,
1557            resolve_response_client_id(cmd.client_id, used_client_id),
1558            cmd.instrument_ids.clone(),
1559            Vec::new(),
1560            Some(start),
1561            Some(end),
1562            ts_init,
1563            cmd.params.clone(),
1564        )),
1565        RequestCommand::Trades(cmd) => DataResponse::Trades(TradesResponse::new(
1566            cmd.request_id,
1567            resolve_response_client_id(cmd.client_id, used_client_id),
1568            cmd.instrument_ids.clone(),
1569            Vec::new(),
1570            Some(start),
1571            Some(end),
1572            ts_init,
1573            cmd.params.clone(),
1574        )),
1575        RequestCommand::FundingRates(cmd) => DataResponse::FundingRates(FundingRatesResponse::new(
1576            cmd.request_id,
1577            resolve_response_client_id(cmd.client_id, used_client_id),
1578            cmd.single_instrument_id()?,
1579            Vec::new(),
1580            Some(start),
1581            Some(end),
1582            ts_init,
1583            cmd.params.clone(),
1584        )),
1585        RequestCommand::Bars(cmd) => DataResponse::Bars(BarsResponse::new(
1586            cmd.request_id,
1587            resolve_response_client_id(cmd.client_id, used_client_id),
1588            cmd.single_bar_type()?,
1589            Vec::new(),
1590            Some(start),
1591            Some(end),
1592            ts_init,
1593            cmd.params.clone(),
1594        )),
1595        RequestCommand::BookDeltas(cmd) => DataResponse::BookDeltas(BookDeltasResponse::new(
1596            cmd.request_id,
1597            resolve_response_client_id(cmd.client_id, used_client_id),
1598            cmd.single_instrument_id()?,
1599            Vec::new(),
1600            Some(start),
1601            Some(end),
1602            ts_init,
1603            cmd.params.clone(),
1604        )),
1605        RequestCommand::BookDepth(cmd) => DataResponse::BookDepth(BookDepthResponse::new(
1606            cmd.request_id,
1607            resolve_response_client_id(cmd.client_id, used_client_id),
1608            cmd.single_instrument_id()?,
1609            Vec::new(),
1610            Some(start),
1611            Some(end),
1612            ts_init,
1613            cmd.params.clone(),
1614        )),
1615        _ => {
1616            anyhow::bail!("Cannot build empty catalog response for non-catalog-eligible request")
1617        }
1618    };
1619
1620    Ok(response)
1621}
1622
1623fn build_quotes_catalog_response(
1624    cmd: &RequestQuotes,
1625    data: BatchView<QuoteTick>,
1626    start: UnixNanos,
1627    end: UnixNanos,
1628    used_client_id: Option<ClientId>,
1629    ts_init: UnixNanos,
1630) -> DataResponse {
1631    let params = catalog_response_params(cmd.params.as_ref());
1632    DataResponse::Quotes(QuotesResponse::new(
1633        cmd.request_id,
1634        resolve_response_client_id(cmd.client_id, used_client_id),
1635        cmd.instrument_ids.clone(),
1636        data,
1637        Some(start),
1638        Some(end),
1639        ts_init,
1640        Some(params),
1641    ))
1642}
1643
1644fn build_trades_catalog_response(
1645    cmd: &RequestTrades,
1646    data: BatchView<TradeTick>,
1647    start: UnixNanos,
1648    end: UnixNanos,
1649    used_client_id: Option<ClientId>,
1650    ts_init: UnixNanos,
1651) -> DataResponse {
1652    let params = catalog_response_params(cmd.params.as_ref());
1653    DataResponse::Trades(TradesResponse::new(
1654        cmd.request_id,
1655        resolve_response_client_id(cmd.client_id, used_client_id),
1656        cmd.instrument_ids.clone(),
1657        data,
1658        Some(start),
1659        Some(end),
1660        ts_init,
1661        Some(params),
1662    ))
1663}
1664
1665fn build_funding_rates_catalog_response(
1666    cmd: &RequestFundingRates,
1667    data: BatchView<FundingRateUpdate>,
1668    start: UnixNanos,
1669    end: UnixNanos,
1670    used_client_id: Option<ClientId>,
1671    ts_init: UnixNanos,
1672) -> anyhow::Result<DataResponse> {
1673    let params = catalog_response_params(cmd.params.as_ref());
1674    Ok(DataResponse::FundingRates(FundingRatesResponse::new(
1675        cmd.request_id,
1676        resolve_response_client_id(cmd.client_id, used_client_id),
1677        cmd.single_instrument_id()?,
1678        data,
1679        Some(start),
1680        Some(end),
1681        ts_init,
1682        Some(params),
1683    )))
1684}
1685
1686fn build_bars_catalog_response(
1687    cmd: &RequestBars,
1688    data: BatchView<Bar>,
1689    start: UnixNanos,
1690    end: UnixNanos,
1691    used_client_id: Option<ClientId>,
1692    ts_init: UnixNanos,
1693) -> anyhow::Result<DataResponse> {
1694    let params = catalog_response_params(cmd.params.as_ref());
1695    Ok(DataResponse::Bars(BarsResponse::new(
1696        cmd.request_id,
1697        resolve_response_client_id(cmd.client_id, used_client_id),
1698        cmd.single_bar_type()?,
1699        data,
1700        Some(start),
1701        Some(end),
1702        ts_init,
1703        Some(params),
1704    )))
1705}
1706
1707fn build_custom_data_catalog_response(
1708    cmd: &RequestCustomData,
1709    data: Vec<CustomData>,
1710    start: UnixNanos,
1711    end: UnixNanos,
1712    ts_init: UnixNanos,
1713) -> DataResponse {
1714    let params = catalog_response_params(cmd.params.as_ref());
1715    DataResponse::Data(CustomDataResponse::new(
1716        cmd.request_id,
1717        cmd.client_id,
1718        None,
1719        cmd.data_type.clone(),
1720        data,
1721        Some(start),
1722        Some(end),
1723        ts_init,
1724        Some(params),
1725    ))
1726}
1727
1728fn build_book_deltas_catalog_response(
1729    cmd: &RequestBookDeltas,
1730    data: BatchView<OrderBookDelta>,
1731    start: UnixNanos,
1732    end: UnixNanos,
1733    used_client_id: Option<ClientId>,
1734    ts_init: UnixNanos,
1735) -> anyhow::Result<DataResponse> {
1736    let params = catalog_response_params(cmd.params.as_ref());
1737    Ok(DataResponse::BookDeltas(BookDeltasResponse::new(
1738        cmd.request_id,
1739        resolve_response_client_id(cmd.client_id, used_client_id),
1740        cmd.single_instrument_id()?,
1741        data,
1742        Some(start),
1743        Some(end),
1744        ts_init,
1745        Some(params),
1746    )))
1747}
1748
1749fn build_book_depth_catalog_response(
1750    cmd: &RequestBookDepth,
1751    data: BatchView<OrderBookDepth>,
1752    start: UnixNanos,
1753    end: UnixNanos,
1754    used_client_id: Option<ClientId>,
1755    ts_init: UnixNanos,
1756) -> anyhow::Result<DataResponse> {
1757    let params = catalog_response_params(cmd.params.as_ref());
1758    Ok(DataResponse::BookDepth(BookDepthResponse::new(
1759        cmd.request_id,
1760        resolve_response_client_id(cmd.client_id, used_client_id),
1761        cmd.single_instrument_id()?,
1762        data,
1763        Some(start),
1764        Some(end),
1765        ts_init,
1766        Some(params),
1767    )))
1768}
1769
1770fn catalog_response_params(existing: Option<&Params>) -> Params {
1771    let mut params = existing.map_or_default(|params| IndexMap::clone(params));
1772    params.insert(PARAM_UPDATE_CATALOG.to_string(), Value::Bool(false));
1773    Params::new(params)
1774}
1775
1776fn custom_data_from_query_result(data: DataBatch) -> anyhow::Result<Vec<CustomData>> {
1777    CustomData::from_batch(data)
1778}
1779
1780fn instrument_only_last(params: Option<&Params>) -> bool {
1781    params
1782        .and_then(|params| params.get_bool("only_last"))
1783        .unwrap_or(true)
1784}
1785
1786fn latest_instruments(data: Vec<InstrumentAny>) -> Vec<InstrumentAny> {
1787    let mut instruments: AHashMap<_, InstrumentAny> = AHashMap::new();
1788
1789    for instrument in data {
1790        let id = instrument.id();
1791        match instruments.get(&id) {
1792            Some(existing) if existing.ts_init() >= instrument.ts_init() => {}
1793            _ => {
1794                instruments.insert(id, instrument);
1795            }
1796        }
1797    }
1798
1799    let mut data: Vec<_> = instruments.into_values().collect();
1800    data.sort_by_key(|instrument| instrument.id().to_string());
1801    data
1802}
1803
1804fn instrument_response_venue(request_venue: Option<Venue>, data: &[InstrumentAny]) -> Venue {
1805    request_venue.unwrap_or_else(|| {
1806        data.iter()
1807            .map(Instrument::venue)
1808            .min_by_key(std::string::ToString::to_string)
1809            .unwrap_or_else(|| Venue::from(CATALOG_CLIENT_ID))
1810    })
1811}
1812
1813fn resolve_response_client_id(
1814    request_client_id: Option<ClientId>,
1815    used_client_id: Option<ClientId>,
1816) -> ClientId {
1817    request_client_id
1818        .or(used_client_id)
1819        .unwrap_or_else(|| ClientId::new(CATALOG_CLIENT_ID))
1820}
1821
1822#[cfg(test)]
1823mod tests {
1824    use std::sync::Arc;
1825
1826    use nautilus_common::messages::data::RequestJoin;
1827    use rstest::rstest;
1828
1829    use super::*;
1830
1831    #[rstest]
1832    #[case::full(0, 3)]
1833    #[case::subrange(1, 2)]
1834    fn test_catalog_quote_response_uses_selected_batch_view(
1835        #[case] start: usize,
1836        #[case] end: usize,
1837    ) {
1838        let first = QuoteTick {
1839            ts_init: UnixNanos::from(11),
1840            ..Default::default()
1841        };
1842
1843        let mut second = first;
1844        second.ts_init = UnixNanos::from(23);
1845        let mut third = first;
1846        third.ts_init = UnixNanos::from(37);
1847        let original = Arc::new(vec![first, second, third]);
1848        let result = quote_ticks_from_query_result(DataBatch::Quote(BatchView::new(
1849            Arc::clone(&original),
1850            start..end,
1851        )))
1852        .unwrap();
1853        assert_eq!(result.as_ref(), &original[start..end]);
1854        assert_eq!(result.range(), start..end);
1855        assert!(Arc::ptr_eq(result.arc(), &original));
1856    }
1857
1858    #[rstest]
1859    fn test_response_batches_preserves_narrowed_payloads(#[values(false, true)] empty: bool) {
1860        use nautilus_model::data::stubs::{stub_delta, stub_depth10};
1861
1862        macro_rules! check {
1863            ($response:ident, $variant:ident, $batch:ident, $key:expr, $seed:expr) => {{
1864                let key = $key;
1865                let mut row = $seed;
1866                row.ts_init = UnixNanos::from(23);
1867                let mut before = row.clone();
1868                before.ts_init = UnixNanos::from(11);
1869                let mut after = row.clone();
1870                after.ts_init = UnixNanos::from(37);
1871                let backing = Arc::new(vec![before, row.clone(), after]);
1872                let end = if empty { 1 } else { 2 };
1873                let response = DataResponse::$variant($response::new(
1874                    UUID4::new(),
1875                    ClientId::from("SOURCE"),
1876                    key,
1877                    BatchView::new(Arc::clone(&backing), 1..end),
1878                    None,
1879                    None,
1880                    UnixNanos::from(41),
1881                    None,
1882                ));
1883
1884                let batches = response_batches(&response).unwrap();
1885
1886                assert_eq!(batches.len(), 1);
1887                assert_eq!(batches[0].0.as_deref(), Some(key.to_string().as_str()));
1888                let DataBatch::$batch(view) = &batches[0].1 else {
1889                    panic!("Unexpected batch {:?}", batches[0].1);
1890                };
1891                assert_eq!(view.range(), 1..end);
1892                assert!(Arc::ptr_eq(view.arc(), &backing));
1893                assert_eq!(
1894                    view.as_ref(),
1895                    if empty {
1896                        &[]
1897                    } else {
1898                        std::slice::from_ref(&row)
1899                    }
1900                );
1901            }};
1902        }
1903
1904        let delta = stub_delta();
1905        let depth = stub_depth10();
1906        let quote = QuoteTick::default();
1907        let trade = TradeTick::default();
1908        let bar = Bar::default();
1909
1910        let funding = FundingRateUpdate::new(
1911            quote.instrument_id,
1912            "0.001".parse().unwrap(),
1913            Some(60),
1914            Some(UnixNanos::from(47)),
1915            UnixNanos::from(17),
1916            UnixNanos::from(23),
1917        );
1918
1919        check!(
1920            BookDeltasResponse,
1921            BookDeltas,
1922            BookDelta,
1923            delta.instrument_id,
1924            delta
1925        );
1926        check!(
1927            BookDepthResponse,
1928            BookDepth,
1929            BookDepth,
1930            depth.instrument_id,
1931            depth
1932        );
1933        check!(QuotesResponse, Quotes, Quote, quote.instrument_id, quote);
1934        check!(TradesResponse, Trades, Trade, trade.instrument_id, trade);
1935        check!(
1936            FundingRatesResponse,
1937            FundingRates,
1938            FundingRate,
1939            funding.instrument_id,
1940            funding
1941        );
1942        check!(BarsResponse, Bars, Bar, bar.bar_type, bar);
1943    }
1944
1945    #[rstest]
1946    #[case::deltas(NautilusDataType::OrderBookDelta)]
1947    #[case::depth(NautilusDataType::OrderBookDepth)]
1948    #[case::quotes(NautilusDataType::QuoteTick)]
1949    #[case::trades(NautilusDataType::TradeTick)]
1950    #[case::bars(NautilusDataType::Bar)]
1951    #[case::funding(NautilusDataType::FundingRateUpdate)]
1952    fn test_request_contains_batch_uses_visible_requested_members(
1953        #[case] kind: NautilusDataType,
1954        #[values(false, true)] remapped: bool,
1955        #[values(false, true)] empty: bool,
1956    ) {
1957        use nautilus_model::data::stubs::{stub_delta, stub_depth10};
1958
1959        let requested = InstrumentId::from("AUD/USD.SIM");
1960
1961        let actual = if remapped {
1962            InstrumentId::from("GBP/USD.SIM")
1963        } else {
1964            requested
1965        };
1966
1967        let requested_bar = format!("{requested}-1-MINUTE-LAST-EXTERNAL")
1968            .parse::<BarType>()
1969            .unwrap();
1970        let actual_bar = format!("{actual}-1-MINUTE-LAST-EXTERNAL")
1971            .parse::<BarType>()
1972            .unwrap();
1973
1974        macro_rules! request {
1975            ($variant:ident, $request:ident, $id:expr $(, $depth:expr)?) => {
1976                RequestCommand::$variant($request::new(
1977                    $id, None, None, None, $($depth,)? None, UUID4::new(), UnixNanos::from(37), None,
1978                ))
1979            };
1980        }
1981
1982        macro_rules! response {
1983            ($variant:ident, $response:ident, $key:expr, $row:expr) => {{
1984                let data = BatchView::new(Arc::new(vec![$row]), 0..usize::from(!empty));
1985                DataResponse::$variant($response::new(
1986                    UUID4::new(),
1987                    ClientId::from("SOURCE"),
1988                    $key,
1989                    data,
1990                    None,
1991                    None,
1992                    UnixNanos::from(41),
1993                    None,
1994                ))
1995            }};
1996        }
1997
1998        let (request, response) = match kind {
1999            NautilusDataType::OrderBookDelta => {
2000                let row = OrderBookDelta {
2001                    instrument_id: actual,
2002                    ..stub_delta()
2003                };
2004
2005                (
2006                    request!(BookDeltas, RequestBookDeltas, requested),
2007                    response!(BookDeltas, BookDeltasResponse, requested, row),
2008                )
2009            }
2010            NautilusDataType::OrderBookDepth => {
2011                let row = OrderBookDepth {
2012                    instrument_id: actual,
2013                    ..stub_depth10()
2014                };
2015
2016                (
2017                    request!(BookDepth, RequestBookDepth, requested, None),
2018                    response!(BookDepth, BookDepthResponse, requested, row),
2019                )
2020            }
2021            NautilusDataType::QuoteTick => {
2022                let row = QuoteTick {
2023                    instrument_id: actual,
2024                    ..Default::default()
2025                };
2026
2027                (
2028                    request!(Quotes, RequestQuotes, requested),
2029                    response!(Quotes, QuotesResponse, requested, row),
2030                )
2031            }
2032            NautilusDataType::TradeTick => {
2033                let row = TradeTick {
2034                    instrument_id: actual,
2035                    ..Default::default()
2036                };
2037
2038                (
2039                    request!(Trades, RequestTrades, requested),
2040                    response!(Trades, TradesResponse, requested, row),
2041                )
2042            }
2043            NautilusDataType::Bar => {
2044                let row = Bar {
2045                    bar_type: actual_bar,
2046                    ..Default::default()
2047                };
2048
2049                (
2050                    request!(Bars, RequestBars, requested_bar),
2051                    response!(Bars, BarsResponse, requested_bar, row),
2052                )
2053            }
2054            NautilusDataType::FundingRateUpdate => {
2055                let row = FundingRateUpdate::new(
2056                    actual,
2057                    "0.001".parse().unwrap(),
2058                    Some(60),
2059                    Some(UnixNanos::from(47)),
2060                    UnixNanos::from(17),
2061                    UnixNanos::from(23),
2062                );
2063                (
2064                    request!(FundingRates, RequestFundingRates, requested),
2065                    response!(FundingRates, FundingRatesResponse, requested, row),
2066                )
2067            }
2068            _ => panic!("unexpected data type"),
2069        };
2070
2071        let batches = response_batches(&response).unwrap();
2072        let (identifier, batch) = &batches[0];
2073
2074        assert_eq!(batches.len(), 1);
2075        assert_eq!(
2076            request_contains_batch(&request, identifier.as_deref(), batch),
2077            empty || !remapped,
2078        );
2079    }
2080
2081    #[rstest]
2082    #[case::quotes(true)]
2083    #[case::trades(false)]
2084    fn test_response_batches_keep_expanded_parent_instruments(#[case] quotes: bool) {
2085        let requested = [
2086            InstrumentId::from("ES.FUT.GLBX"),
2087            InstrumentId::from("NQ.FUT.GLBX"),
2088        ];
2089        let first = InstrumentId::from("ESH24.GLBX");
2090        let second = InstrumentId::from("NQH24.GLBX");
2091
2092        let mut response = if quotes {
2093            let rows = vec![
2094                QuoteTick {
2095                    instrument_id: first,
2096                    ..Default::default()
2097                },
2098                QuoteTick {
2099                    instrument_id: second,
2100                    ..Default::default()
2101                },
2102            ];
2103            DataResponse::Quotes(QuotesResponse::new(
2104                UUID4::new(),
2105                ClientId::from("SOURCE"),
2106                requested,
2107                rows,
2108                None,
2109                None,
2110                UnixNanos::from(3),
2111                None,
2112            ))
2113        } else {
2114            let rows = vec![
2115                TradeTick {
2116                    instrument_id: first,
2117                    ..Default::default()
2118                },
2119                TradeTick {
2120                    instrument_id: second,
2121                    ..Default::default()
2122                },
2123            ];
2124            DataResponse::Trades(TradesResponse::new(
2125                UUID4::new(),
2126                ClientId::from("SOURCE"),
2127                requested,
2128                rows,
2129                None,
2130                None,
2131                UnixNanos::from(3),
2132                None,
2133            ))
2134        };
2135
2136        macro_rules! narrow {
2137            ($response:expr, $row:ident) => {{
2138                let mut rows = $response.data.to_vec();
2139                let hidden = $row {
2140                    instrument_id: InstrumentId::from("HIDDEN.GLBX"),
2141                    ..Default::default()
2142                };
2143                rows.insert(0, hidden);
2144                rows.push(hidden);
2145                $response.data = BatchView::new(Arc::new(rows), 1..3);
2146            }};
2147        }
2148
2149        match &mut response {
2150            DataResponse::Quotes(r) => narrow!(r, QuoteTick),
2151            DataResponse::Trades(r) => narrow!(r, TradeTick),
2152            _ => unreachable!(),
2153        }
2154
2155        let batches = response_batches(&response).unwrap();
2156
2157        assert_eq!(batches.len(), 2);
2158
2159        for ((identifier, batch), id) in batches.iter().zip([first, second]) {
2160            assert_eq!(identifier.as_deref(), Some(id.to_string().as_str()));
2161
2162            match batch {
2163                DataBatch::Quote(rows) => assert_eq!(
2164                    &rows[..],
2165                    &[QuoteTick {
2166                        instrument_id: id,
2167                        ..Default::default()
2168                    }]
2169                ),
2170                DataBatch::Trade(rows) => assert_eq!(
2171                    &rows[..],
2172                    &[TradeTick {
2173                        instrument_id: id,
2174                        ..Default::default()
2175                    }]
2176                ),
2177                other => panic!("Unexpected batch {other:?}"),
2178            }
2179        }
2180    }
2181
2182    #[rstest]
2183    fn test_build_empty_response_rejects_non_catalog_variant() {
2184        let request = RequestCommand::Join(RequestJoin::new(
2185            vec![UUID4::new()],
2186            None,
2187            None,
2188            UUID4::new(),
2189            UnixNanos::default(),
2190            None,
2191            None,
2192        ));
2193
2194        let result = build_empty_response(
2195            &request,
2196            UnixNanos::from(1u64),
2197            UnixNanos::from(2u64),
2198            None,
2199            UnixNanos::from(3u64),
2200        );
2201
2202        assert_eq!(
2203            result.unwrap_err().to_string(),
2204            "Cannot build empty catalog response for non-catalog-eligible request"
2205        );
2206    }
2207}