Skip to main content

nautilus_data/engine/
streaming.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use ahash::{AHashMap, AHashSet};
17use indexmap::IndexMap;
18use jiff::Timestamp;
19use nautilus_common::messages::data::{
20    BarsResponse, BookDeltasResponse, BookDepthResponse, CustomDataResponse, DataRequestFailed,
21    DataResponse, FundingRatesResponse, InstrumentResponse, InstrumentsResponse, QuotesResponse,
22    RequestBars, RequestBookDeltas, RequestBookDepth, RequestCommand, RequestCustomData,
23    RequestFundingRates, RequestInstrument, RequestInstruments, RequestQuotes, RequestTrades,
24    SubscribeBars, SubscribeCommand, SubscribeCustomData, SubscribeQuotes, SubscribeTrades,
25    TradesResponse,
26};
27use nautilus_core::{
28    Params, UUID4, UnixNanos,
29    correctness::{FAILED, check_key_not_in_map},
30};
31use nautilus_model::{
32    data::{
33        Bar, BarType, BatchView, CustomData, DataBatch, FromDataBatch, FundingRateUpdate,
34        NautilusDataType, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick,
35    },
36    identifiers::{ClientId, InstrumentId, Venue},
37    instruments::{Instrument, InstrumentAny},
38};
39use nautilus_persistence::catalog::traits::{CatalogInstrumentQuery, CatalogQuery, DataCatalog};
40use serde_json::Value;
41use ustr::Ustr;
42
43use super::{DataEngine, requests::request_params};
44
45const PARAM_SKIP_CATALOG_DATA: &str = "skip_catalog_data";
46pub(super) const PARAM_UPDATE_CATALOG: &str = "update_catalog";
47const PARAM_FORCE_INSTRUMENT_UPDATE: &str = "force_instrument_update";
48const PARAM_SUBSCRIPTION_NAME: &str = "subscription_name";
49const PARAM_FROM_DAY_START: &str = "from_day_start";
50const CATALOG_CLIENT_ID: &str = "CATALOG";
51
52pub(crate) type CatalogMap = AHashMap<Ustr, DataCatalog>;
53
54impl DataEngine {
55    /// Registers the `catalog` with the engine with an optional specific `name`.
56    ///
57    /// # Panics
58    ///
59    /// Panics if a catalog with the same `name` has already been registered.
60    pub fn register_catalog(&mut self, catalog: DataCatalog, name: Option<&str>) {
61        let name = Ustr::from(name.unwrap_or("catalog_0"));
62
63        check_key_not_in_map(&name, &self.catalogs, "name", "catalogs").expect(FAILED);
64
65        self.catalogs.insert(name, catalog);
66        log::info!("Registered catalog <{name}>");
67    }
68
69    pub(super) fn subscribe_command_with_prefilled_start_ns(
70        &mut self,
71        cmd: SubscribeCommand,
72    ) -> anyhow::Result<SubscribeCommand> {
73        match cmd {
74            SubscribeCommand::Quotes(cmd)
75                if cmd.instrument_ids.len() == 1
76                    && Self::is_start_ns_missing(cmd.params.as_ref()) =>
77            {
78                let identifier = cmd.instrument_ids[0].to_string();
79                let params = self.params_with_prefilled_start_ns(
80                    cmd.params.as_ref(),
81                    &NautilusDataType::QuoteTick,
82                    &identifier,
83                )?;
84                Ok(SubscribeCommand::Quotes(SubscribeQuotes { params, ..cmd }))
85            }
86            SubscribeCommand::Trades(cmd)
87                if cmd.instrument_ids.len() == 1
88                    && Self::is_start_ns_missing(cmd.params.as_ref()) =>
89            {
90                let identifier = cmd.instrument_ids[0].to_string();
91                let params = self.params_with_prefilled_start_ns(
92                    cmd.params.as_ref(),
93                    &NautilusDataType::TradeTick,
94                    &identifier,
95                )?;
96                Ok(SubscribeCommand::Trades(SubscribeTrades { params, ..cmd }))
97            }
98            SubscribeCommand::Bars(cmd)
99                if cmd.bar_types.len() == 1
100                    && cmd.single_bar_type()?.is_externally_aggregated()
101                    && Self::is_start_ns_missing(cmd.params.as_ref()) =>
102            {
103                let identifier = cmd.single_bar_type()?.to_string();
104                let params = self.params_with_prefilled_start_ns(
105                    cmd.params.as_ref(),
106                    &NautilusDataType::Bar,
107                    &identifier,
108                )?;
109                Ok(SubscribeCommand::Bars(SubscribeBars { params, ..cmd }))
110            }
111            SubscribeCommand::Data(cmd) if Self::is_start_ns_missing(cmd.params.as_ref()) => {
112                let type_name = cmd.data_type.type_name().to_string();
113                let identifier = cmd.data_type.identifier().map(String::from);
114                let params = self.params_with_custom_data_prefilled_start_ns(
115                    cmd.params.as_ref(),
116                    &type_name,
117                    identifier.as_deref(),
118                )?;
119                Ok(SubscribeCommand::Data(SubscribeCustomData {
120                    params,
121                    ..cmd
122                }))
123            }
124            _ => Ok(cmd),
125        }
126    }
127
128    fn is_start_ns_missing(params: Option<&Params>) -> bool {
129        params.is_none_or(|params| !params.contains_key("start_ns"))
130    }
131
132    fn params_with_prefilled_start_ns(
133        &mut self,
134        params: Option<&Params>,
135        data_type: &NautilusDataType,
136        identifier: &str,
137    ) -> anyhow::Result<Option<Params>> {
138        let last_timestamp = self.catalog_last_timestamp(data_type, identifier)?;
139
140        Ok(Some(Self::params_with_start_ns(params, last_timestamp)))
141    }
142
143    fn params_with_custom_data_prefilled_start_ns(
144        &mut self,
145        params: Option<&Params>,
146        type_name: &str,
147        identifier: Option<&str>,
148    ) -> anyhow::Result<Option<Params>> {
149        let last_timestamp = self.catalog_custom_data_last_timestamp(type_name, identifier)?;
150
151        Ok(Some(Self::params_with_start_ns(params, last_timestamp)))
152    }
153
154    fn params_with_start_ns(params: Option<&Params>, last_timestamp: Option<u64>) -> Params {
155        let start_ns = last_timestamp.map_or(Value::Null, |last_timestamp| {
156            Value::from(last_timestamp.saturating_add(1))
157        });
158
159        let mut params = params
160            .map(|params| IndexMap::clone(params))
161            .unwrap_or_default();
162
163        params.insert("start_ns".to_string(), start_ns);
164
165        Params::new(params)
166    }
167
168    fn catalog_last_timestamp(
169        &mut self,
170        data_type: &NautilusDataType,
171        identifier: &str,
172    ) -> anyhow::Result<Option<u64>> {
173        for catalog in self.catalogs.values_mut() {
174            if let Some(last_timestamp) =
175                catalog.query_last_timestamp(data_type.clone(), Some(identifier))?
176            {
177                return Ok(Some(last_timestamp));
178            }
179        }
180
181        Ok(None)
182    }
183
184    fn catalog_custom_data_last_timestamp(
185        &mut self,
186        type_name: &str,
187        identifier: Option<&str>,
188    ) -> anyhow::Result<Option<u64>> {
189        // `make_path_custom_data` / `get_directory_intervals` are inherent on
190        // `ParquetDataCatalog` and not exposed through `Catalog`. Use the
191        // trait-level `query_last_timestamp` with `NautilusDataType::Custom` so the
192        // path works for any backend that implements the trait.
193        let data_type = NautilusDataType::Custom {
194            type_name: type_name.to_string(),
195        };
196
197        for catalog in self.catalogs.values_mut() {
198            if let Some(last_timestamp) =
199                catalog.query_last_timestamp(data_type.clone(), identifier)?
200            {
201                return Ok(Some(last_timestamp));
202            }
203        }
204
205        Ok(None)
206    }
207
208    /// Rejects a request asking for catalog write-back (`update_catalog`) that cannot get one:
209    /// no catalog is registered, or the request carries a source schema or a record limit.
210    pub(super) fn validate_catalog_update(&self, request: &RequestCommand) -> anyhow::Result<()> {
211        if request_params(request).and_then(|p| p.get_bool(PARAM_UPDATE_CATALOG)) != Some(true) {
212            return Ok(());
213        }
214
215        anyhow::ensure!(
216            !self.catalogs.is_empty(),
217            "Catalog write-back requires a registered catalog"
218        );
219        anyhow::ensure!(
220            !request_has_schema(request),
221            "Catalog write-back is unsupported for requests with a source schema"
222        );
223        anyhow::ensure!(
224            request_limit(request).is_none(),
225            "Catalog write-back is unsupported for requests with a record limit"
226        );
227        Ok(())
228    }
229
230    /// Writes `response` back to a catalog when `request` asks for it (`update_catalog`).
231    /// `validate_catalog_update` has accepted the request.
232    ///
233    /// # Errors
234    ///
235    /// Returns an error if the requested write-back fails, so the response data is not
236    /// persisted: an identifier's coverage is unverified, or a write fails.
237    pub(super) fn update_catalog_from_response(
238        &mut self,
239        response: &DataResponse,
240        request: &RequestCommand,
241    ) -> anyhow::Result<()> {
242        let Some(params) =
243            request_params(request).filter(|p| p.get_bool(PARAM_UPDATE_CATALOG).unwrap_or(false))
244        else {
245            return Ok(());
246        };
247
248        if response
249            .params()
250            .is_some_and(|p| p.get_bool(PARAM_UPDATE_CATALOG) == Some(false))
251        {
252            return Ok(());
253        }
254
255        if let DataResponse::Instrument(value) = response {
256            return self.write_response_instruments(std::slice::from_ref(&value.data), params);
257        }
258
259        if let DataResponse::Instruments(value) = response {
260            return self.write_response_instruments(&value.data, params);
261        }
262
263        let batches = match response_batches(response) {
264            Ok(batches) => batches,
265            Err(e) => {
266                log::error!("Cannot prepare response catalog write: {e}");
267                anyhow::bail!(
268                    "Catalog write-back failed: cannot prepare the response batches: {e}"
269                );
270            }
271        };
272
273        let bounds = response_write_bounds(request, response);
274        let data_type = response.data_kind();
275        let mut failures = Vec::new();
276
277        for (identifier, batch) in batches {
278            if !request_contains_batch(request, identifier.as_deref(), &batch) {
279                log::warn!(
280                    "Skipping catalog write-back because identifier coverage is unverified, request={}, identifier={identifier:?}",
281                    request.request_id()
282                );
283                failures.push(format!(
284                    "Catalog write-back failed: identifier coverage is unverified, identifier={identifier:?}"
285                ));
286                continue;
287            }
288
289            if let Err(e) =
290                self.write_response_batch(&batch, &data_type, identifier.as_deref(), params, bounds)
291            {
292                log::error!("Catalog write-back failed for {identifier:?}: {e}");
293                failures.push(format!("Catalog write-back failed for {identifier:?}: {e}"));
294            }
295        }
296
297        if !failures.is_empty() {
298            anyhow::bail!(failures.join("; "));
299        }
300        Ok(())
301    }
302
303    fn write_response_batch(
304        &mut self,
305        batch: &DataBatch,
306        data_type: &NautilusDataType,
307        identifier: Option<&str>,
308        params: &Params,
309        bounds: (Option<UnixNanos>, Option<UnixNanos>),
310    ) -> anyhow::Result<()> {
311        let name = self.catalog_for_update(data_type, identifier, params)?;
312        let catalog = self
313            .catalogs
314            .get_mut(&name)
315            .expect("selected catalog is registered");
316        let (start, end) = bounds;
317        if batch.is_empty() {
318            if let (Some(start), Some(end)) = (start, end) {
319                catalog.record_empty_coverage(data_type.clone(), identifier, start, end)?;
320            }
321
322            return Ok(());
323        }
324
325        catalog.write_data_batch(batch, start, end, Some(params.clone()))
326    }
327
328    fn write_response_instruments(
329        &mut self,
330        instruments: &[InstrumentAny],
331        params: &Params,
332    ) -> anyhow::Result<()> {
333        let force = params
334            .get_bool(PARAM_FORCE_INSTRUMENT_UPDATE)
335            .unwrap_or(false);
336        let mut failures = Vec::new();
337
338        for instrument in instruments {
339            let identifier = instrument.id().to_string();
340
341            let result = (|| {
342                let name = self.catalog_for_update(
343                    &NautilusDataType::Instrument,
344                    Some(&identifier),
345                    params,
346                )?;
347                let catalog = self
348                    .catalogs
349                    .get_mut(&name)
350                    .expect("selected catalog is registered");
351
352                if !force {
353                    let query = CatalogInstrumentQuery::new()
354                        .with_instrument_ids(Some(vec![identifier.clone()]));
355
356                    if !catalog.instruments(&query)?.is_empty() {
357                        return Ok(());
358                    }
359                }
360
361                catalog.write_instruments(std::slice::from_ref(instrument))
362            })();
363
364            if let Err(e) = result {
365                log::error!("Instrument catalog write-back failed for {identifier}: {e}");
366                failures.push(format!(
367                    "Catalog write-back failed for instrument {identifier}: {e}"
368                ));
369            }
370        }
371
372        if !failures.is_empty() {
373            anyhow::bail!(failures.join("; "));
374        }
375        Ok(())
376    }
377
378    fn catalog_for_update(
379        &mut self,
380        data_type: &NautilusDataType,
381        identifier: Option<&str>,
382        params: &Params,
383    ) -> anyhow::Result<Ustr> {
384        if let Some(name) = params.get_str("catalog_name").map(Ustr::from) {
385            anyhow::ensure!(
386                self.catalogs.contains_key(&name),
387                "Catalog '{name}' is not registered"
388            );
389            return Ok(name);
390        }
391
392        let mut names: Vec<_> = self.catalogs.keys().copied().collect();
393        names.sort_by(|a, b| a.as_str().cmp(b.as_str()));
394        for name in &names {
395            let catalog = self.catalogs.get_mut(name).expect("catalog key exists");
396            if catalog
397                .query_last_timestamp(data_type.clone(), identifier)?
398                .is_some()
399            {
400                return Ok(*name);
401            }
402        }
403
404        names
405            .first()
406            .copied()
407            .ok_or_else(|| anyhow::anyhow!("No catalog registered"))
408    }
409
410    pub(super) fn catalogs_registered(&self) -> bool {
411        !self.catalogs.is_empty()
412    }
413
414    // Bounds the request window, walks the catalogs to find one whose missing
415    // intervals differ from the full requested range, then fans the parent out via
416    // the pipeline with one catalog leg plus one client leg per missing interval.
417    // With no catalog match and no resolvable client, the engine emits an empty
418    // response keyed by the parent request ID.
419    pub(super) fn dispatch_date_range_request(
420        &mut self,
421        req: RequestCommand,
422    ) -> anyhow::Result<()> {
423        if request_has_schema(&req) {
424            return self.dispatch_request_to_client(req).map(|_| ());
425        }
426
427        if matches!(
428            req,
429            RequestCommand::Instrument(_) | RequestCommand::Instruments(_)
430        ) {
431            return self.dispatch_instrument_catalog_request(req);
432        }
433
434        if matches!(&req, RequestCommand::Quotes(r) if r.instrument_ids.len() > 1)
435            || matches!(&req, RequestCommand::Trades(r) if r.instrument_ids.len() > 1)
436        {
437            return self.dispatch_batch_catalog_request(req);
438        }
439
440        let Some(key) = request_identifier(&req) else {
441            return self.dispatch_request_to_client(req).map(|_| ());
442        };
443
444        let now_ns = self.clock.borrow().timestamp_ns();
445        let now_dt = now_ns.to_datetime_utc();
446        let query_past_data = request_params(&req)
447            .and_then(|p| p.get(PARAM_SUBSCRIPTION_NAME))
448            .is_none();
449
450        let (start_dt, end_dt) = bound_request_dates(
451            request_start(&req),
452            request_end(&req),
453            now_dt,
454            query_past_data,
455        );
456        let start_ns = datetime_to_unix_nanos_or_zero(start_dt);
457        let end_ns = datetime_to_unix_nanos_or_zero(end_dt);
458
459        if start_ns > end_ns {
460            anyhow::bail!(
461                "Cannot dispatch request, start {start_ns} was greater than end {end_ns}"
462            );
463        }
464
465        // Floor the catalog window to the UTC day boundary so the day-start F_SNAPSHOT frame is
466        // selected and read for the snapshot replay; client gaps keep the original window.
467        // The parent request keeps its original start, so the merged response trims back to it.
468        let (catalog_start_dt, catalog_start_ns) = if matches!(req, RequestCommand::BookDeltas(_))
469            && request_params(&req)
470                .and_then(|p| p.get_bool(PARAM_FROM_DAY_START))
471                .unwrap_or(true)
472        {
473            let floored = floor_to_utc_day(start_dt);
474            (floored, datetime_to_unix_nanos_or_zero(floored))
475        } else {
476            (start_dt, start_ns)
477        };
478
479        let query_interval = vec![(start_ns.as_u64(), end_ns.as_u64())];
480        let mut missing_intervals = query_interval.clone();
481        let coverage = self.catalog_coverage(&key, catalog_start_ns.as_u64(), end_ns.as_u64())?;
482
483        let winning_catalog = if let Some(CatalogCoverage {
484            catalog: name,
485            missing: intervals,
486        }) = coverage
487        {
488            // Client legs fill only the requested window, not the pre-start range
489            missing_intervals = if catalog_start_ns == start_ns {
490                intervals
491            } else {
492                catalog_missing_intervals(
493                    self.catalogs
494                        .get_mut(&name)
495                        .expect("selected catalog is registered"),
496                    start_ns.as_u64(),
497                    end_ns.as_u64(),
498                    &key,
499                )?
500            };
501
502            Some(name)
503        } else {
504            None
505        };
506
507        let skip_catalog_data = request_params(&req)
508            .and_then(|p| p.get_bool(PARAM_SKIP_CATALOG_DATA))
509            .unwrap_or(false);
510
511        // When `skip_catalog_data` is set the client must serve the full parent window;
512        // dropping the catalog leg without resetting the missing intervals would leave
513        // the catalog-covered range unanswered.
514        if skip_catalog_data {
515            missing_intervals = query_interval;
516        }
517
518        let client_id = req.client_id().copied();
519        let venue = req.venue().copied();
520        let has_client = self
521            .get_client(client_id.as_ref(), venue.as_ref())
522            .is_some();
523
524        if !has_client {
525            missing_intervals.clear();
526        }
527
528        let mut plan = Vec::new();
529
530        if !skip_catalog_data && let Some(catalog) = winning_catalog {
531            let request =
532                with_dates_for_pipeline(&req, Some(catalog_start_dt), Some(end_dt), now_ns);
533            plan.push(RequestLeg::Catalog { request, catalog });
534        }
535
536        for (start, end) in missing_intervals {
537            let start = UnixNanos::from(start).to_datetime_utc();
538            let end = UnixNanos::from(end).to_datetime_utc();
539            let request = with_dates_for_pipeline(&req, Some(start), Some(end), now_ns);
540            plan.push(RequestLeg::Client(request));
541        }
542
543        self.dispatch_request_plan(req, plan)
544    }
545
546    fn dispatch_batch_catalog_request(&mut self, parent: RequestCommand) -> anyhow::Result<()> {
547        let ids = match &parent {
548            RequestCommand::Quotes(r) => r.instrument_ids.clone(),
549            RequestCommand::Trades(r) => r.instrument_ids.clone(),
550            _ => unreachable!("only quote and trade lists use catalog batch planning"),
551        };
552
553        let now_ns = self.clock.borrow().timestamp_ns();
554        let query_past = request_params(&parent)
555            .and_then(|p| p.get(PARAM_SUBSCRIPTION_NAME))
556            .is_none();
557        let (start, end) = bound_request_dates(
558            request_start(&parent),
559            request_end(&parent),
560            now_ns.to_datetime_utc(),
561            query_past,
562        );
563        let start_ns = datetime_to_unix_nanos_or_zero(start);
564        let end_ns = datetime_to_unix_nanos_or_zero(end);
565        anyhow::ensure!(start_ns <= end_ns, "Request start exceeds end");
566        let skip_catalog = request_params(&parent)
567            .and_then(|p| p.get_bool(PARAM_SKIP_CATALOG_DATA))
568            .unwrap_or(false);
569        let full = vec![(start_ns.as_u64(), end_ns.as_u64())];
570        let mut clients: IndexMap<(u64, u64), Vec<InstrumentId>> = IndexMap::new();
571        let mut plan = Vec::new();
572
573        for id in ids {
574            let mut member = with_request_instrument_ids(&parent, vec![id]);
575            let key = request_identifier(&member).expect("single quote or trade identifier");
576            let mut missing = full.clone();
577            if !skip_catalog
578                && let Some(CatalogCoverage {
579                    catalog,
580                    missing: intervals,
581                }) = self.catalog_coverage(&key, start_ns.as_u64(), end_ns.as_u64())?
582            {
583                missing = intervals;
584                member = with_dates_for_pipeline(&member, Some(start), Some(end), now_ns);
585                plan.push(RequestLeg::Catalog {
586                    request: member,
587                    catalog,
588                });
589            }
590
591            for interval in missing {
592                clients.entry(interval).or_default().push(id);
593            }
594        }
595
596        for ((start, end), ids) in clients {
597            let member = with_request_instrument_ids(&parent, ids);
598            let client_id = member.client_id().copied();
599            let venue = member.venue().copied();
600
601            if self
602                .get_client(client_id.as_ref(), venue.as_ref())
603                .is_none()
604            {
605                continue;
606            }
607
608            let start = UnixNanos::from(start).to_datetime_utc();
609            let end = UnixNanos::from(end).to_datetime_utc();
610            let request = with_dates_for_pipeline(&member, Some(start), Some(end), now_ns);
611            plan.push(RequestLeg::Client(request));
612        }
613
614        self.dispatch_request_plan(parent, plan)
615    }
616
617    fn catalog_coverage(
618        &mut self,
619        key: &RequestCatalogKey,
620        start: u64,
621        end: u64,
622    ) -> anyhow::Result<Option<CatalogCoverage>> {
623        for (name, catalog) in &mut self.catalogs {
624            let missing = catalog_missing_intervals(catalog, start, end, key)?;
625            if missing != [(start, end)] {
626                return Ok(Some(CatalogCoverage {
627                    catalog: *name,
628                    missing,
629                }));
630            }
631        }
632
633        Ok(None)
634    }
635
636    fn dispatch_request_plan(
637        &mut self,
638        parent: RequestCommand,
639        plan: Vec<RequestLeg>,
640    ) -> anyhow::Result<()> {
641        let parent_id = *parent.request_id();
642
643        if plan.is_empty() {
644            let now = self.clock.borrow().timestamp_ns();
645            let (start, end) = bound_request_dates(
646                request_start(&parent),
647                request_end(&parent),
648                now.to_datetime_utc(),
649                true,
650            );
651            let response = build_empty_response(
652                &parent,
653                datetime_to_unix_nanos_or_zero(start),
654                datetime_to_unix_nanos_or_zero(end),
655                None,
656                now,
657            )?;
658            self.response(response);
659            return Ok(());
660        }
661
662        self.new_request_pipeline(parent, plan.len());
663        for leg in &plan {
664            self.register_request_pipeline_leg(leg.request_id(), parent_id);
665        }
666
667        for leg in plan {
668            let request_id = leg.request_id();
669
670            let result = match leg {
671                RequestLeg::Catalog { request, catalog } => {
672                    let start = request_start(&request)
673                        .map(datetime_to_unix_nanos_or_zero)
674                        .expect("catalog leg has bounded start");
675                    let end = request_end(&request)
676                        .map(datetime_to_unix_nanos_or_zero)
677                        .expect("catalog leg has bounded end");
678                    let now = self.clock.borrow().timestamp_ns();
679                    let client_id = request.client_id().copied();
680                    let venue = request.venue().copied();
681                    let client = self
682                        .get_client(client_id.as_ref(), venue.as_ref())
683                        .map(|value| value.client_id());
684                    self.query_catalog_leg(&request, catalog, start, end, client, now)
685                        .map(|response| self.response(response))
686                }
687                RequestLeg::Client(request) => self.dispatch_request_to_client(request).map(|_| ()),
688                RequestLeg::Response(response) => {
689                    self.response(response);
690                    Ok(())
691                }
692            };
693
694            if let Err(e) = result {
695                self.request_failed(DataRequestFailed {
696                    request_id,
697                    reason: e.to_string(),
698                    response: None,
699                });
700            }
701        }
702
703        Ok(())
704    }
705
706    fn query_catalog_leg(
707        &mut self,
708        leg: &RequestCommand,
709        catalog_name: Ustr,
710        start_ns: UnixNanos,
711        end_ns: UnixNanos,
712        used_client_id: Option<ClientId>,
713        ts_init: UnixNanos,
714    ) -> anyhow::Result<DataResponse> {
715        let catalog = self.catalogs.get_mut(&catalog_name).ok_or_else(|| {
716            anyhow::anyhow!("Catalog {catalog_name} disappeared between intervals query and read")
717        })?;
718
719        match leg {
720            RequestCommand::Quotes(cmd) => {
721                let data = quote_ticks_from_query_result(
722                    catalog.query_batch(
723                        &CatalogQuery::new(NautilusDataType::QuoteTick)
724                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
725                            .with_range(Some(start_ns), Some(end_ns)),
726                    )?,
727                )?;
728                Ok(build_quotes_catalog_response(
729                    cmd,
730                    data,
731                    start_ns,
732                    end_ns,
733                    used_client_id,
734                    ts_init,
735                ))
736            }
737            RequestCommand::Trades(cmd) => {
738                let data = trade_ticks_from_query_result(
739                    catalog.query_batch(
740                        &CatalogQuery::new(NautilusDataType::TradeTick)
741                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
742                            .with_range(Some(start_ns), Some(end_ns)),
743                    )?,
744                )?;
745                Ok(build_trades_catalog_response(
746                    cmd,
747                    data,
748                    start_ns,
749                    end_ns,
750                    used_client_id,
751                    ts_init,
752                ))
753            }
754            RequestCommand::FundingRates(cmd) => {
755                let data = funding_rates_from_query_result(
756                    catalog.query_batch(
757                        &CatalogQuery::new(NautilusDataType::FundingRateUpdate)
758                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
759                            .with_range(Some(start_ns), Some(end_ns)),
760                    )?,
761                )?;
762                build_funding_rates_catalog_response(
763                    cmd,
764                    data,
765                    start_ns,
766                    end_ns,
767                    used_client_id,
768                    ts_init,
769                )
770            }
771            RequestCommand::Bars(cmd) => {
772                let data = bars_from_query_result(
773                    catalog.query_batch(
774                        &CatalogQuery::new(NautilusDataType::Bar)
775                            .with_identifiers(Some(vec![cmd.single_bar_type()?.to_string()]))
776                            .with_range(Some(start_ns), Some(end_ns)),
777                    )?,
778                )?;
779                build_bars_catalog_response(cmd, data, start_ns, end_ns, used_client_id, ts_init)
780            }
781            RequestCommand::Data(cmd) => {
782                let identifiers = cmd
783                    .data_type
784                    .identifier()
785                    .map(|identifier| vec![identifier.to_string()]);
786                let where_clause = request_filter_expr(cmd.params.as_ref());
787                let data = catalog.query_batch(
788                    &CatalogQuery::new(NautilusDataType::Custom {
789                        type_name: cmd.data_type.type_name().to_string(),
790                    })
791                    .with_identifiers(identifiers)
792                    .with_range(Some(start_ns), Some(end_ns))
793                    .with_where_clause(where_clause),
794                )?;
795                Ok(build_custom_data_catalog_response(
796                    cmd,
797                    custom_data_from_query_result(data)?,
798                    start_ns,
799                    end_ns,
800                    ts_init,
801                ))
802            }
803            RequestCommand::BookDeltas(cmd) => {
804                let data = order_book_deltas_from_query_result(
805                    catalog.query_batch(
806                        &CatalogQuery::new(NautilusDataType::OrderBookDelta)
807                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
808                            .with_range(Some(start_ns), Some(end_ns)),
809                    )?,
810                )?;
811                build_book_deltas_catalog_response(
812                    cmd,
813                    data,
814                    start_ns,
815                    end_ns,
816                    used_client_id,
817                    ts_init,
818                )
819            }
820            RequestCommand::BookDepth(cmd) => {
821                let data = order_book_depths_from_query_result(
822                    catalog.query_batch(
823                        &CatalogQuery::new(NautilusDataType::OrderBookDepth)
824                            .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
825                            .with_range(Some(start_ns), Some(end_ns)),
826                    )?,
827                )?;
828                build_book_depth_catalog_response(
829                    cmd,
830                    data,
831                    start_ns,
832                    end_ns,
833                    used_client_id,
834                    ts_init,
835                )
836            }
837            _ => {
838                anyhow::bail!("query_catalog_leg called with non-catalog-eligible variant {leg:?}")
839            }
840        }
841    }
842
843    fn dispatch_instrument_catalog_request(&mut self, req: RequestCommand) -> anyhow::Result<()> {
844        match req {
845            RequestCommand::Instrument(cmd) => self.dispatch_instrument_request(cmd),
846            RequestCommand::Instruments(cmd) => self.dispatch_instruments_request(cmd),
847            _ => self.dispatch_request_to_client(req).map(|_| ()),
848        }
849    }
850
851    fn dispatch_instrument_request(&mut self, cmd: RequestInstrument) -> anyhow::Result<()> {
852        if cmd.instrument_ids.len() > 1 {
853            return self.dispatch_instrument_list(cmd);
854        }
855
856        let force_instrument_update = cmd
857            .params
858            .as_ref()
859            .and_then(|params| params.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
860            .unwrap_or(false);
861
862        if force_instrument_update {
863            return self
864                .dispatch_request_to_client(RequestCommand::Instrument(cmd))
865                .map(|_| ());
866        }
867
868        let identifier = cmd.single_instrument_id()?.to_string();
869        let now_ns = self.clock.borrow().timestamp_ns();
870        let used_client_id = self
871            .get_client(
872                cmd.client_id.as_ref(),
873                Some(&cmd.single_instrument_id()?.venue),
874            )
875            .map(|client| client.client_id());
876        let (start_dt, end_dt) =
877            bound_request_dates(cmd.start, cmd.end, now_ns.to_datetime_utc(), true);
878        let start_ns = datetime_to_unix_nanos_or_zero(start_dt);
879        let end_ns = datetime_to_unix_nanos_or_zero(end_dt);
880        let start = Some(start_ns);
881        let end = cmd.end.map(datetime_to_unix_nanos_or_zero);
882
883        for catalog in self.catalogs.values_mut() {
884            let data = latest_instruments(
885                catalog.instruments(
886                    &CatalogInstrumentQuery::new()
887                        .with_instrument_ids(Some(vec![identifier.clone()]))
888                        .with_range(start, end),
889                )?,
890            );
891
892            if let Some(instrument) = data.into_iter().next() {
893                let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
894                    cmd.request_id,
895                    resolve_response_client_id(cmd.client_id, used_client_id),
896                    cmd.single_instrument_id()?,
897                    instrument,
898                    Some(start_ns),
899                    Some(end_ns),
900                    now_ns,
901                    Some(catalog_response_params(cmd.params.as_ref())),
902                )));
903                self.response(response);
904                return Ok(());
905            }
906        }
907
908        self.dispatch_request_to_client(RequestCommand::Instrument(cmd))
909            .map(|_| ())
910    }
911
912    fn dispatch_instrument_list(&mut self, command: RequestInstrument) -> anyhow::Result<()> {
913        let parent = RequestCommand::Instrument(command.clone());
914        let force = command
915            .params
916            .as_ref()
917            .and_then(|p| p.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
918            .unwrap_or(false);
919
920        if force {
921            return self.dispatch_request_to_client(parent).map(|_| ());
922        }
923
924        let now = self.clock.borrow().timestamp_ns();
925        let (start, end) =
926            bound_request_dates(command.start, command.end, now.to_datetime_utc(), true);
927        let start_ns = datetime_to_unix_nanos_or_zero(start);
928        let end_ns = datetime_to_unix_nanos_or_zero(end);
929        let mut missing = Vec::new();
930        let mut plan = Vec::new();
931
932        for id in &command.instrument_ids {
933            let mut instrument = None;
934
935            for catalog in self.catalogs.values_mut() {
936                let query = CatalogInstrumentQuery::new()
937                    .with_instrument_ids(Some(vec![id.to_string()]))
938                    .with_range(
939                        Some(start_ns),
940                        command.end.map(datetime_to_unix_nanos_or_zero),
941                    );
942                instrument = latest_instruments(catalog.instruments(&query)?)
943                    .into_iter()
944                    .find(|value| value.id() == *id);
945
946                if instrument.is_some() {
947                    break;
948                }
949            }
950
951            if let Some(instrument) = instrument {
952                let response = InstrumentResponse::new(
953                    UUID4::new(),
954                    command
955                        .client_id
956                        .unwrap_or_else(|| ClientId::new(CATALOG_CLIENT_ID)),
957                    *id,
958                    instrument,
959                    Some(start_ns),
960                    Some(end_ns),
961                    now,
962                    Some(catalog_response_params(command.params.as_ref())),
963                );
964                plan.push(RequestLeg::Response(DataResponse::Instrument(Box::new(
965                    response,
966                ))));
967            } else {
968                missing.push(*id);
969            }
970        }
971
972        if !missing.is_empty() {
973            let mut child = command;
974            child.request_id = UUID4::new();
975            child.instrument_ids = missing;
976            plan.push(RequestLeg::Client(RequestCommand::Instrument(child)));
977        }
978
979        self.dispatch_request_plan(parent, plan)
980    }
981
982    fn dispatch_instruments_request(&mut self, cmd: RequestInstruments) -> anyhow::Result<()> {
983        let update_catalog = cmd
984            .params
985            .as_ref()
986            .and_then(|params| params.get_bool(PARAM_UPDATE_CATALOG))
987            .unwrap_or(false);
988        let force_instrument_update = cmd
989            .params
990            .as_ref()
991            .and_then(|params| params.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
992            .unwrap_or(false);
993
994        if update_catalog || force_instrument_update {
995            return self
996                .dispatch_request_to_client(RequestCommand::Instruments(cmd))
997                .map(|_| ());
998        }
999
1000        let now_ns = self.clock.borrow().timestamp_ns();
1001        let used_client_id = self
1002            .get_client(cmd.client_id.as_ref(), cmd.venue.as_ref())
1003            .map(|client| client.client_id());
1004        let (start_dt, _) = bound_request_dates(cmd.start, cmd.end, now_ns.to_datetime_utc(), true);
1005        let start = Some(datetime_to_unix_nanos_or_zero(start_dt));
1006        let end = cmd.end.map(datetime_to_unix_nanos_or_zero);
1007        let mut data = Vec::new();
1008
1009        for catalog in self.catalogs.values_mut() {
1010            data.extend(
1011                catalog.instruments(
1012                    &CatalogInstrumentQuery::new()
1013                        .with_range(start, end)
1014                        .with_where_clause(request_filter_expr(cmd.params.as_ref())),
1015                )?,
1016            );
1017        }
1018
1019        if let Some(venue) = cmd.venue {
1020            data.retain(|instrument| instrument.venue() == venue);
1021        }
1022
1023        if instrument_only_last(cmd.params.as_ref()) {
1024            data = latest_instruments(data);
1025        }
1026
1027        let response = DataResponse::Instruments(InstrumentsResponse::new(
1028            cmd.request_id,
1029            resolve_response_client_id(cmd.client_id, used_client_id),
1030            instrument_response_venue(cmd.venue, &data),
1031            data,
1032            // Report the queried window: results sorted by ID cannot be trimmed by `ts_init`
1033            start,
1034            end,
1035            now_ns,
1036            Some(catalog_response_params(cmd.params.as_ref())),
1037        ));
1038        self.response(response);
1039        Ok(())
1040    }
1041}
1042
1043struct RequestCatalogKey {
1044    data_type: NautilusDataType,
1045    identifier: Option<String>,
1046}
1047
1048fn response_write_bounds(
1049    request: &RequestCommand,
1050    response: &DataResponse,
1051) -> (Option<UnixNanos>, Option<UnixNanos>) {
1052    (
1053        request_start(request)
1054            .map(datetime_to_unix_nanos_or_zero)
1055            .or(response.start()),
1056        request_end(request)
1057            .map(datetime_to_unix_nanos_or_zero)
1058            .or(response.end()),
1059    )
1060}
1061
1062fn response_batches(response: &DataResponse) -> anyhow::Result<Vec<(Option<String>, DataBatch)>> {
1063    let batches = match response {
1064        DataResponse::Quotes(r) => {
1065            let ids = response_instrument_ids(
1066                &r.instrument_ids,
1067                r.data.iter().map(|row| row.instrument_id),
1068            );
1069            let single = ids.len() == 1;
1070            ids.into_iter()
1071                .map(|id| {
1072                    let data = if single {
1073                        r.data.clone()
1074                    } else {
1075                        r.data
1076                            .iter()
1077                            .filter(|row| row.instrument_id == id)
1078                            .copied()
1079                            .collect::<Vec<_>>()
1080                            .into()
1081                    };
1082
1083                    (Some(id.to_string()), DataBatch::Quote(data))
1084                })
1085                .collect()
1086        }
1087        DataResponse::Trades(r) => {
1088            let ids = response_instrument_ids(
1089                &r.instrument_ids,
1090                r.data.iter().map(|row| row.instrument_id),
1091            );
1092            let single = ids.len() == 1;
1093            ids.into_iter()
1094                .map(|id| {
1095                    let data = if single {
1096                        r.data.clone()
1097                    } else {
1098                        r.data
1099                            .iter()
1100                            .filter(|row| row.instrument_id == id)
1101                            .copied()
1102                            .collect::<Vec<_>>()
1103                            .into()
1104                    };
1105
1106                    (Some(id.to_string()), DataBatch::Trade(data))
1107                })
1108                .collect()
1109        }
1110        DataResponse::BookDeltas(r) => vec![(
1111            Some(r.instrument_id.to_string()),
1112            DataBatch::BookDelta(r.data.clone()),
1113        )],
1114        DataResponse::BookDepth(r) => vec![(
1115            Some(r.instrument_id.to_string()),
1116            DataBatch::BookDepth(r.data.clone()),
1117        )],
1118        DataResponse::FundingRates(r) => vec![(
1119            Some(r.instrument_id.to_string()),
1120            DataBatch::FundingRate(r.data.clone()),
1121        )],
1122        DataResponse::Bars(r) => {
1123            vec![(Some(r.bar_type.to_string()), DataBatch::Bar(r.data.clone()))]
1124        }
1125        DataResponse::Data(r) => {
1126            let values = if let Some(data) = r.data.downcast_ref::<Vec<CustomData>>() {
1127                data.clone()
1128            } else if let Some(data) = r.data.downcast_ref::<CustomData>() {
1129                vec![data.clone()]
1130            } else {
1131                anyhow::bail!("Custom response payload is not catalog data");
1132            };
1133
1134            vec![(
1135                r.data_type.identifier().map(String::from),
1136                DataBatch::Custom(values.into()),
1137            )]
1138        }
1139        _ => Vec::new(),
1140    };
1141
1142    Ok(batches)
1143}
1144
1145struct CatalogCoverage {
1146    catalog: Ustr,
1147    missing: Vec<(u64, u64)>,
1148}
1149
1150enum RequestLeg {
1151    Catalog {
1152        request: RequestCommand,
1153        catalog: Ustr,
1154    },
1155    Client(RequestCommand),
1156    Response(DataResponse),
1157}
1158
1159impl RequestLeg {
1160    fn request_id(&self) -> UUID4 {
1161        match self {
1162            Self::Catalog { request, .. } | Self::Client(request) => *request.request_id(),
1163            Self::Response(response) => *response.correlation_id(),
1164        }
1165    }
1166}
1167
1168fn with_request_instrument_ids(parent: &RequestCommand, ids: Vec<InstrumentId>) -> RequestCommand {
1169    let mut request = parent.clone();
1170    match &mut request {
1171        RequestCommand::Quotes(r) => r.instrument_ids = ids,
1172        RequestCommand::Trades(r) => r.instrument_ids = ids,
1173        _ => unreachable!("catalog batch planning only supports quotes and trades"),
1174    }
1175
1176    request
1177}
1178
1179pub(super) fn is_date_range_variant(req: &RequestCommand) -> bool {
1180    matches!(
1181        req,
1182        RequestCommand::Data(_)
1183            | RequestCommand::Instrument(_)
1184            | RequestCommand::Instruments(_)
1185            | RequestCommand::Quotes(_)
1186            | RequestCommand::Trades(_)
1187            | RequestCommand::FundingRates(_)
1188            | RequestCommand::Bars(_)
1189            | RequestCommand::BookDeltas(_)
1190            | RequestCommand::BookDepth(_)
1191    )
1192}
1193
1194fn request_identifier(req: &RequestCommand) -> Option<RequestCatalogKey> {
1195    match req {
1196        RequestCommand::Data(cmd) => Some(RequestCatalogKey {
1197            data_type: NautilusDataType::Custom {
1198                type_name: cmd.data_type.type_name().to_string(),
1199            },
1200            identifier: cmd.data_type.identifier().map(String::from),
1201        }),
1202        RequestCommand::Quotes(cmd) => Some(RequestCatalogKey::new(
1203            NautilusDataType::QuoteTick,
1204            Some(cmd.single_instrument_id().ok()?.to_string()),
1205        )),
1206        RequestCommand::Trades(cmd) => Some(RequestCatalogKey::new(
1207            NautilusDataType::TradeTick,
1208            Some(cmd.single_instrument_id().ok()?.to_string()),
1209        )),
1210        RequestCommand::FundingRates(cmd) => Some(RequestCatalogKey::new(
1211            NautilusDataType::FundingRateUpdate,
1212            Some(cmd.single_instrument_id().ok()?.to_string()),
1213        )),
1214        RequestCommand::Bars(cmd) => Some(RequestCatalogKey::new(
1215            NautilusDataType::Bar,
1216            Some(cmd.single_bar_type().ok()?.to_string()),
1217        )),
1218        RequestCommand::BookDeltas(cmd) => Some(RequestCatalogKey::new(
1219            NautilusDataType::OrderBookDelta,
1220            Some(cmd.single_instrument_id().ok()?.to_string()),
1221        )),
1222        RequestCommand::BookDepth(cmd) => Some(RequestCatalogKey::new(
1223            NautilusDataType::OrderBookDepth,
1224            Some(cmd.single_instrument_id().ok()?.to_string()),
1225        )),
1226        _ => None,
1227    }
1228}
1229
1230impl RequestCatalogKey {
1231    fn new(data_type: NautilusDataType, identifier: Option<String>) -> Self {
1232        Self {
1233            data_type,
1234            identifier,
1235        }
1236    }
1237
1238    fn catalog_data_type(&self) -> NautilusDataType {
1239        self.data_type.clone()
1240    }
1241}
1242
1243fn catalog_missing_intervals(
1244    catalog: &mut DataCatalog,
1245    start: u64,
1246    end: u64,
1247    key: &RequestCatalogKey,
1248) -> anyhow::Result<Vec<(u64, u64)>> {
1249    catalog.get_missing_intervals_for_request(
1250        UnixNanos::from(start),
1251        UnixNanos::from(end),
1252        key.catalog_data_type(),
1253        key.identifier.as_deref(),
1254    )
1255}
1256
1257fn quote_ticks_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<QuoteTick>> {
1258    match data {
1259        DataBatch::Quote(data) => Ok(data),
1260        _ => anyhow::bail!("expected a Quote batch, found another data batch variant"),
1261    }
1262}
1263
1264fn trade_ticks_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<TradeTick>> {
1265    match data {
1266        DataBatch::Trade(data) => Ok(data),
1267        _ => anyhow::bail!("expected a Trade batch, found another data batch variant"),
1268    }
1269}
1270
1271fn funding_rates_from_query_result(
1272    data: DataBatch,
1273) -> anyhow::Result<BatchView<FundingRateUpdate>> {
1274    match data {
1275        DataBatch::FundingRate(data) => Ok(data),
1276        _ => anyhow::bail!("expected a FundingRate batch, found another data batch variant"),
1277    }
1278}
1279
1280fn bars_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<Bar>> {
1281    match data {
1282        DataBatch::Bar(data) => Ok(data),
1283        _ => anyhow::bail!("expected a Bar batch, found another data batch variant"),
1284    }
1285}
1286
1287fn order_book_deltas_from_query_result(
1288    data: DataBatch,
1289) -> anyhow::Result<BatchView<OrderBookDelta>> {
1290    match data {
1291        DataBatch::BookDelta(data) => Ok(data),
1292        _ => anyhow::bail!("expected a BookDelta batch, found another data batch variant"),
1293    }
1294}
1295
1296fn order_book_depths_from_query_result(
1297    data: DataBatch,
1298) -> anyhow::Result<BatchView<OrderBookDepth>> {
1299    match data {
1300        DataBatch::BookDepth(data) => Ok(data),
1301        _ => anyhow::bail!("expected a BookDepth batch, found another data batch variant"),
1302    }
1303}
1304
1305fn request_limit(request: &RequestCommand) -> Option<std::num::NonZeroUsize> {
1306    match request {
1307        RequestCommand::Quotes(r) => r.limit,
1308        RequestCommand::Trades(r) => r.limit,
1309        RequestCommand::Bars(r) => r.limit,
1310        RequestCommand::FundingRates(r) => r.limit,
1311        RequestCommand::BookDeltas(r) => r.limit,
1312        RequestCommand::BookDepth(r) => r.limit,
1313        RequestCommand::Data(r) => r.limit,
1314        _ => None,
1315    }
1316}
1317
1318fn request_has_schema(request: &RequestCommand) -> bool {
1319    matches!(
1320        request,
1321        RequestCommand::Quotes(_) | RequestCommand::Trades(_)
1322    ) && request_params(request).is_some_and(|params| params.contains_key("schema"))
1323}
1324
1325fn request_contains_batch(
1326    request: &RequestCommand,
1327    identifier: Option<&str>,
1328    batch: &DataBatch,
1329) -> bool {
1330    let contains = |ids: &[InstrumentId]| {
1331        identifier
1332            .and_then(|value| value.parse::<InstrumentId>().ok())
1333            .is_some_and(|id| ids.contains(&id))
1334    };
1335
1336    match (request, batch) {
1337        (RequestCommand::Quotes(r), DataBatch::Quote(_)) => contains(&r.instrument_ids),
1338        (RequestCommand::Trades(r), DataBatch::Trade(_)) => contains(&r.instrument_ids),
1339        (RequestCommand::BookDeltas(r), DataBatch::BookDelta(rows)) => {
1340            contains(&r.instrument_ids)
1341                && rows
1342                    .iter()
1343                    .all(|row| r.instrument_ids.contains(&row.instrument_id))
1344        }
1345        (RequestCommand::BookDepth(r), DataBatch::BookDepth(rows)) => {
1346            contains(&r.instrument_ids)
1347                && rows
1348                    .iter()
1349                    .all(|row| r.instrument_ids.contains(&row.instrument_id))
1350        }
1351        (RequestCommand::FundingRates(r), DataBatch::FundingRate(rows)) => {
1352            contains(&r.instrument_ids)
1353                && rows
1354                    .iter()
1355                    .all(|row| r.instrument_ids.contains(&row.instrument_id))
1356        }
1357        (RequestCommand::Bars(r), DataBatch::Bar(rows)) => {
1358            identifier
1359                .and_then(|value| value.parse::<BarType>().ok())
1360                .is_some_and(|bar_type| r.bar_types.contains(&bar_type))
1361                && rows.iter().all(|row| r.bar_types.contains(&row.bar_type))
1362        }
1363        (RequestCommand::Data(_), DataBatch::Custom(_)) => {
1364            request_identifier(request).is_some_and(|key| key.identifier.as_deref() == identifier)
1365        }
1366        _ => false,
1367    }
1368}
1369
1370fn response_instrument_ids(
1371    requested: &[InstrumentId],
1372    actual: impl Iterator<Item = InstrumentId>,
1373) -> Vec<InstrumentId> {
1374    let actual = actual.collect::<AHashSet<_>>();
1375    if actual.iter().all(|id| requested.contains(id)) {
1376        return requested.to_vec();
1377    }
1378
1379    let mut ids = actual.into_iter().collect::<Vec<_>>();
1380    ids.sort_unstable();
1381    ids
1382}
1383
1384fn request_start(req: &RequestCommand) -> Option<Timestamp> {
1385    match req {
1386        RequestCommand::Data(cmd) => cmd.start,
1387        RequestCommand::Instrument(cmd) => cmd.start,
1388        RequestCommand::Instruments(cmd) => cmd.start,
1389        RequestCommand::Quotes(cmd) => cmd.start,
1390        RequestCommand::Trades(cmd) => cmd.start,
1391        RequestCommand::FundingRates(cmd) => cmd.start,
1392        RequestCommand::Bars(cmd) => cmd.start,
1393        RequestCommand::BookDeltas(cmd) => cmd.start,
1394        RequestCommand::BookDepth(cmd) => cmd.start,
1395        _ => None,
1396    }
1397}
1398
1399fn request_end(req: &RequestCommand) -> Option<Timestamp> {
1400    match req {
1401        RequestCommand::Data(cmd) => cmd.end,
1402        RequestCommand::Instrument(cmd) => cmd.end,
1403        RequestCommand::Instruments(cmd) => cmd.end,
1404        RequestCommand::Quotes(cmd) => cmd.end,
1405        RequestCommand::Trades(cmd) => cmd.end,
1406        RequestCommand::FundingRates(cmd) => cmd.end,
1407        RequestCommand::Bars(cmd) => cmd.end,
1408        RequestCommand::BookDeltas(cmd) => cmd.end,
1409        RequestCommand::BookDepth(cmd) => cmd.end,
1410        _ => None,
1411    }
1412}
1413
1414fn request_filter_expr(params: Option<&Params>) -> Option<String> {
1415    params
1416        .and_then(|params| params.get_str("filter_expr"))
1417        .filter(|filter_expr| !filter_expr.is_empty())
1418        .map(ToString::to_string)
1419}
1420
1421fn bound_request_dates(
1422    start: Option<Timestamp>,
1423    end: Option<Timestamp>,
1424    now: Timestamp,
1425    query_past_data: bool,
1426) -> (Timestamp, Timestamp) {
1427    let zero = Timestamp::UNIX_EPOCH;
1428    let mut start = start.unwrap_or(zero);
1429    let mut end = end.unwrap_or(now);
1430
1431    if query_past_data {
1432        if start > now {
1433            start = now;
1434        }
1435
1436        if end > now {
1437            end = now;
1438        }
1439    }
1440
1441    (start, end)
1442}
1443
1444fn datetime_to_unix_nanos_or_zero(dt: Timestamp) -> UnixNanos {
1445    UnixNanos::from(u64::try_from(dt.as_nanosecond().max(0)).unwrap_or(0))
1446}
1447
1448fn floor_to_utc_day(dt: Timestamp) -> Timestamp {
1449    let midnight = jiff::tz::Offset::UTC.to_datetime(dt).date().at(0, 0, 0, 0);
1450    jiff::tz::Offset::UTC
1451        .to_timestamp(midnight)
1452        .expect("midnight UTC is always valid")
1453}
1454
1455fn with_dates_for_pipeline(
1456    req: &RequestCommand,
1457    start: Option<Timestamp>,
1458    end: Option<Timestamp>,
1459    ts_init: UnixNanos,
1460) -> RequestCommand {
1461    let new_id = UUID4::new();
1462
1463    match req {
1464        RequestCommand::Quotes(cmd) => RequestCommand::Quotes(RequestQuotes {
1465            instrument_ids: cmd.instrument_ids.clone(),
1466            start,
1467            end,
1468            limit: cmd.limit,
1469            client_id: cmd.client_id,
1470            request_id: new_id,
1471            ts_init,
1472            params: cmd.params.clone(),
1473        }),
1474        RequestCommand::Trades(cmd) => RequestCommand::Trades(RequestTrades {
1475            instrument_ids: cmd.instrument_ids.clone(),
1476            start,
1477            end,
1478            limit: cmd.limit,
1479            client_id: cmd.client_id,
1480            request_id: new_id,
1481            ts_init,
1482            params: cmd.params.clone(),
1483        }),
1484        RequestCommand::FundingRates(cmd) => RequestCommand::FundingRates(RequestFundingRates {
1485            instrument_ids: cmd.instrument_ids.clone(),
1486            start,
1487            end,
1488            limit: cmd.limit,
1489            client_id: cmd.client_id,
1490            request_id: new_id,
1491            ts_init,
1492            params: cmd.params.clone(),
1493        }),
1494        RequestCommand::BookDeltas(cmd) => RequestCommand::BookDeltas(RequestBookDeltas {
1495            instrument_ids: cmd.instrument_ids.clone(),
1496            start,
1497            end,
1498            limit: cmd.limit,
1499            client_id: cmd.client_id,
1500            request_id: new_id,
1501            ts_init,
1502            params: cmd.params.clone(),
1503        }),
1504        RequestCommand::BookDepth(cmd) => RequestCommand::BookDepth(RequestBookDepth {
1505            instrument_ids: cmd.instrument_ids.clone(),
1506            start,
1507            end,
1508            limit: cmd.limit,
1509            depth: cmd.depth,
1510            client_id: cmd.client_id,
1511            request_id: new_id,
1512            ts_init,
1513            params: cmd.params.clone(),
1514        }),
1515        RequestCommand::Data(cmd) => RequestCommand::Data(RequestCustomData {
1516            client_id: cmd.client_id,
1517            data_type: cmd.data_type.clone(),
1518            start,
1519            end,
1520            limit: cmd.limit,
1521            request_id: new_id,
1522            ts_init,
1523            params: cmd.params.clone(),
1524        }),
1525        RequestCommand::Bars(cmd) => RequestCommand::Bars(RequestBars {
1526            bar_types: cmd.bar_types.clone(),
1527            start,
1528            end,
1529            limit: cmd.limit,
1530            client_id: cmd.client_id,
1531            request_id: new_id,
1532            ts_init,
1533            params: cmd.params.clone(),
1534        }),
1535        // `Join` and the non-date-range variants should never reach this path; the dispatcher
1536        // gates on `is_date_range_variant` first. Cloning preserves behavior if a caller
1537        // reaches this arm.
1538        _ => req.clone(),
1539    }
1540}
1541
1542fn build_empty_response(
1543    req: &RequestCommand,
1544    start: UnixNanos,
1545    end: UnixNanos,
1546    used_client_id: Option<ClientId>,
1547    ts_init: UnixNanos,
1548) -> anyhow::Result<DataResponse> {
1549    let response = match req {
1550        RequestCommand::Data(cmd) => DataResponse::Data(CustomDataResponse::new(
1551            cmd.request_id,
1552            cmd.client_id,
1553            None,
1554            cmd.data_type.clone(),
1555            Vec::<CustomData>::new(),
1556            Some(start),
1557            Some(end),
1558            ts_init,
1559            cmd.params.clone(),
1560        )),
1561        RequestCommand::Quotes(cmd) => DataResponse::Quotes(QuotesResponse::new(
1562            cmd.request_id,
1563            resolve_response_client_id(cmd.client_id, used_client_id),
1564            cmd.instrument_ids.clone(),
1565            Vec::new(),
1566            Some(start),
1567            Some(end),
1568            ts_init,
1569            cmd.params.clone(),
1570        )),
1571        RequestCommand::Trades(cmd) => DataResponse::Trades(TradesResponse::new(
1572            cmd.request_id,
1573            resolve_response_client_id(cmd.client_id, used_client_id),
1574            cmd.instrument_ids.clone(),
1575            Vec::new(),
1576            Some(start),
1577            Some(end),
1578            ts_init,
1579            cmd.params.clone(),
1580        )),
1581        RequestCommand::FundingRates(cmd) => DataResponse::FundingRates(FundingRatesResponse::new(
1582            cmd.request_id,
1583            resolve_response_client_id(cmd.client_id, used_client_id),
1584            cmd.single_instrument_id()?,
1585            Vec::new(),
1586            Some(start),
1587            Some(end),
1588            ts_init,
1589            cmd.params.clone(),
1590        )),
1591        RequestCommand::Bars(cmd) => DataResponse::Bars(BarsResponse::new(
1592            cmd.request_id,
1593            resolve_response_client_id(cmd.client_id, used_client_id),
1594            cmd.single_bar_type()?,
1595            Vec::new(),
1596            Some(start),
1597            Some(end),
1598            ts_init,
1599            cmd.params.clone(),
1600        )),
1601        RequestCommand::BookDeltas(cmd) => DataResponse::BookDeltas(BookDeltasResponse::new(
1602            cmd.request_id,
1603            resolve_response_client_id(cmd.client_id, used_client_id),
1604            cmd.single_instrument_id()?,
1605            Vec::new(),
1606            Some(start),
1607            Some(end),
1608            ts_init,
1609            cmd.params.clone(),
1610        )),
1611        RequestCommand::BookDepth(cmd) => DataResponse::BookDepth(BookDepthResponse::new(
1612            cmd.request_id,
1613            resolve_response_client_id(cmd.client_id, used_client_id),
1614            cmd.single_instrument_id()?,
1615            Vec::new(),
1616            Some(start),
1617            Some(end),
1618            ts_init,
1619            cmd.params.clone(),
1620        )),
1621        _ => {
1622            anyhow::bail!("Cannot build empty catalog response for non-catalog-eligible request")
1623        }
1624    };
1625
1626    Ok(response)
1627}
1628
1629fn build_quotes_catalog_response(
1630    cmd: &RequestQuotes,
1631    data: BatchView<QuoteTick>,
1632    start: UnixNanos,
1633    end: UnixNanos,
1634    used_client_id: Option<ClientId>,
1635    ts_init: UnixNanos,
1636) -> DataResponse {
1637    let params = catalog_response_params(cmd.params.as_ref());
1638    DataResponse::Quotes(QuotesResponse::new(
1639        cmd.request_id,
1640        resolve_response_client_id(cmd.client_id, used_client_id),
1641        cmd.instrument_ids.clone(),
1642        data,
1643        Some(start),
1644        Some(end),
1645        ts_init,
1646        Some(params),
1647    ))
1648}
1649
1650fn build_trades_catalog_response(
1651    cmd: &RequestTrades,
1652    data: BatchView<TradeTick>,
1653    start: UnixNanos,
1654    end: UnixNanos,
1655    used_client_id: Option<ClientId>,
1656    ts_init: UnixNanos,
1657) -> DataResponse {
1658    let params = catalog_response_params(cmd.params.as_ref());
1659    DataResponse::Trades(TradesResponse::new(
1660        cmd.request_id,
1661        resolve_response_client_id(cmd.client_id, used_client_id),
1662        cmd.instrument_ids.clone(),
1663        data,
1664        Some(start),
1665        Some(end),
1666        ts_init,
1667        Some(params),
1668    ))
1669}
1670
1671fn build_funding_rates_catalog_response(
1672    cmd: &RequestFundingRates,
1673    data: BatchView<FundingRateUpdate>,
1674    start: UnixNanos,
1675    end: UnixNanos,
1676    used_client_id: Option<ClientId>,
1677    ts_init: UnixNanos,
1678) -> anyhow::Result<DataResponse> {
1679    let params = catalog_response_params(cmd.params.as_ref());
1680    Ok(DataResponse::FundingRates(FundingRatesResponse::new(
1681        cmd.request_id,
1682        resolve_response_client_id(cmd.client_id, used_client_id),
1683        cmd.single_instrument_id()?,
1684        data,
1685        Some(start),
1686        Some(end),
1687        ts_init,
1688        Some(params),
1689    )))
1690}
1691
1692fn build_bars_catalog_response(
1693    cmd: &RequestBars,
1694    data: BatchView<Bar>,
1695    start: UnixNanos,
1696    end: UnixNanos,
1697    used_client_id: Option<ClientId>,
1698    ts_init: UnixNanos,
1699) -> anyhow::Result<DataResponse> {
1700    let params = catalog_response_params(cmd.params.as_ref());
1701    Ok(DataResponse::Bars(BarsResponse::new(
1702        cmd.request_id,
1703        resolve_response_client_id(cmd.client_id, used_client_id),
1704        cmd.single_bar_type()?,
1705        data,
1706        Some(start),
1707        Some(end),
1708        ts_init,
1709        Some(params),
1710    )))
1711}
1712
1713fn build_custom_data_catalog_response(
1714    cmd: &RequestCustomData,
1715    data: Vec<CustomData>,
1716    start: UnixNanos,
1717    end: UnixNanos,
1718    ts_init: UnixNanos,
1719) -> DataResponse {
1720    let params = catalog_response_params(cmd.params.as_ref());
1721    DataResponse::Data(CustomDataResponse::new(
1722        cmd.request_id,
1723        cmd.client_id,
1724        None,
1725        cmd.data_type.clone(),
1726        data,
1727        Some(start),
1728        Some(end),
1729        ts_init,
1730        Some(params),
1731    ))
1732}
1733
1734fn build_book_deltas_catalog_response(
1735    cmd: &RequestBookDeltas,
1736    data: BatchView<OrderBookDelta>,
1737    start: UnixNanos,
1738    end: UnixNanos,
1739    used_client_id: Option<ClientId>,
1740    ts_init: UnixNanos,
1741) -> anyhow::Result<DataResponse> {
1742    let params = catalog_response_params(cmd.params.as_ref());
1743    Ok(DataResponse::BookDeltas(BookDeltasResponse::new(
1744        cmd.request_id,
1745        resolve_response_client_id(cmd.client_id, used_client_id),
1746        cmd.single_instrument_id()?,
1747        data,
1748        Some(start),
1749        Some(end),
1750        ts_init,
1751        Some(params),
1752    )))
1753}
1754
1755fn build_book_depth_catalog_response(
1756    cmd: &RequestBookDepth,
1757    data: BatchView<OrderBookDepth>,
1758    start: UnixNanos,
1759    end: UnixNanos,
1760    used_client_id: Option<ClientId>,
1761    ts_init: UnixNanos,
1762) -> anyhow::Result<DataResponse> {
1763    let params = catalog_response_params(cmd.params.as_ref());
1764    Ok(DataResponse::BookDepth(BookDepthResponse::new(
1765        cmd.request_id,
1766        resolve_response_client_id(cmd.client_id, used_client_id),
1767        cmd.single_instrument_id()?,
1768        data,
1769        Some(start),
1770        Some(end),
1771        ts_init,
1772        Some(params),
1773    )))
1774}
1775
1776fn catalog_response_params(existing: Option<&Params>) -> Params {
1777    let mut params = existing
1778        .map(|params| IndexMap::clone(params))
1779        .unwrap_or_default();
1780    params.insert(PARAM_UPDATE_CATALOG.to_string(), Value::Bool(false));
1781    Params::new(params)
1782}
1783
1784fn custom_data_from_query_result(data: DataBatch) -> anyhow::Result<Vec<CustomData>> {
1785    CustomData::from_batch(data)
1786}
1787
1788fn instrument_only_last(params: Option<&Params>) -> bool {
1789    params
1790        .and_then(|params| params.get_bool("only_last"))
1791        .unwrap_or(true)
1792}
1793
1794fn latest_instruments(data: Vec<InstrumentAny>) -> Vec<InstrumentAny> {
1795    let mut instruments: AHashMap<_, InstrumentAny> = AHashMap::new();
1796
1797    for instrument in data {
1798        let id = instrument.id();
1799        match instruments.get(&id) {
1800            Some(existing) if existing.ts_init() >= instrument.ts_init() => {}
1801            _ => {
1802                instruments.insert(id, instrument);
1803            }
1804        }
1805    }
1806
1807    let mut data: Vec<_> = instruments.into_values().collect();
1808    data.sort_by_key(|instrument| instrument.id().to_string());
1809    data
1810}
1811
1812fn instrument_response_venue(request_venue: Option<Venue>, data: &[InstrumentAny]) -> Venue {
1813    request_venue.unwrap_or_else(|| {
1814        data.iter()
1815            .map(Instrument::venue)
1816            .min_by_key(std::string::ToString::to_string)
1817            .unwrap_or_else(|| Venue::from(CATALOG_CLIENT_ID))
1818    })
1819}
1820
1821fn resolve_response_client_id(
1822    request_client_id: Option<ClientId>,
1823    used_client_id: Option<ClientId>,
1824) -> ClientId {
1825    request_client_id
1826        .or(used_client_id)
1827        .unwrap_or_else(|| ClientId::new(CATALOG_CLIENT_ID))
1828}
1829
1830#[cfg(test)]
1831mod tests {
1832    use std::sync::Arc;
1833
1834    use nautilus_common::messages::data::RequestJoin;
1835    use rstest::rstest;
1836
1837    use super::*;
1838
1839    #[rstest]
1840    #[case::full(0, 3)]
1841    #[case::subrange(1, 2)]
1842    fn test_catalog_quote_response_uses_selected_batch_view(
1843        #[case] start: usize,
1844        #[case] end: usize,
1845    ) {
1846        let first = QuoteTick {
1847            ts_init: UnixNanos::from(11),
1848            ..Default::default()
1849        };
1850
1851        let mut second = first;
1852        second.ts_init = UnixNanos::from(23);
1853        let mut third = first;
1854        third.ts_init = UnixNanos::from(37);
1855        let original = Arc::new(vec![first, second, third]);
1856        let result = quote_ticks_from_query_result(DataBatch::Quote(BatchView::new(
1857            Arc::clone(&original),
1858            start..end,
1859        )))
1860        .unwrap();
1861        assert_eq!(result.as_ref(), &original[start..end]);
1862        assert_eq!(result.range(), start..end);
1863        assert!(Arc::ptr_eq(result.arc(), &original));
1864    }
1865
1866    #[rstest]
1867    fn test_response_batches_preserves_narrowed_payloads(#[values(false, true)] empty: bool) {
1868        use nautilus_model::data::stubs::{stub_delta, stub_depth10};
1869
1870        macro_rules! check {
1871            ($response:ident, $variant:ident, $batch:ident, $key:expr, $seed:expr) => {{
1872                let key = $key;
1873                let mut row = $seed;
1874                row.ts_init = UnixNanos::from(23);
1875                let mut before = row.clone();
1876                before.ts_init = UnixNanos::from(11);
1877                let mut after = row.clone();
1878                after.ts_init = UnixNanos::from(37);
1879                let backing = Arc::new(vec![before, row.clone(), after]);
1880                let end = if empty { 1 } else { 2 };
1881                let response = DataResponse::$variant($response::new(
1882                    UUID4::new(),
1883                    ClientId::from("SOURCE"),
1884                    key,
1885                    BatchView::new(Arc::clone(&backing), 1..end),
1886                    None,
1887                    None,
1888                    UnixNanos::from(41),
1889                    None,
1890                ));
1891
1892                let batches = response_batches(&response).unwrap();
1893
1894                assert_eq!(batches.len(), 1);
1895                assert_eq!(batches[0].0.as_deref(), Some(key.to_string().as_str()));
1896                let DataBatch::$batch(view) = &batches[0].1 else {
1897                    panic!("Unexpected batch {:?}", batches[0].1);
1898                };
1899                assert_eq!(view.range(), 1..end);
1900                assert!(Arc::ptr_eq(view.arc(), &backing));
1901                assert_eq!(
1902                    view.as_ref(),
1903                    if empty {
1904                        &[]
1905                    } else {
1906                        std::slice::from_ref(&row)
1907                    }
1908                );
1909            }};
1910        }
1911
1912        let delta = stub_delta();
1913        let depth = stub_depth10();
1914        let quote = QuoteTick::default();
1915        let trade = TradeTick::default();
1916        let bar = Bar::default();
1917
1918        let funding = FundingRateUpdate::new(
1919            quote.instrument_id,
1920            "0.001".parse().unwrap(),
1921            Some(60),
1922            Some(UnixNanos::from(47)),
1923            UnixNanos::from(17),
1924            UnixNanos::from(23),
1925        );
1926
1927        check!(
1928            BookDeltasResponse,
1929            BookDeltas,
1930            BookDelta,
1931            delta.instrument_id,
1932            delta
1933        );
1934        check!(
1935            BookDepthResponse,
1936            BookDepth,
1937            BookDepth,
1938            depth.instrument_id,
1939            depth
1940        );
1941        check!(QuotesResponse, Quotes, Quote, quote.instrument_id, quote);
1942        check!(TradesResponse, Trades, Trade, trade.instrument_id, trade);
1943        check!(
1944            FundingRatesResponse,
1945            FundingRates,
1946            FundingRate,
1947            funding.instrument_id,
1948            funding
1949        );
1950        check!(BarsResponse, Bars, Bar, bar.bar_type, bar);
1951    }
1952
1953    #[rstest]
1954    #[case::deltas(NautilusDataType::OrderBookDelta)]
1955    #[case::depth(NautilusDataType::OrderBookDepth)]
1956    #[case::quotes(NautilusDataType::QuoteTick)]
1957    #[case::trades(NautilusDataType::TradeTick)]
1958    #[case::bars(NautilusDataType::Bar)]
1959    #[case::funding(NautilusDataType::FundingRateUpdate)]
1960    fn test_request_contains_batch_uses_visible_requested_members(
1961        #[case] kind: NautilusDataType,
1962        #[values(false, true)] remapped: bool,
1963        #[values(false, true)] empty: bool,
1964    ) {
1965        use nautilus_model::data::stubs::{stub_delta, stub_depth10};
1966
1967        let requested = InstrumentId::from("AUD/USD.SIM");
1968
1969        let actual = if remapped {
1970            InstrumentId::from("GBP/USD.SIM")
1971        } else {
1972            requested
1973        };
1974
1975        let requested_bar = format!("{requested}-1-MINUTE-LAST-EXTERNAL")
1976            .parse::<BarType>()
1977            .unwrap();
1978        let actual_bar = format!("{actual}-1-MINUTE-LAST-EXTERNAL")
1979            .parse::<BarType>()
1980            .unwrap();
1981
1982        macro_rules! request {
1983            ($variant:ident, $request:ident, $id:expr $(, $depth:expr)?) => {
1984                RequestCommand::$variant($request::new(
1985                    $id, None, None, None, $($depth,)? None, UUID4::new(), UnixNanos::from(37), None,
1986                ))
1987            };
1988        }
1989
1990        macro_rules! response {
1991            ($variant:ident, $response:ident, $key:expr, $row:expr) => {{
1992                let data = BatchView::new(Arc::new(vec![$row]), 0..usize::from(!empty));
1993                DataResponse::$variant($response::new(
1994                    UUID4::new(),
1995                    ClientId::from("SOURCE"),
1996                    $key,
1997                    data,
1998                    None,
1999                    None,
2000                    UnixNanos::from(41),
2001                    None,
2002                ))
2003            }};
2004        }
2005
2006        let (request, response) = match kind {
2007            NautilusDataType::OrderBookDelta => {
2008                let row = OrderBookDelta {
2009                    instrument_id: actual,
2010                    ..stub_delta()
2011                };
2012
2013                (
2014                    request!(BookDeltas, RequestBookDeltas, requested),
2015                    response!(BookDeltas, BookDeltasResponse, requested, row),
2016                )
2017            }
2018            NautilusDataType::OrderBookDepth => {
2019                let row = OrderBookDepth {
2020                    instrument_id: actual,
2021                    ..stub_depth10()
2022                };
2023
2024                (
2025                    request!(BookDepth, RequestBookDepth, requested, None),
2026                    response!(BookDepth, BookDepthResponse, requested, row),
2027                )
2028            }
2029            NautilusDataType::QuoteTick => {
2030                let row = QuoteTick {
2031                    instrument_id: actual,
2032                    ..Default::default()
2033                };
2034
2035                (
2036                    request!(Quotes, RequestQuotes, requested),
2037                    response!(Quotes, QuotesResponse, requested, row),
2038                )
2039            }
2040            NautilusDataType::TradeTick => {
2041                let row = TradeTick {
2042                    instrument_id: actual,
2043                    ..Default::default()
2044                };
2045
2046                (
2047                    request!(Trades, RequestTrades, requested),
2048                    response!(Trades, TradesResponse, requested, row),
2049                )
2050            }
2051            NautilusDataType::Bar => {
2052                let row = Bar {
2053                    bar_type: actual_bar,
2054                    ..Default::default()
2055                };
2056
2057                (
2058                    request!(Bars, RequestBars, requested_bar),
2059                    response!(Bars, BarsResponse, requested_bar, row),
2060                )
2061            }
2062            NautilusDataType::FundingRateUpdate => {
2063                let row = FundingRateUpdate::new(
2064                    actual,
2065                    "0.001".parse().unwrap(),
2066                    Some(60),
2067                    Some(UnixNanos::from(47)),
2068                    UnixNanos::from(17),
2069                    UnixNanos::from(23),
2070                );
2071                (
2072                    request!(FundingRates, RequestFundingRates, requested),
2073                    response!(FundingRates, FundingRatesResponse, requested, row),
2074                )
2075            }
2076            _ => panic!("unexpected data type"),
2077        };
2078
2079        let batches = response_batches(&response).unwrap();
2080        let (identifier, batch) = &batches[0];
2081
2082        assert_eq!(batches.len(), 1);
2083        assert_eq!(
2084            request_contains_batch(&request, identifier.as_deref(), batch),
2085            empty || !remapped,
2086        );
2087    }
2088
2089    #[rstest]
2090    #[case::quotes(true)]
2091    #[case::trades(false)]
2092    fn test_response_batches_keep_expanded_parent_instruments(#[case] quotes: bool) {
2093        let requested = [
2094            InstrumentId::from("ES.FUT.GLBX"),
2095            InstrumentId::from("NQ.FUT.GLBX"),
2096        ];
2097        let first = InstrumentId::from("ESH24.GLBX");
2098        let second = InstrumentId::from("NQH24.GLBX");
2099
2100        let mut response = if quotes {
2101            let rows = vec![
2102                QuoteTick {
2103                    instrument_id: first,
2104                    ..Default::default()
2105                },
2106                QuoteTick {
2107                    instrument_id: second,
2108                    ..Default::default()
2109                },
2110            ];
2111            DataResponse::Quotes(QuotesResponse::new(
2112                UUID4::new(),
2113                ClientId::from("SOURCE"),
2114                requested,
2115                rows,
2116                None,
2117                None,
2118                UnixNanos::from(3),
2119                None,
2120            ))
2121        } else {
2122            let rows = vec![
2123                TradeTick {
2124                    instrument_id: first,
2125                    ..Default::default()
2126                },
2127                TradeTick {
2128                    instrument_id: second,
2129                    ..Default::default()
2130                },
2131            ];
2132            DataResponse::Trades(TradesResponse::new(
2133                UUID4::new(),
2134                ClientId::from("SOURCE"),
2135                requested,
2136                rows,
2137                None,
2138                None,
2139                UnixNanos::from(3),
2140                None,
2141            ))
2142        };
2143
2144        macro_rules! narrow {
2145            ($response:expr, $row:ident) => {{
2146                let mut rows = $response.data.to_vec();
2147                let hidden = $row {
2148                    instrument_id: InstrumentId::from("HIDDEN.GLBX"),
2149                    ..Default::default()
2150                };
2151                rows.insert(0, hidden);
2152                rows.push(hidden);
2153                $response.data = BatchView::new(Arc::new(rows), 1..3);
2154            }};
2155        }
2156
2157        match &mut response {
2158            DataResponse::Quotes(r) => narrow!(r, QuoteTick),
2159            DataResponse::Trades(r) => narrow!(r, TradeTick),
2160            _ => unreachable!(),
2161        }
2162
2163        let batches = response_batches(&response).unwrap();
2164
2165        assert_eq!(batches.len(), 2);
2166
2167        for ((identifier, batch), id) in batches.iter().zip([first, second]) {
2168            assert_eq!(identifier.as_deref(), Some(id.to_string().as_str()));
2169
2170            match batch {
2171                DataBatch::Quote(rows) => assert_eq!(
2172                    &rows[..],
2173                    &[QuoteTick {
2174                        instrument_id: id,
2175                        ..Default::default()
2176                    }]
2177                ),
2178                DataBatch::Trade(rows) => assert_eq!(
2179                    &rows[..],
2180                    &[TradeTick {
2181                        instrument_id: id,
2182                        ..Default::default()
2183                    }]
2184                ),
2185                other => panic!("Unexpected batch {other:?}"),
2186            }
2187        }
2188    }
2189
2190    #[rstest]
2191    fn test_build_empty_response_rejects_non_catalog_variant() {
2192        let request = RequestCommand::Join(RequestJoin::new(
2193            vec![UUID4::new()],
2194            None,
2195            None,
2196            UUID4::new(),
2197            UnixNanos::default(),
2198            None,
2199            None,
2200        ));
2201
2202        let result = build_empty_response(
2203            &request,
2204            UnixNanos::from(1u64),
2205            UnixNanos::from(2u64),
2206            None,
2207            UnixNanos::from(3u64),
2208        );
2209
2210        assert_eq!(
2211            result.unwrap_err().to_string(),
2212            "Cannot build empty catalog response for non-catalog-eligible request"
2213        );
2214    }
2215}