1use ahash::{AHashMap, AHashSet};
17use indexmap::IndexMap;
18use jiff::Timestamp;
19use nautilus_common::messages::data::{
20 BarsResponse, BookDeltasResponse, BookDepthResponse, CustomDataResponse, DataRequestFailed,
21 DataResponse, FundingRatesResponse, InstrumentResponse, InstrumentsResponse, QuotesResponse,
22 RequestBars, RequestBookDeltas, RequestBookDepth, RequestCommand, RequestCustomData,
23 RequestFundingRates, RequestInstrument, RequestInstruments, RequestQuotes, RequestTrades,
24 SubscribeBars, SubscribeCommand, SubscribeCustomData, SubscribeQuotes, SubscribeTrades,
25 TradesResponse,
26};
27use nautilus_core::{
28 Params, UUID4, UnixNanos,
29 correctness::{FAILED, check_key_not_in_map},
30};
31use nautilus_model::{
32 data::{
33 Bar, BarType, BatchView, CustomData, DataBatch, FromDataBatch, FundingRateUpdate,
34 NautilusDataType, OrderBookDelta, OrderBookDepth, QuoteTick, TradeTick,
35 },
36 identifiers::{ClientId, InstrumentId, Venue},
37 instruments::{Instrument, InstrumentAny},
38};
39use nautilus_persistence::catalog::traits::{CatalogInstrumentQuery, CatalogQuery, DataCatalog};
40use serde_json::Value;
41use ustr::Ustr;
42
43use super::{DataEngine, requests::request_params};
44
45const PARAM_SKIP_CATALOG_DATA: &str = "skip_catalog_data";
46pub(super) const PARAM_UPDATE_CATALOG: &str = "update_catalog";
47const PARAM_FORCE_INSTRUMENT_UPDATE: &str = "force_instrument_update";
48const PARAM_SUBSCRIPTION_NAME: &str = "subscription_name";
49const PARAM_FROM_DAY_START: &str = "from_day_start";
50const CATALOG_CLIENT_ID: &str = "CATALOG";
51
52pub(crate) type CatalogMap = AHashMap<Ustr, DataCatalog>;
53
54impl DataEngine {
55 pub fn register_catalog(&mut self, catalog: DataCatalog, name: Option<&str>) {
61 let name = Ustr::from(name.unwrap_or("catalog_0"));
62
63 check_key_not_in_map(&name, &self.catalogs, "name", "catalogs").expect(FAILED);
64
65 self.catalogs.insert(name, catalog);
66 log::info!("Registered catalog <{name}>");
67 }
68
69 pub(super) fn subscribe_command_with_prefilled_start_ns(
70 &mut self,
71 cmd: SubscribeCommand,
72 ) -> anyhow::Result<SubscribeCommand> {
73 match cmd {
74 SubscribeCommand::Quotes(cmd)
75 if cmd.instrument_ids.len() == 1
76 && Self::is_start_ns_missing(cmd.params.as_ref()) =>
77 {
78 let identifier = cmd.instrument_ids[0].to_string();
79 let params = self.params_with_prefilled_start_ns(
80 cmd.params.as_ref(),
81 &NautilusDataType::QuoteTick,
82 &identifier,
83 )?;
84 Ok(SubscribeCommand::Quotes(SubscribeQuotes { params, ..cmd }))
85 }
86 SubscribeCommand::Trades(cmd)
87 if cmd.instrument_ids.len() == 1
88 && Self::is_start_ns_missing(cmd.params.as_ref()) =>
89 {
90 let identifier = cmd.instrument_ids[0].to_string();
91 let params = self.params_with_prefilled_start_ns(
92 cmd.params.as_ref(),
93 &NautilusDataType::TradeTick,
94 &identifier,
95 )?;
96 Ok(SubscribeCommand::Trades(SubscribeTrades { params, ..cmd }))
97 }
98 SubscribeCommand::Bars(cmd)
99 if cmd.bar_types.len() == 1
100 && cmd.single_bar_type()?.is_externally_aggregated()
101 && Self::is_start_ns_missing(cmd.params.as_ref()) =>
102 {
103 let identifier = cmd.single_bar_type()?.to_string();
104 let params = self.params_with_prefilled_start_ns(
105 cmd.params.as_ref(),
106 &NautilusDataType::Bar,
107 &identifier,
108 )?;
109 Ok(SubscribeCommand::Bars(SubscribeBars { params, ..cmd }))
110 }
111 SubscribeCommand::Data(cmd) if Self::is_start_ns_missing(cmd.params.as_ref()) => {
112 let type_name = cmd.data_type.type_name().to_string();
113 let identifier = cmd.data_type.identifier().map(String::from);
114 let params = self.params_with_custom_data_prefilled_start_ns(
115 cmd.params.as_ref(),
116 &type_name,
117 identifier.as_deref(),
118 )?;
119 Ok(SubscribeCommand::Data(SubscribeCustomData {
120 params,
121 ..cmd
122 }))
123 }
124 _ => Ok(cmd),
125 }
126 }
127
128 fn is_start_ns_missing(params: Option<&Params>) -> bool {
129 params.is_none_or(|params| !params.contains_key("start_ns"))
130 }
131
132 fn params_with_prefilled_start_ns(
133 &mut self,
134 params: Option<&Params>,
135 data_type: &NautilusDataType,
136 identifier: &str,
137 ) -> anyhow::Result<Option<Params>> {
138 let last_timestamp = self.catalog_last_timestamp(data_type, identifier)?;
139
140 Ok(Some(Self::params_with_start_ns(params, last_timestamp)))
141 }
142
143 fn params_with_custom_data_prefilled_start_ns(
144 &mut self,
145 params: Option<&Params>,
146 type_name: &str,
147 identifier: Option<&str>,
148 ) -> anyhow::Result<Option<Params>> {
149 let last_timestamp = self.catalog_custom_data_last_timestamp(type_name, identifier)?;
150
151 Ok(Some(Self::params_with_start_ns(params, last_timestamp)))
152 }
153
154 fn params_with_start_ns(params: Option<&Params>, last_timestamp: Option<u64>) -> Params {
155 let start_ns = last_timestamp.map_or(Value::Null, |last_timestamp| {
156 Value::from(last_timestamp.saturating_add(1))
157 });
158
159 let mut params = params.map_or_default(|params| IndexMap::clone(params));
160
161 params.insert("start_ns".to_string(), start_ns);
162
163 Params::new(params)
164 }
165
166 fn catalog_last_timestamp(
167 &mut self,
168 data_type: &NautilusDataType,
169 identifier: &str,
170 ) -> anyhow::Result<Option<u64>> {
171 for catalog in self.catalogs.values_mut() {
172 if let Some(last_timestamp) =
173 catalog.query_last_timestamp(data_type.clone(), Some(identifier))?
174 {
175 return Ok(Some(last_timestamp));
176 }
177 }
178
179 Ok(None)
180 }
181
182 fn catalog_custom_data_last_timestamp(
183 &mut self,
184 type_name: &str,
185 identifier: Option<&str>,
186 ) -> anyhow::Result<Option<u64>> {
187 let data_type = NautilusDataType::Custom {
192 type_name: type_name.to_string(),
193 };
194
195 for catalog in self.catalogs.values_mut() {
196 if let Some(last_timestamp) =
197 catalog.query_last_timestamp(data_type.clone(), identifier)?
198 {
199 return Ok(Some(last_timestamp));
200 }
201 }
202
203 Ok(None)
204 }
205
206 pub(super) fn validate_catalog_update(&self, request: &RequestCommand) -> anyhow::Result<()> {
209 if request_params(request).and_then(|p| p.get_bool(PARAM_UPDATE_CATALOG)) != Some(true) {
210 return Ok(());
211 }
212
213 anyhow::ensure!(
214 !self.catalogs.is_empty(),
215 "Catalog write-back requires a registered catalog"
216 );
217 anyhow::ensure!(
218 !request_has_schema(request),
219 "Catalog write-back is unsupported for requests with a source schema"
220 );
221 anyhow::ensure!(
222 request_limit(request).is_none(),
223 "Catalog write-back is unsupported for requests with a record limit"
224 );
225 Ok(())
226 }
227
228 pub(super) fn update_catalog_from_response(
236 &mut self,
237 response: &DataResponse,
238 request: &RequestCommand,
239 ) -> anyhow::Result<()> {
240 let Some(params) =
241 request_params(request).filter(|p| p.get_bool(PARAM_UPDATE_CATALOG).unwrap_or(false))
242 else {
243 return Ok(());
244 };
245
246 if response
247 .params()
248 .is_some_and(|p| p.get_bool(PARAM_UPDATE_CATALOG) == Some(false))
249 {
250 return Ok(());
251 }
252
253 if let DataResponse::Instrument(value) = response {
254 return self.write_response_instruments(std::slice::from_ref(&value.data), params);
255 }
256
257 if let DataResponse::Instruments(value) = response {
258 return self.write_response_instruments(&value.data, params);
259 }
260
261 let batches = match response_batches(response) {
262 Ok(batches) => batches,
263 Err(e) => {
264 log::error!("Cannot prepare response catalog write: {e}");
265 anyhow::bail!(
266 "Catalog write-back failed: cannot prepare the response batches: {e}"
267 );
268 }
269 };
270
271 let bounds = response_write_bounds(request, response);
272 let data_type = response.data_kind();
273 let mut failures = Vec::new();
274
275 for (identifier, batch) in batches {
276 if !request_contains_batch(request, identifier.as_deref(), &batch) {
277 log::warn!(
278 "Skipping catalog write-back because identifier coverage is unverified, request={}, identifier={identifier:?}",
279 request.request_id()
280 );
281 failures.push(format!(
282 "Catalog write-back failed: identifier coverage is unverified, identifier={identifier:?}"
283 ));
284 continue;
285 }
286
287 if let Err(e) =
288 self.write_response_batch(&batch, &data_type, identifier.as_deref(), params, bounds)
289 {
290 log::error!("Catalog write-back failed for {identifier:?}: {e}");
291 failures.push(format!("Catalog write-back failed for {identifier:?}: {e}"));
292 }
293 }
294
295 if !failures.is_empty() {
296 anyhow::bail!(failures.join("; "));
297 }
298 Ok(())
299 }
300
301 fn write_response_batch(
302 &mut self,
303 batch: &DataBatch,
304 data_type: &NautilusDataType,
305 identifier: Option<&str>,
306 params: &Params,
307 bounds: (Option<UnixNanos>, Option<UnixNanos>),
308 ) -> anyhow::Result<()> {
309 let name = self.catalog_for_update(data_type, identifier, params)?;
310 let catalog = self
311 .catalogs
312 .get_mut(&name)
313 .expect("selected catalog is registered");
314 let (start, end) = bounds;
315 if batch.is_empty() {
316 if let (Some(start), Some(end)) = (start, end) {
317 catalog.record_empty_coverage(data_type.clone(), identifier, start, end)?;
318 }
319
320 return Ok(());
321 }
322
323 catalog.write_data_batch(batch, start, end, Some(params.clone()))
324 }
325
326 fn write_response_instruments(
327 &mut self,
328 instruments: &[InstrumentAny],
329 params: &Params,
330 ) -> anyhow::Result<()> {
331 let force = params
332 .get_bool(PARAM_FORCE_INSTRUMENT_UPDATE)
333 .unwrap_or(false);
334 let mut failures = Vec::new();
335
336 for instrument in instruments {
337 let identifier = instrument.id().to_string();
338
339 let result = (|| {
340 let name = self.catalog_for_update(
341 &NautilusDataType::Instrument,
342 Some(&identifier),
343 params,
344 )?;
345 let catalog = self
346 .catalogs
347 .get_mut(&name)
348 .expect("selected catalog is registered");
349
350 if !force {
351 let query = CatalogInstrumentQuery::new()
352 .with_instrument_ids(Some(vec![identifier.clone()]));
353
354 if !catalog.instruments(&query)?.is_empty() {
355 return Ok(());
356 }
357 }
358
359 catalog.write_instruments(std::slice::from_ref(instrument))
360 })();
361
362 if let Err(e) = result {
363 log::error!("Instrument catalog write-back failed for {identifier}: {e}");
364 failures.push(format!(
365 "Catalog write-back failed for instrument {identifier}: {e}"
366 ));
367 }
368 }
369
370 if !failures.is_empty() {
371 anyhow::bail!(failures.join("; "));
372 }
373 Ok(())
374 }
375
376 fn catalog_for_update(
377 &mut self,
378 data_type: &NautilusDataType,
379 identifier: Option<&str>,
380 params: &Params,
381 ) -> anyhow::Result<Ustr> {
382 if let Some(name) = params.get_str("catalog_name").map(Ustr::from) {
383 anyhow::ensure!(
384 self.catalogs.contains_key(&name),
385 "Catalog '{name}' is not registered"
386 );
387 return Ok(name);
388 }
389
390 let mut names: Vec<_> = self.catalogs.keys().copied().collect();
391 names.sort_by(|a, b| a.as_str().cmp(b.as_str()));
392 for name in &names {
393 let catalog = self.catalogs.get_mut(name).expect("catalog key exists");
394 if catalog
395 .query_last_timestamp(data_type.clone(), identifier)?
396 .is_some()
397 {
398 return Ok(*name);
399 }
400 }
401
402 names
403 .first()
404 .copied()
405 .ok_or_else(|| anyhow::anyhow!("No catalog registered"))
406 }
407
408 pub(super) fn catalogs_registered(&self) -> bool {
409 !self.catalogs.is_empty()
410 }
411
412 pub(super) fn dispatch_date_range_request(
418 &mut self,
419 req: RequestCommand,
420 ) -> anyhow::Result<()> {
421 if request_has_schema(&req) {
422 return self.dispatch_request_to_client(req).map(|_| ());
423 }
424
425 if matches!(
426 req,
427 RequestCommand::Instrument(_) | RequestCommand::Instruments(_)
428 ) {
429 return self.dispatch_instrument_catalog_request(req);
430 }
431
432 if matches!(&req, RequestCommand::Quotes(r) if r.instrument_ids.len() > 1)
433 || matches!(&req, RequestCommand::Trades(r) if r.instrument_ids.len() > 1)
434 {
435 return self.dispatch_batch_catalog_request(req);
436 }
437
438 let Some(key) = request_identifier(&req) else {
439 return self.dispatch_request_to_client(req).map(|_| ());
440 };
441
442 let now_ns = self.clock.borrow().timestamp_ns();
443 let now_dt = now_ns.to_datetime_utc();
444 let query_past_data = request_params(&req)
445 .and_then(|p| p.get(PARAM_SUBSCRIPTION_NAME))
446 .is_none();
447
448 let (start_dt, end_dt) = bound_request_dates(
449 request_start(&req),
450 request_end(&req),
451 now_dt,
452 query_past_data,
453 );
454 let start_ns = datetime_to_unix_nanos_or_zero(start_dt);
455 let end_ns = datetime_to_unix_nanos_or_zero(end_dt);
456
457 if start_ns > end_ns {
458 anyhow::bail!(
459 "Cannot dispatch request, start {start_ns} was greater than end {end_ns}"
460 );
461 }
462
463 let catalog_start_dt = catalog_request_start(&req, start_dt);
464 let catalog_start_ns = datetime_to_unix_nanos_or_zero(catalog_start_dt);
465 let query_interval = vec![(start_ns.as_u64(), end_ns.as_u64())];
466 let coverage = self.catalog_coverage_for_request(
467 &key,
468 start_ns.as_u64(),
469 end_ns.as_u64(),
470 catalog_start_ns.as_u64(),
471 )?;
472
473 let (winning_catalog, mut missing_intervals) = match coverage {
474 Some(coverage) => (Some(coverage.catalog), coverage.missing),
475 None => (None, query_interval.clone()),
476 };
477
478 let skip_catalog_data = request_params(&req)
479 .and_then(|p| p.get_bool(PARAM_SKIP_CATALOG_DATA))
480 .unwrap_or(false);
481
482 if skip_catalog_data {
486 missing_intervals = query_interval;
487 }
488
489 let client_id = req.client_id().copied();
490 let venue = req.venue().copied();
491 let has_client = self
492 .get_client(client_id.as_ref(), venue.as_ref())
493 .is_some();
494
495 if !has_client {
496 missing_intervals.clear();
497 }
498
499 let mut plan = Vec::new();
500
501 if !skip_catalog_data && let Some(catalog) = winning_catalog {
502 let request =
503 with_dates_for_pipeline(&req, Some(catalog_start_dt), Some(end_dt), now_ns);
504 plan.push(RequestLeg::Catalog { request, catalog });
505 }
506
507 for (start, end) in missing_intervals {
508 let start = UnixNanos::from(start).to_datetime_utc();
509 let end = UnixNanos::from(end).to_datetime_utc();
510 let request = with_dates_for_pipeline(&req, Some(start), Some(end), now_ns);
511 plan.push(RequestLeg::Client(request));
512 }
513
514 self.dispatch_request_plan(req, plan)
515 }
516
517 fn dispatch_batch_catalog_request(&mut self, parent: RequestCommand) -> anyhow::Result<()> {
518 let ids = match &parent {
519 RequestCommand::Quotes(r) => r.instrument_ids.clone(),
520 RequestCommand::Trades(r) => r.instrument_ids.clone(),
521 _ => unreachable!("only quote and trade lists use catalog batch planning"),
522 };
523
524 let now_ns = self.clock.borrow().timestamp_ns();
525 let query_past = request_params(&parent)
526 .and_then(|p| p.get(PARAM_SUBSCRIPTION_NAME))
527 .is_none();
528 let (start, end) = bound_request_dates(
529 request_start(&parent),
530 request_end(&parent),
531 now_ns.to_datetime_utc(),
532 query_past,
533 );
534 let start_ns = datetime_to_unix_nanos_or_zero(start);
535 let end_ns = datetime_to_unix_nanos_or_zero(end);
536 anyhow::ensure!(start_ns <= end_ns, "Request start exceeds end");
537 let skip_catalog = request_params(&parent)
538 .and_then(|p| p.get_bool(PARAM_SKIP_CATALOG_DATA))
539 .unwrap_or(false);
540 let full = vec![(start_ns.as_u64(), end_ns.as_u64())];
541 let mut clients: IndexMap<(u64, u64), Vec<InstrumentId>> = IndexMap::new();
542 let mut plan = Vec::new();
543
544 for id in ids {
545 let mut member = with_request_instrument_ids(&parent, vec![id]);
546 let key = request_identifier(&member).expect("single quote or trade identifier");
547 let mut missing = full.clone();
548 if !skip_catalog
549 && let Some(CatalogCoverage {
550 catalog,
551 missing: intervals,
552 }) = self.catalog_coverage(&key, start_ns.as_u64(), end_ns.as_u64())?
553 {
554 missing = intervals;
555 member = with_dates_for_pipeline(&member, Some(start), Some(end), now_ns);
556 plan.push(RequestLeg::Catalog {
557 request: member,
558 catalog,
559 });
560 }
561
562 for interval in missing {
563 clients.entry(interval).or_default().push(id);
564 }
565 }
566
567 for ((start, end), ids) in clients {
568 let member = with_request_instrument_ids(&parent, ids);
569 let client_id = member.client_id().copied();
570 let venue = member.venue().copied();
571
572 if self
573 .get_client(client_id.as_ref(), venue.as_ref())
574 .is_none()
575 {
576 continue;
577 }
578
579 let start = UnixNanos::from(start).to_datetime_utc();
580 let end = UnixNanos::from(end).to_datetime_utc();
581 let request = with_dates_for_pipeline(&member, Some(start), Some(end), now_ns);
582 plan.push(RequestLeg::Client(request));
583 }
584
585 self.dispatch_request_plan(parent, plan)
586 }
587
588 fn catalog_coverage_for_request(
589 &mut self,
590 key: &RequestCatalogKey,
591 start: u64,
592 end: u64,
593 catalog_start: u64,
594 ) -> anyhow::Result<Option<CatalogCoverage>> {
595 let Some(mut coverage) = self.catalog_coverage(key, catalog_start, end)? else {
596 return Ok(None);
597 };
598
599 if catalog_start != start {
601 let catalog = self
602 .catalogs
603 .get_mut(&coverage.catalog)
604 .expect("selected catalog is registered");
605 coverage.missing = catalog_missing_intervals(catalog, start, end, key)?;
606 }
607
608 Ok(Some(coverage))
609 }
610
611 fn catalog_coverage(
612 &mut self,
613 key: &RequestCatalogKey,
614 start: u64,
615 end: u64,
616 ) -> anyhow::Result<Option<CatalogCoverage>> {
617 for (name, catalog) in &mut self.catalogs {
618 let missing = catalog_missing_intervals(catalog, start, end, key)?;
619 if missing != [(start, end)] {
620 return Ok(Some(CatalogCoverage {
621 catalog: *name,
622 missing,
623 }));
624 }
625 }
626
627 Ok(None)
628 }
629
630 fn dispatch_request_plan(
631 &mut self,
632 parent: RequestCommand,
633 plan: Vec<RequestLeg>,
634 ) -> anyhow::Result<()> {
635 let parent_id = *parent.request_id();
636
637 if plan.is_empty() {
638 let now = self.clock.borrow().timestamp_ns();
639 let (start, end) = bound_request_dates(
640 request_start(&parent),
641 request_end(&parent),
642 now.to_datetime_utc(),
643 true,
644 );
645 let response = build_empty_response(
646 &parent,
647 datetime_to_unix_nanos_or_zero(start),
648 datetime_to_unix_nanos_or_zero(end),
649 None,
650 now,
651 )?;
652 self.response(response);
653 return Ok(());
654 }
655
656 self.new_request_pipeline(parent, plan.len());
657 for leg in &plan {
658 self.register_request_pipeline_leg(leg.request_id(), parent_id);
659 }
660
661 for leg in plan {
662 let request_id = leg.request_id();
663
664 let result = match leg {
665 RequestLeg::Catalog { request, catalog } => self
666 .query_catalog_leg(&request, catalog)
667 .map(|response| self.response(response)),
668 RequestLeg::Client(request) => self.dispatch_request_to_client(request).map(|_| ()),
669 RequestLeg::Response(response) => {
670 self.response(response);
671 Ok(())
672 }
673 };
674
675 if let Err(e) = result {
676 self.request_failed(DataRequestFailed {
677 request_id,
678 reason: e.to_string(),
679 response: None,
680 });
681 }
682 }
683
684 Ok(())
685 }
686
687 fn query_catalog_leg(
688 &mut self,
689 leg: &RequestCommand,
690 catalog_name: Ustr,
691 ) -> anyhow::Result<DataResponse> {
692 let start_ns = request_start(leg)
693 .map(datetime_to_unix_nanos_or_zero)
694 .expect("catalog leg has bounded start");
695 let end_ns = request_end(leg)
696 .map(datetime_to_unix_nanos_or_zero)
697 .expect("catalog leg has bounded end");
698 let ts_init = self.clock.borrow().timestamp_ns();
699 let client_id = leg.client_id().copied();
700 let venue = leg.venue().copied();
701 let used_client_id = self
702 .get_client(client_id.as_ref(), venue.as_ref())
703 .map(|value| value.client_id());
704
705 let catalog = self.catalogs.get_mut(&catalog_name).ok_or_else(|| {
706 anyhow::anyhow!("Catalog {catalog_name} disappeared between intervals query and read")
707 })?;
708
709 match leg {
710 RequestCommand::Quotes(cmd) => {
711 let data = quote_ticks_from_query_result(
712 catalog.query_batch(
713 &CatalogQuery::new(NautilusDataType::QuoteTick)
714 .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
715 .with_range(Some(start_ns), Some(end_ns)),
716 )?,
717 )?;
718 Ok(build_quotes_catalog_response(
719 cmd,
720 data,
721 start_ns,
722 end_ns,
723 used_client_id,
724 ts_init,
725 ))
726 }
727 RequestCommand::Trades(cmd) => {
728 let data = trade_ticks_from_query_result(
729 catalog.query_batch(
730 &CatalogQuery::new(NautilusDataType::TradeTick)
731 .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
732 .with_range(Some(start_ns), Some(end_ns)),
733 )?,
734 )?;
735 Ok(build_trades_catalog_response(
736 cmd,
737 data,
738 start_ns,
739 end_ns,
740 used_client_id,
741 ts_init,
742 ))
743 }
744 RequestCommand::FundingRates(cmd) => {
745 let data = funding_rates_from_query_result(
746 catalog.query_batch(
747 &CatalogQuery::new(NautilusDataType::FundingRateUpdate)
748 .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
749 .with_range(Some(start_ns), Some(end_ns)),
750 )?,
751 )?;
752 build_funding_rates_catalog_response(
753 cmd,
754 data,
755 start_ns,
756 end_ns,
757 used_client_id,
758 ts_init,
759 )
760 }
761 RequestCommand::Bars(cmd) => {
762 let data = bars_from_query_result(
763 catalog.query_batch(
764 &CatalogQuery::new(NautilusDataType::Bar)
765 .with_identifiers(Some(vec![cmd.single_bar_type()?.to_string()]))
766 .with_range(Some(start_ns), Some(end_ns)),
767 )?,
768 )?;
769 build_bars_catalog_response(cmd, data, start_ns, end_ns, used_client_id, ts_init)
770 }
771 RequestCommand::Data(cmd) => {
772 let identifiers = cmd
773 .data_type
774 .identifier()
775 .map(|identifier| vec![identifier.to_string()]);
776 let where_clause = request_filter_expr(cmd.params.as_ref());
777 let data = catalog.query_batch(
778 &CatalogQuery::new(NautilusDataType::Custom {
779 type_name: cmd.data_type.type_name().to_string(),
780 })
781 .with_identifiers(identifiers)
782 .with_range(Some(start_ns), Some(end_ns))
783 .with_where_clause(where_clause),
784 )?;
785 Ok(build_custom_data_catalog_response(
786 cmd,
787 custom_data_from_query_result(data)?,
788 start_ns,
789 end_ns,
790 ts_init,
791 ))
792 }
793 RequestCommand::BookDeltas(cmd) => {
794 let data = order_book_deltas_from_query_result(
795 catalog.query_batch(
796 &CatalogQuery::new(NautilusDataType::OrderBookDelta)
797 .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
798 .with_range(Some(start_ns), Some(end_ns)),
799 )?,
800 )?;
801 build_book_deltas_catalog_response(
802 cmd,
803 data,
804 start_ns,
805 end_ns,
806 used_client_id,
807 ts_init,
808 )
809 }
810 RequestCommand::BookDepth(cmd) => {
811 let data = order_book_depths_from_query_result(
812 catalog.query_batch(
813 &CatalogQuery::new(NautilusDataType::OrderBookDepth)
814 .with_identifiers(Some(vec![cmd.single_instrument_id()?.to_string()]))
815 .with_range(Some(start_ns), Some(end_ns)),
816 )?,
817 )?;
818 build_book_depth_catalog_response(
819 cmd,
820 data,
821 start_ns,
822 end_ns,
823 used_client_id,
824 ts_init,
825 )
826 }
827 _ => {
828 anyhow::bail!("query_catalog_leg called with non-catalog-eligible variant {leg:?}")
829 }
830 }
831 }
832
833 fn dispatch_instrument_catalog_request(&mut self, req: RequestCommand) -> anyhow::Result<()> {
834 match req {
835 RequestCommand::Instrument(cmd) => self.dispatch_instrument_request(cmd),
836 RequestCommand::Instruments(cmd) => self.dispatch_instruments_request(cmd),
837 _ => self.dispatch_request_to_client(req).map(|_| ()),
838 }
839 }
840
841 fn dispatch_instrument_request(&mut self, cmd: RequestInstrument) -> anyhow::Result<()> {
842 if cmd.instrument_ids.len() > 1 {
843 return self.dispatch_instrument_list(cmd);
844 }
845
846 let force_instrument_update = cmd
847 .params
848 .as_ref()
849 .and_then(|params| params.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
850 .unwrap_or(false);
851
852 if force_instrument_update {
853 return self
854 .dispatch_request_to_client(RequestCommand::Instrument(cmd))
855 .map(|_| ());
856 }
857
858 let identifier = cmd.single_instrument_id()?.to_string();
859 let now_ns = self.clock.borrow().timestamp_ns();
860 let used_client_id = self
861 .get_client(
862 cmd.client_id.as_ref(),
863 Some(&cmd.single_instrument_id()?.venue),
864 )
865 .map(|client| client.client_id());
866 let (start_dt, end_dt) =
867 bound_request_dates(cmd.start, cmd.end, now_ns.to_datetime_utc(), true);
868 let start_ns = datetime_to_unix_nanos_or_zero(start_dt);
869 let end_ns = datetime_to_unix_nanos_or_zero(end_dt);
870 let start = Some(start_ns);
871 let end = cmd.end.map(datetime_to_unix_nanos_or_zero);
872
873 for catalog in self.catalogs.values_mut() {
874 let data = latest_instruments(
875 catalog.instruments(
876 &CatalogInstrumentQuery::new()
877 .with_instrument_ids(Some(vec![identifier.clone()]))
878 .with_range(start, end),
879 )?,
880 );
881
882 if let Some(instrument) = data.into_iter().next() {
883 let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
884 cmd.request_id,
885 resolve_response_client_id(cmd.client_id, used_client_id),
886 cmd.single_instrument_id()?,
887 instrument,
888 Some(start_ns),
889 Some(end_ns),
890 now_ns,
891 Some(catalog_response_params(cmd.params.as_ref())),
892 )));
893 self.response(response);
894 return Ok(());
895 }
896 }
897
898 self.dispatch_request_to_client(RequestCommand::Instrument(cmd))
899 .map(|_| ())
900 }
901
902 fn dispatch_instrument_list(&mut self, command: RequestInstrument) -> anyhow::Result<()> {
903 let parent = RequestCommand::Instrument(command.clone());
904 let force = command
905 .params
906 .as_ref()
907 .and_then(|p| p.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
908 .unwrap_or(false);
909
910 if force {
911 return self.dispatch_request_to_client(parent).map(|_| ());
912 }
913
914 let now = self.clock.borrow().timestamp_ns();
915 let (start, end) =
916 bound_request_dates(command.start, command.end, now.to_datetime_utc(), true);
917 let start_ns = datetime_to_unix_nanos_or_zero(start);
918 let end_ns = datetime_to_unix_nanos_or_zero(end);
919 let mut missing = Vec::new();
920 let mut plan = Vec::new();
921
922 for id in &command.instrument_ids {
923 let mut instrument = None;
924
925 for catalog in self.catalogs.values_mut() {
926 let query = CatalogInstrumentQuery::new()
927 .with_instrument_ids(Some(vec![id.to_string()]))
928 .with_range(
929 Some(start_ns),
930 command.end.map(datetime_to_unix_nanos_or_zero),
931 );
932 instrument = latest_instruments(catalog.instruments(&query)?)
933 .into_iter()
934 .find(|value| value.id() == *id);
935
936 if instrument.is_some() {
937 break;
938 }
939 }
940
941 if let Some(instrument) = instrument {
942 let response = InstrumentResponse::new(
943 UUID4::new(),
944 command
945 .client_id
946 .unwrap_or_else(|| ClientId::new(CATALOG_CLIENT_ID)),
947 *id,
948 instrument,
949 Some(start_ns),
950 Some(end_ns),
951 now,
952 Some(catalog_response_params(command.params.as_ref())),
953 );
954 plan.push(RequestLeg::Response(DataResponse::Instrument(Box::new(
955 response,
956 ))));
957 } else {
958 missing.push(*id);
959 }
960 }
961
962 if !missing.is_empty() {
963 let mut child = command;
964 child.request_id = UUID4::new();
965 child.instrument_ids = missing;
966 plan.push(RequestLeg::Client(RequestCommand::Instrument(child)));
967 }
968
969 self.dispatch_request_plan(parent, plan)
970 }
971
972 fn dispatch_instruments_request(&mut self, cmd: RequestInstruments) -> anyhow::Result<()> {
973 let update_catalog = cmd
974 .params
975 .as_ref()
976 .and_then(|params| params.get_bool(PARAM_UPDATE_CATALOG))
977 .unwrap_or(false);
978 let force_instrument_update = cmd
979 .params
980 .as_ref()
981 .and_then(|params| params.get_bool(PARAM_FORCE_INSTRUMENT_UPDATE))
982 .unwrap_or(false);
983
984 if update_catalog || force_instrument_update {
985 return self
986 .dispatch_request_to_client(RequestCommand::Instruments(cmd))
987 .map(|_| ());
988 }
989
990 let now_ns = self.clock.borrow().timestamp_ns();
991 let used_client_id = self
992 .get_client(cmd.client_id.as_ref(), cmd.venue.as_ref())
993 .map(|client| client.client_id());
994 let (start_dt, _) = bound_request_dates(cmd.start, cmd.end, now_ns.to_datetime_utc(), true);
995 let start = Some(datetime_to_unix_nanos_or_zero(start_dt));
996 let end = cmd.end.map(datetime_to_unix_nanos_or_zero);
997 let mut data = Vec::new();
998
999 for catalog in self.catalogs.values_mut() {
1000 data.extend(
1001 catalog.instruments(
1002 &CatalogInstrumentQuery::new()
1003 .with_range(start, end)
1004 .with_where_clause(request_filter_expr(cmd.params.as_ref())),
1005 )?,
1006 );
1007 }
1008
1009 if let Some(venue) = cmd.venue {
1010 data.retain(|instrument| instrument.venue() == venue);
1011 }
1012
1013 if instrument_only_last(cmd.params.as_ref()) {
1014 data = latest_instruments(data);
1015 }
1016
1017 let response = DataResponse::Instruments(InstrumentsResponse::new(
1018 cmd.request_id,
1019 resolve_response_client_id(cmd.client_id, used_client_id),
1020 instrument_response_venue(cmd.venue, &data),
1021 data,
1022 start,
1024 end,
1025 now_ns,
1026 Some(catalog_response_params(cmd.params.as_ref())),
1027 ));
1028 self.response(response);
1029 Ok(())
1030 }
1031}
1032
1033struct RequestCatalogKey {
1034 data_type: NautilusDataType,
1035 identifier: Option<String>,
1036}
1037
1038fn response_write_bounds(
1039 request: &RequestCommand,
1040 response: &DataResponse,
1041) -> (Option<UnixNanos>, Option<UnixNanos>) {
1042 (
1043 request_start(request)
1044 .map(datetime_to_unix_nanos_or_zero)
1045 .or(response.start()),
1046 request_end(request)
1047 .map(datetime_to_unix_nanos_or_zero)
1048 .or(response.end()),
1049 )
1050}
1051
1052fn response_batches(response: &DataResponse) -> anyhow::Result<Vec<(Option<String>, DataBatch)>> {
1053 let batches = match response {
1054 DataResponse::Quotes(r) => response_instrument_batches(
1055 &r.instrument_ids,
1056 &r.data,
1057 |row| row.instrument_id,
1058 DataBatch::Quote,
1059 ),
1060 DataResponse::Trades(r) => response_instrument_batches(
1061 &r.instrument_ids,
1062 &r.data,
1063 |row| row.instrument_id,
1064 DataBatch::Trade,
1065 ),
1066 DataResponse::BookDeltas(r) => vec![(
1067 Some(r.instrument_id.to_string()),
1068 DataBatch::BookDelta(r.data.clone()),
1069 )],
1070 DataResponse::BookDepth(r) => vec![(
1071 Some(r.instrument_id.to_string()),
1072 DataBatch::BookDepth(r.data.clone()),
1073 )],
1074 DataResponse::FundingRates(r) => vec![(
1075 Some(r.instrument_id.to_string()),
1076 DataBatch::FundingRate(r.data.clone()),
1077 )],
1078 DataResponse::Bars(r) => {
1079 vec![(Some(r.bar_type.to_string()), DataBatch::Bar(r.data.clone()))]
1080 }
1081 DataResponse::Data(r) => {
1082 let values = if let Some(data) = r.data.downcast_ref::<Vec<CustomData>>() {
1083 data.clone()
1084 } else if let Some(data) = r.data.downcast_ref::<CustomData>() {
1085 vec![data.clone()]
1086 } else {
1087 anyhow::bail!("Custom response payload is not catalog data");
1088 };
1089
1090 vec![(
1091 r.data_type.identifier().map(String::from),
1092 DataBatch::Custom(values.into()),
1093 )]
1094 }
1095 _ => Vec::new(),
1096 };
1097
1098 Ok(batches)
1099}
1100
1101fn response_instrument_batches<T: Copy>(
1102 requested: &[InstrumentId],
1103 data: &BatchView<T>,
1104 instrument_id: impl Fn(&T) -> InstrumentId,
1105 into_batch: impl Fn(BatchView<T>) -> DataBatch,
1106) -> Vec<(Option<String>, DataBatch)> {
1107 let ids = response_instrument_ids(requested, data.iter().map(&instrument_id));
1108 let single = ids.len() == 1;
1109 ids.into_iter()
1110 .map(|id| {
1111 let data = if single {
1112 data.clone()
1113 } else {
1114 data.iter()
1115 .filter(|row| instrument_id(row) == id)
1116 .copied()
1117 .collect::<Vec<_>>()
1118 .into()
1119 };
1120
1121 (Some(id.to_string()), into_batch(data))
1122 })
1123 .collect()
1124}
1125
1126struct CatalogCoverage {
1127 catalog: Ustr,
1128 missing: Vec<(u64, u64)>,
1129}
1130
1131enum RequestLeg {
1132 Catalog {
1133 request: RequestCommand,
1134 catalog: Ustr,
1135 },
1136 Client(RequestCommand),
1137 Response(DataResponse),
1138}
1139
1140impl RequestLeg {
1141 fn request_id(&self) -> UUID4 {
1142 match self {
1143 Self::Catalog { request, .. } | Self::Client(request) => *request.request_id(),
1144 Self::Response(response) => *response.correlation_id(),
1145 }
1146 }
1147}
1148
1149fn with_request_instrument_ids(parent: &RequestCommand, ids: Vec<InstrumentId>) -> RequestCommand {
1150 let mut request = parent.clone();
1151 match &mut request {
1152 RequestCommand::Quotes(r) => r.instrument_ids = ids,
1153 RequestCommand::Trades(r) => r.instrument_ids = ids,
1154 _ => unreachable!("catalog batch planning only supports quotes and trades"),
1155 }
1156
1157 request
1158}
1159
1160pub(super) fn is_date_range_variant(req: &RequestCommand) -> bool {
1161 matches!(
1162 req,
1163 RequestCommand::Data(_)
1164 | RequestCommand::Instrument(_)
1165 | RequestCommand::Instruments(_)
1166 | RequestCommand::Quotes(_)
1167 | RequestCommand::Trades(_)
1168 | RequestCommand::FundingRates(_)
1169 | RequestCommand::Bars(_)
1170 | RequestCommand::BookDeltas(_)
1171 | RequestCommand::BookDepth(_)
1172 )
1173}
1174
1175fn request_identifier(req: &RequestCommand) -> Option<RequestCatalogKey> {
1176 match req {
1177 RequestCommand::Data(cmd) => Some(RequestCatalogKey {
1178 data_type: NautilusDataType::Custom {
1179 type_name: cmd.data_type.type_name().to_string(),
1180 },
1181 identifier: cmd.data_type.identifier().map(String::from),
1182 }),
1183 RequestCommand::Quotes(cmd) => Some(RequestCatalogKey::new(
1184 NautilusDataType::QuoteTick,
1185 Some(cmd.single_instrument_id().ok()?.to_string()),
1186 )),
1187 RequestCommand::Trades(cmd) => Some(RequestCatalogKey::new(
1188 NautilusDataType::TradeTick,
1189 Some(cmd.single_instrument_id().ok()?.to_string()),
1190 )),
1191 RequestCommand::FundingRates(cmd) => Some(RequestCatalogKey::new(
1192 NautilusDataType::FundingRateUpdate,
1193 Some(cmd.single_instrument_id().ok()?.to_string()),
1194 )),
1195 RequestCommand::Bars(cmd) => Some(RequestCatalogKey::new(
1196 NautilusDataType::Bar,
1197 Some(cmd.single_bar_type().ok()?.to_string()),
1198 )),
1199 RequestCommand::BookDeltas(cmd) => Some(RequestCatalogKey::new(
1200 NautilusDataType::OrderBookDelta,
1201 Some(cmd.single_instrument_id().ok()?.to_string()),
1202 )),
1203 RequestCommand::BookDepth(cmd) => Some(RequestCatalogKey::new(
1204 NautilusDataType::OrderBookDepth,
1205 Some(cmd.single_instrument_id().ok()?.to_string()),
1206 )),
1207 _ => None,
1208 }
1209}
1210
1211impl RequestCatalogKey {
1212 fn new(data_type: NautilusDataType, identifier: Option<String>) -> Self {
1213 Self {
1214 data_type,
1215 identifier,
1216 }
1217 }
1218
1219 fn catalog_data_type(&self) -> NautilusDataType {
1220 self.data_type.clone()
1221 }
1222}
1223
1224fn catalog_missing_intervals(
1225 catalog: &mut DataCatalog,
1226 start: u64,
1227 end: u64,
1228 key: &RequestCatalogKey,
1229) -> anyhow::Result<Vec<(u64, u64)>> {
1230 catalog.get_missing_intervals_for_request(
1231 UnixNanos::from(start),
1232 UnixNanos::from(end),
1233 key.catalog_data_type(),
1234 key.identifier.as_deref(),
1235 )
1236}
1237
1238fn quote_ticks_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<QuoteTick>> {
1239 match data {
1240 DataBatch::Quote(data) => Ok(data),
1241 _ => anyhow::bail!("expected a Quote batch, found another data batch variant"),
1242 }
1243}
1244
1245fn trade_ticks_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<TradeTick>> {
1246 match data {
1247 DataBatch::Trade(data) => Ok(data),
1248 _ => anyhow::bail!("expected a Trade batch, found another data batch variant"),
1249 }
1250}
1251
1252fn funding_rates_from_query_result(
1253 data: DataBatch,
1254) -> anyhow::Result<BatchView<FundingRateUpdate>> {
1255 match data {
1256 DataBatch::FundingRate(data) => Ok(data),
1257 _ => anyhow::bail!("expected a FundingRate batch, found another data batch variant"),
1258 }
1259}
1260
1261fn bars_from_query_result(data: DataBatch) -> anyhow::Result<BatchView<Bar>> {
1262 match data {
1263 DataBatch::Bar(data) => Ok(data),
1264 _ => anyhow::bail!("expected a Bar batch, found another data batch variant"),
1265 }
1266}
1267
1268fn order_book_deltas_from_query_result(
1269 data: DataBatch,
1270) -> anyhow::Result<BatchView<OrderBookDelta>> {
1271 match data {
1272 DataBatch::BookDelta(data) => Ok(data),
1273 _ => anyhow::bail!("expected a BookDelta batch, found another data batch variant"),
1274 }
1275}
1276
1277fn order_book_depths_from_query_result(
1278 data: DataBatch,
1279) -> anyhow::Result<BatchView<OrderBookDepth>> {
1280 match data {
1281 DataBatch::BookDepth(data) => Ok(data),
1282 _ => anyhow::bail!("expected a BookDepth batch, found another data batch variant"),
1283 }
1284}
1285
1286fn request_limit(request: &RequestCommand) -> Option<std::num::NonZeroUsize> {
1287 match request {
1288 RequestCommand::Quotes(r) => r.limit,
1289 RequestCommand::Trades(r) => r.limit,
1290 RequestCommand::Bars(r) => r.limit,
1291 RequestCommand::FundingRates(r) => r.limit,
1292 RequestCommand::BookDeltas(r) => r.limit,
1293 RequestCommand::BookDepth(r) => r.limit,
1294 RequestCommand::Data(r) => r.limit,
1295 _ => None,
1296 }
1297}
1298
1299fn request_has_schema(request: &RequestCommand) -> bool {
1300 matches!(
1301 request,
1302 RequestCommand::Quotes(_) | RequestCommand::Trades(_)
1303 ) && request_params(request).is_some_and(|params| params.contains_key("schema"))
1304}
1305
1306fn request_contains_batch(
1307 request: &RequestCommand,
1308 identifier: Option<&str>,
1309 batch: &DataBatch,
1310) -> bool {
1311 let contains = |ids: &[InstrumentId]| {
1312 identifier
1313 .and_then(|value| value.parse::<InstrumentId>().ok())
1314 .is_some_and(|id| ids.contains(&id))
1315 };
1316
1317 match (request, batch) {
1318 (RequestCommand::Quotes(r), DataBatch::Quote(_)) => contains(&r.instrument_ids),
1319 (RequestCommand::Trades(r), DataBatch::Trade(_)) => contains(&r.instrument_ids),
1320 (RequestCommand::BookDeltas(r), DataBatch::BookDelta(rows)) => {
1321 contains(&r.instrument_ids)
1322 && rows
1323 .iter()
1324 .all(|row| r.instrument_ids.contains(&row.instrument_id))
1325 }
1326 (RequestCommand::BookDepth(r), DataBatch::BookDepth(rows)) => {
1327 contains(&r.instrument_ids)
1328 && rows
1329 .iter()
1330 .all(|row| r.instrument_ids.contains(&row.instrument_id))
1331 }
1332 (RequestCommand::FundingRates(r), DataBatch::FundingRate(rows)) => {
1333 contains(&r.instrument_ids)
1334 && rows
1335 .iter()
1336 .all(|row| r.instrument_ids.contains(&row.instrument_id))
1337 }
1338 (RequestCommand::Bars(r), DataBatch::Bar(rows)) => {
1339 identifier
1340 .and_then(|value| value.parse::<BarType>().ok())
1341 .is_some_and(|bar_type| r.bar_types.contains(&bar_type))
1342 && rows.iter().all(|row| r.bar_types.contains(&row.bar_type))
1343 }
1344 (RequestCommand::Data(_), DataBatch::Custom(_)) => {
1345 request_identifier(request).is_some_and(|key| key.identifier.as_deref() == identifier)
1346 }
1347 _ => false,
1348 }
1349}
1350
1351fn response_instrument_ids(
1352 requested: &[InstrumentId],
1353 actual: impl Iterator<Item = InstrumentId>,
1354) -> Vec<InstrumentId> {
1355 let actual = actual.collect::<AHashSet<_>>();
1356 if actual.iter().all(|id| requested.contains(id)) {
1357 return requested.to_vec();
1358 }
1359
1360 let mut ids = actual.into_iter().collect::<Vec<_>>();
1361 ids.sort_unstable();
1362 ids
1363}
1364
1365fn request_start(req: &RequestCommand) -> Option<Timestamp> {
1366 match req {
1367 RequestCommand::Data(cmd) => cmd.start,
1368 RequestCommand::Instrument(cmd) => cmd.start,
1369 RequestCommand::Instruments(cmd) => cmd.start,
1370 RequestCommand::Quotes(cmd) => cmd.start,
1371 RequestCommand::Trades(cmd) => cmd.start,
1372 RequestCommand::FundingRates(cmd) => cmd.start,
1373 RequestCommand::Bars(cmd) => cmd.start,
1374 RequestCommand::BookDeltas(cmd) => cmd.start,
1375 RequestCommand::BookDepth(cmd) => cmd.start,
1376 _ => None,
1377 }
1378}
1379
1380fn request_end(req: &RequestCommand) -> Option<Timestamp> {
1381 match req {
1382 RequestCommand::Data(cmd) => cmd.end,
1383 RequestCommand::Instrument(cmd) => cmd.end,
1384 RequestCommand::Instruments(cmd) => cmd.end,
1385 RequestCommand::Quotes(cmd) => cmd.end,
1386 RequestCommand::Trades(cmd) => cmd.end,
1387 RequestCommand::FundingRates(cmd) => cmd.end,
1388 RequestCommand::Bars(cmd) => cmd.end,
1389 RequestCommand::BookDeltas(cmd) => cmd.end,
1390 RequestCommand::BookDepth(cmd) => cmd.end,
1391 _ => None,
1392 }
1393}
1394
1395fn request_filter_expr(params: Option<&Params>) -> Option<String> {
1396 params
1397 .and_then(|params| params.get_str("filter_expr"))
1398 .filter(|filter_expr| !filter_expr.is_empty())
1399 .map(ToString::to_string)
1400}
1401
1402fn bound_request_dates(
1403 start: Option<Timestamp>,
1404 end: Option<Timestamp>,
1405 now: Timestamp,
1406 query_past_data: bool,
1407) -> (Timestamp, Timestamp) {
1408 let zero = Timestamp::UNIX_EPOCH;
1409 let mut start = start.unwrap_or(zero);
1410 let mut end = end.unwrap_or(now);
1411
1412 if query_past_data {
1413 if start > now {
1414 start = now;
1415 }
1416
1417 if end > now {
1418 end = now;
1419 }
1420 }
1421
1422 (start, end)
1423}
1424
1425fn datetime_to_unix_nanos_or_zero(dt: Timestamp) -> UnixNanos {
1426 UnixNanos::from(u64::try_from(dt.as_nanosecond().max(0)).unwrap_or(0))
1427}
1428
1429fn catalog_request_start(request: &RequestCommand, start: Timestamp) -> Timestamp {
1430 if matches!(request, RequestCommand::BookDeltas(_))
1432 && request_params(request)
1433 .and_then(|p| p.get_bool(PARAM_FROM_DAY_START))
1434 .unwrap_or(true)
1435 {
1436 floor_to_utc_day(start)
1437 } else {
1438 start
1439 }
1440}
1441
1442fn floor_to_utc_day(dt: Timestamp) -> Timestamp {
1443 let midnight = jiff::tz::Offset::UTC.to_datetime(dt).date().at(0, 0, 0, 0);
1444 jiff::tz::Offset::UTC
1445 .to_timestamp(midnight)
1446 .expect("midnight UTC is always valid")
1447}
1448
1449fn with_dates_for_pipeline(
1450 req: &RequestCommand,
1451 start: Option<Timestamp>,
1452 end: Option<Timestamp>,
1453 ts_init: UnixNanos,
1454) -> RequestCommand {
1455 let new_id = UUID4::new();
1456
1457 match req {
1458 RequestCommand::Quotes(cmd) => RequestCommand::Quotes(RequestQuotes {
1459 instrument_ids: cmd.instrument_ids.clone(),
1460 start,
1461 end,
1462 limit: cmd.limit,
1463 client_id: cmd.client_id,
1464 request_id: new_id,
1465 ts_init,
1466 params: cmd.params.clone(),
1467 }),
1468 RequestCommand::Trades(cmd) => RequestCommand::Trades(RequestTrades {
1469 instrument_ids: cmd.instrument_ids.clone(),
1470 start,
1471 end,
1472 limit: cmd.limit,
1473 client_id: cmd.client_id,
1474 request_id: new_id,
1475 ts_init,
1476 params: cmd.params.clone(),
1477 }),
1478 RequestCommand::FundingRates(cmd) => RequestCommand::FundingRates(RequestFundingRates {
1479 instrument_ids: cmd.instrument_ids.clone(),
1480 start,
1481 end,
1482 limit: cmd.limit,
1483 client_id: cmd.client_id,
1484 request_id: new_id,
1485 ts_init,
1486 params: cmd.params.clone(),
1487 }),
1488 RequestCommand::BookDeltas(cmd) => RequestCommand::BookDeltas(RequestBookDeltas {
1489 instrument_ids: cmd.instrument_ids.clone(),
1490 start,
1491 end,
1492 limit: cmd.limit,
1493 client_id: cmd.client_id,
1494 request_id: new_id,
1495 ts_init,
1496 params: cmd.params.clone(),
1497 }),
1498 RequestCommand::BookDepth(cmd) => RequestCommand::BookDepth(RequestBookDepth {
1499 instrument_ids: cmd.instrument_ids.clone(),
1500 start,
1501 end,
1502 limit: cmd.limit,
1503 depth: cmd.depth,
1504 client_id: cmd.client_id,
1505 request_id: new_id,
1506 ts_init,
1507 params: cmd.params.clone(),
1508 }),
1509 RequestCommand::Data(cmd) => RequestCommand::Data(RequestCustomData {
1510 client_id: cmd.client_id,
1511 data_type: cmd.data_type.clone(),
1512 start,
1513 end,
1514 limit: cmd.limit,
1515 request_id: new_id,
1516 ts_init,
1517 params: cmd.params.clone(),
1518 }),
1519 RequestCommand::Bars(cmd) => RequestCommand::Bars(RequestBars {
1520 bar_types: cmd.bar_types.clone(),
1521 start,
1522 end,
1523 limit: cmd.limit,
1524 client_id: cmd.client_id,
1525 request_id: new_id,
1526 ts_init,
1527 params: cmd.params.clone(),
1528 }),
1529 _ => req.clone(),
1533 }
1534}
1535
1536fn build_empty_response(
1537 req: &RequestCommand,
1538 start: UnixNanos,
1539 end: UnixNanos,
1540 used_client_id: Option<ClientId>,
1541 ts_init: UnixNanos,
1542) -> anyhow::Result<DataResponse> {
1543 let response = match req {
1544 RequestCommand::Data(cmd) => DataResponse::Data(CustomDataResponse::new(
1545 cmd.request_id,
1546 cmd.client_id,
1547 None,
1548 cmd.data_type.clone(),
1549 Vec::<CustomData>::new(),
1550 Some(start),
1551 Some(end),
1552 ts_init,
1553 cmd.params.clone(),
1554 )),
1555 RequestCommand::Quotes(cmd) => DataResponse::Quotes(QuotesResponse::new(
1556 cmd.request_id,
1557 resolve_response_client_id(cmd.client_id, used_client_id),
1558 cmd.instrument_ids.clone(),
1559 Vec::new(),
1560 Some(start),
1561 Some(end),
1562 ts_init,
1563 cmd.params.clone(),
1564 )),
1565 RequestCommand::Trades(cmd) => DataResponse::Trades(TradesResponse::new(
1566 cmd.request_id,
1567 resolve_response_client_id(cmd.client_id, used_client_id),
1568 cmd.instrument_ids.clone(),
1569 Vec::new(),
1570 Some(start),
1571 Some(end),
1572 ts_init,
1573 cmd.params.clone(),
1574 )),
1575 RequestCommand::FundingRates(cmd) => DataResponse::FundingRates(FundingRatesResponse::new(
1576 cmd.request_id,
1577 resolve_response_client_id(cmd.client_id, used_client_id),
1578 cmd.single_instrument_id()?,
1579 Vec::new(),
1580 Some(start),
1581 Some(end),
1582 ts_init,
1583 cmd.params.clone(),
1584 )),
1585 RequestCommand::Bars(cmd) => DataResponse::Bars(BarsResponse::new(
1586 cmd.request_id,
1587 resolve_response_client_id(cmd.client_id, used_client_id),
1588 cmd.single_bar_type()?,
1589 Vec::new(),
1590 Some(start),
1591 Some(end),
1592 ts_init,
1593 cmd.params.clone(),
1594 )),
1595 RequestCommand::BookDeltas(cmd) => DataResponse::BookDeltas(BookDeltasResponse::new(
1596 cmd.request_id,
1597 resolve_response_client_id(cmd.client_id, used_client_id),
1598 cmd.single_instrument_id()?,
1599 Vec::new(),
1600 Some(start),
1601 Some(end),
1602 ts_init,
1603 cmd.params.clone(),
1604 )),
1605 RequestCommand::BookDepth(cmd) => DataResponse::BookDepth(BookDepthResponse::new(
1606 cmd.request_id,
1607 resolve_response_client_id(cmd.client_id, used_client_id),
1608 cmd.single_instrument_id()?,
1609 Vec::new(),
1610 Some(start),
1611 Some(end),
1612 ts_init,
1613 cmd.params.clone(),
1614 )),
1615 _ => {
1616 anyhow::bail!("Cannot build empty catalog response for non-catalog-eligible request")
1617 }
1618 };
1619
1620 Ok(response)
1621}
1622
1623fn build_quotes_catalog_response(
1624 cmd: &RequestQuotes,
1625 data: BatchView<QuoteTick>,
1626 start: UnixNanos,
1627 end: UnixNanos,
1628 used_client_id: Option<ClientId>,
1629 ts_init: UnixNanos,
1630) -> DataResponse {
1631 let params = catalog_response_params(cmd.params.as_ref());
1632 DataResponse::Quotes(QuotesResponse::new(
1633 cmd.request_id,
1634 resolve_response_client_id(cmd.client_id, used_client_id),
1635 cmd.instrument_ids.clone(),
1636 data,
1637 Some(start),
1638 Some(end),
1639 ts_init,
1640 Some(params),
1641 ))
1642}
1643
1644fn build_trades_catalog_response(
1645 cmd: &RequestTrades,
1646 data: BatchView<TradeTick>,
1647 start: UnixNanos,
1648 end: UnixNanos,
1649 used_client_id: Option<ClientId>,
1650 ts_init: UnixNanos,
1651) -> DataResponse {
1652 let params = catalog_response_params(cmd.params.as_ref());
1653 DataResponse::Trades(TradesResponse::new(
1654 cmd.request_id,
1655 resolve_response_client_id(cmd.client_id, used_client_id),
1656 cmd.instrument_ids.clone(),
1657 data,
1658 Some(start),
1659 Some(end),
1660 ts_init,
1661 Some(params),
1662 ))
1663}
1664
1665fn build_funding_rates_catalog_response(
1666 cmd: &RequestFundingRates,
1667 data: BatchView<FundingRateUpdate>,
1668 start: UnixNanos,
1669 end: UnixNanos,
1670 used_client_id: Option<ClientId>,
1671 ts_init: UnixNanos,
1672) -> anyhow::Result<DataResponse> {
1673 let params = catalog_response_params(cmd.params.as_ref());
1674 Ok(DataResponse::FundingRates(FundingRatesResponse::new(
1675 cmd.request_id,
1676 resolve_response_client_id(cmd.client_id, used_client_id),
1677 cmd.single_instrument_id()?,
1678 data,
1679 Some(start),
1680 Some(end),
1681 ts_init,
1682 Some(params),
1683 )))
1684}
1685
1686fn build_bars_catalog_response(
1687 cmd: &RequestBars,
1688 data: BatchView<Bar>,
1689 start: UnixNanos,
1690 end: UnixNanos,
1691 used_client_id: Option<ClientId>,
1692 ts_init: UnixNanos,
1693) -> anyhow::Result<DataResponse> {
1694 let params = catalog_response_params(cmd.params.as_ref());
1695 Ok(DataResponse::Bars(BarsResponse::new(
1696 cmd.request_id,
1697 resolve_response_client_id(cmd.client_id, used_client_id),
1698 cmd.single_bar_type()?,
1699 data,
1700 Some(start),
1701 Some(end),
1702 ts_init,
1703 Some(params),
1704 )))
1705}
1706
1707fn build_custom_data_catalog_response(
1708 cmd: &RequestCustomData,
1709 data: Vec<CustomData>,
1710 start: UnixNanos,
1711 end: UnixNanos,
1712 ts_init: UnixNanos,
1713) -> DataResponse {
1714 let params = catalog_response_params(cmd.params.as_ref());
1715 DataResponse::Data(CustomDataResponse::new(
1716 cmd.request_id,
1717 cmd.client_id,
1718 None,
1719 cmd.data_type.clone(),
1720 data,
1721 Some(start),
1722 Some(end),
1723 ts_init,
1724 Some(params),
1725 ))
1726}
1727
1728fn build_book_deltas_catalog_response(
1729 cmd: &RequestBookDeltas,
1730 data: BatchView<OrderBookDelta>,
1731 start: UnixNanos,
1732 end: UnixNanos,
1733 used_client_id: Option<ClientId>,
1734 ts_init: UnixNanos,
1735) -> anyhow::Result<DataResponse> {
1736 let params = catalog_response_params(cmd.params.as_ref());
1737 Ok(DataResponse::BookDeltas(BookDeltasResponse::new(
1738 cmd.request_id,
1739 resolve_response_client_id(cmd.client_id, used_client_id),
1740 cmd.single_instrument_id()?,
1741 data,
1742 Some(start),
1743 Some(end),
1744 ts_init,
1745 Some(params),
1746 )))
1747}
1748
1749fn build_book_depth_catalog_response(
1750 cmd: &RequestBookDepth,
1751 data: BatchView<OrderBookDepth>,
1752 start: UnixNanos,
1753 end: UnixNanos,
1754 used_client_id: Option<ClientId>,
1755 ts_init: UnixNanos,
1756) -> anyhow::Result<DataResponse> {
1757 let params = catalog_response_params(cmd.params.as_ref());
1758 Ok(DataResponse::BookDepth(BookDepthResponse::new(
1759 cmd.request_id,
1760 resolve_response_client_id(cmd.client_id, used_client_id),
1761 cmd.single_instrument_id()?,
1762 data,
1763 Some(start),
1764 Some(end),
1765 ts_init,
1766 Some(params),
1767 )))
1768}
1769
1770fn catalog_response_params(existing: Option<&Params>) -> Params {
1771 let mut params = existing.map_or_default(|params| IndexMap::clone(params));
1772 params.insert(PARAM_UPDATE_CATALOG.to_string(), Value::Bool(false));
1773 Params::new(params)
1774}
1775
1776fn custom_data_from_query_result(data: DataBatch) -> anyhow::Result<Vec<CustomData>> {
1777 CustomData::from_batch(data)
1778}
1779
1780fn instrument_only_last(params: Option<&Params>) -> bool {
1781 params
1782 .and_then(|params| params.get_bool("only_last"))
1783 .unwrap_or(true)
1784}
1785
1786fn latest_instruments(data: Vec<InstrumentAny>) -> Vec<InstrumentAny> {
1787 let mut instruments: AHashMap<_, InstrumentAny> = AHashMap::new();
1788
1789 for instrument in data {
1790 let id = instrument.id();
1791 match instruments.get(&id) {
1792 Some(existing) if existing.ts_init() >= instrument.ts_init() => {}
1793 _ => {
1794 instruments.insert(id, instrument);
1795 }
1796 }
1797 }
1798
1799 let mut data: Vec<_> = instruments.into_values().collect();
1800 data.sort_by_key(|instrument| instrument.id().to_string());
1801 data
1802}
1803
1804fn instrument_response_venue(request_venue: Option<Venue>, data: &[InstrumentAny]) -> Venue {
1805 request_venue.unwrap_or_else(|| {
1806 data.iter()
1807 .map(Instrument::venue)
1808 .min_by_key(std::string::ToString::to_string)
1809 .unwrap_or_else(|| Venue::from(CATALOG_CLIENT_ID))
1810 })
1811}
1812
1813fn resolve_response_client_id(
1814 request_client_id: Option<ClientId>,
1815 used_client_id: Option<ClientId>,
1816) -> ClientId {
1817 request_client_id
1818 .or(used_client_id)
1819 .unwrap_or_else(|| ClientId::new(CATALOG_CLIENT_ID))
1820}
1821
1822#[cfg(test)]
1823mod tests {
1824 use std::sync::Arc;
1825
1826 use nautilus_common::messages::data::RequestJoin;
1827 use rstest::rstest;
1828
1829 use super::*;
1830
1831 #[rstest]
1832 #[case::full(0, 3)]
1833 #[case::subrange(1, 2)]
1834 fn test_catalog_quote_response_uses_selected_batch_view(
1835 #[case] start: usize,
1836 #[case] end: usize,
1837 ) {
1838 let first = QuoteTick {
1839 ts_init: UnixNanos::from(11),
1840 ..Default::default()
1841 };
1842
1843 let mut second = first;
1844 second.ts_init = UnixNanos::from(23);
1845 let mut third = first;
1846 third.ts_init = UnixNanos::from(37);
1847 let original = Arc::new(vec![first, second, third]);
1848 let result = quote_ticks_from_query_result(DataBatch::Quote(BatchView::new(
1849 Arc::clone(&original),
1850 start..end,
1851 )))
1852 .unwrap();
1853 assert_eq!(result.as_ref(), &original[start..end]);
1854 assert_eq!(result.range(), start..end);
1855 assert!(Arc::ptr_eq(result.arc(), &original));
1856 }
1857
1858 #[rstest]
1859 fn test_response_batches_preserves_narrowed_payloads(#[values(false, true)] empty: bool) {
1860 use nautilus_model::data::stubs::{stub_delta, stub_depth10};
1861
1862 macro_rules! check {
1863 ($response:ident, $variant:ident, $batch:ident, $key:expr, $seed:expr) => {{
1864 let key = $key;
1865 let mut row = $seed;
1866 row.ts_init = UnixNanos::from(23);
1867 let mut before = row.clone();
1868 before.ts_init = UnixNanos::from(11);
1869 let mut after = row.clone();
1870 after.ts_init = UnixNanos::from(37);
1871 let backing = Arc::new(vec![before, row.clone(), after]);
1872 let end = if empty { 1 } else { 2 };
1873 let response = DataResponse::$variant($response::new(
1874 UUID4::new(),
1875 ClientId::from("SOURCE"),
1876 key,
1877 BatchView::new(Arc::clone(&backing), 1..end),
1878 None,
1879 None,
1880 UnixNanos::from(41),
1881 None,
1882 ));
1883
1884 let batches = response_batches(&response).unwrap();
1885
1886 assert_eq!(batches.len(), 1);
1887 assert_eq!(batches[0].0.as_deref(), Some(key.to_string().as_str()));
1888 let DataBatch::$batch(view) = &batches[0].1 else {
1889 panic!("Unexpected batch {:?}", batches[0].1);
1890 };
1891 assert_eq!(view.range(), 1..end);
1892 assert!(Arc::ptr_eq(view.arc(), &backing));
1893 assert_eq!(
1894 view.as_ref(),
1895 if empty {
1896 &[]
1897 } else {
1898 std::slice::from_ref(&row)
1899 }
1900 );
1901 }};
1902 }
1903
1904 let delta = stub_delta();
1905 let depth = stub_depth10();
1906 let quote = QuoteTick::default();
1907 let trade = TradeTick::default();
1908 let bar = Bar::default();
1909
1910 let funding = FundingRateUpdate::new(
1911 quote.instrument_id,
1912 "0.001".parse().unwrap(),
1913 Some(60),
1914 Some(UnixNanos::from(47)),
1915 UnixNanos::from(17),
1916 UnixNanos::from(23),
1917 );
1918
1919 check!(
1920 BookDeltasResponse,
1921 BookDeltas,
1922 BookDelta,
1923 delta.instrument_id,
1924 delta
1925 );
1926 check!(
1927 BookDepthResponse,
1928 BookDepth,
1929 BookDepth,
1930 depth.instrument_id,
1931 depth
1932 );
1933 check!(QuotesResponse, Quotes, Quote, quote.instrument_id, quote);
1934 check!(TradesResponse, Trades, Trade, trade.instrument_id, trade);
1935 check!(
1936 FundingRatesResponse,
1937 FundingRates,
1938 FundingRate,
1939 funding.instrument_id,
1940 funding
1941 );
1942 check!(BarsResponse, Bars, Bar, bar.bar_type, bar);
1943 }
1944
1945 #[rstest]
1946 #[case::deltas(NautilusDataType::OrderBookDelta)]
1947 #[case::depth(NautilusDataType::OrderBookDepth)]
1948 #[case::quotes(NautilusDataType::QuoteTick)]
1949 #[case::trades(NautilusDataType::TradeTick)]
1950 #[case::bars(NautilusDataType::Bar)]
1951 #[case::funding(NautilusDataType::FundingRateUpdate)]
1952 fn test_request_contains_batch_uses_visible_requested_members(
1953 #[case] kind: NautilusDataType,
1954 #[values(false, true)] remapped: bool,
1955 #[values(false, true)] empty: bool,
1956 ) {
1957 use nautilus_model::data::stubs::{stub_delta, stub_depth10};
1958
1959 let requested = InstrumentId::from("AUD/USD.SIM");
1960
1961 let actual = if remapped {
1962 InstrumentId::from("GBP/USD.SIM")
1963 } else {
1964 requested
1965 };
1966
1967 let requested_bar = format!("{requested}-1-MINUTE-LAST-EXTERNAL")
1968 .parse::<BarType>()
1969 .unwrap();
1970 let actual_bar = format!("{actual}-1-MINUTE-LAST-EXTERNAL")
1971 .parse::<BarType>()
1972 .unwrap();
1973
1974 macro_rules! request {
1975 ($variant:ident, $request:ident, $id:expr $(, $depth:expr)?) => {
1976 RequestCommand::$variant($request::new(
1977 $id, None, None, None, $($depth,)? None, UUID4::new(), UnixNanos::from(37), None,
1978 ))
1979 };
1980 }
1981
1982 macro_rules! response {
1983 ($variant:ident, $response:ident, $key:expr, $row:expr) => {{
1984 let data = BatchView::new(Arc::new(vec![$row]), 0..usize::from(!empty));
1985 DataResponse::$variant($response::new(
1986 UUID4::new(),
1987 ClientId::from("SOURCE"),
1988 $key,
1989 data,
1990 None,
1991 None,
1992 UnixNanos::from(41),
1993 None,
1994 ))
1995 }};
1996 }
1997
1998 let (request, response) = match kind {
1999 NautilusDataType::OrderBookDelta => {
2000 let row = OrderBookDelta {
2001 instrument_id: actual,
2002 ..stub_delta()
2003 };
2004
2005 (
2006 request!(BookDeltas, RequestBookDeltas, requested),
2007 response!(BookDeltas, BookDeltasResponse, requested, row),
2008 )
2009 }
2010 NautilusDataType::OrderBookDepth => {
2011 let row = OrderBookDepth {
2012 instrument_id: actual,
2013 ..stub_depth10()
2014 };
2015
2016 (
2017 request!(BookDepth, RequestBookDepth, requested, None),
2018 response!(BookDepth, BookDepthResponse, requested, row),
2019 )
2020 }
2021 NautilusDataType::QuoteTick => {
2022 let row = QuoteTick {
2023 instrument_id: actual,
2024 ..Default::default()
2025 };
2026
2027 (
2028 request!(Quotes, RequestQuotes, requested),
2029 response!(Quotes, QuotesResponse, requested, row),
2030 )
2031 }
2032 NautilusDataType::TradeTick => {
2033 let row = TradeTick {
2034 instrument_id: actual,
2035 ..Default::default()
2036 };
2037
2038 (
2039 request!(Trades, RequestTrades, requested),
2040 response!(Trades, TradesResponse, requested, row),
2041 )
2042 }
2043 NautilusDataType::Bar => {
2044 let row = Bar {
2045 bar_type: actual_bar,
2046 ..Default::default()
2047 };
2048
2049 (
2050 request!(Bars, RequestBars, requested_bar),
2051 response!(Bars, BarsResponse, requested_bar, row),
2052 )
2053 }
2054 NautilusDataType::FundingRateUpdate => {
2055 let row = FundingRateUpdate::new(
2056 actual,
2057 "0.001".parse().unwrap(),
2058 Some(60),
2059 Some(UnixNanos::from(47)),
2060 UnixNanos::from(17),
2061 UnixNanos::from(23),
2062 );
2063 (
2064 request!(FundingRates, RequestFundingRates, requested),
2065 response!(FundingRates, FundingRatesResponse, requested, row),
2066 )
2067 }
2068 _ => panic!("unexpected data type"),
2069 };
2070
2071 let batches = response_batches(&response).unwrap();
2072 let (identifier, batch) = &batches[0];
2073
2074 assert_eq!(batches.len(), 1);
2075 assert_eq!(
2076 request_contains_batch(&request, identifier.as_deref(), batch),
2077 empty || !remapped,
2078 );
2079 }
2080
2081 #[rstest]
2082 #[case::quotes(true)]
2083 #[case::trades(false)]
2084 fn test_response_batches_keep_expanded_parent_instruments(#[case] quotes: bool) {
2085 let requested = [
2086 InstrumentId::from("ES.FUT.GLBX"),
2087 InstrumentId::from("NQ.FUT.GLBX"),
2088 ];
2089 let first = InstrumentId::from("ESH24.GLBX");
2090 let second = InstrumentId::from("NQH24.GLBX");
2091
2092 let mut response = if quotes {
2093 let rows = vec![
2094 QuoteTick {
2095 instrument_id: first,
2096 ..Default::default()
2097 },
2098 QuoteTick {
2099 instrument_id: second,
2100 ..Default::default()
2101 },
2102 ];
2103 DataResponse::Quotes(QuotesResponse::new(
2104 UUID4::new(),
2105 ClientId::from("SOURCE"),
2106 requested,
2107 rows,
2108 None,
2109 None,
2110 UnixNanos::from(3),
2111 None,
2112 ))
2113 } else {
2114 let rows = vec![
2115 TradeTick {
2116 instrument_id: first,
2117 ..Default::default()
2118 },
2119 TradeTick {
2120 instrument_id: second,
2121 ..Default::default()
2122 },
2123 ];
2124 DataResponse::Trades(TradesResponse::new(
2125 UUID4::new(),
2126 ClientId::from("SOURCE"),
2127 requested,
2128 rows,
2129 None,
2130 None,
2131 UnixNanos::from(3),
2132 None,
2133 ))
2134 };
2135
2136 macro_rules! narrow {
2137 ($response:expr, $row:ident) => {{
2138 let mut rows = $response.data.to_vec();
2139 let hidden = $row {
2140 instrument_id: InstrumentId::from("HIDDEN.GLBX"),
2141 ..Default::default()
2142 };
2143 rows.insert(0, hidden);
2144 rows.push(hidden);
2145 $response.data = BatchView::new(Arc::new(rows), 1..3);
2146 }};
2147 }
2148
2149 match &mut response {
2150 DataResponse::Quotes(r) => narrow!(r, QuoteTick),
2151 DataResponse::Trades(r) => narrow!(r, TradeTick),
2152 _ => unreachable!(),
2153 }
2154
2155 let batches = response_batches(&response).unwrap();
2156
2157 assert_eq!(batches.len(), 2);
2158
2159 for ((identifier, batch), id) in batches.iter().zip([first, second]) {
2160 assert_eq!(identifier.as_deref(), Some(id.to_string().as_str()));
2161
2162 match batch {
2163 DataBatch::Quote(rows) => assert_eq!(
2164 &rows[..],
2165 &[QuoteTick {
2166 instrument_id: id,
2167 ..Default::default()
2168 }]
2169 ),
2170 DataBatch::Trade(rows) => assert_eq!(
2171 &rows[..],
2172 &[TradeTick {
2173 instrument_id: id,
2174 ..Default::default()
2175 }]
2176 ),
2177 other => panic!("Unexpected batch {other:?}"),
2178 }
2179 }
2180 }
2181
2182 #[rstest]
2183 fn test_build_empty_response_rejects_non_catalog_variant() {
2184 let request = RequestCommand::Join(RequestJoin::new(
2185 vec![UUID4::new()],
2186 None,
2187 None,
2188 UUID4::new(),
2189 UnixNanos::default(),
2190 None,
2191 None,
2192 ));
2193
2194 let result = build_empty_response(
2195 &request,
2196 UnixNanos::from(1u64),
2197 UnixNanos::from(2u64),
2198 None,
2199 UnixNanos::from(3u64),
2200 );
2201
2202 assert_eq!(
2203 result.unwrap_err().to_string(),
2204 "Cannot build empty catalog response for non-catalog-eligible request"
2205 );
2206 }
2207}