1use ahash::{AHashMap, AHashSet};
17use databento::dbn::{self, PitSymbolMap, SType};
18use dbn::{Publisher, Record};
19use indexmap::IndexMap;
20use nautilus_model::identifiers::{
21 InstrumentId, Symbol, Venue,
22 futures::{
23 FuturesYearDigits, format_futures_symbol, parse_futures_symbol, resolve_futures_year,
24 resolve_futures_year_from_expiry,
25 },
26};
27
28use super::types::PublisherId;
29
30#[derive(Debug)]
31pub struct MetadataCache {
32 metadata: dbn::Metadata,
33 date_metadata_map: AHashMap<time::Date, PitSymbolMap>,
34}
35
36impl MetadataCache {
37 #[must_use]
38 pub fn new(metadata: dbn::Metadata) -> Self {
39 Self {
40 metadata,
41 date_metadata_map: AHashMap::new(),
42 }
43 }
44
45 pub fn symbol_map_for_date(&mut self, date: time::Date) -> dbn::Result<&PitSymbolMap> {
49 if !self.date_metadata_map.contains_key(&date) {
50 let map = self.metadata.symbol_map_for_date(date)?;
51 self.date_metadata_map.insert(date, map);
52 }
53
54 self.date_metadata_map
55 .get(&date)
56 .ok_or_else(|| dbn::Error::decode(format!("metadata cache missing for date {date}")))
57 }
58}
59
60pub fn instrument_id_to_symbol_string(
61 instrument_id: InstrumentId,
62 symbol_venue_map: &mut AHashMap<Symbol, Venue>,
63) -> String {
64 symbol_venue_map
65 .entry(instrument_id.symbol)
66 .or_insert(instrument_id.venue);
67 instrument_id.symbol.to_string()
68}
69
70pub fn decode_nautilus_instrument_id(
82 record: &dbn::RecordRef,
83 metadata: &mut MetadataCache,
84 publisher_venue_map: &IndexMap<PublisherId, Venue>,
85 symbol_venue_map: &AHashMap<Symbol, Venue>,
86 futures_year_digits: FuturesYearDigits,
87 futures_symbols: &AHashMap<(PublisherId, u32, time::Date), Symbol>,
88 requested_symbols: Option<&AHashSet<Symbol>>,
89) -> anyhow::Result<InstrumentId> {
90 let publisher = record
91 .publisher()
92 .map_err(|e| anyhow::anyhow!("Invalid `publisher` for record: {e}"))?;
93 let publisher_id = publisher as PublisherId;
94 let venue = publisher_venue_map
95 .get(&publisher_id)
96 .ok_or_else(|| anyhow::anyhow!("`Venue` not found for `publisher_id` {publisher_id}"))?;
97 let mut instrument_id = get_nautilus_instrument_id_for_record(
98 record,
99 metadata,
100 *venue,
101 futures_year_digits,
102 futures_symbols,
103 symbol_venue_map,
104 requested_symbols,
105 )?;
106
107 if publisher == Publisher::GlbxMdp3Glbx
108 && let Some(venue) = symbol_venue_map.get(&instrument_id.symbol)
109 {
110 instrument_id.venue = *venue;
111 }
112
113 Ok(instrument_id)
114}
115
116pub fn get_nautilus_instrument_id_for_record(
132 record: &dbn::RecordRef,
133 metadata: &mut MetadataCache,
134 venue: Venue,
135 futures_year_digits: FuturesYearDigits,
136 futures_symbols: &AHashMap<(PublisherId, u32, time::Date), Symbol>,
137 symbol_venue_map: &AHashMap<Symbol, Venue>,
138 requested_symbols: Option<&AHashSet<Symbol>>,
139) -> anyhow::Result<InstrumentId> {
140 if record.get::<dbn::MboMsg>().is_none()
141 && record.get::<dbn::TradeMsg>().is_none()
142 && record.get::<dbn::Mbp1Msg>().is_none()
143 && record.get::<dbn::Bbo1SMsg>().is_none()
144 && record.get::<dbn::Bbo1MMsg>().is_none()
145 && record.get::<dbn::Mbp10Msg>().is_none()
146 && record.get::<dbn::OhlcvMsg>().is_none()
147 && record.get::<dbn::StatusMsg>().is_none()
148 && record.get::<dbn::ImbalanceMsg>().is_none()
149 && record.get::<dbn::StatMsg>().is_none()
150 && record.get::<dbn::InstrumentDefMsg>().is_none()
151 && record.get::<dbn::Cmbp1Msg>().is_none()
152 && record.get::<dbn::CbboMsg>().is_none()
153 {
154 anyhow::bail!("DBN message type is not currently supported");
155 }
156
157 let instrument_id = record.header().instrument_id;
158 let nanoseconds = record.raw_index_ts();
159 let duration = time::Duration::nanoseconds(nanoseconds as i64);
160 let datetime = time::OffsetDateTime::UNIX_EPOCH
161 .checked_add(duration)
162 .ok_or_else(|| anyhow::anyhow!("Timestamp overflow for record"))?;
163 let date = datetime.date();
164 let symbol_map = metadata.symbol_map_for_date(date)?;
165 let raw_symbol = symbol_map
166 .get(instrument_id)
167 .ok_or_else(|| anyhow::anyhow!("No raw symbol found for {instrument_id}"))?;
168
169 if futures_year_digits == FuturesYearDigits::Two
170 && record.get::<dbn::InstrumentDefMsg>().is_none()
171 && let Some(symbol) =
172 futures_symbols.get(&(record.header().publisher_id, instrument_id, date))
173 {
174 return Ok(InstrumentId::new(*symbol, venue));
175 }
176
177 let symbol = Symbol::from_str_unchecked(raw_symbol);
178 let instrument_id = InstrumentId::new(symbol, venue);
179 resolve_futures_instrument_id(
180 record,
181 instrument_id,
182 futures_year_digits,
183 symbol_venue_map,
184 requested_symbols,
185 )
186}
187
188#[must_use]
189pub fn infer_symbology_type(symbol: &str) -> SType {
190 if symbol.ends_with(".FUT") || symbol.ends_with(".OPT") {
191 return SType::Parent;
192 }
193
194 let parts: Vec<&str> = symbol.split('.').collect();
195 if parts.len() == 3 && parts[2].chars().all(|c| c.is_ascii_digit()) {
196 return SType::Continuous;
197 }
198
199 if symbol.chars().all(|c| c.is_ascii_digit()) {
200 return SType::InstrumentId;
201 }
202
203 SType::RawSymbol
204}
205
206pub fn check_consistent_symbology(symbols: &[&str]) -> anyhow::Result<()> {
210 if symbols.is_empty() {
211 anyhow::bail!("No symbols provided");
212 }
213 let first_symbol = symbols[0];
214 let first_stype = infer_symbology_type(first_symbol);
215
216 for symbol in symbols {
217 let next_stype = infer_symbology_type(symbol);
218 if next_stype != first_stype {
219 anyhow::bail!(
220 "Inconsistent symbology types: '{first_stype}' for {first_symbol} vs '{next_stype}' for {symbol}"
221 );
222 }
223 }
224
225 Ok(())
226}
227
228pub(crate) fn dataset_uses_futures_symbology(dataset: &str) -> bool {
229 matches!(
230 dataset,
231 "GLBX.MDP3"
232 | "IFEU.IMPACT"
233 | "NDEX.IMPACT"
234 | "IFUS.IMPACT"
235 | "IFLL.IMPACT"
236 | "XEUR.EOBI"
237 | "XEEE.EOBI"
238 | "XCBF.PITCH"
239 | "XOSE.ITCH"
240 )
241}
242
243pub(crate) fn raw_symbol(
244 symbol: &str,
245 dataset: &str,
246 futures_year_digits: FuturesYearDigits,
247) -> String {
248 if futures_year_digits == FuturesYearDigits::Two && dataset_uses_futures_symbology(dataset) {
249 rewrite_futures_symbol(symbol, 0, 1, false).unwrap_or_else(|| symbol.to_string())
250 } else {
251 symbol.to_string()
252 }
253}
254
255fn rewrite_futures_symbol(
256 symbol: &str,
257 reference_year: i32,
258 year_digits: u8,
259 from_definition: bool,
260) -> Option<String> {
261 let (code, suffix) = symbol
262 .split_once(' ')
263 .map_or((symbol, ""), |(code, suffix)| (code, suffix));
264
265 let legs = code
266 .split('-')
267 .map(|leg| {
268 let parsed = parse_futures_symbol(leg)?;
269 let year = if year_digits == 1 {
270 parsed.year_digits.parse::<i32>().ok()?
271 } else if from_definition {
272 resolve_futures_year_from_expiry(parsed.year_digits, reference_year)?
273 } else {
274 resolve_futures_year(parsed.year_digits, reference_year)?
275 };
276
277 format_futures_symbol(parsed.root, parsed.month_code, year, year_digits)
278 })
279 .collect::<Option<Vec<_>>>()?;
280
281 let code = legs.join("-");
282 Some(if suffix.is_empty() {
283 code
284 } else {
285 format!("{code} {suffix}")
286 })
287}
288
289pub(crate) fn resolve_futures_instrument_id(
290 record: &dbn::RecordRef,
291 instrument_id: InstrumentId,
292 futures_year_digits: FuturesYearDigits,
293 symbol_venue_map: &AHashMap<Symbol, Venue>,
294 requested_symbols: Option<&AHashSet<Symbol>>,
295) -> anyhow::Result<InstrumentId> {
296 if futures_year_digits != FuturesYearDigits::Two
297 || !record
298 .publisher()
299 .is_ok_and(|publisher| dataset_uses_futures_symbology(publisher.dataset().as_str()))
300 {
301 return Ok(instrument_id);
302 }
303
304 let mut timestamp = record.raw_index_ts();
305 let definition = record.get::<dbn::InstrumentDefMsg>();
306 let maturity_year = definition
307 .filter(|msg| msg.instrument_class as u8 != b'S')
308 .map(|msg| msg.maturity_year)
309 .filter(|year| !matches!(*year, 0 | u16::MAX))
310 .map(i32::from);
311
312 if let Some(msg) = definition {
313 if !matches!(msg.instrument_class as u8 as char, 'F' | 'S' | 'C' | 'P') {
314 return Ok(instrument_id);
315 }
316
317 if msg.expiration == dbn::UNDEF_TIMESTAMP && maturity_year.is_none() {
318 log::warn!(
319 "Keeping raw futures symbol for {instrument_id}: definition has no expiration"
320 );
321 return Ok(instrument_id);
322 }
323
324 timestamp = msg.expiration;
325 }
326
327 let reference_year = if let Some(year) = maturity_year {
328 year
329 } else {
330 time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(timestamp))
331 .map_err(|e| anyhow::anyhow!("Invalid futures reference timestamp {timestamp}: {e}"))?
332 .year()
333 };
334
335 let mut symbol = rewrite_futures_symbol(
336 instrument_id.symbol.as_str(),
337 reference_year,
338 2,
339 definition.is_some(),
340 );
341
342 if definition.is_none() {
343 symbol = select_requested_futures_symbol(
344 symbol,
345 instrument_id.symbol.as_str(),
346 reference_year,
347 symbol_venue_map,
348 requested_symbols,
349 );
350 }
351
352 Ok(symbol.map_or(instrument_id, |symbol| {
353 InstrumentId::new(Symbol::from(symbol.as_str()), instrument_id.venue)
354 }))
355}
356
357fn select_requested_futures_symbol(
358 inferred: Option<String>,
359 native: &str,
360 reference_year: i32,
361 symbol_venue_map: &AHashMap<Symbol, Venue>,
362 requested_symbols: Option<&AHashSet<Symbol>>,
363) -> Option<String> {
364 let hints_empty =
365 requested_symbols.map_or_else(|| symbol_venue_map.is_empty(), |symbols| symbols.is_empty());
366
367 let contains = |symbol: &str| {
368 let symbol = Symbol::from(symbol);
369 requested_symbols.map_or_else(
370 || symbol_venue_map.contains_key(&symbol),
371 |symbols| symbols.contains(&symbol),
372 )
373 };
374
375 if hints_empty || inferred.as_ref().is_none_or(|symbol| contains(symbol)) {
376 return inferred;
377 }
378
379 rewrite_futures_symbol(native, reference_year - 1, 2, false)
380 .filter(|symbol| contains(symbol))
381 .or(inferred)
382}
383
384#[cfg(test)]
385mod tests {
386 use rstest::*;
387
388 use super::*;
389
390 #[rstest]
391 #[case("1", "instrument_id")]
392 #[case("123456789", "instrument_id")]
393 #[case("AAPL", "raw_symbol")]
394 #[case("ESM4", "raw_symbol")]
395 #[case("BRN FMM0024!", "raw_symbol")]
396 #[case("BRN 99 5617289", "raw_symbol")]
397 #[case("SPY 240319P00511000", "raw_symbol")]
398 #[case("ES.FUT", "parent")]
399 #[case("ES.OPT", "parent")]
400 #[case("BRN.FUT", "parent")]
401 #[case("SPX.OPT", "parent")]
402 #[case("ES.c.0", "continuous")]
403 #[case("SPX.n.0", "continuous")]
404 fn test_infer_symbology_type(#[case] symbol: String, #[case] expected: SType) {
405 let result = infer_symbology_type(&symbol);
406 assert_eq!(result, expected);
407 }
408
409 #[rstest]
410 fn test_check_consistent_symbology_when_empty_symbols() {
411 let symbols: Vec<&str> = vec![];
412 assert!(check_consistent_symbology(&symbols).is_err());
413 }
414
415 #[rstest]
416 fn test_instrument_id_to_symbol_string_updates_map() {
417 let symbol = Symbol::from("TEST");
418 let venue = Venue::from("XNAS");
419 let instrument_id = InstrumentId::new(symbol, venue);
420 let mut map: AHashMap<Symbol, Venue> = AHashMap::new();
421
422 let sym_str = instrument_id_to_symbol_string(instrument_id, &mut map);
424 assert_eq!(sym_str, "TEST");
425 assert_eq!(map.get(&Symbol::from("TEST")), Some(&Venue::from("XNAS")));
426
427 let other = Venue::from("XLON");
429 let inst2 = InstrumentId::new(Symbol::from("TEST"), other);
430 let sym_str2 = instrument_id_to_symbol_string(inst2, &mut map);
431 assert_eq!(sym_str2, "TEST");
432
433 assert_eq!(map.get(&Symbol::from("TEST")), Some(&Venue::from("XNAS")));
435 }
436
437 #[rstest]
438 fn test_check_consistent_symbology_when_inconsistent() {
439 let symbols = vec!["ESM4", "ES.OPT"];
440 let result = check_consistent_symbology(&symbols);
441 assert!(result.is_err());
442 assert_eq!(
443 result.err().unwrap().to_string(),
444 "Inconsistent symbology types: 'raw_symbol' for ESM4 vs 'parent' for ES.OPT"
445 );
446 }
447
448 #[rstest]
449 #[case(vec!["AAPL,MSFT"])]
450 #[case(vec!["ES.OPT,ES.FUT"])]
451 #[case(vec!["ES.c.0,ES.c.1"])]
452 fn test_check_consistent_symbology_when_consistent(#[case] symbols: Vec<&str>) {
453 let result = check_consistent_symbology(&symbols);
454 assert!(result.is_ok());
455 }
456
457 #[rstest]
458 #[case("ESZ26", "GLBX.MDP3", 2, "ESZ6")]
459 #[case("ESZ95", "GLBX.MDP3", 2, "ESZ5")]
460 #[case("ESZ26 P5000", "GLBX.MDP3", 2, "ESZ6 P5000")]
461 #[case("ESU26-ESM27", "GLBX.MDP3", 2, "ESU6-ESM7")]
462 #[case("CLZ29-CLF30", "GLBX.MDP3", 2, "CLZ9-CLF0")]
463 #[case("ESZ26", "XNAS.ITCH", 2, "ESZ26")]
464 #[case("ESZ26", "GLBX.MDP3", 1, "ESZ26")]
465 #[case("ES.c.0", "GLBX.MDP3", 2, "ES.c.0")]
466 #[case("ES.FUT", "GLBX.MDP3", 2, "ES.FUT")]
467 #[case("BRN FMM0024!", "IFEU.IMPACT", 2, "BRN FMM0024!")]
468 fn test_raw_symbol_preserves_venue_format(
469 #[case] symbol: &str,
470 #[case] dataset: &str,
471 #[case] digits: u8,
472 #[case] expected: &str,
473 ) {
474 assert_eq!(
475 raw_symbol(symbol, dataset, digits.try_into().unwrap()),
476 expected
477 );
478 }
479
480 #[rstest]
481 #[case("ESZ6", "2026-12-18T00:00:00Z", "ESZ26.XCME")]
482 #[case("ESZ9", "2029-12-21T00:00:00Z", "ESZ29.XCME")]
483 #[case("ESH0", "2020-03-20T00:00:00Z", "ESH20.XCME")]
484 #[case("ESH0", "2030-03-15T00:00:00Z", "ESH30.XCME")]
485 #[case("CLF6", "2025-12-19T00:00:00Z", "CLF26.XCME")]
486 #[case("CLF0", "2029-12-19T00:00:00Z", "CLF30.XCME")]
487 #[case("SR3Z6", "2027-03-16T00:00:00Z", "SR3Z26.XCME")]
488 #[case("SR3Z9", "2030-03-19T00:00:00Z", "SR3Z29.XCME")]
489 #[case("SR3Z6-SR3H7", "2027-03-16T00:00:00Z", "SR3Z26-SR3H27.XCME")]
490 #[case("SR3Z6-SR3Z3", "2027-03-16T00:00:00Z", "SR3Z26-SR3Z33.XCME")]
491 #[case("CLZ9-CLF0", "2029-11-20T00:00:00Z", "CLZ29-CLF30.XCME")]
492 fn test_futures_definition_uses_contract_year(
493 #[case] symbol: &str,
494 #[case] expiry: &str,
495 #[case] expected: InstrumentId,
496 ) {
497 let expiration = expiry.parse::<jiff::Timestamp>().unwrap().as_nanosecond() as u64;
498
499 let msg = dbn::InstrumentDefMsg {
500 hd: dbn::RecordHeader::new::<dbn::InstrumentDefMsg>(
501 dbn::rtype::INSTRUMENT_DEF,
502 1,
503 42,
504 0,
505 ),
506 instrument_class: if symbol.contains('-') { b'S' } else { b'F' } as std::ffi::c_char,
507 expiration,
508 ..Default::default()
509 };
510
511 let id = InstrumentId::new(Symbol::from(symbol), Venue::from("XCME"));
512 let record = dbn::RecordRef::from(&msg);
513 assert_eq!(
514 resolve_futures_instrument_id(
515 &record,
516 id,
517 FuturesYearDigits::Two,
518 &AHashMap::new(),
519 None
520 )
521 .unwrap(),
522 expected
523 );
524 assert_eq!(
525 resolve_futures_instrument_id(
526 &record,
527 id,
528 FuturesYearDigits::One,
529 &AHashMap::new(),
530 None
531 )
532 .unwrap(),
533 id
534 );
535 }
536
537 #[rstest]
538 #[case(dbn::UNDEF_TIMESTAMP)]
539 #[case(1_000_000_000)]
540 fn test_bbo_year_uses_receive_timestamp(#[case] ts_event: u64) {
541 let ts_recv = "2029-12-20T00:00:00Z"
542 .parse::<jiff::Timestamp>()
543 .unwrap()
544 .as_nanosecond() as u64;
545 let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
546 msg.hd.publisher_id = 1;
547 msg.hd.instrument_id = 42;
548 msg.hd.ts_event = ts_event;
549 msg.ts_recv = ts_recv;
550 let id = InstrumentId::from("CLF0.XNYM");
551 assert_eq!(
552 resolve_futures_instrument_id(
553 &dbn::RecordRef::from(&msg),
554 id,
555 FuturesYearDigits::Two,
556 &AHashMap::new(),
557 None
558 )
559 .unwrap(),
560 InstrumentId::from("CLF30.XNYM")
561 );
562 }
563
564 #[rstest]
565 #[case(2, "SR3Z6", "SR3Z26", "SR3Z26.XCME")]
566 #[case(2, "SR3Z6", "SR3Z36", "SR3Z36.XCME")]
567 #[case(2, "SR3Z6", "CLF26", "SR3Z36.XCME")]
568 #[case(2, "SR3Z6-SR3H7", "SR3Z26-SR3H27", "SR3Z26-SR3H27.XCME")]
569 #[case(1, "SR3Z6", "SR3Z26", "SR3Z6.XCME")]
570 fn test_market_record_uses_requested_contract_year(
571 #[case] digits: u8,
572 #[case] native: &str,
573 #[case] requested: &str,
574 #[case] expected: InstrumentId,
575 ) {
576 let mut msg = dbn::BboMsg::default_for_schema(dbn::Schema::Bbo1S);
577 msg.hd.publisher_id = 1;
578 msg.ts_recv = "2027-02-01T00:00:00Z"
579 .parse::<jiff::Timestamp>()
580 .unwrap()
581 .as_nanosecond() as u64;
582 let id = InstrumentId::new(Symbol::from(native), Venue::from("XCME"));
583 let symbols = AHashMap::from_iter([(Symbol::from(requested), Venue::from("XCME"))]);
584
585 let result = resolve_futures_instrument_id(
586 &dbn::RecordRef::from(&msg),
587 id,
588 digits.try_into().unwrap(),
589 &symbols,
590 None,
591 )
592 .unwrap();
593
594 assert_eq!(result, expected);
595 }
596
597 #[rstest]
598 #[case(2026, "CLF26.XCME")]
599 #[case(0, "CLF6.XCME")]
600 #[case(u16::MAX, "CLF6.XCME")]
601 fn test_definition_maturity_year_resolves_without_expiration(
602 #[case] maturity_year: u16,
603 #[case] expected: InstrumentId,
604 ) {
605 let msg = dbn::InstrumentDefMsg {
606 hd: dbn::RecordHeader::new::<dbn::InstrumentDefMsg>(
607 dbn::rtype::INSTRUMENT_DEF,
608 1,
609 42,
610 0,
611 ),
612 instrument_class: b'F' as std::ffi::c_char,
613 maturity_year,
614 expiration: dbn::UNDEF_TIMESTAMP,
615 ..Default::default()
616 };
617
618 let id = InstrumentId::from("CLF6.XCME");
619 let result = resolve_futures_instrument_id(
620 &dbn::RecordRef::from(&msg),
621 id,
622 FuturesYearDigits::Two,
623 &AHashMap::new(),
624 None,
625 )
626 .unwrap();
627 assert_eq!(result, expected);
628 assert_eq!(
629 resolve_futures_instrument_id(
630 &dbn::RecordRef::from(&msg),
631 id,
632 FuturesYearDigits::One,
633 &AHashMap::new(),
634 None
635 )
636 .unwrap(),
637 id
638 );
639 }
640}