1use std::{collections::HashMap, num::NonZeroUsize, path::Path, sync::Arc};
19
20use anyhow::Context;
21use jiff::civil::Date;
22use nautilus_core::{UnixNanos, string::secret::SecretString, time::get_atomic_clock_realtime};
23use nautilus_model::{
24 data::{Bar, BarType, OrderBookDelta, QuoteTick, TradeTick},
25 enums::AggregationSource,
26 identifiers::InstrumentId,
27 instruments::{Instrument, InstrumentAny},
28 orderbook::OrderBook,
29};
30use parking_lot::{Mutex, RwLock};
31use ustr::Ustr;
32
33use crate::{
34 common::credential::resolve_api_key,
35 config::{CoinApiInstrumentSelection, validate_price_precisions, validate_selections},
36 csv,
37 http::{
38 client::{CoinApiHttpClient, CoinApiHttpConfig, HistoryPages},
39 parse::{
40 parse_bar, parse_instruments_with_precisions, parse_order_book, parse_quote_tick,
41 parse_trade_tick,
42 },
43 query::{BookQuery, HistoryRange},
44 },
45 websocket::parse::bar_spec_to_period_id,
46};
47
48#[derive(Debug)]
58pub struct CoinApiDataLoader {
59 api_key: Option<SecretString>,
60 http_config: CoinApiHttpConfig,
61 http: Mutex<Option<Arc<CoinApiHttpClient>>>,
62 instruments: RwLock<HashMap<InstrumentId, LoadedInstrument>>,
63}
64
65impl CoinApiDataLoader {
66 #[must_use]
68 pub fn new(api_key: Option<SecretString>) -> Self {
69 Self::with_http_config(api_key, CoinApiHttpConfig::default())
70 }
71
72 #[must_use]
74 pub fn with_http_config(api_key: Option<SecretString>, http_config: CoinApiHttpConfig) -> Self {
75 Self {
76 api_key,
77 http_config,
78 http: Mutex::new(None),
79 instruments: RwLock::new(HashMap::new()),
80 }
81 }
82
83 pub fn add_instrument(
91 &self,
92 instrument: InstrumentAny,
93 symbol_id: Option<Ustr>,
94 symbol_id_int: Option<u64>,
95 ) -> anyhow::Result<()> {
96 let mut instruments = self.instruments.write();
97 let id = instrument.id();
98
99 let loaded = LoadedInstrument {
100 instrument,
101 symbol_id,
102 symbol_id_int,
103 };
104
105 insert_instrument(&mut instruments, id, loaded)
106 }
107
108 pub fn instrument(&self, instrument_id: InstrumentId) -> anyhow::Result<InstrumentAny> {
114 Ok(self.loaded(instrument_id)?.instrument)
115 }
116
117 pub async fn request_instruments(
124 &self,
125 instruments: &[CoinApiInstrumentSelection],
126 price_precisions: &HashMap<InstrumentId, u8>,
127 ) -> anyhow::Result<Vec<InstrumentAny>> {
128 validate_selections(instruments)?;
129 validate_price_precisions(instruments, price_precisions)?;
130 anyhow::ensure!(
131 !instruments.is_empty(),
132 "CoinAPI loader requires instrument selections"
133 );
134 let http = self.http()?;
135 let now = get_atomic_clock_realtime().get_time_ns();
136 let mut loaded = Vec::new();
137
138 for selection in instruments {
139 let symbols = http
140 .get_symbols(&selection.exchange_id, &selection.query)
141 .await?;
142 let parsed = parse_instruments_with_precisions(&symbols, price_precisions, now);
143 for id in &selection.instrument_ids {
144 let item = parsed
145 .iter()
146 .find(|item| item.instrument.id() == *id)
147 .with_context(|| format!("CoinAPI returned no instrument {id}"))?;
148 let symbol_id_int = symbols
149 .iter()
150 .find(|symbol| symbol.symbol_id == item.symbol_id)
151 .and_then(|symbol| symbol.symbol_id_int);
152 loaded.push(LoadedInstrument {
153 instrument: item.instrument.clone(),
154 symbol_id: Some(item.symbol_id),
155 symbol_id_int,
156 });
157 }
158 }
159
160 let mut instruments = self.instruments.write();
161 let mut candidate = instruments.clone();
162 for item in &loaded {
163 insert_instrument(&mut candidate, item.instrument.id(), item.clone())?;
164 }
165
166 *instruments = candidate;
167 Ok(loaded.into_iter().map(|item| item.instrument).collect())
168 }
169
170 pub async fn download_flat_file(
181 &self,
182 bucket: &str,
183 object_key: &str,
184 filepath: impl AsRef<Path>,
185 ) -> anyhow::Result<()> {
186 self.http()?
187 .download_flat_file(bucket, object_key, filepath.as_ref())
188 .await?;
189 Ok(())
190 }
191
192 pub fn load_book_deltas(
201 &self,
202 path: impl AsRef<Path>,
203 instrument_id: InstrumentId,
204 date: Option<Date>,
205 limit: Option<usize>,
206 ) -> anyhow::Result<Vec<OrderBookDelta>> {
207 let instrument = self.instrument(instrument_id)?;
208 csv::load_deltas(
209 path,
210 instrument_id,
211 date,
212 Some(instrument.price_precision()),
213 Some(instrument.size_precision()),
214 limit,
215 )
216 }
217
218 pub fn stream_book_deltas<P: AsRef<Path>>(
228 &self,
229 path: P,
230 instrument_id: InstrumentId,
231 date: Option<Date>,
232 chunk_size: usize,
233 limit: Option<usize>,
234 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<Vec<OrderBookDelta>>> + use<P>> {
235 let instrument = self.instrument(instrument_id)?;
236 csv::stream_deltas(
237 path,
238 instrument_id,
239 date,
240 chunk_size,
241 Some(instrument.price_precision()),
242 Some(instrument.size_precision()),
243 limit,
244 )
245 }
246
247 pub fn load_quotes(
253 &self,
254 path: impl AsRef<Path>,
255 instrument_id: InstrumentId,
256 limit: Option<usize>,
257 ) -> anyhow::Result<Vec<QuoteTick>> {
258 let instrument = self.instrument(instrument_id)?;
259 csv::load_quotes(
260 path,
261 instrument_id,
262 Some(instrument.price_precision()),
263 Some(instrument.size_precision()),
264 limit,
265 )
266 }
267
268 pub fn stream_quotes<P: AsRef<Path>>(
275 &self,
276 path: P,
277 instrument_id: InstrumentId,
278 chunk_size: usize,
279 limit: Option<usize>,
280 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<Vec<QuoteTick>>> + use<P>> {
281 let instrument = self.instrument(instrument_id)?;
282 csv::stream_quotes(
283 path,
284 instrument_id,
285 chunk_size,
286 Some(instrument.price_precision()),
287 Some(instrument.size_precision()),
288 limit,
289 )
290 }
291
292 pub fn load_trades(
298 &self,
299 path: impl AsRef<Path>,
300 instrument_id: InstrumentId,
301 limit: Option<usize>,
302 ) -> anyhow::Result<Vec<TradeTick>> {
303 let instrument = self.instrument(instrument_id)?;
304 csv::load_trades(
305 path,
306 instrument_id,
307 Some(instrument.price_precision()),
308 Some(instrument.size_precision()),
309 limit,
310 )
311 }
312
313 pub fn stream_trades<P: AsRef<Path>>(
320 &self,
321 path: P,
322 instrument_id: InstrumentId,
323 chunk_size: usize,
324 limit: Option<usize>,
325 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<Vec<TradeTick>>> + use<P>> {
326 let instrument = self.instrument(instrument_id)?;
327 csv::stream_trades(
328 path,
329 instrument_id,
330 chunk_size,
331 Some(instrument.price_precision()),
332 Some(instrument.size_precision()),
333 limit,
334 )
335 }
336
337 pub fn load_bars(
346 &self,
347 path: impl AsRef<Path>,
348 period_id: Option<&str>,
349 limit: Option<usize>,
350 ) -> anyhow::Result<Vec<Bar>> {
351 let instruments = self.instruments.read();
352
353 let ids = instruments
354 .values()
355 .filter_map(|item| {
356 Some((
357 item.symbol_id_int?,
358 (item.instrument.id(), item.instrument.price_precision()),
359 ))
360 })
361 .collect::<HashMap<_, _>>();
362
363 anyhow::ensure!(
364 !ids.is_empty(),
365 "CoinAPI bars require numeric symbol mappings"
366 );
367 drop(instruments);
368 csv::load_bars(path, &ids, period_id, limit)
369 }
370
371 pub async fn request_book_snapshot(
377 &self,
378 instrument_id: InstrumentId,
379 depth: Option<u32>,
380 ) -> anyhow::Result<OrderBook> {
381 let item = self.loaded(instrument_id)?;
382 let symbol_id = item
383 .symbol_id
384 .context("CoinAPI instrument has no REST symbol mapping")?;
385
386 let query = BookQuery {
387 limit_levels: depth,
388 };
389
390 let http = self.http()?;
391 let source = http.get_book(symbol_id.as_str(), &query).await?;
392 parse_order_book(&source, &item.instrument)
393 }
394
395 pub async fn request_quotes(
404 &self,
405 instrument_id: InstrumentId,
406 start: UnixNanos,
407 end: UnixNanos,
408 limit: Option<NonZeroUsize>,
409 ) -> anyhow::Result<Vec<QuoteTick>> {
410 let item = self.loaded(instrument_id)?;
411 let symbol_id = item
412 .symbol_id
413 .context("CoinAPI instrument has no REST symbol mapping")?;
414
415 let range = HistoryRange {
416 time_start: start.to_datetime_utc(),
417 time_end: end.to_datetime_utc(),
418 limit,
419 };
420
421 let http = self.http()?;
422 let pages = http.get_quotes_range(symbol_id.as_str(), &range).await?;
423 check_complete(&pages, http.credits_remaining())?;
424 let records = pages.records.iter();
425 records
426 .map(|row| parse_quote_tick(row, &item.instrument))
427 .collect()
428 }
429
430 pub async fn request_trades(
439 &self,
440 instrument_id: InstrumentId,
441 start: UnixNanos,
442 end: UnixNanos,
443 limit: Option<NonZeroUsize>,
444 ) -> anyhow::Result<Vec<TradeTick>> {
445 let item = self.loaded(instrument_id)?;
446 let symbol_id = item
447 .symbol_id
448 .context("CoinAPI instrument has no REST symbol mapping")?;
449
450 let range = HistoryRange {
451 time_start: start.to_datetime_utc(),
452 time_end: end.to_datetime_utc(),
453 limit,
454 };
455
456 let http = self.http()?;
457 let pages = http.get_trades_range(symbol_id.as_str(), &range).await?;
458 check_complete(&pages, http.credits_remaining())?;
459 let records = pages.records.iter();
460 records
461 .map(|row| parse_trade_tick(row, &item.instrument))
462 .collect()
463 }
464
465 pub async fn request_bars(
474 &self,
475 bar_type: BarType,
476 start: UnixNanos,
477 end: UnixNanos,
478 limit: Option<NonZeroUsize>,
479 ) -> anyhow::Result<Vec<Bar>> {
480 anyhow::ensure!(
481 bar_type.aggregation_source() == AggregationSource::External,
482 "CoinAPI provides only external bars"
483 );
484 let period = bar_spec_to_period_id(bar_type.spec())?;
485 let instrument_id = bar_type.instrument_id();
486 let item = self.loaded(instrument_id)?;
487 let symbol_id = item
488 .symbol_id
489 .context("CoinAPI instrument has no REST symbol mapping")?;
490
491 let range = HistoryRange {
492 time_start: start.to_datetime_utc(),
493 time_end: end.to_datetime_utc(),
494 limit,
495 };
496
497 let http = self.http()?;
498 let pages = http
499 .get_ohlcv_range(symbol_id.as_str(), period, &range)
500 .await?;
501 check_complete(&pages, http.credits_remaining())?;
502 let closed_by = end
503 .to_datetime_utc()
504 .min(get_atomic_clock_realtime().get_time_ns().to_datetime_utc());
505 let records = pages
506 .records
507 .iter()
508 .filter(|row| row.time_period_end <= closed_by);
509 records
510 .map(|row| parse_bar(row, bar_type, &item.instrument))
511 .collect()
512 }
513
514 fn loaded(&self, instrument_id: InstrumentId) -> anyhow::Result<LoadedInstrument> {
515 self.instruments
516 .read()
517 .get(&instrument_id)
518 .cloned()
519 .with_context(|| format!("No loaded instrument {instrument_id}"))
520 }
521
522 fn http(&self) -> anyhow::Result<Arc<CoinApiHttpClient>> {
523 let mut http = self.http.lock();
524 if let Some(client) = http.as_ref() {
525 return Ok(Arc::clone(client));
526 }
527
528 let key = resolve_api_key(self.api_key.clone())?;
529 let client = CoinApiHttpClient::new(key, self.http_config.clone())?;
530 let client = Arc::new(client);
531 *http = Some(Arc::clone(&client));
532 Ok(client)
533 }
534}
535
536#[derive(Debug, Clone)]
537struct LoadedInstrument {
538 instrument: InstrumentAny,
539 symbol_id: Option<Ustr>,
540 symbol_id_int: Option<u64>,
541}
542
543fn insert_instrument(
544 instruments: &mut HashMap<InstrumentId, LoadedInstrument>,
545 id: InstrumentId,
546 loaded: LoadedInstrument,
547) -> anyhow::Result<()> {
548 anyhow::ensure!(
549 loaded.symbol_id.is_none_or(|symbol| !symbol.is_empty()),
550 "CoinAPI symbol ID must not be empty"
551 );
552
553 for (other_id, other) in instruments.iter() {
554 if *other_id == id {
555 continue;
556 }
557
558 anyhow::ensure!(
559 loaded.symbol_id.is_none() || loaded.symbol_id != other.symbol_id,
560 "CoinAPI REST symbol already belongs to {other_id}"
561 );
562 anyhow::ensure!(
563 loaded.symbol_id_int.is_none() || loaded.symbol_id_int != other.symbol_id_int,
564 "CoinAPI numeric symbol already belongs to {other_id}"
565 );
566 }
567
568 instruments.insert(id, loaded);
569 Ok(())
570}
571
572fn check_complete<T>(pages: &HistoryPages<T>, credits_remaining: u64) -> anyhow::Result<()> {
573 anyhow::ensure!(
574 !pages.is_truncated,
575 "CoinAPI history truncated after {} records with {credits_remaining} credits remaining; no partial result returned; use a shorter window or a new loader with a larger credit_budget",
576 pages.records.len()
577 );
578 Ok(())
579}
580
581#[cfg(test)]
582mod tests {
583 use nautilus_model::{
584 data::OrderBookDelta,
585 enums::{BookAction, RecordFlag},
586 };
587 use rstest::rstest;
588
589 use super::*;
590 use crate::{
591 config::CoinApiInstrumentSelection,
592 http::{models::Symbol, parse::parse_instruments, query::SymbolsQuery},
593 };
594
595 fn instrument() -> InstrumentAny {
596 let symbols: Vec<Symbol> =
597 serde_json::from_str(include_str!("../test_data/symbols_spot.json")).unwrap();
598 let (mut instruments, _) = parse_instruments(&symbols, UnixNanos::default());
599 instruments.remove(0).instrument
600 }
601
602 #[tokio::test]
603 async fn metadata_rejects_unselected_price_override_before_credentials() {
604 let selected = InstrumentId::from("BTCUSDT.BINANCE");
605 let overridden = InstrumentId::from("BTC-USD.COINBASE");
606
607 let loader = CoinApiDataLoader::with_http_config(
608 None,
609 CoinApiHttpConfig {
610 base_url: "invalid-url".to_owned(),
611 ..Default::default()
612 },
613 );
614
615 let instruments = [CoinApiInstrumentSelection {
616 exchange_id: "BINANCE".into(),
617 query: SymbolsQuery::default(),
618 instrument_ids: vec![selected],
619 }];
620
621 let price_precisions = HashMap::from([(overridden, 3)]);
622 let error = loader
623 .request_instruments(&instruments, &price_precisions)
624 .await
625 .unwrap_err();
626
627 assert_eq!(
628 error.to_string(),
629 "price precision for BTC-USD.COINBASE, which is not a selected instrument"
630 );
631 }
632
633 #[rstest]
634 fn local_loads_use_registered_precision_without_credentials() {
635 let loader = CoinApiDataLoader::new(None);
636 let instrument = instrument();
637 let id = instrument.id();
638 loader
639 .add_instrument(instrument.clone(), None, None)
640 .unwrap();
641 let path = Path::new(env!("CARGO_MANIFEST_DIR"))
642 .join("test_data/csv/trades_binance_spot_btc_usdt_20250901.csv");
643 let trades = loader.load_trades(&path, id, Some(2)).unwrap();
644 let streamed: Vec<_> = loader
645 .stream_trades(&path, id, 1, Some(2))
646 .unwrap()
647 .collect::<anyhow::Result<Vec<_>>>()
648 .unwrap()
649 .into_iter()
650 .flatten()
651 .collect();
652 assert_eq!(loader.instrument(id).unwrap(), instrument);
653 assert_eq!(trades, streamed);
654 assert_eq!(trades.len(), 2);
655 assert_eq!(trades[0].price.to_string(), "108246.36");
656 assert_eq!(trades[0].size.as_decimal().to_string(), "0.00023");
657 assert_eq!(trades[0].size.precision, instrument.size_precision());
658 }
659
660 #[rstest]
661 fn missing_metadata_and_duplicate_mappings_preserve_registry() {
662 let loader = CoinApiDataLoader::new(None);
663 let instrument = instrument();
664 let id = instrument.id();
665 let missing = loader.load_trades("absent.csv", id, None).unwrap_err();
666 assert_eq!(missing.to_string(), format!("No loaded instrument {id}"));
667 loader
668 .add_instrument(
669 instrument.clone(),
670 Some("BINANCE_SPOT_BTC_USDT".into()),
671 Some(42),
672 )
673 .unwrap();
674 let mut other = instrument.clone();
675 if let InstrumentAny::CurrencyPair(pair) = &mut other {
676 pair.id = InstrumentId::from("OTHER.BINANCE");
677 }
678
679 let error = loader
680 .add_instrument(other.clone(), None, Some(42))
681 .unwrap_err();
682 assert_eq!(
683 error.to_string(),
684 format!("CoinAPI numeric symbol already belongs to {id}")
685 );
686 assert_eq!(loader.instrument(id).unwrap(), instrument);
687 assert_eq!(
688 loader.instrument(other.id()).unwrap_err().to_string(),
689 format!("No loaded instrument {}", other.id())
690 );
691 }
692
693 #[rstest]
694 fn book_stream_preserves_complete_batches_and_definition_snapshot() {
695 let loader = CoinApiDataLoader::new(None);
696 let instrument = instrument();
697 let id = instrument.id();
698 loader.add_instrument(instrument, None, None).unwrap();
699 let path = Path::new(env!("CARGO_MANIFEST_DIR"))
700 .join("test_data/csv/limitbook_full_binance_spot_btc_usdt_20250901.csv");
701 let stream = loader
702 .stream_book_deltas(&path, id, Some(Date::new(2025, 9, 1).unwrap()), 5, None)
703 .unwrap();
704 drop(loader);
705 let chunks = stream.collect::<anyhow::Result<Vec<_>>>().unwrap();
706 let first: &OrderBookDelta = &chunks[0][0];
707 assert_eq!(first.action, BookAction::Clear);
708 assert_eq!(first.instrument_id, id);
709 assert!(
710 chunks
711 .iter()
712 .all(|chunk| { RecordFlag::F_LAST.matches(chunk.last().unwrap().flags) })
713 );
714 }
715}