1use std::{
19 collections::{HashMap, HashSet},
20 sync::atomic::{AtomicU64, Ordering},
21 time::Duration,
22};
23
24use nautilus_core::string::secret::SecretString;
25use nautilus_network::{
26 ConnectionUrl,
27 backoff::ExponentialBackoff,
28 dst::time,
29 http::{HttpClient, HttpRedirectPolicy, Method},
30 retry::{RetryConfig, RetryManager},
31};
32use serde::{
33 Deserialize, Deserializer, Serialize,
34 de::{DeserializeOwned, Error as SerdeError},
35};
36use serde_json::{Map, Value};
37use tokio_util::sync::CancellationToken;
38use url::Url;
39use ustr::Ustr;
40
41use super::{
42 error::Error,
43 kalshi::{Event as KalshiEvent, EventResponse, Market as KalshiMarket, MarketResponse},
44 models::{
45 ActivityBatch, ActivityCurrent, Book, BookEvent, Exchange, Market, MarketsPage, Ohlcv,
46 OhlcvPeriod, Quote, Trade,
47 },
48 query::{
49 ActiveQuery, ActivityQuery, FlatFileOhlcvQuery, HistoryQuery, ListingsQuery, OhlcvQuery,
50 Pagination,
51 },
52};
53use crate::common::credential::resolve_api_key;
54
55const DURATION_MAX_MS: u64 = 86_400_000;
56
57pub const REST_URL: &str = "https://api.prediction-markets.finfeedapi.com/v1/";
59
60#[derive(Debug, Clone, Serialize, Deserialize, bon::Builder)]
75#[builder(on(SecretString, into))]
76#[serde(default, deny_unknown_fields)]
77pub struct FinFeedHttpConfig {
78 #[builder(default = FinFeedHttpConfig::default().base_url)]
81 pub base_url: SecretString,
82 pub proxy_url: Option<SecretString>,
84 #[builder(default = FinFeedHttpConfig::default().timeout_secs)]
86 pub timeout_secs: u64,
87 #[builder(default = FinFeedHttpConfig::default().max_concurrent_requests)]
89 pub max_concurrent_requests: usize,
90 #[builder(default = FinFeedHttpConfig::default().attempt_budget)]
92 pub attempt_budget: u64,
93 #[builder(default = FinFeedHttpConfig::default().min_request_interval_ms)]
95 pub min_request_interval_ms: u64,
96 #[serde(deserialize_with = "deserialize_retry")]
102 #[builder(default = FinFeedHttpConfig::default().retry)]
103 pub retry: RetryConfig,
104 pub kalshi_metadata_url: Option<SecretString>,
107}
108
109impl Default for FinFeedHttpConfig {
110 fn default() -> Self {
111 Self {
112 base_url: REST_URL.into(),
113 proxy_url: None,
114 timeout_secs: 300,
115 max_concurrent_requests: 1,
116 attempt_budget: 100,
117 min_request_interval_ms: 10_000,
118 retry: RetryConfig {
119 jitter_ms: 0,
120 operation_timeout_ms: Some(330_000),
121 max_elapsed_ms: Some(600_000),
122 ..RetryConfig::default()
123 },
124 kalshi_metadata_url: None,
125 }
126 }
127}
128
129#[derive(Debug)]
135pub struct FinFeedHttpClient {
136 client: HttpClient,
137 base_url: ConnectionUrl,
138 api_key: SecretString,
139 concurrency: tokio::sync::Semaphore,
140 next_request: tokio::sync::Mutex<Option<time::Instant>>,
141 interval: Duration,
142 attempts_remaining: AtomicU64,
143 retry: RetryManager<Error>,
144 cancellation: CancellationToken,
145 kalshi_base_url: ConnectionUrl,
146}
147
148impl FinFeedHttpClient {
149 pub fn new(api_key: Option<SecretString>, config: FinFeedHttpConfig) -> Result<Self, Error> {
156 let api_key = resolve_api_key(api_key)?;
157 let base_url = endpoint_url(&config.base_url, &api_key)?;
158 if let Some(proxy) = &config.proxy_url {
159 let proxy = Url::parse(proxy.expose_secret())
160 .map_err(|_| Error::Validation("invalid proxy URL".into()))?;
161
162 if !matches!(proxy.scheme(), "http" | "https")
163 || proxy.host_str().is_none()
164 || proxy.query().is_some()
165 || proxy.fragment().is_some()
166 || proxy.path() != "/"
167 {
168 return Err(Error::Validation(
169 "proxy must be HTTP(S) without path, query, or fragment".into(),
170 ));
171 }
172 }
173
174 validate_controls(&config)?;
175
176 let kalshi_endpoint = config.kalshi_metadata_url.as_ref().map_or(
177 "https://external-api.kalshi.com/trade-api/v2/",
178 SecretString::expose_secret,
179 );
180 let kalshi_endpoint = SecretString::from(kalshi_endpoint);
181
182 let kalshi_base_url = endpoint_url(&kalshi_endpoint, &api_key).map_err(|_| {
183 Error::Validation("invalid public Kalshi metadata configuration".into())
184 })?;
185
186 let client = HttpClient::builder()
187 .timeout_secs(config.timeout_secs)
188 .redact_url(true)
189 .redirect_policy(HttpRedirectPolicy::Reject)
190 .use_system_proxy(false)
191 .maybe_proxy_url(
192 config
193 .proxy_url
194 .as_ref()
195 .map(|url| url.expose_secret().to_owned()),
196 )
197 .header_keys(vec!["retry-after".into()])
198 .build()?;
199 Ok(Self {
200 client,
201 base_url: base_url.as_str().into(),
202 api_key,
203 concurrency: tokio::sync::Semaphore::new(config.max_concurrent_requests),
204 next_request: tokio::sync::Mutex::new(None),
205 interval: Duration::from_millis(config.min_request_interval_ms),
206 attempts_remaining: AtomicU64::new(config.attempt_budget),
207 retry: RetryManager::new(config.retry),
208 cancellation: CancellationToken::new(),
209 kalshi_base_url: kalshi_base_url.as_str().into(),
210 })
211 }
212
213 pub fn cancel(&self) {
217 self.cancellation.cancel();
218 }
219
220 #[must_use]
222 pub fn attempts_remaining(&self) -> u64 {
223 self.attempts_remaining.load(Ordering::Relaxed)
224 }
225
226 pub async fn get_exchanges(&self) -> Result<Vec<Exchange>, Error> {
234 self.get(&["exchanges"], &()).await
235 }
236
237 pub async fn get_exchange(&self, exchange_id: &str) -> Result<Exchange, Error> {
245 self.get(&["exchanges", exchange_id], &()).await
246 }
247
248 pub async fn get_listings(
257 &self,
258 exchange_id: &str,
259 query: &ListingsQuery,
260 ) -> Result<MarketsPage, Error> {
261 query.validate()?;
262 let page: MarketsPage = self
263 .get(&["markets", exchange_id, "history"], query)
264 .await?;
265 check_page(page.data.as_ref().map_or(0, Vec::len), query.limit)?;
266 Ok(page)
267 }
268
269 pub async fn get_active(
278 &self,
279 exchange_id: &str,
280 query: &ActiveQuery,
281 ) -> Result<Vec<Ustr>, Error> {
282 query.validate()?;
283 let ids: Vec<Ustr> = self.get(&["markets", exchange_id, "active"], query).await?;
284 check_page(ids.len(), query.limit)?;
285 Ok(ids)
286 }
287
288 pub async fn get_market(&self, exchange_id: &str, market_id: &str) -> Result<Market, Error> {
296 self.get(&["markets", exchange_id, market_id], &()).await
297 }
298
299 pub async fn get_current(
307 &self,
308 exchange_id: &str,
309 market_id: &str,
310 ) -> Result<ActivityCurrent, Error> {
311 self.get(&["activity", exchange_id, market_id, "current"], &())
312 .await
313 }
314
315 pub async fn get_latest(
324 &self,
325 exchange_id: &str,
326 market_id: &str,
327 query: &ActivityQuery,
328 ) -> Result<ActivityBatch, Error> {
329 query.validate()?;
330 let batch: ActivityBatch = self
331 .get(&["activity", exchange_id, market_id, "latest"], query)
332 .await?;
333 check_page(batch.trades.as_ref().map_or(0, Vec::len), query.limit)?;
334 check_page(batch.quotes.as_ref().map_or(0, Vec::len), query.limit)?;
335 Ok(batch)
336 }
337
338 pub async fn get_trades_history(
347 &self,
348 exchange_id: &str,
349 market_id: &str,
350 query: &HistoryQuery,
351 ) -> Result<Vec<Trade>, Error> {
352 query.validate_hourly()?;
353 let records: Vec<Trade> = self
354 .get(
355 &["activity", exchange_id, market_id, "history", "trades"],
356 query,
357 )
358 .await?;
359 check_page(records.len(), query.limit)?;
360 Ok(records)
361 }
362
363 pub async fn get_quotes_history(
372 &self,
373 exchange_id: &str,
374 market_id: &str,
375 query: &HistoryQuery,
376 ) -> Result<Vec<Quote>, Error> {
377 query.validate_hourly()?;
378 let records: Vec<Quote> = self
379 .get(
380 &["activity", exchange_id, market_id, "history", "quotes"],
381 query,
382 )
383 .await?;
384 check_page(records.len(), query.limit)?;
385 Ok(records)
386 }
387
388 pub async fn get_book(&self, exchange_id: &str, market_id: &str) -> Result<Book, Error> {
396 self.get(&["orderbook", exchange_id, market_id, "current"], &())
397 .await
398 }
399
400 pub async fn get_book_history(
413 &self,
414 exchange_id: &str,
415 market_id: &str,
416 query: &HistoryQuery,
417 ) -> Result<Vec<BookEvent>, Error> {
418 query.validate_hourly()?;
419 let records: Vec<BookEvent> = self
420 .get(&["orderbook", exchange_id, market_id, "history"], query)
421 .await?;
422 check_page(records.len(), query.limit)?;
423 Ok(records)
424 }
425
426 pub async fn get_periods(&self) -> Result<Vec<OhlcvPeriod>, Error> {
434 self.get(&["ohlcv", "periods"], &()).await
435 }
436
437 pub async fn get_ohlcv_history(
447 &self,
448 exchange_id: &str,
449 market_id: &str,
450 query: &OhlcvQuery,
451 ) -> Result<Vec<Ohlcv>, Error> {
452 query.validate(false)?;
453 let records: Vec<Ohlcv> = self
454 .get(&["ohlcv", exchange_id, market_id, "history"], query)
455 .await?;
456 check_page(records.len(), query.limit)?;
457 Ok(records)
458 }
459
460 pub async fn get_ohlcv_flat_file_history(
474 &self,
475 exchange_id: &str,
476 market_id: &str,
477 query: &FlatFileOhlcvQuery,
478 ) -> Result<Vec<Ohlcv>, Error> {
479 query.validate()?;
480 let records: Vec<Ohlcv> = self
481 .get(
482 &["activity", exchange_id, market_id, "history", "ohlcv"],
483 query,
484 )
485 .await?;
486 check_page(records.len(), query.history.limit)?;
487 Ok(records)
488 }
489
490 pub async fn get_ohlcv_exchange_history(
504 &self,
505 exchange_id: &str,
506 query: &OhlcvQuery,
507 ) -> Result<Vec<Ohlcv>, Error> {
508 query.validate_exchange()?;
509 let records: Vec<Ohlcv> = self.get(&["ohlcv", exchange_id, "history"], query).await?;
510 check_page(records.len(), query.limit)?;
511 Ok(records)
512 }
513
514 pub async fn get_ohlcv_latest(
523 &self,
524 exchange_id: &str,
525 market_id: &str,
526 query: &OhlcvQuery,
527 ) -> Result<Vec<Ohlcv>, Error> {
528 query.validate(true)?;
529 let records: Vec<Ohlcv> = self
530 .get(&["ohlcv", exchange_id, market_id, "latest"], query)
531 .await?;
532 check_page(records.len(), query.limit)?;
533 Ok(records)
534 }
535
536 pub async fn get_listings_all(
544 &self,
545 exchange_id: &str,
546 query: &ListingsQuery,
547 bounds: Pagination,
548 ) -> Result<Vec<Market>, Error> {
549 query.validate()?;
550 bounds.validate()?;
551 let mut query = query.clone();
552 let mut cursors = HashSet::new();
553 if let Some(cursor) = &query.cursor {
554 cursors.insert(cursor.clone());
555 }
556
557 let mut records = Vec::new();
558
559 for _ in 0..bounds.max_pages {
560 let page = self.get_listings(exchange_id, &query).await?;
561 let data = page.data.unwrap_or_default();
562 if data.len() > bounds.max_records.saturating_sub(records.len()) {
563 return Err(Error::Response("listing record bound exceeded"));
564 }
565
566 records.extend(data);
567 match page.next_cursor {
568 None => return Ok(records),
569 Some(cursor) if cursor.is_empty() || !cursors.insert(cursor.clone()) => {
570 return Err(Error::Response("listing cursor is empty or repeated"));
571 }
572 Some(cursor) => query.cursor = Some(cursor),
573 }
574 }
575
576 Err(Error::Response("listing page bound exceeded"))
577 }
578
579 pub async fn get_active_all(
589 &self,
590 exchange_id: &str,
591 query: &ActiveQuery,
592 bounds: Pagination,
593 ) -> Result<Vec<Ustr>, Error> {
594 query.validate()?;
595 bounds.validate()?;
596 let mut query = query.clone();
597 let mut seen = HashSet::new();
598 let mut ids = Vec::new();
599
600 for _ in 0..bounds.max_pages {
601 let page = self.get_active(exchange_id, &query).await?;
602 if page.is_empty() {
603 return Ok(ids);
604 }
605
606 if page.len() > bounds.max_records.saturating_sub(ids.len()) {
607 return Err(Error::Response("active ID record bound exceeded"));
608 }
609
610 for id in page {
611 if id.is_empty() || !seen.insert(id) {
612 return Err(Error::Response("active ID is empty or repeated"));
613 }
614
615 ids.push(id);
616 }
617
618 query.page = query
619 .page
620 .checked_add(1)
621 .filter(|page| i32::try_from(*page).is_ok())
622 .ok_or(Error::Response("active page overflow"))?;
623 }
624
625 Err(Error::Response("active page bound exceeded"))
626 }
627
628 pub(crate) async fn kalshi_metadata(
629 &self,
630 ticker: &str,
631 event_ticker: &str,
632 ) -> Result<(KalshiMarket, KalshiEvent), Error> {
633 let market = match self
634 .get_from::<MarketResponse, _>(&["markets", ticker], &(), false)
635 .await
636 {
637 Err(Error::Http { status: 404, .. }) => {
638 self.get_from(&["historical", "markets", ticker], &(), false)
639 .await
640 }
641 result => result,
642 }
643 .map_err(|_| Error::KalshiMetadata)?;
644
645 let event: EventResponse = self
646 .get_from(&["events", event_ticker], &(), false)
647 .await
648 .map_err(|_| Error::KalshiMetadata)?;
649 Ok((market.market, event.event))
650 }
651
652 async fn get<T: DeserializeOwned, Q: Serialize>(
653 &self,
654 path: &[&str],
655 query: &Q,
656 ) -> Result<T, Error> {
657 self.get_from(path, query, true).await
658 }
659
660 async fn get_from<T: DeserializeOwned, Q: Serialize>(
661 &self,
662 path: &[&str],
663 query: &Q,
664 finfeed: bool,
665 ) -> Result<T, Error> {
666 if path
667 .iter()
668 .any(|part| part.trim().is_empty() || matches!(*part, "." | ".."))
669 {
670 return Err(Error::Validation(
671 "path identifiers must not be empty or dot segments".into(),
672 ));
673 }
674
675 let base_url = if finfeed {
676 &self.base_url
677 } else {
678 &self.kalshi_base_url
679 };
680
681 let mut url = Url::parse(base_url.expose())
682 .map_err(|_| Error::Validation("invalid base URL".into()))?;
683 url.path_segments_mut()
684 .map_err(|()| Error::Validation("invalid base URL path".into()))?
685 .pop_if_empty()
686 .extend(path);
687 self.retry
688 .invocation(
689 if finfeed {
690 "FinFeed GET"
691 } else {
692 "Public Kalshi metadata GET"
693 },
694 || async {
695 let _permit = self
696 .concurrency
697 .acquire()
698 .await
699 .map_err(|_| Error::Response("admission closed"))?;
700 let mut next = self.next_request.lock().await;
701 if let Some(deadline) = *next {
702 time::sleep_until(deadline).await;
703 }
704
705 if finfeed {
706 self.attempts_remaining
707 .try_update(Ordering::Relaxed, Ordering::Relaxed, |remaining| {
708 remaining.checked_sub(1)
709 })
710 .map_err(|_| Error::AttemptBudget)?;
711 }
712
713 *next = Some(time::Instant::now() + self.interval);
714 drop(next);
715 let mut headers = HashMap::from([("Accept".into(), "application/json".into())]);
716 if finfeed {
717 headers.insert(
718 "Authorization".into(),
719 self.api_key.expose_secret().to_owned(),
720 );
721 headers.insert("X-Referrer".into(), "nautilus".into());
722 }
723
724 let response = self
725 .client
726 .request_with_params(
727 Method::GET,
728 url.to_string(),
729 Some(query),
730 Some(headers),
731 None,
732 None,
733 None,
734 )
735 .await?;
736
737 if !response.status.is_success() {
738 return Err(Error::Http {
739 status: response.status.as_u16(),
740 retry_after_secs: response
741 .headers
742 .get("retry-after")
743 .and_then(|value| value.parse().ok()),
744 });
745 }
746
747 serde_json::from_slice(&response.body).map_err(Error::from)
748 },
749 Error::is_retryable,
750 Error::Retry,
751 )
752 .retry_delay(&Error::retry_delay)
753 .cancellation_token(&self.cancellation)
754 .execute()
755 .await
756 }
757}
758
759fn endpoint_url(endpoint: &SecretString, api_key: &SecretString) -> Result<Url, Error> {
760 let mut base_url = Url::parse(endpoint.expose_secret())
761 .map_err(|_| Error::Validation("invalid base URL".into()))?;
762
763 let loopback = base_url.host_str().is_some_and(|host| {
764 host == "localhost"
765 || host
766 .trim_matches(['[', ']'])
767 .parse::<std::net::IpAddr>()
768 .is_ok_and(|ip| ip.is_loopback())
769 });
770
771 if !(base_url.scheme() == "https" || (base_url.scheme() == "http" && loopback))
772 || base_url.host_str().is_none()
773 || !base_url.username().is_empty()
774 || base_url.password().is_some()
775 || base_url.query().is_some()
776 || base_url.fragment().is_some()
777 || endpoint.expose_secret().contains(api_key.expose_secret())
778 {
779 return Err(Error::Validation(
780 "base URL must be HTTPS or loopback HTTP without credentials, query, or fragment"
781 .into(),
782 ));
783 }
784
785 if !base_url.path().ends_with('/') {
786 base_url.set_path(&format!("{}/", base_url.path()));
787 }
788
789 Ok(base_url)
790}
791
792fn deserialize_retry<'de, D>(deserializer: D) -> Result<RetryConfig, D::Error>
793where
794 D: Deserializer<'de>,
795{
796 let overrides = Map::<String, Value>::deserialize(deserializer)?;
797 let value =
798 serde_json::to_value(FinFeedHttpConfig::default().retry).map_err(SerdeError::custom)?;
799 let mut defaults = Map::deserialize(value).map_err(SerdeError::custom)?;
800 defaults.extend(overrides);
801 RetryConfig::deserialize(Value::Object(defaults)).map_err(SerdeError::custom)
802}
803
804fn validate_controls(config: &FinFeedHttpConfig) -> Result<(), Error> {
805 let invalid_control = [
806 (
807 "http_timeout_secs (http.timeout_secs)",
808 config.timeout_secs == 0 || config.timeout_secs > DURATION_MAX_MS / 1_000,
809 ),
810 (
811 "max_concurrent_requests",
812 config.max_concurrent_requests == 0
813 || config.max_concurrent_requests > tokio::sync::Semaphore::MAX_PERMITS,
814 ),
815 ("attempt_budget", config.attempt_budget == 0),
816 (
817 "min_request_interval_ms",
818 config.min_request_interval_ms == 0 || config.min_request_interval_ms > DURATION_MAX_MS,
819 ),
820 (
821 "retry_max_elapsed_ms (http.retry.max_elapsed_ms)",
822 config
823 .retry
824 .max_elapsed_ms
825 .is_none_or(|ms| ms == 0 || ms > DURATION_MAX_MS),
826 ),
827 (
828 "retry_operation_timeout_ms (http.retry.operation_timeout_ms)",
829 config
830 .retry
831 .operation_timeout_ms
832 .is_none_or(|ms| ms == 0 || ms > DURATION_MAX_MS),
833 ),
834 ("max_retries", config.retry.max_retries == u32::MAX),
835 (
836 "retry_delay_max_ms (http.retry.max_delay_ms)",
837 config.retry.max_delay_ms > DURATION_MAX_MS,
838 ),
839 ]
840 .into_iter()
841 .find_map(|(field, invalid)| invalid.then_some(field));
842
843 if let Some(field) = invalid_control {
844 return Err(Error::Validation(format!(
845 "request controls must be positive and bounded within one day: {field}"
846 )));
847 }
848
849 ExponentialBackoff::new(
850 Duration::from_millis(config.retry.initial_delay_ms),
851 Duration::from_millis(config.retry.max_delay_ms),
852 config.retry.backoff_factor,
853 config.retry.jitter_ms,
854 config.retry.immediate_first,
855 )
856 .map_err(|e| Error::Validation(format!("invalid retry backoff: {e}")))?;
857 Ok(())
858}
859
860fn check_page(records: usize, limit: u32) -> Result<(), Error> {
861 if records > limit as usize {
862 return Err(Error::Response("response exceeds requested record limit"));
863 }
864
865 Ok(())
866}