1use std::{
19 collections::{HashMap, HashSet},
20 sync::atomic::{AtomicU64, Ordering},
21 time::Duration,
22};
23
24use nautilus_core::string::secret::SecretString;
25use nautilus_network::{
26 ConnectionUrl,
27 backoff::ExponentialBackoff,
28 dst::time,
29 http::{HttpClient, HttpRedirectPolicy, Method},
30 retry::{RetryConfig, RetryManager},
31};
32use serde::{
33 Deserialize, Deserializer, Serialize,
34 de::{DeserializeOwned, Error as SerdeError},
35};
36use serde_json::{Map, Value};
37use tokio_util::sync::CancellationToken;
38use url::Url;
39use ustr::Ustr;
40
41use super::{
42 error::Error,
43 models::{
44 ActivityBatch, ActivityCurrent, Book, BookEvent, Exchange, Market, MarketsPage, Ohlcv,
45 OhlcvPeriod, Quote, Trade,
46 },
47 query::{
48 ActiveQuery, ActivityQuery, FlatFileOhlcvQuery, HistoryQuery, ListingsQuery, OhlcvQuery,
49 Pagination,
50 },
51};
52use crate::common::credential::resolve_api_key;
53
54const DURATION_MAX_MS: u64 = 86_400_000;
55
56pub const REST_URL: &str = "https://api.prediction-markets.finfeedapi.com/v1/";
58
59#[derive(Debug, Clone, Serialize, Deserialize, bon::Builder)]
73#[builder(on(SecretString, into))]
74#[serde(default, deny_unknown_fields)]
75pub struct FinFeedHttpConfig {
76 #[builder(default = FinFeedHttpConfig::default().base_url)]
79 pub base_url: SecretString,
80 pub proxy_url: Option<SecretString>,
82 #[builder(default = FinFeedHttpConfig::default().timeout_secs)]
84 pub timeout_secs: u64,
85 #[builder(default = FinFeedHttpConfig::default().max_concurrent_requests)]
87 pub max_concurrent_requests: usize,
88 #[builder(default = FinFeedHttpConfig::default().attempt_budget)]
90 pub attempt_budget: u64,
91 #[builder(default = FinFeedHttpConfig::default().min_request_interval_ms)]
93 pub min_request_interval_ms: u64,
94 #[serde(deserialize_with = "deserialize_retry")]
100 #[builder(default = FinFeedHttpConfig::default().retry)]
101 pub retry: RetryConfig,
102}
103
104impl Default for FinFeedHttpConfig {
105 fn default() -> Self {
106 Self {
107 base_url: REST_URL.into(),
108 proxy_url: None,
109 timeout_secs: 300,
110 max_concurrent_requests: 1,
111 attempt_budget: 100,
112 min_request_interval_ms: 10_000,
113 retry: RetryConfig {
114 jitter_ms: 0,
115 operation_timeout_ms: Some(330_000),
116 max_elapsed_ms: Some(600_000),
117 ..RetryConfig::default()
118 },
119 }
120 }
121}
122
123#[derive(Debug)]
129pub struct FinFeedHttpClient {
130 client: HttpClient,
131 base_url: ConnectionUrl,
132 api_key: SecretString,
133 concurrency: tokio::sync::Semaphore,
134 next_request: tokio::sync::Mutex<Option<time::Instant>>,
135 interval: Duration,
136 attempts_remaining: AtomicU64,
137 retry: RetryManager<Error>,
138 cancellation: CancellationToken,
139}
140
141impl FinFeedHttpClient {
142 pub fn new(api_key: Option<SecretString>, config: FinFeedHttpConfig) -> Result<Self, Error> {
149 let api_key = resolve_api_key(api_key)?;
150 let mut base_url = Url::parse(config.base_url.expose_secret())
151 .map_err(|_| Error::Validation("invalid base URL".into()))?;
152
153 let loopback = base_url.host_str().is_some_and(|host| {
154 host == "localhost"
155 || host
156 .trim_matches(['[', ']'])
157 .parse::<std::net::IpAddr>()
158 .is_ok_and(|ip| ip.is_loopback())
159 });
160
161 if !(base_url.scheme() == "https" || (base_url.scheme() == "http" && loopback))
162 || base_url.host_str().is_none()
163 || !base_url.username().is_empty()
164 || base_url.password().is_some()
165 || base_url.query().is_some()
166 || base_url.fragment().is_some()
167 || config
168 .base_url
169 .expose_secret()
170 .contains(api_key.expose_secret())
171 {
172 return Err(Error::Validation(
173 "base URL must be HTTPS or loopback HTTP without credentials, query, or fragment"
174 .into(),
175 ));
176 }
177
178 if !base_url.path().ends_with('/') {
179 base_url.set_path(&format!("{}/", base_url.path()));
180 }
181
182 if let Some(proxy) = &config.proxy_url {
183 let proxy = Url::parse(proxy.expose_secret())
184 .map_err(|_| Error::Validation("invalid proxy URL".into()))?;
185
186 if !matches!(proxy.scheme(), "http" | "https")
187 || proxy.host_str().is_none()
188 || proxy.query().is_some()
189 || proxy.fragment().is_some()
190 || proxy.path() != "/"
191 {
192 return Err(Error::Validation(
193 "proxy must be HTTP(S) without path, query, or fragment".into(),
194 ));
195 }
196 }
197
198 validate_controls(&config)?;
199 let client = HttpClient::builder()
200 .timeout_secs(config.timeout_secs)
201 .redact_url(true)
202 .redirect_policy(HttpRedirectPolicy::Reject)
203 .use_system_proxy(false)
204 .maybe_proxy_url(
205 config
206 .proxy_url
207 .as_ref()
208 .map(|url| url.expose_secret().to_owned()),
209 )
210 .header_keys(vec!["retry-after".into()])
211 .build()?;
212 Ok(Self {
213 client,
214 base_url: base_url.as_str().into(),
215 api_key,
216 concurrency: tokio::sync::Semaphore::new(config.max_concurrent_requests),
217 next_request: tokio::sync::Mutex::new(None),
218 interval: Duration::from_millis(config.min_request_interval_ms),
219 attempts_remaining: AtomicU64::new(config.attempt_budget),
220 retry: RetryManager::new(config.retry),
221 cancellation: CancellationToken::new(),
222 })
223 }
224
225 pub fn cancel(&self) {
229 self.cancellation.cancel();
230 }
231
232 #[must_use]
234 pub fn attempts_remaining(&self) -> u64 {
235 self.attempts_remaining.load(Ordering::Relaxed)
236 }
237
238 pub async fn get_exchanges(&self) -> Result<Vec<Exchange>, Error> {
246 self.get(&["exchanges"], &()).await
247 }
248
249 pub async fn get_exchange(&self, exchange_id: &str) -> Result<Exchange, Error> {
257 self.get(&["exchanges", exchange_id], &()).await
258 }
259
260 pub async fn get_listings(
269 &self,
270 exchange_id: &str,
271 query: &ListingsQuery,
272 ) -> Result<MarketsPage, Error> {
273 query.validate()?;
274 let page: MarketsPage = self
275 .get(&["markets", exchange_id, "history"], query)
276 .await?;
277 check_page(page.data.as_ref().map_or(0, Vec::len), query.limit)?;
278 Ok(page)
279 }
280
281 pub async fn get_active(
290 &self,
291 exchange_id: &str,
292 query: &ActiveQuery,
293 ) -> Result<Vec<Ustr>, Error> {
294 query.validate()?;
295 let ids: Vec<Ustr> = self.get(&["markets", exchange_id, "active"], query).await?;
296 check_page(ids.len(), query.limit)?;
297 Ok(ids)
298 }
299
300 pub async fn get_market(&self, exchange_id: &str, market_id: &str) -> Result<Market, Error> {
308 self.get(&["markets", exchange_id, market_id], &()).await
309 }
310
311 pub async fn get_current(
319 &self,
320 exchange_id: &str,
321 market_id: &str,
322 ) -> Result<ActivityCurrent, Error> {
323 self.get(&["activity", exchange_id, market_id, "current"], &())
324 .await
325 }
326
327 pub async fn get_latest(
336 &self,
337 exchange_id: &str,
338 market_id: &str,
339 query: &ActivityQuery,
340 ) -> Result<ActivityBatch, Error> {
341 query.validate()?;
342 let batch: ActivityBatch = self
343 .get(&["activity", exchange_id, market_id, "latest"], query)
344 .await?;
345 check_page(batch.trades.as_ref().map_or(0, Vec::len), query.limit)?;
346 check_page(batch.quotes.as_ref().map_or(0, Vec::len), query.limit)?;
347 Ok(batch)
348 }
349
350 pub async fn get_trades_history(
359 &self,
360 exchange_id: &str,
361 market_id: &str,
362 query: &HistoryQuery,
363 ) -> Result<Vec<Trade>, Error> {
364 query.validate_hourly()?;
365 let records: Vec<Trade> = self
366 .get(
367 &["activity", exchange_id, market_id, "history", "trades"],
368 query,
369 )
370 .await?;
371 check_page(records.len(), query.limit)?;
372 Ok(records)
373 }
374
375 pub async fn get_quotes_history(
384 &self,
385 exchange_id: &str,
386 market_id: &str,
387 query: &HistoryQuery,
388 ) -> Result<Vec<Quote>, Error> {
389 query.validate_hourly()?;
390 let records: Vec<Quote> = self
391 .get(
392 &["activity", exchange_id, market_id, "history", "quotes"],
393 query,
394 )
395 .await?;
396 check_page(records.len(), query.limit)?;
397 Ok(records)
398 }
399
400 pub async fn get_book(&self, exchange_id: &str, market_id: &str) -> Result<Book, Error> {
408 self.get(&["orderbook", exchange_id, market_id, "current"], &())
409 .await
410 }
411
412 pub async fn get_book_history(
425 &self,
426 exchange_id: &str,
427 market_id: &str,
428 query: &HistoryQuery,
429 ) -> Result<Vec<BookEvent>, Error> {
430 query.validate_hourly()?;
431 let records: Vec<BookEvent> = self
432 .get(&["orderbook", exchange_id, market_id, "history"], query)
433 .await?;
434 check_page(records.len(), query.limit)?;
435 Ok(records)
436 }
437
438 pub async fn get_periods(&self) -> Result<Vec<OhlcvPeriod>, Error> {
446 self.get(&["ohlcv", "periods"], &()).await
447 }
448
449 pub async fn get_ohlcv_history(
459 &self,
460 exchange_id: &str,
461 market_id: &str,
462 query: &OhlcvQuery,
463 ) -> Result<Vec<Ohlcv>, Error> {
464 query.validate(false)?;
465 let records: Vec<Ohlcv> = self
466 .get(&["ohlcv", exchange_id, market_id, "history"], query)
467 .await?;
468 check_page(records.len(), query.limit)?;
469 Ok(records)
470 }
471
472 pub async fn get_ohlcv_flat_file_history(
486 &self,
487 exchange_id: &str,
488 market_id: &str,
489 query: &FlatFileOhlcvQuery,
490 ) -> Result<Vec<Ohlcv>, Error> {
491 query.validate()?;
492 let records: Vec<Ohlcv> = self
493 .get(
494 &["activity", exchange_id, market_id, "history", "ohlcv"],
495 query,
496 )
497 .await?;
498 check_page(records.len(), query.history.limit)?;
499 Ok(records)
500 }
501
502 pub async fn get_ohlcv_exchange_history(
516 &self,
517 exchange_id: &str,
518 query: &OhlcvQuery,
519 ) -> Result<Vec<Ohlcv>, Error> {
520 query.validate_exchange()?;
521 let records: Vec<Ohlcv> = self.get(&["ohlcv", exchange_id, "history"], query).await?;
522 check_page(records.len(), query.limit)?;
523 Ok(records)
524 }
525
526 pub async fn get_ohlcv_latest(
535 &self,
536 exchange_id: &str,
537 market_id: &str,
538 query: &OhlcvQuery,
539 ) -> Result<Vec<Ohlcv>, Error> {
540 query.validate(true)?;
541 let records: Vec<Ohlcv> = self
542 .get(&["ohlcv", exchange_id, market_id, "latest"], query)
543 .await?;
544 check_page(records.len(), query.limit)?;
545 Ok(records)
546 }
547
548 pub async fn get_listings_all(
556 &self,
557 exchange_id: &str,
558 query: &ListingsQuery,
559 bounds: Pagination,
560 ) -> Result<Vec<Market>, Error> {
561 query.validate()?;
562 bounds.validate()?;
563 let mut query = query.clone();
564 let mut cursors = HashSet::new();
565 if let Some(cursor) = &query.cursor {
566 cursors.insert(cursor.clone());
567 }
568
569 let mut records = Vec::new();
570
571 for _ in 0..bounds.max_pages {
572 let page = self.get_listings(exchange_id, &query).await?;
573 let data = page.data.unwrap_or_default();
574 if data.len() > bounds.max_records.saturating_sub(records.len()) {
575 return Err(Error::Response("listing record bound exceeded"));
576 }
577
578 records.extend(data);
579 match page.next_cursor {
580 None => return Ok(records),
581 Some(cursor) if cursor.is_empty() || !cursors.insert(cursor.clone()) => {
582 return Err(Error::Response("listing cursor is empty or repeated"));
583 }
584 Some(cursor) => query.cursor = Some(cursor),
585 }
586 }
587
588 Err(Error::Response("listing page bound exceeded"))
589 }
590
591 pub async fn get_active_all(
601 &self,
602 exchange_id: &str,
603 query: &ActiveQuery,
604 bounds: Pagination,
605 ) -> Result<Vec<Ustr>, Error> {
606 query.validate()?;
607 bounds.validate()?;
608 let mut query = query.clone();
609 let mut seen = HashSet::new();
610 let mut ids = Vec::new();
611
612 for _ in 0..bounds.max_pages {
613 let page = self.get_active(exchange_id, &query).await?;
614 if page.is_empty() {
615 return Ok(ids);
616 }
617
618 if page.len() > bounds.max_records.saturating_sub(ids.len()) {
619 return Err(Error::Response("active ID record bound exceeded"));
620 }
621
622 for id in page {
623 if id.is_empty() || !seen.insert(id) {
624 return Err(Error::Response("active ID is empty or repeated"));
625 }
626
627 ids.push(id);
628 }
629
630 query.page = query
631 .page
632 .checked_add(1)
633 .filter(|page| i32::try_from(*page).is_ok())
634 .ok_or(Error::Response("active page overflow"))?;
635 }
636
637 Err(Error::Response("active page bound exceeded"))
638 }
639
640 async fn get<T: DeserializeOwned, Q: Serialize>(
641 &self,
642 path: &[&str],
643 query: &Q,
644 ) -> Result<T, Error> {
645 if path
646 .iter()
647 .any(|part| part.trim().is_empty() || matches!(*part, "." | ".."))
648 {
649 return Err(Error::Validation(
650 "path identifiers must not be empty or dot segments".into(),
651 ));
652 }
653
654 let mut url = Url::parse(self.base_url.expose())
655 .map_err(|_| Error::Validation("invalid base URL".into()))?;
656 url.path_segments_mut()
657 .map_err(|()| Error::Validation("invalid base URL path".into()))?
658 .pop_if_empty()
659 .extend(path);
660 self.retry
661 .invocation(
662 "FinFeed GET",
663 || async {
664 let _permit = self
665 .concurrency
666 .acquire()
667 .await
668 .map_err(|_| Error::Response("admission closed"))?;
669 let mut next = self.next_request.lock().await;
670 if let Some(deadline) = *next {
671 time::sleep_until(deadline).await;
672 }
673
674 self.attempts_remaining
675 .try_update(Ordering::Relaxed, Ordering::Relaxed, |remaining| {
676 remaining.checked_sub(1)
677 })
678 .map_err(|_| Error::AttemptBudget)?;
679
680 *next = Some(time::Instant::now() + self.interval);
681 drop(next);
682 let headers = HashMap::from([
683 (
684 "Authorization".into(),
685 self.api_key.expose_secret().to_owned(),
686 ),
687 ("X-Referrer".into(), "nautilus".into()),
688 ("Accept".into(), "application/json".into()),
689 ]);
690 let response = self
691 .client
692 .request_with_params(
693 Method::GET,
694 url.to_string(),
695 Some(query),
696 Some(headers),
697 None,
698 None,
699 None,
700 )
701 .await?;
702
703 if !response.status.is_success() {
704 return Err(Error::Http {
705 status: response.status.as_u16(),
706 retry_after_secs: response
707 .headers
708 .get("retry-after")
709 .and_then(|value| value.parse().ok()),
710 });
711 }
712
713 serde_json::from_slice(&response.body).map_err(Error::from)
714 },
715 Error::is_retryable,
716 Error::Retry,
717 )
718 .retry_delay(&Error::retry_delay)
719 .cancellation_token(&self.cancellation)
720 .execute()
721 .await
722 }
723}
724
725fn deserialize_retry<'de, D>(deserializer: D) -> Result<RetryConfig, D::Error>
726where
727 D: Deserializer<'de>,
728{
729 let overrides = Map::<String, Value>::deserialize(deserializer)?;
730 let value =
731 serde_json::to_value(FinFeedHttpConfig::default().retry).map_err(SerdeError::custom)?;
732 let mut defaults = Map::deserialize(value).map_err(SerdeError::custom)?;
733 defaults.extend(overrides);
734 RetryConfig::deserialize(Value::Object(defaults)).map_err(SerdeError::custom)
735}
736
737fn validate_controls(config: &FinFeedHttpConfig) -> Result<(), Error> {
738 let invalid_control = [
739 (
740 "http_timeout_secs (http.timeout_secs)",
741 config.timeout_secs == 0 || config.timeout_secs > DURATION_MAX_MS / 1_000,
742 ),
743 (
744 "max_concurrent_requests",
745 config.max_concurrent_requests == 0
746 || config.max_concurrent_requests > tokio::sync::Semaphore::MAX_PERMITS,
747 ),
748 ("attempt_budget", config.attempt_budget == 0),
749 (
750 "min_request_interval_ms",
751 config.min_request_interval_ms == 0 || config.min_request_interval_ms > DURATION_MAX_MS,
752 ),
753 (
754 "retry_max_elapsed_ms (http.retry.max_elapsed_ms)",
755 config
756 .retry
757 .max_elapsed_ms
758 .is_none_or(|ms| ms == 0 || ms > DURATION_MAX_MS),
759 ),
760 (
761 "retry_operation_timeout_ms (http.retry.operation_timeout_ms)",
762 config
763 .retry
764 .operation_timeout_ms
765 .is_none_or(|ms| ms == 0 || ms > DURATION_MAX_MS),
766 ),
767 ("max_retries", config.retry.max_retries == u32::MAX),
768 (
769 "retry_delay_max_ms (http.retry.max_delay_ms)",
770 config.retry.max_delay_ms > DURATION_MAX_MS,
771 ),
772 ]
773 .into_iter()
774 .find_map(|(field, invalid)| invalid.then_some(field));
775
776 if let Some(field) = invalid_control {
777 return Err(Error::Validation(format!(
778 "request controls must be positive and bounded within one day: {field}"
779 )));
780 }
781
782 ExponentialBackoff::new(
783 Duration::from_millis(config.retry.initial_delay_ms),
784 Duration::from_millis(config.retry.max_delay_ms),
785 config.retry.backoff_factor,
786 config.retry.jitter_ms,
787 config.retry.immediate_first,
788 )
789 .map_err(|e| Error::Validation(format!("invalid retry backoff: {e}")))?;
790 Ok(())
791}
792
793fn check_page(records: usize, limit: u32) -> Result<(), Error> {
794 if records > limit as usize {
795 return Err(Error::Response("response exceeds requested record limit"));
796 }
797
798 Ok(())
799}