Skip to main content

nautilus_finfeed/
data.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Data client with generation-owned requests, polling subscriptions, and bounded shutdown.
17
18mod poll;
19
20use std::{collections::HashMap, future::Future, num::NonZeroUsize, sync::Arc, time::Duration};
21
22use anyhow::Context;
23use async_trait::async_trait;
24use jiff::Timestamp;
25use nautilus_common::{
26    clients::{DataClient, execute_bar_subscriptions, execute_instrument_subscriptions},
27    live::{dst::time, sender::EventSender},
28    messages::{DataEvent, DataResponse, data::*},
29};
30use nautilus_core::{
31    Params,
32    datetime::datetime_to_unix_nanos,
33    string::secret::SecretString,
34    time::{AtomicTime, get_atomic_clock_realtime},
35};
36use nautilus_live::task::{TaskGroup, TaskGroupGuard};
37use nautilus_model::{
38    data::depth::DEPTH10_LEN,
39    enums::BookType,
40    identifiers::{ClientId, InstrumentId, Venue},
41    instruments::{BinaryOption, InstrumentAny},
42};
43use parking_lot::Mutex;
44use serde::{Deserialize, de::IntoDeserializer};
45
46use crate::{
47    book,
48    common::{consts::FINFEED, credential::resolve_api_key},
49    config::FinFeedDataClientConfig,
50    history,
51    http::{client::FinFeedHttpClient, error::Error, models::MarketStatus, query::PAGE_LIMIT_MAX},
52    instruments::{FinFeedInstrumentProvider, selected_ids},
53    parse,
54};
55
56const TASK_GRACE: Duration = Duration::from_secs(1);
57const TASK_FORCE: Duration = Duration::from_secs(2);
58
59/// Prediction-market data client for requests and book, quote, trade, and bar polling subscriptions.
60/// Execution is unsupported. The injected sender supports standalone clients and runtime dispatch.
61///
62/// # Connection Lifecycle
63///
64/// Startup publishes every selected definition before readiness. Disconnect closes task admission
65/// and response publication before draining work. Reconnect uses fresh transport and instrument
66/// state after the previous generation drains.
67///
68/// # Polling Subscriptions
69///
70/// - Quotes and trades share sampled latest-activity polls.
71/// - L2 MBP deltas and depth share validated current-book snapshots.
72/// - Bars publish periods closing after subscribe time, with a cutoff taken before each request.
73///   Their publication watermark survives reconnects. Provider OHLCV maps to `LAST` for compatibility;
74///   trade-only price provenance is unverified.
75///
76/// Polling guarantees neither completeness nor backfill.
77///
78/// # Market Status and Definition Refreshes
79///
80/// Fresh definition requests apply provider lifecycle metadata and publish instrument updates and
81/// correlated responses.
82///
83/// - `Closed` and `Resolved` end all polling subscriptions and reject new ones for that outcome.
84/// - `Open` and `Suspended` keep polling.
85///
86/// If any selected definition fails to refresh, list refreshes preserve loaded definitions and
87/// outcome ownership. Historical requests remain available for loaded terminal definitions.
88/// The client infers no `InstrumentStatus` or `InstrumentClose` event.
89///
90/// # Historical Requests
91///
92/// Historical series require a start and share the configured attempt budget.
93///
94/// - Trades and quotes retrieve covering UTC hours with checked overlapping continuation, then filter
95///   and order records by exposed timestamps. Completeness under provider processing-time selection
96///   remains unverified.
97/// - Quotes use entry time for `ts_event` and receive time for `ts_init`.
98/// - Trades use their sole timestamp for both fields and infer no aggressor side.
99/// - Bars continue from validated period ends, including short nonempty pages, and emit only complete
100///   external OHLCV periods contained in the requested window.
101///
102/// # Historical Books
103///
104/// Historical L2 books use hour-boundary selectors with checked overlapping continuation and replay
105/// returned records from a snapshot seed assumed to be complete. Source coverage, bounded timestamp
106/// delay, and continuity remain assumptions.
107///
108/// - Numeric actions `0`, `1`, and `3` mean relative ADD, absolute SET, and replacement snapshot.
109/// - Source order and exchange timestamps are preserved. Each completed group's derived `ts_init`
110///   is the cumulative maximum of processing clocks when present and exchange clocks otherwise.
111///   Seed placement and window bounds use this ordering clock, not measured receipt time.
112/// - Series begin with a generated replacement at start. Point snapshots select an `end` timestamp
113///   in request params.
114/// - Changed or noncontiguous overlap, nonprogress, exhausted allowance, missing seeds, rewound lower
115///   selectors at or past their hourly upper bound, and unsupported actions fail without
116///   partial delivery.
117///
118/// # Validation and Failures
119///
120/// Current books are validated atomically. Prices and raw sizes must fit the loaded definition
121/// exactly; OHLCV volumes allow bounded normalization of numeric tails at size precision.
122/// Asynchronous failures log an error and publish no response. Synchronous validation failures
123/// return an error.
124#[derive(Debug)]
125pub struct FinFeedDataClient {
126    clock: &'static AtomicTime,
127    client_id: ClientId,
128    config: FinFeedDataClientConfig,
129    api_key: SecretString,
130    http: Arc<FinFeedHttpClient>,
131    provider: Arc<tokio::sync::Mutex<FinFeedInstrumentProvider>>,
132    session_tasks: TaskGroup,
133    command_tasks: TaskGroup,
134    publication: Arc<Mutex<Publication>>,
135    sender: EventSender<DataEvent>,
136    poll_changed: Arc<tokio::sync::Notify>,
137}
138
139impl FinFeedDataClient {
140    /// Constructs a disconnected client with explicit event plumbing.
141    ///
142    /// The bootstrap deadline must be in `[1, 86400]` seconds. Poll spacing must be in
143    /// `[100, 86400000]` milliseconds and the per-list poll limit in `[1, 10000]`.
144    ///
145    /// # Errors
146    ///
147    /// Returns an error if:
148    /// - Selections, credentials, REST controls, the bootstrap deadline, or polling bounds are invalid.
149    /// - Neither an explicit key nor `FINFEED_API_KEY` is available.
150    pub fn new(
151        client_id: ClientId,
152        config: FinFeedDataClientConfig,
153        sender: EventSender<DataEvent>,
154    ) -> anyhow::Result<Self> {
155        selected_ids(&config.instruments)?;
156        anyhow::ensure!(
157            (1..=86_400).contains(&config.connect_timeout_secs),
158            "invalid FinFeed connect deadline: connect_timeout_secs must be in [1, 86400], was {}",
159            config.connect_timeout_secs
160        );
161        anyhow::ensure!(
162            (100..=86_400_000).contains(&config.poll_interval_ms),
163            "invalid FinFeed polling controls: poll_interval_ms must be in [100, 86400000], was {}",
164            config.poll_interval_ms
165        );
166        anyhow::ensure!(
167            (1..=PAGE_LIMIT_MAX).contains(&config.poll_limit),
168            "invalid FinFeed polling controls: poll_limit must be in [1, {PAGE_LIMIT_MAX}], was {}",
169            config.poll_limit
170        );
171        let api_key = resolve_api_key(config.api_key.clone())?;
172
173        let http = Arc::new(FinFeedHttpClient::new(
174            Some(api_key.clone()),
175            config.http.clone(),
176        )?);
177        let provider = FinFeedInstrumentProvider::new(Arc::clone(&http));
178        Ok(Self {
179            clock: get_atomic_clock_realtime(),
180            client_id,
181            config,
182            api_key,
183            http,
184            provider: Arc::new(tokio::sync::Mutex::new(provider)),
185            session_tasks: TaskGroup::new(),
186            command_tasks: TaskGroup::new(),
187            publication: Arc::new(Mutex::new(Publication::default())),
188            sender,
189            poll_changed: Arc::new(tokio::sync::Notify::new()),
190        })
191    }
192}
193
194#[async_trait(?Send)]
195impl DataClient for FinFeedDataClient {
196    fn client_id(&self) -> ClientId {
197        self.client_id
198    }
199
200    fn venue(&self) -> Option<Venue> {
201        None
202    }
203
204    fn start(&mut self) -> anyhow::Result<()> {
205        Ok(())
206    }
207
208    fn stop(&mut self) -> anyhow::Result<()> {
209        self.begin_shutdown();
210        self.session_tasks.abort();
211        self.command_tasks.abort();
212        Ok(())
213    }
214
215    fn reset(&mut self) -> anyhow::Result<()> {
216        self.stop()?;
217        let mut state = self.publication.lock();
218        state.instruments.clear();
219        state.polling = poll::Polling::default();
220        Ok(())
221    }
222
223    fn dispose(&mut self) -> anyhow::Result<()> {
224        self.reset()
225    }
226
227    fn is_connected(&self) -> bool {
228        self.publication.lock().connected
229    }
230
231    fn is_disconnected(&self) -> bool {
232        !self.is_connected()
233    }
234
235    async fn connect(&mut self) -> anyhow::Result<()> {
236        if self.is_connected() {
237            return Ok(());
238        }
239
240        if !self.session_tasks.is_open() || !self.command_tasks.is_open() {
241            self.begin_shutdown();
242            self.finish_shutdown().await?;
243
244            self.http = Arc::new(FinFeedHttpClient::new(
245                Some(self.api_key.clone()),
246                self.config.http.clone(),
247            )?);
248            self.session_tasks.start_generation()?;
249            self.command_tasks.start_generation()?;
250        }
251
252        let http = Arc::clone(&self.http);
253        let publication = Arc::clone(&self.publication);
254        let guard = TaskGroupGuard::new(&[&self.session_tasks, &self.command_tasks], move || {
255            publication.lock().connected = false;
256            http.cancel();
257        });
258
259        let mut provider = FinFeedInstrumentProvider::new(Arc::clone(&self.http));
260        let intended = self.publication.lock().polling.instrument_ids();
261
262        let loaded = time::timeout(
263            Duration::from_secs(self.config.connect_timeout_secs),
264            Box::pin(async {
265                let mut loaded = provider
266                    .load(&self.config.instruments, self.clock.get_time_ns())
267                    .await?;
268
269                for id in intended {
270                    if !loaded.iter().any(|instrument| instrument.id == id) {
271                        loaded.push(
272                            provider
273                                .request_instrument(id, self.clock.get_time_ns())
274                                .await?,
275                        );
276                    }
277                }
278
279                Ok::<_, crate::http::error::Error>(loaded)
280            }),
281        )
282        .await
283        .context("FinFeed bootstrap timed out")?;
284
285        if let Err(Error::Http { status, .. }) = &loaded {
286            log::error!("FinFeed bootstrap failed with HTTP status {status}");
287        }
288
289        let loaded = loaded?;
290        {
291            let mut state = self.publication.lock();
292
293            for instrument in &loaded {
294                let instrument = InstrumentAny::from(instrument.clone());
295                send(&self.sender, DataEvent::Data(instrument.into()))?;
296            }
297
298            state.instruments.clear();
299            for instrument in loaded {
300                state.update(instrument);
301            }
302
303            self.provider = Arc::new(tokio::sync::Mutex::new(provider));
304            state.connected = true;
305            state.polling.restore();
306        }
307
308        let publication = Arc::clone(&self.publication);
309        let sender = self.sender.clone();
310        let http = Arc::clone(&self.http);
311        let changed = Arc::clone(&self.poll_changed);
312        let cancellation = self.session_tasks.cancellation_token();
313        let interval = Duration::from_millis(self.config.poll_interval_ms);
314        let limit = self.config.poll_limit;
315
316        self.session_tasks.spawn(async move {
317            poll::run(
318                publication,
319                sender,
320                http,
321                changed,
322                cancellation,
323                interval,
324                limit,
325            )
326            .await;
327        })?;
328
329        guard.disarm();
330
331        log::info!("Connected {}", self.client_id);
332        Ok(())
333    }
334
335    async fn disconnect(&mut self) -> anyhow::Result<()> {
336        self.begin_shutdown();
337        self.finish_shutdown().await?;
338        self.publication.lock().instruments.clear();
339
340        log::info!("Disconnected {}", self.client_id);
341        Ok(())
342    }
343
344    fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
345        let instrument_id = request.single_instrument_id()?;
346        crate::common::symbol::from_instrument_id(instrument_id)?;
347        let provider = Arc::clone(&self.provider);
348        let state = Arc::clone(&self.publication);
349        let sender = self.sender.clone();
350        let clock = self.clock;
351        let changed = Arc::clone(&self.poll_changed);
352        let client_id = request.client_id.unwrap_or(self.client_id);
353        self.spawn_request("FinFeed instrument request", async move {
354            let mut provider = provider.lock().await;
355            let instrument = provider
356                .request_instrument(instrument_id, clock.get_time_ns())
357                .await?;
358            let start = datetime_to_unix_nanos(request.start);
359            let end = datetime_to_unix_nanos(request.end);
360            let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
361                request.request_id,
362                client_id,
363                instrument_id,
364                instrument.clone().into(),
365                start,
366                end,
367                clock.get_time_ns(),
368                request.params,
369            )));
370            publish(&state, &sender, response, vec![instrument])?;
371            changed.notify_one();
372            Ok(())
373        })
374    }
375
376    fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
377        let ids: Vec<_> = selected_ids(&self.config.instruments)?
378            .into_iter()
379            .filter(|id| request.venue.is_none_or(|venue| id.venue == venue))
380            .collect();
381        let provider = Arc::clone(&self.provider);
382        let state = Arc::clone(&self.publication);
383        let sender = self.sender.clone();
384        let clock = self.clock;
385        let client_id = request.client_id.unwrap_or(self.client_id);
386        let venue = request.venue.unwrap_or_else(|| Venue::from(FINFEED));
387        let changed = Arc::clone(&self.poll_changed);
388        self.spawn_request("FinFeed instruments request", async move {
389            let mut provider = provider.lock().await;
390            let mut refreshed = provider.clone();
391            let mut loaded = Vec::with_capacity(ids.len());
392            for id in ids {
393                loaded.push(
394                    refreshed
395                        .request_instrument(id, clock.get_time_ns())
396                        .await?,
397                );
398            }
399
400            let data = loaded.iter().cloned().map(Into::into).collect();
401            let start = datetime_to_unix_nanos(request.start);
402            let end = datetime_to_unix_nanos(request.end);
403            let response = DataResponse::Instruments(InstrumentsResponse::new(
404                request.request_id,
405                client_id,
406                venue,
407                data,
408                start,
409                end,
410                clock.get_time_ns(),
411                request.params,
412            ));
413            publish(&state, &sender, response, loaded)?;
414            *provider = refreshed;
415            changed.notify_one();
416            Ok(())
417        })
418    }
419
420    fn request_book_snapshot(&self, request: RequestBookSnapshot) -> anyhow::Result<()> {
421        let instrument_id = request.single_instrument_id()?;
422        let time = book_time(request.params.as_ref())?;
423        let window = time
424            .map(|time| history::Window::new(Some(time), Some(time), None, time))
425            .transpose()?;
426        let instrument = self.loaded(instrument_id)?;
427        let (exchange, id) = crate::common::symbol::from_instrument_id(instrument.id)?;
428        let http = Arc::clone(&self.http);
429        let state = Arc::clone(&self.publication);
430        let sender = self.sender.clone();
431        let clock = self.clock;
432        let client_id = request.client_id.unwrap_or(self.client_id);
433        self.spawn_request("FinFeed book request", async move {
434            let book = if let Some(window) = window {
435                let batches = history::books(&http, &instrument, &window).await?;
436                book::snapshot(&batches, &instrument, request.depth.map(NonZeroUsize::get))?
437            } else {
438                let source = http.get_book(&exchange, &id).await?;
439                parse::book(&source, &instrument, request.depth.map(NonZeroUsize::get))?
440            };
441
442            let bound = datetime_to_unix_nanos(time);
443            let response = DataResponse::Book(BookResponse::new(
444                request.request_id,
445                client_id,
446                instrument_id,
447                book,
448                bound,
449                bound,
450                clock.get_time_ns(),
451                request.params,
452            ));
453            publish(&state, &sender, response, Vec::new())
454        })
455    }
456
457    fn request_quotes(&self, request: RequestQuotes) -> anyhow::Result<()> {
458        let instrument_id = request.single_instrument_id()?;
459        let instrument = self.loaded(instrument_id)?;
460
461        let window = history::Window::new(
462            request.start,
463            request.end,
464            request.limit,
465            self.clock.get_time_ns().to_datetime_utc(),
466        )?;
467        let http = Arc::clone(&self.http);
468        let state = Arc::clone(&self.publication);
469        let sender = self.sender.clone();
470        let clock = self.clock;
471        let client_id = request.client_id.unwrap_or(self.client_id);
472        self.spawn_request("FinFeed quotes request", async move {
473            let quotes = history::quotes(&http, &instrument, &window).await?;
474            let start = datetime_to_unix_nanos(request.start);
475            let end = datetime_to_unix_nanos(request.end);
476            let response = DataResponse::Quotes(QuotesResponse::new(
477                request.request_id,
478                client_id,
479                instrument_id,
480                quotes,
481                start,
482                end,
483                clock.get_time_ns(),
484                request.params,
485            ));
486            publish(&state, &sender, response, Vec::new())
487        })
488    }
489
490    fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
491        let instrument_id = request.single_instrument_id()?;
492        let instrument = self.loaded(instrument_id)?;
493
494        let window = history::Window::new(
495            request.start,
496            request.end,
497            request.limit,
498            self.clock.get_time_ns().to_datetime_utc(),
499        )?;
500        let http = Arc::clone(&self.http);
501        let state = Arc::clone(&self.publication);
502        let sender = self.sender.clone();
503        let clock = self.clock;
504        let client_id = request.client_id.unwrap_or(self.client_id);
505        self.spawn_request("FinFeed trades request", async move {
506            let trades = history::trades(&http, &instrument, &window).await?;
507            let start = datetime_to_unix_nanos(request.start);
508            let end = datetime_to_unix_nanos(request.end);
509            let response = DataResponse::Trades(TradesResponse::new(
510                request.request_id,
511                client_id,
512                instrument_id,
513                trades,
514                start,
515                end,
516                clock.get_time_ns(),
517                request.params,
518            ));
519            publish(&state, &sender, response, Vec::new())
520        })
521    }
522
523    fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
524        let bar_type = request.single_bar_type()?;
525        let period_length = history::period_length(bar_type)?;
526        let instrument = self.loaded(bar_type.instrument_id())?;
527
528        let window = history::Window::new(
529            request.start,
530            request.end,
531            request.limit,
532            self.clock.get_time_ns().to_datetime_utc(),
533        )?;
534        let http = Arc::clone(&self.http);
535        let state = Arc::clone(&self.publication);
536        let sender = self.sender.clone();
537        let clock = self.clock;
538        let client_id = request.client_id.unwrap_or(self.client_id);
539        self.spawn_request("FinFeed bars request", async move {
540            let bars = history::bars(
541                &http,
542                &instrument,
543                bar_type,
544                &window,
545                clock.get_time_ns().to_datetime_utc(),
546                period_length,
547            )
548            .await?;
549            let start = datetime_to_unix_nanos(request.start);
550            let end = datetime_to_unix_nanos(request.end);
551            let response = DataResponse::Bars(BarsResponse::new(
552                request.request_id,
553                client_id,
554                bar_type,
555                bars,
556                start,
557                end,
558                clock.get_time_ns(),
559                request.params,
560            ));
561            publish(&state, &sender, response, Vec::new())
562        })
563    }
564
565    fn subscribe_bars(&mut self, cmd: SubscribeBars) -> anyhow::Result<()> {
566        execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
567            history::period_length(bar_type)?;
568            let mut state = self.publication.lock();
569            state.check_subscription(bar_type.instrument_id())?;
570
571            if state
572                .polling
573                .subscribe_bars(bar_type, self.clock.get_time_ns().to_datetime_utc())?
574            {
575                self.poll_changed.notify_one();
576            }
577
578            Ok(())
579        })
580    }
581
582    fn unsubscribe_bars(&mut self, cmd: &UnsubscribeBars) -> anyhow::Result<()> {
583        execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
584            if self.publication.lock().polling.unsubscribe_bars(bar_type) {
585                self.poll_changed.notify_one();
586            }
587
588            Ok(())
589        })
590    }
591
592    fn subscribe_book_deltas(&mut self, cmd: SubscribeBookDeltas) -> anyhow::Result<()> {
593        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
594            self.subscribe_book(
595                instrument_id,
596                cmd.book_type,
597                true,
598                cmd.depth.map(NonZeroUsize::get),
599            )
600        })
601    }
602
603    fn subscribe_book_depth(&mut self, cmd: SubscribeBookDepth) -> anyhow::Result<()> {
604        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
605            let depth = cmd.depth.map_or(DEPTH10_LEN, NonZeroUsize::get);
606            self.subscribe_book(instrument_id, cmd.book_type, false, Some(depth))
607        })
608    }
609
610    fn unsubscribe_book_deltas(&mut self, cmd: &UnsubscribeBookDeltas) -> anyhow::Result<()> {
611        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
612            if self
613                .publication
614                .lock()
615                .polling
616                .unsubscribe_book(instrument_id, true)
617            {
618                self.poll_changed.notify_one();
619            }
620
621            Ok(())
622        })
623    }
624
625    fn unsubscribe_book_depth(&mut self, cmd: &UnsubscribeBookDepth) -> anyhow::Result<()> {
626        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
627            if self
628                .publication
629                .lock()
630                .polling
631                .unsubscribe_book(instrument_id, false)
632            {
633                self.poll_changed.notify_one();
634            }
635
636            Ok(())
637        })
638    }
639
640    fn subscribe_quotes(&mut self, cmd: SubscribeQuotes) -> anyhow::Result<()> {
641        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
642            self.subscribe_activity(instrument_id, true)
643        })
644    }
645
646    fn subscribe_trades(&mut self, cmd: SubscribeTrades) -> anyhow::Result<()> {
647        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
648            self.subscribe_activity(instrument_id, false)
649        })
650    }
651
652    fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<()> {
653        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
654            if self
655                .publication
656                .lock()
657                .polling
658                .unsubscribe(instrument_id, true)
659            {
660                self.poll_changed.notify_one();
661            }
662
663            Ok(())
664        })
665    }
666
667    fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> anyhow::Result<()> {
668        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
669            if self
670                .publication
671                .lock()
672                .polling
673                .unsubscribe(instrument_id, false)
674            {
675                self.poll_changed.notify_one();
676            }
677
678            Ok(())
679        })
680    }
681
682    fn request_data(&self, _request: RequestCustomData) -> anyhow::Result<()> {
683        anyhow::bail!("FinFeed custom data requests are unsupported")
684    }
685
686    fn request_book_depth(&self, request: RequestBookDepth) -> anyhow::Result<()> {
687        let instrument_id = request.single_instrument_id()?;
688        book_params(request.params.as_ref())?;
689        let depth = request.depth.map_or(DEPTH10_LEN, NonZeroUsize::get);
690        anyhow::ensure!(
691            depth <= DEPTH10_LEN,
692            "FinFeed book depth exceeds {DEPTH10_LEN}"
693        );
694        let instrument = self.loaded(instrument_id)?;
695
696        let window = history::Window::new(
697            request.start,
698            request.end,
699            request.limit,
700            self.clock.get_time_ns().to_datetime_utc(),
701        )?;
702        let http = Arc::clone(&self.http);
703        let state = Arc::clone(&self.publication);
704        let sender = self.sender.clone();
705        let clock = self.clock;
706        let client_id = request.client_id.unwrap_or(self.client_id);
707        self.spawn_request("FinFeed book depth request", async move {
708            let batches = history::books(&http, &instrument, &window).await?;
709            let depths = book::depths(&batches, &instrument, depth, request.limit)?;
710            let start = datetime_to_unix_nanos(request.start);
711            let end = datetime_to_unix_nanos(request.end);
712            let response = DataResponse::BookDepth(BookDepthResponse::new(
713                request.request_id,
714                client_id,
715                instrument_id,
716                depths,
717                start,
718                end,
719                clock.get_time_ns(),
720                request.params,
721            ));
722            publish(&state, &sender, response, Vec::new())
723        })
724    }
725
726    fn request_book_deltas(&self, request: RequestBookDeltas) -> anyhow::Result<()> {
727        let instrument_id = request.single_instrument_id()?;
728        book_params(request.params.as_ref())?;
729        let instrument = self.loaded(instrument_id)?;
730
731        let window = history::Window::new(
732            request.start,
733            request.end,
734            request.limit,
735            self.clock.get_time_ns().to_datetime_utc(),
736        )?;
737        let http = Arc::clone(&self.http);
738        let state = Arc::clone(&self.publication);
739        let sender = self.sender.clone();
740        let clock = self.clock;
741        let client_id = request.client_id.unwrap_or(self.client_id);
742        self.spawn_request("FinFeed book deltas request", async move {
743            let batches = history::books(&http, &instrument, &window).await?;
744            let deltas = book::deltas(batches, request.limit)?;
745            let start = datetime_to_unix_nanos(request.start);
746            let end = datetime_to_unix_nanos(request.end);
747            let response = DataResponse::BookDeltas(BookDeltasResponse::new(
748                request.request_id,
749                client_id,
750                instrument_id,
751                deltas,
752                start,
753                end,
754                clock.get_time_ns(),
755                request.params,
756            ));
757            publish(&state, &sender, response, Vec::new())
758        })
759    }
760
761    fn request_funding_rates(&self, _request: RequestFundingRates) -> anyhow::Result<()> {
762        anyhow::bail!("FinFeed funding rates are unsupported")
763    }
764
765    fn request_option_chain_reference_price(
766        &self,
767        _request: RequestOptionChainReferencePrice,
768    ) -> anyhow::Result<()> {
769        anyhow::bail!("FinFeed option chain requests are unsupported")
770    }
771}
772
773impl FinFeedDataClient {
774    fn loaded(&self, id: InstrumentId) -> anyhow::Result<BinaryOption> {
775        self.publication.lock().loaded(id).cloned()
776    }
777
778    fn spawn_request<F>(&self, description: &'static str, future: F) -> anyhow::Result<()>
779    where
780        F: Future<Output = anyhow::Result<()>> + Send + 'static,
781    {
782        let state = self.publication.lock();
783        anyhow::ensure!(state.connected, "FinFeed client is disconnected");
784        let cancellation = self.command_tasks.cancellation_token();
785
786        self.command_tasks.spawn(async move {
787            tokio::select! {
788                biased;
789                () = cancellation.cancelled() => {}
790                result = future => {
791                    if let Err(e) = result {
792                        log::error!("{description} failed: {e:#}");
793                    }
794                }
795            }
796        })?;
797
798        Ok(())
799    }
800
801    fn begin_shutdown(&self) {
802        let mut state = self.publication.lock();
803        state.connected = false;
804        self.session_tasks.begin_shutdown();
805        self.command_tasks.begin_shutdown();
806        self.http.cancel();
807    }
808
809    async fn finish_shutdown(&self) -> anyhow::Result<()> {
810        let (session, command) = tokio::join!(
811            self.session_tasks.finish_shutdown(TASK_GRACE, TASK_FORCE),
812            self.command_tasks.finish_shutdown(TASK_GRACE, TASK_FORCE),
813        );
814        session.context("failed to stop FinFeed polling tasks")?;
815        command.context("failed to stop FinFeed request tasks")?;
816        Ok(())
817    }
818
819    fn subscribe_activity(&self, id: InstrumentId, quotes: bool) -> anyhow::Result<()> {
820        let mut state = self.publication.lock();
821        state.check_subscription(id)?;
822
823        if state
824            .polling
825            .subscribe(id, quotes, self.config.poll_limit)?
826        {
827            self.poll_changed.notify_one();
828        }
829
830        Ok(())
831    }
832
833    fn subscribe_book(
834        &self,
835        id: InstrumentId,
836        book_type: BookType,
837        deltas: bool,
838        depth: Option<usize>,
839    ) -> anyhow::Result<()> {
840        let mut state = self.publication.lock();
841        state.check_subscription(id)?;
842        anyhow::ensure!(
843            book_type == BookType::L2_MBP,
844            "FinFeed book subscriptions require L2_MBP"
845        );
846
847        if state.polling.subscribe_book(id, deltas, depth)? {
848            self.poll_changed.notify_one();
849        }
850
851        Ok(())
852    }
853}
854
855impl Drop for FinFeedDataClient {
856    fn drop(&mut self) {
857        self.begin_shutdown();
858    }
859}
860
861#[derive(Debug, Default)]
862struct Publication {
863    connected: bool,
864    instruments: HashMap<InstrumentId, BinaryOption>,
865    polling: poll::Polling,
866}
867
868impl Publication {
869    fn loaded(&self, id: InstrumentId) -> anyhow::Result<&BinaryOption> {
870        anyhow::ensure!(self.connected, "FinFeed client is disconnected");
871        self.instruments
872            .get(&id)
873            .with_context(|| format!("{id} is not a loaded FinFeed instrument"))
874    }
875
876    fn check_subscription(&self, id: InstrumentId) -> anyhow::Result<()> {
877        let instrument = self.loaded(id)?;
878        anyhow::ensure!(
879            !terminal(instrument),
880            "FinFeed instrument {id} is inactive or resolved"
881        );
882        Ok(())
883    }
884
885    fn update(&mut self, instrument: BinaryOption) {
886        if terminal(&instrument) {
887            self.polling.deactivate(instrument.id);
888        } else {
889            self.polling.refresh(instrument.id);
890        }
891
892        self.instruments.insert(instrument.id, instrument);
893    }
894}
895
896fn terminal(instrument: &BinaryOption) -> bool {
897    instrument
898        .info
899        .as_ref()
900        .and_then(|info| info.get_str("status"))
901        .and_then(|status| {
902            let decoded: Result<MarketStatus, serde::de::value::Error> =
903                MarketStatus::deserialize(status.into_deserializer());
904            decoded.ok()
905        })
906        .is_some_and(|status| matches!(status, MarketStatus::Closed | MarketStatus::Resolved))
907}
908
909fn publish(
910    state: &Mutex<Publication>,
911    sender: &EventSender<DataEvent>,
912    response: DataResponse,
913    instruments: Vec<BinaryOption>,
914) -> anyhow::Result<()> {
915    let mut state = state.lock();
916    anyhow::ensure!(state.connected, "FinFeed request generation is closed");
917
918    for instrument in &instruments {
919        let instrument = InstrumentAny::from(instrument.clone());
920        send(sender, DataEvent::Data(instrument.into()))?;
921    }
922
923    send(sender, DataEvent::Response(response))?;
924
925    for instrument in instruments {
926        state.update(instrument);
927    }
928
929    Ok(())
930}
931
932fn send(sender: &EventSender<DataEvent>, event: DataEvent) -> anyhow::Result<()> {
933    sender
934        .send(event)
935        .map_err(|_| anyhow::anyhow!("FinFeed data receiver is closed"))
936}
937
938fn book_time(params: Option<&Params>) -> anyhow::Result<Option<Timestamp>> {
939    book_params(params)?;
940
941    let Some(params) = params else {
942        return Ok(None);
943    };
944
945    anyhow::ensure!(
946        !["date", "start", "time_start", "time_end"]
947            .iter()
948            .any(|key| params.contains_key(*key)),
949        "FinFeed historical point snapshots require an end timestamp"
950    );
951    params
952        .get("end")
953        .map(|_| {
954            params
955                .get_str("end")
956                .context("FinFeed book end must be a timestamp string")?
957                .parse()
958                .context("invalid FinFeed book end timestamp")
959        })
960        .transpose()
961}
962
963fn book_params(params: Option<&Params>) -> anyhow::Result<()> {
964    if let Some(params) = params
965        && params.contains_key("book_type")
966    {
967        anyhow::ensure!(
968            params.get_str("book_type") == Some(BookType::L2_MBP.as_ref()),
969            "FinFeed books require L2_MBP"
970        );
971    }
972
973    Ok(())
974}
975
976#[cfg(test)]
977mod tests {
978    use nautilus_core::{UUID4, UnixNanos};
979    use nautilus_model::data::{BarType, DataType};
980    use rstest::rstest;
981
982    use super::*;
983    use crate::{
984        http::{models::Market, parse::parse_instrument},
985        instruments::FinFeedInstrumentSelection,
986    };
987
988    #[rstest]
989    #[case(Some(serde_json::json!("Closed")), true)]
990    #[case(Some(serde_json::json!("Resolved")), true)]
991    #[case(Some(serde_json::json!("Open")), false)]
992    #[case(Some(serde_json::json!("Suspended")), false)]
993    #[case(Some(serde_json::json!("Unknown")), false)]
994    #[case(Some(serde_json::json!("FutureStatus")), false)]
995    #[case(Some(serde_json::json!("closed")), false)]
996    #[case(Some(serde_json::json!({"Closed": null})), false)]
997    #[case(Some(serde_json::json!(7)), false)]
998    #[case(Some(serde_json::Value::Null), false)]
999    #[case(None, false)]
1000    fn terminal_status_requires_a_recognized_string(
1001        #[case] status: Option<serde_json::Value>,
1002        #[case] expected: bool,
1003    ) {
1004        let market =
1005            serde_json::from_str(include_str!("../test_data/polymarket-market.json")).unwrap();
1006        let mut instrument = parse_instrument(&market, UnixNanos::default()).unwrap();
1007        let mut info = serde_json::json!({});
1008        if let Some(status) = status {
1009            info["status"] = status;
1010        }
1011
1012        instrument.info = Some(serde_json::from_value(info).unwrap());
1013
1014        assert_eq!(terminal(&instrument), expected);
1015    }
1016
1017    #[rstest]
1018    #[case(serde_json::json!({}), true)]
1019    #[case(serde_json::json!({"book_type": "L2_MBP"}), true)]
1020    #[case(serde_json::json!({"book_type": "l2_mbp"}), false)]
1021    #[case(serde_json::json!({"book_type": "L1_MBP"}), false)]
1022    #[case(serde_json::json!({"book_type": "L3_MBO"}), false)]
1023    #[case(serde_json::json!({"book_type": 2}), false)]
1024    #[case(serde_json::json!({"book_type": null}), false)]
1025    fn book_type_params_preserve_exact_wire_validation(
1026        #[case] value: serde_json::Value,
1027        #[case] valid: bool,
1028    ) {
1029        let params: Params = serde_json::from_value(value).unwrap();
1030
1031        let expected = if valid {
1032            Ok(())
1033        } else {
1034            Err("FinFeed books require L2_MBP".to_owned())
1035        };
1036
1037        assert_eq!(
1038            book_params(Some(&params)).map_err(|e| e.to_string()),
1039            expected
1040        );
1041    }
1042
1043    #[rstest]
1044    #[case("instrument")]
1045    #[case("snapshot")]
1046    #[case("quotes")]
1047    #[case("trades")]
1048    #[case("bars")]
1049    #[case("depth")]
1050    #[case("deltas")]
1051    fn requests_reject_multiple_members_before_dispatch(#[case] family: &str) {
1052        let client = subscription_client();
1053        let ids = [
1054            InstrumentId::from("first_NO.FINFEED_POLYMARKET"),
1055            InstrumentId::from("last_NO.FINFEED_POLYMARKET"),
1056        ];
1057        let client_id = Some(client.client_id());
1058        let request_id = UUID4::new();
1059        let ts_init = UnixNanos::from(19);
1060
1061        let result = match family {
1062            "instrument" => client.request_instrument(RequestInstrument::new(
1063                ids, None, None, client_id, request_id, ts_init, None,
1064            )),
1065            "snapshot" => client.request_book_snapshot(RequestBookSnapshot::new(
1066                ids, None, client_id, request_id, ts_init, None,
1067            )),
1068            "quotes" => client.request_quotes(RequestQuotes::new(
1069                ids, None, None, None, client_id, request_id, ts_init, None,
1070            )),
1071            "trades" => client.request_trades(RequestTrades::new(
1072                ids, None, None, None, client_id, request_id, ts_init, None,
1073            )),
1074            "bars" => {
1075                let bar_types =
1076                    ids.map(|id| BarType::from(format!("{id}-1-MINUTE-LAST-EXTERNAL").as_str()));
1077                client.request_bars(RequestBars::new(
1078                    bar_types, None, None, None, client_id, request_id, ts_init, None,
1079                ))
1080            }
1081            "depth" => client.request_book_depth(RequestBookDepth::new(
1082                ids, None, None, None, None, client_id, request_id, ts_init, None,
1083            )),
1084            "deltas" => client.request_book_deltas(RequestBookDeltas::new(
1085                ids, None, None, None, client_id, request_id, ts_init, None,
1086            )),
1087            _ => unreachable!(),
1088        };
1089
1090        let error = result.unwrap_err();
1091
1092        let kind = if family == "bars" {
1093            "bar type"
1094        } else {
1095            "instrument ID"
1096        };
1097
1098        assert!(error.downcast_ref::<IdentifierCountError>().is_some());
1099        assert_eq!(
1100            error.to_string(),
1101            format!("request requires one {kind} on this adapter path, received 2")
1102        );
1103    }
1104
1105    #[rstest]
1106    #[case("data", "FinFeed custom data requests are unsupported")]
1107    #[case("funding", "FinFeed funding rates are unsupported")]
1108    #[case("option_chain", "FinFeed option chain requests are unsupported")]
1109    fn unsupported_requests_fail_explicitly(#[case] family: &str, #[case] expected: &str) {
1110        let client = subscription_client();
1111        let id = InstrumentId::from("first_NO.FINFEED_POLYMARKET");
1112        let client_id = client.client_id();
1113        let request_id = UUID4::new();
1114        let ts_init = UnixNanos::from(29);
1115
1116        let result = match family {
1117            "data" => client.request_data(RequestCustomData::new(
1118                client_id,
1119                DataType::new("FinFeedCustom", None, None),
1120                None,
1121                None,
1122                None,
1123                request_id,
1124                ts_init,
1125                None,
1126            )),
1127            "funding" => client.request_funding_rates(RequestFundingRates::new(
1128                id,
1129                None,
1130                None,
1131                None,
1132                Some(client_id),
1133                request_id,
1134                ts_init,
1135                None,
1136            )),
1137            "option_chain" => {
1138                client.request_option_chain_reference_price(RequestOptionChainReferencePrice::new(
1139                    "DERIBIT:BTC:BTC:1700000000000000000".parse().unwrap(),
1140                    id,
1141                    Some(client_id),
1142                    request_id,
1143                    ts_init,
1144                    None,
1145                ))
1146            }
1147            _ => unreachable!(),
1148        };
1149
1150        assert_eq!(result.unwrap_err().to_string(), expected);
1151    }
1152
1153    #[rstest]
1154    #[case("quotes")]
1155    #[case("trades")]
1156    #[case("deltas")]
1157    #[case("depth")]
1158    fn instrument_batches_continue_after_invalid_member_and_release_every_scope(
1159        #[case] feed: &str,
1160    ) {
1161        let mut client = subscription_client();
1162        let first = InstrumentId::from("first_NO.FINFEED_POLYMARKET");
1163        let missing = InstrumentId::from("missing_NO.FINFEED_POLYMARKET");
1164        let last = InstrumentId::from("last_NO.FINFEED_POLYMARKET");
1165        let ids = [first, missing, last];
1166        let client_id = Some(client.client_id());
1167        let command_id = UUID4::new();
1168        let ts_init = UnixNanos::from(17);
1169        let depth = NonZeroUsize::new(5);
1170
1171        let result = match feed {
1172            "quotes" => client.subscribe_quotes(SubscribeQuotes::new(
1173                ids, client_id, None, command_id, ts_init, None, None,
1174            )),
1175            "trades" => client.subscribe_trades(SubscribeTrades::new(
1176                ids, client_id, None, command_id, ts_init, None, None,
1177            )),
1178            "deltas" => client.subscribe_book_deltas(SubscribeBookDeltas::new(
1179                ids,
1180                BookType::L2_MBP,
1181                client_id,
1182                None,
1183                command_id,
1184                ts_init,
1185                depth,
1186                false,
1187                None,
1188                None,
1189            )),
1190            "depth" => client.subscribe_book_depth(SubscribeBookDepth::new(
1191                ids,
1192                BookType::L2_MBP,
1193                client_id,
1194                None,
1195                command_id,
1196                ts_init,
1197                depth,
1198                false,
1199                None,
1200                None,
1201            )),
1202            _ => unreachable!(),
1203        };
1204
1205        let error = result.unwrap_err();
1206        let partial = error.downcast_ref::<SubscriptionError>().unwrap();
1207        assert_eq!(partial.instrument_ids_accepted, vec![first, last]);
1208        assert_eq!(partial.bar_types_accepted, Vec::<BarType>::new());
1209        assert_eq!(
1210            partial.source.to_string(),
1211            format!("{missing} is not a loaded FinFeed instrument")
1212        );
1213
1214        {
1215            let mut state = client.publication.lock();
1216            assert_eq!(state.polling.instrument_ids(), vec![first, last]);
1217
1218            for id in [first, last] {
1219                let changed = match feed {
1220                    "quotes" | "trades" => state
1221                        .polling
1222                        .subscribe(id, feed == "quotes", client.config.poll_limit)
1223                        .unwrap(),
1224                    "deltas" | "depth" => state
1225                        .polling
1226                        .subscribe_book(id, feed == "deltas", Some(5))
1227                        .unwrap(),
1228                    _ => unreachable!(),
1229                };
1230
1231                assert!(!changed);
1232            }
1233        }
1234
1235        match feed {
1236            "quotes" => client.unsubscribe_quotes(&UnsubscribeQuotes::new(
1237                ids, client_id, None, command_id, ts_init, None, None,
1238            )),
1239            "trades" => client.unsubscribe_trades(&UnsubscribeTrades::new(
1240                ids, client_id, None, command_id, ts_init, None, None,
1241            )),
1242            "deltas" => client.unsubscribe_book_deltas(&UnsubscribeBookDeltas::new(
1243                ids, client_id, None, command_id, ts_init, None, None,
1244            )),
1245            "depth" => client.unsubscribe_book_depth(&UnsubscribeBookDepth::new(
1246                ids, client_id, None, command_id, ts_init, None, None,
1247            )),
1248            _ => unreachable!(),
1249        }
1250        .unwrap();
1251
1252        assert_eq!(
1253            client.publication.lock().polling.instrument_ids(),
1254            Vec::<InstrumentId>::new()
1255        );
1256    }
1257
1258    #[rstest]
1259    fn bar_batches_preserve_full_types_and_release_every_scope_after_invalid_member() {
1260        let mut client = subscription_client();
1261        let first = BarType::from("first_NO.FINFEED_POLYMARKET-1-MINUTE-LAST-EXTERNAL");
1262        let invalid = BarType::from("first_NO.FINFEED_POLYMARKET-1-MINUTE-LAST-INTERNAL");
1263        let second = BarType::from("first_NO.FINFEED_POLYMARKET-1-HOUR-LAST-EXTERNAL");
1264        let last = BarType::from("last_NO.FINFEED_POLYMARKET-1-DAY-LAST-EXTERNAL");
1265        let bar_types = [first, invalid, second, last];
1266        let client_id = Some(client.client_id());
1267        let command_id = UUID4::new();
1268        let ts_init = UnixNanos::from(23);
1269        let error = client
1270            .subscribe_bars(SubscribeBars::new(
1271                bar_types, client_id, None, command_id, ts_init, None, None,
1272            ))
1273            .unwrap_err();
1274        let partial = error.downcast_ref::<SubscriptionError>().unwrap();
1275        assert_eq!(partial.instrument_ids_accepted, Vec::<InstrumentId>::new());
1276        assert_eq!(partial.bar_types_accepted, vec![first, second, last]);
1277        assert_eq!(
1278            partial.source.to_string(),
1279            "FinFeed provides external OHLCV bars only"
1280        );
1281
1282        {
1283            let mut state = client.publication.lock();
1284            assert_eq!(
1285                state.polling.instrument_ids(),
1286                vec![first.instrument_id(), last.instrument_id()]
1287            );
1288
1289            for bar_type in [first, second, last] {
1290                assert!(
1291                    !state
1292                        .polling
1293                        .subscribe_bars(bar_type, Timestamp::UNIX_EPOCH)
1294                        .unwrap()
1295                );
1296            }
1297        }
1298
1299        client
1300            .unsubscribe_bars(&UnsubscribeBars::new(
1301                bar_types, client_id, None, command_id, ts_init, None, None,
1302            ))
1303            .unwrap();
1304        assert_eq!(
1305            client.publication.lock().polling.instrument_ids(),
1306            Vec::<InstrumentId>::new()
1307        );
1308    }
1309
1310    pub(super) fn subscription_client() -> FinFeedDataClient {
1311        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel();
1312
1313        let config = FinFeedDataClientConfig {
1314            api_key: Some(SecretString::from("test-api-key")),
1315            instruments: vec![FinFeedInstrumentSelection {
1316                exchange_id: "POLYMARKET".into(),
1317                market_ids: vec!["first_NO".into(), "last_NO".into()],
1318            }],
1319            ..Default::default()
1320        };
1321
1322        let client =
1323            FinFeedDataClient::new(ClientId::from("FINFEED"), config, sender.into()).unwrap();
1324        let source: Market =
1325            serde_json::from_str(include_str!("../test_data/polymarket-market.json")).unwrap();
1326        let instrument = parse_instrument(&source, UnixNanos::from(0)).unwrap();
1327        {
1328            let mut state = client.publication.lock();
1329            state.connected = true;
1330
1331            for id in ["first_NO.FINFEED_POLYMARKET", "last_NO.FINFEED_POLYMARKET"] {
1332                let mut instrument = instrument.clone();
1333                instrument.id = InstrumentId::from(id);
1334                state.instruments.insert(instrument.id, instrument);
1335            }
1336        }
1337
1338        client
1339    }
1340
1341    #[rstest]
1342    fn drop_closes_publication_for_captured_request_state() {
1343        let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel();
1344
1345        let config = FinFeedDataClientConfig {
1346            api_key: Some(SecretString::from("test-api-key")),
1347            instruments: vec![FinFeedInstrumentSelection {
1348                exchange_id: "POLYMARKET".into(),
1349                market_ids: vec!["contract_NO".into()],
1350            }],
1351            ..Default::default()
1352        };
1353
1354        let client =
1355            FinFeedDataClient::new(ClientId::from("FINFEED"), config, sender.into()).unwrap();
1356        let state = Arc::clone(&client.publication);
1357        let sender = client.sender.clone();
1358        state.lock().connected = true;
1359        let response = DataResponse::Instruments(InstrumentsResponse::new(
1360            UUID4::new(),
1361            ClientId::from("FINFEED"),
1362            Venue::from(FINFEED),
1363            Vec::new(),
1364            None,
1365            None,
1366            UnixNanos::from(1),
1367            None,
1368        ));
1369        publish(&state, &sender, response.clone(), Vec::new()).unwrap();
1370        assert!(matches!(
1371            receiver.try_recv().unwrap(),
1372            DataEvent::Response(_)
1373        ));
1374        drop(client);
1375        assert!(!state.lock().connected);
1376        let error = publish(&state, &sender, response, Vec::new()).unwrap_err();
1377        assert_eq!(error.to_string(), "FinFeed request generation is closed");
1378        assert!(receiver.try_recv().is_err());
1379    }
1380}