Skip to main content

nautilus_interactive_brokers/data/
core.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Core data client implementation for Interactive Brokers.
17
18use std::{
19    collections::HashMap,
20    fmt::Debug,
21    sync::{
22        Arc, Mutex,
23        atomic::{AtomicBool, Ordering},
24    },
25    time::Duration,
26};
27
28use ahash::AHashMap;
29use anyhow::Context;
30use ibapi::{
31    contracts::{Contract, Currency as IBCurrency, Exchange as IBExchange, SecurityType, Symbol},
32    market_data::IgnoreSize,
33    prelude::{StreamExt, SubscriptionItemStreamExt},
34};
35use nautilus_common::{
36    clients::{DataClient, execute_bar_subscriptions, execute_instrument_subscriptions},
37    live::{runner::get_data_event_sender, sender::EventSender},
38    messages::{
39        DataEvent, DataResponse,
40        data::{
41            BarsResponse, InstrumentResponse, InstrumentsResponse, QuotesResponse, RequestBars,
42            RequestInstrument, RequestInstruments, RequestQuotes, RequestTrades, SubscribeBars,
43            SubscribeBookDeltas, SubscribeIndexPrices, SubscribeOptionGreeks, SubscribeQuotes,
44            SubscribeTrades, TradesResponse, UnsubscribeBars, UnsubscribeBookDeltas,
45            UnsubscribeIndexPrices, UnsubscribeOptionGreeks, UnsubscribeQuotes, UnsubscribeTrades,
46        },
47    },
48};
49use nautilus_core::{
50    UnixNanos,
51    params::Params,
52    time::{AtomicTime, get_atomic_clock_realtime},
53};
54use nautilus_live::task::{TaskGroup, TaskJoinOutcome, TaskSlot, finish_task};
55use nautilus_model::{
56    data::BarType,
57    enums::BookType,
58    identifiers::{ClientId, InstrumentId, Venue},
59    instruments::{Instrument, any::InstrumentAny},
60};
61use tokio_util::sync::CancellationToken;
62
63use super::{
64    cache::{OptionGreeksCache, QuoteCache},
65    convert::{
66        IbPriceConverter, bar_request_range, bar_type_to_ib_bar_size, extend_historical_tick_batch,
67        historical_tick_request_end, ib_bar_to_nautilus_bar, jiff_to_ib_datetime,
68        price_type_to_ib_realtime_what_to_show_for_security,
69        price_type_to_ib_what_to_show_for_security, retain_historical_ticks_in_range,
70        should_continue_historical_tick_pagination,
71    },
72    parse::log_tick_parse_error,
73    streams::{
74        DataFarmConnectionState, StreamConfig, handle_historical_bars_subscription,
75        handle_index_price_subscription, handle_market_depth_subscription,
76        handle_option_greeks_subscription, handle_quote_subscription,
77        handle_realtime_bars_subscription, handle_tick_by_tick_quote_subscription,
78        handle_trade_subscription, monitor_data_farm_notices,
79    },
80};
81use crate::{
82    common::{
83        consts::IB_VENUE,
84        contracts::parse_contract_from_json,
85        shared_client::SharedClientHandle,
86        spreads::{is_spread_instrument_id, parse_spread_instrument_id_to_legs},
87        symbology::is_crypto_contract,
88    },
89    config::{InteractiveBrokersDataClientConfig, MarketDataType},
90    data_types::register_ib_custom_data,
91    providers::instruments::InteractiveBrokersInstrumentProvider,
92};
93
94/// Interactive Brokers data client.
95///
96/// This client provides market data functionality using the `rust-ibapi` library.
97/// It manages subscriptions, handles historical data requests, and streams
98/// market data to NautilusTrader.
99#[cfg_attr(
100    feature = "python",
101    pyo3::pyclass(module = "nautilus_trader.adapters.interactive_brokers")
102)]
103pub struct InteractiveBrokersDataClient {
104    client_id: ClientId,
105    config: InteractiveBrokersDataClientConfig,
106    instrument_provider: Arc<InteractiveBrokersInstrumentProvider>,
107    is_connected: Arc<AtomicBool>,
108    cancellation_token: CancellationToken,
109    session_tasks: TaskGroup,
110    command_tasks: TaskGroup,
111    data_sender: EventSender<DataEvent>,
112    subscriptions: Arc<Mutex<AHashMap<SubscriptionKey, SubscriptionInfo>>>,
113    option_greeks_subscriptions: Arc<Mutex<AHashMap<InstrumentId, CancellationToken>>>,
114    quote_cache: Arc<tokio::sync::Mutex<QuoteCache>>,
115    option_greeks_cache: Arc<tokio::sync::Mutex<OptionGreeksCache>>,
116    clock: &'static AtomicTime,
117    ib_client: Option<SharedClientHandle>,
118    last_bars: Arc<tokio::sync::Mutex<AHashMap<String, ibapi::market_data::realtime::Bar>>>,
119    bar_timeout_tasks: Arc<tokio::sync::Mutex<AHashMap<String, TaskSlot<()>>>>,
120    data_farm_state: Arc<DataFarmConnectionState>,
121}
122
123/// Information about an active subscription.
124#[derive(Debug)]
125struct SubscriptionInfo {
126    /// Cancellation token for this specific subscription.
127    cancellation_token: CancellationToken,
128    /// Unique id distinguishing this registration from later ones for the same key.
129    id: u64,
130}
131
132/// Type of subscription.
133#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
134enum SubscriptionType {
135    /// Quote subscription.
136    Quotes,
137    /// Index price subscription.
138    IndexPrices,
139    /// Trade subscription.
140    Trades,
141    /// Bar subscription, one stream per bar type.
142    Bars(BarType),
143    /// Order book delta subscription.
144    BookDeltas,
145}
146
147type SubscriptionKey = (InstrumentId, SubscriptionType);
148
149static SUBSCRIPTION_ID_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
150
151fn next_subscription_id() -> u64 {
152    SUBSCRIPTION_ID_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
153}
154
155/// Removes a subscription's registry entry when its stream task ends permanently,
156/// so a later re-subscribe is not blocked by a dead registration. The id check
157/// keeps a finishing task from removing a newer registration for the same key.
158struct SubscriptionGuard {
159    subscriptions: Arc<Mutex<AHashMap<SubscriptionKey, SubscriptionInfo>>>,
160    key: SubscriptionKey,
161    id: u64,
162}
163
164impl SubscriptionGuard {
165    fn release(self) {
166        // Dropping performs the removal; a panicking stream task reaches the same
167        // path through Drop, so a dead registration can never block a re-subscribe.
168    }
169}
170
171impl Drop for SubscriptionGuard {
172    fn drop(&mut self) {
173        if let Ok(mut subscriptions) = self.subscriptions.lock()
174            && subscriptions
175                .get(&self.key)
176                .is_some_and(|info| info.id == self.id)
177        {
178            subscriptions.remove(&self.key);
179        }
180    }
181}
182
183fn lock<T>(mutex: &Mutex<T>) -> anyhow::Result<std::sync::MutexGuard<'_, T>> {
184    mutex
185        .lock()
186        .map_err(|_| anyhow::anyhow!("Failed to lock IB data client state"))
187}
188
189fn parse_start_ns(params: Option<&nautilus_core::Params>) -> Option<UnixNanos> {
190    params
191        .and_then(|params| params.get_u64("start_ns"))
192        .or_else(|| {
193            params
194                .and_then(|params| params.get_str("start_ns"))
195                .and_then(|value| value.parse::<u64>().ok())
196        })
197        .map(UnixNanos::from)
198}
199
200fn parse_bool_param_value(value: &str) -> bool {
201    matches!(value, "true" | "True" | "1")
202}
203
204fn params_to_string_filters(params: Option<&Params>) -> Option<HashMap<String, String>> {
205    let filters: HashMap<String, String> = params?
206        .iter()
207        .filter_map(|(key, value)| value.as_str().map(|value| (key.clone(), value.to_string())))
208        .collect();
209    (!filters.is_empty()).then_some(filters)
210}
211
212fn datetime_to_unix_nanos(dt: jiff::Timestamp) -> UnixNanos {
213    UnixNanos::from(u64::try_from(dt.as_nanosecond()).unwrap_or_default())
214}
215
216fn request_trading_hours(use_regular_trading_hours: bool) -> ibapi::market_data::TradingHours {
217    if use_regular_trading_hours {
218        ibapi::market_data::TradingHours::Regular
219    } else {
220        ibapi::market_data::TradingHours::Extended
221    }
222}
223
224const HISTORICAL_TICK_DEFAULT_LIMIT: usize = 10_000;
225
226impl InteractiveBrokersDataClient {
227    /// Create a new `InteractiveBrokersDataClient`.
228    ///
229    /// # Errors
230    ///
231    /// Returns an error if client creation fails.
232    pub fn new(
233        client_id: ClientId,
234        config: InteractiveBrokersDataClientConfig,
235        instrument_provider: Arc<InteractiveBrokersInstrumentProvider>,
236    ) -> anyhow::Result<Self> {
237        let clock = get_atomic_clock_realtime();
238        config.validate()?;
239        register_ib_custom_data();
240        let data_sender = get_data_event_sender();
241
242        let session_tasks = TaskGroup::new();
243        let command_tasks = TaskGroup::new();
244
245        Ok(Self {
246            client_id,
247            config,
248            instrument_provider,
249            is_connected: Arc::new(AtomicBool::new(false)),
250            cancellation_token: session_tasks.cancellation_token(),
251            session_tasks,
252            command_tasks,
253            data_sender,
254            subscriptions: Arc::new(Mutex::new(AHashMap::new())),
255            option_greeks_subscriptions: Arc::new(Mutex::new(AHashMap::new())),
256            quote_cache: Arc::new(tokio::sync::Mutex::new(QuoteCache::new())),
257            option_greeks_cache: Arc::new(tokio::sync::Mutex::new(OptionGreeksCache::new())),
258            clock,
259            ib_client: None,
260            last_bars: Arc::new(tokio::sync::Mutex::new(AHashMap::new())),
261            bar_timeout_tasks: Arc::new(tokio::sync::Mutex::new(AHashMap::new())),
262            data_farm_state: Arc::new(DataFarmConnectionState::default()),
263        })
264    }
265
266    fn cancel_active_subscriptions(&self) -> anyhow::Result<()> {
267        {
268            let mut subscriptions = lock(&self.subscriptions)?;
269            for subscription in subscriptions.values() {
270                subscription.cancellation_token.cancel();
271            }
272
273            subscriptions.clear();
274        }
275
276        {
277            let mut subscriptions = lock(&self.option_greeks_subscriptions)?;
278            for cancellation_token in subscriptions.values() {
279                cancellation_token.cancel();
280            }
281
282            subscriptions.clear();
283        }
284
285        Ok(())
286    }
287
288    fn spawn_command<F>(&self, future: F)
289    where
290        F: std::future::Future<Output = ()> + Send + 'static,
291    {
292        if let Err(e) = self.command_tasks.spawn(future) {
293            tracing::warn!("Skipping IB data command after shutdown began: {e}");
294        }
295    }
296
297    async fn finish_tasks(&self) -> anyhow::Result<()> {
298        let (session_result, command_result) = tokio::join!(
299            self.session_tasks
300                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2)),
301            self.command_tasks
302                .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2)),
303        );
304        self.finish_bar_timeout_tasks().await;
305        session_result.context("failed to finish IB data session tasks")?;
306        command_result.context("failed to finish IB data command tasks")?;
307        Ok(())
308    }
309
310    async fn prepare_task_groups(&mut self) -> anyhow::Result<()> {
311        if !self.session_tasks.is_open() || !self.command_tasks.is_open() {
312            self.session_tasks.begin_shutdown();
313            self.command_tasks.begin_shutdown();
314            self.finish_tasks().await?;
315            self.session_tasks
316                .start_generation()
317                .context("failed to start IB data session task generation")?;
318            self.command_tasks
319                .start_generation()
320                .context("failed to start IB data command task generation")?;
321            self.cancellation_token = self.session_tasks.cancellation_token();
322        }
323
324        Ok(())
325    }
326
327    async fn finish_bar_timeout_tasks(&self) {
328        let mut tasks = self.bar_timeout_tasks.lock().await;
329        let bar_types = tasks.keys().cloned().collect::<Vec<_>>();
330        for bar_type in bar_types {
331            let handle = tasks
332                .get_mut(&bar_type)
333                .expect("bar timeout task key collected from map");
334
335            match finish_task(handle, Duration::ZERO, Duration::from_secs(1)).await {
336                None => {}
337                Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => {}
338                Some(TaskJoinOutcome::Failed(e)) => {
339                    tracing::warn!("IB bar timeout task join failed: {e}");
340                }
341                Some(TaskJoinOutcome::Incomplete) => {
342                    tracing::warn!("IB bar timeout task did not stop within one second");
343                }
344            }
345
346            if handle.is_none() {
347                tasks.remove(&bar_type);
348            }
349        }
350    }
351
352    /// Get a reference to the IB client if connected.
353    /// This is used internally for provider method calls.
354    #[allow(dead_code)] // Library API - may be used by other modules or PyO3 bindings
355    pub(crate) fn get_ib_client(&self) -> Option<&Arc<ibapi::Client>> {
356        self.ib_client.as_ref().map(|h| h.as_arc())
357    }
358
359    /// Get a reference to the instrument provider.
360    #[allow(dead_code)] // Library API - may be used by other modules or PyO3 bindings
361    pub(crate) fn instrument_provider(&self) -> Arc<InteractiveBrokersInstrumentProvider> {
362        Arc::clone(&self.instrument_provider)
363    }
364
365    /// Batch load multiple instrument IDs using the internal IB client.
366    ///
367    /// This method calls the provider's batch_load with the data client's IB client.
368    ///
369    /// # Errors
370    ///
371    /// Returns an error if:
372    /// - The client is not connected.
373    /// - The provider batch_load fails.
374    pub async fn batch_load_instruments(
375        &self,
376        instrument_ids: Vec<InstrumentId>,
377    ) -> anyhow::Result<Vec<InstrumentId>> {
378        log::debug!(
379            "Batch loading {} IB instruments through data client",
380            instrument_ids.len()
381        );
382        let client = self
383            .ib_client
384            .as_ref()
385            .context("IB client not connected. Call connect() first")?;
386
387        let loaded = self
388            .instrument_provider
389            .batch_load(client, instrument_ids, None)
390            .await?;
391        log::debug!("Batch loaded {} IB instruments", loaded.len());
392        Ok(loaded)
393    }
394
395    /// Fetch option chain for an underlying contract with expiry filtering.
396    ///
397    /// This method calls the provider's fetch_option_chain_by_range with the data client's IB client.
398    ///
399    /// # Errors
400    ///
401    /// Returns an error if:
402    /// - The client is not connected.
403    /// - The provider method fails.
404    pub async fn fetch_option_chain_by_range(
405        &self,
406        underlying_symbol: &str,
407        exchange: Option<&str>,
408        currency: Option<&str>,
409        expiry_min: Option<&str>,
410        expiry_max: Option<&str>,
411    ) -> anyhow::Result<usize> {
412        log::debug!(
413            "Fetching IB option chain by range (symbol={underlying_symbol}, exchange={exchange:?}, currency={currency:?}, expiry_min={expiry_min:?}, expiry_max={expiry_max:?})"
414        );
415        let client = self
416            .ib_client
417            .as_ref()
418            .context("IB client not connected. Call connect() first")?;
419
420        let underlying = Contract {
421            contract_id: 0,
422            symbol: Symbol::from(underlying_symbol.to_string()),
423            security_type: SecurityType::Stock,
424            last_trade_date_or_contract_month: String::new(),
425            strike: f64::MAX,
426            right: None,
427            multiplier: String::new(),
428            exchange: IBExchange::from(exchange.unwrap_or("SMART")),
429            currency: IBCurrency::from(currency.unwrap_or("USD")),
430            local_symbol: String::new(),
431            primary_exchange: IBExchange::from(""),
432            trading_class: String::new(),
433            include_expired: false,
434            security_id_type: None,
435            security_id: String::new(),
436            combo_legs_description: String::new(),
437            combo_legs: Vec::new(),
438            delta_neutral_contract: None,
439            issuer_id: String::new(),
440            description: String::new(),
441            last_trade_date: None,
442        };
443
444        let count = self
445            .instrument_provider
446            .fetch_option_chain_by_range(client, &underlying, expiry_min, expiry_max, None)
447            .await?;
448        log::debug!("Fetched {count} IB option instruments for {underlying_symbol}");
449        Ok(count)
450    }
451
452    /// Fetch futures chain for a given underlying symbol.
453    ///
454    /// This method calls the provider's fetch_futures_chain with the data client's IB client.
455    ///
456    /// # Errors
457    ///
458    /// Returns an error if:
459    /// - The client is not connected.
460    /// - The provider method fails.
461    pub async fn fetch_futures_chain(
462        &self,
463        symbol: &str,
464        exchange: Option<&str>,
465        currency: Option<&str>,
466        min_expiry_days: Option<u32>,
467        max_expiry_days: Option<u32>,
468    ) -> anyhow::Result<usize> {
469        log::debug!(
470            "Fetching IB futures chain (symbol={symbol}, exchange={exchange:?}, currency={currency:?}, min_days={min_expiry_days:?}, max_days={max_expiry_days:?})"
471        );
472        let client = self
473            .ib_client
474            .as_ref()
475            .context("IB client not connected. Call connect() first")?;
476
477        let count = self
478            .instrument_provider
479            .fetch_futures_chain(
480                client,
481                symbol,
482                exchange.unwrap_or(""),
483                currency.unwrap_or("USD"),
484                None,
485                false,
486                min_expiry_days,
487                max_expiry_days,
488            )
489            .await?;
490        log::debug!("Fetched {count} IB futures instruments for {symbol}");
491        Ok(count)
492    }
493
494    /// Fetch BAG (spread) contract details.
495    ///
496    /// This method calls the provider's fetch_bag_contract with the data client's IB client.
497    ///
498    /// # Errors
499    ///
500    /// Returns an error if:
501    /// - The client is not connected.
502    /// - The provider method fails.
503    pub async fn fetch_bag_contract(
504        &self,
505        bag_contract: &ibapi::contracts::Contract,
506    ) -> anyhow::Result<usize> {
507        log::debug!(
508            "Fetching IB BAG contract details (contract_id={}, exchange={}, symbol={})",
509            bag_contract.contract_id,
510            bag_contract.exchange.as_str(),
511            bag_contract.symbol.as_str()
512        );
513        let client = self
514            .ib_client
515            .as_ref()
516            .context("IB client not connected. Call connect() first")?;
517
518        let count = self
519            .instrument_provider
520            .fetch_bag_contract(client, bag_contract)
521            .await?;
522        log::debug!("Fetched {count} BAG instruments");
523        Ok(count)
524    }
525}
526
527impl Debug for InteractiveBrokersDataClient {
528    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
529        f.debug_struct(stringify!(InteractiveBrokersDataClient))
530            .field("client_id", &self.client_id)
531            .field("config", &self.config)
532            .field("is_connected", &self.is_connected.load(Ordering::Relaxed))
533            .field("has_ib_client", &self.ib_client.is_some())
534            .finish_non_exhaustive()
535    }
536}
537
538#[async_trait::async_trait(?Send)]
539impl DataClient for InteractiveBrokersDataClient {
540    fn client_id(&self) -> ClientId {
541        self.client_id
542    }
543
544    fn venue(&self) -> Option<Venue> {
545        // Interactive Brokers is a multi-venue adapter (SMART, IDEALPRO, ZEROHASH, CME, etc.),
546        // so the data client must register for default routing rather than a single venue:
547        // subscriptions and requests are routed by the command's venue (derived from the
548        // instrument ID), which would otherwise never match and be dropped by the data engine.
549        // Mirrors the other multi-venue adapters (Tardis, Databento).
550        None
551    }
552
553    fn start(&mut self) -> anyhow::Result<()> {
554        tracing::info!(
555            client_id = %self.client_id,
556            "Starting Interactive Brokers data client"
557        );
558        Ok(())
559    }
560
561    fn stop(&mut self) -> anyhow::Result<()> {
562        tracing::info!(
563            "Stopping Interactive Brokers data client {id}",
564            id = self.client_id
565        );
566        self.session_tasks.begin_shutdown();
567        self.command_tasks.begin_shutdown();
568        self.cancel_active_subscriptions()?;
569        self.is_connected.store(false, Ordering::Relaxed);
570
571        self.clear_bar_tracking_state();
572
573        Ok(())
574    }
575
576    fn reset(&mut self) -> anyhow::Result<()> {
577        tracing::debug!(
578            "Resetting Interactive Brokers data client {id}",
579            id = self.client_id
580        );
581        self.cancel_active_subscriptions()?;
582        self.session_tasks.begin_shutdown();
583        self.command_tasks.begin_shutdown();
584        self.is_connected.store(false, Ordering::Relaxed);
585        self.clear_bar_tracking_state();
586        self.quote_cache = Arc::new(tokio::sync::Mutex::new(QuoteCache::new()));
587        self.option_greeks_cache = Arc::new(tokio::sync::Mutex::new(OptionGreeksCache::new()));
588        self.cancellation_token = CancellationToken::new();
589
590        Ok(())
591    }
592
593    fn dispose(&mut self) -> anyhow::Result<()> {
594        self.stop()
595    }
596
597    async fn connect(&mut self) -> anyhow::Result<()> {
598        tracing::debug!("Connecting Interactive Brokers data client...");
599
600        self.prepare_task_groups().await?;
601
602        let handle = crate::common::shared_client::get_or_connect(
603            &self.config.host,
604            self.config.port,
605            self.config.client_id,
606            self.config.connection_timeout,
607            self.config.max_reconnect_attempts,
608        )
609        .await
610        .context("Failed to connect to IB Gateway/TWS")?;
611
612        let client = Arc::clone(handle.as_arc());
613
614        tracing::info!(
615            "Connected to IB Gateway/TWS at {}:{} (client_id: {})",
616            self.config.host,
617            self.config.port,
618            self.config.client_id
619        );
620
621        // Set market data type if not default
622        let ib_data_type = (self.config.market_data_type != MarketDataType::Realtime)
623            .then(|| self.config.market_data_type.into());
624
625        if let Some(ib_data_type) = ib_data_type {
626            client
627                .switch_market_data_type(ib_data_type)
628                .await
629                .context("Failed to switch market data type")?;
630            tracing::info!("Set market data type to {:?}", self.config.market_data_type);
631        }
632
633        // Initialize provider and load instruments from cache/config if configured
634        tracing::debug!("Initializing IB data instrument provider");
635
636        if let Err(e) = self
637            .instrument_provider
638            .initialize_with_client(client.as_ref())
639            .await
640        {
641            if !self.config.instrument_provider.load_ids.is_empty()
642                || !self.config.instrument_provider.load_contracts.is_empty()
643            {
644                return Err(e).context("Failed to load configured IB instruments on startup");
645            }
646
647            tracing::warn!("Failed to load instruments on startup: {}", e);
648        }
649
650        self.ib_client = Some(handle);
651
652        let data_farm_state = Arc::clone(&self.data_farm_state);
653        let is_connected = Arc::clone(&self.is_connected);
654        let cancellation_token = self.cancellation_token.child_token();
655
656        if let Err(e) = self.session_tasks.spawn(async move {
657            if let Err(e) = monitor_data_farm_notices(
658                client,
659                data_farm_state,
660                ib_data_type,
661                cancellation_token,
662                is_connected,
663            )
664            .await
665            {
666                tracing::warn!("IB data farm notice monitor stopped: {e:?}");
667            }
668        }) {
669            self.session_tasks.begin_shutdown();
670            self.command_tasks.begin_shutdown();
671            self.ib_client = None;
672
673            if let Err(teardown_error) = self.finish_tasks().await {
674                return Err(anyhow::Error::new(e)
675                    .context(format!("IB data startup teardown failed: {teardown_error}")));
676            }
677
678            return Err(anyhow::Error::new(e).context("failed to register IB data farm monitor"));
679        }
680
681        self.is_connected.store(true, Ordering::Relaxed);
682
683        let instrument_count = self.instrument_provider.count();
684        if instrument_count > 0 {
685            tracing::debug!(
686                "Data client connected with {} instruments in provider cache",
687                instrument_count
688            );
689
690            for instrument in self.instrument_provider.get_all() {
691                if let Err(e) = self.data_sender.send(DataEvent::Data(instrument.into())) {
692                    tracing::warn!("Failed to publish startup-loaded instrument: {e}");
693                    break;
694                }
695            }
696        }
697
698        tracing::info!("Connected Interactive Brokers data client");
699        Ok(())
700    }
701
702    async fn disconnect(&mut self) -> anyhow::Result<()> {
703        tracing::debug!("Disconnecting Interactive Brokers data client...");
704
705        self.session_tasks.begin_shutdown();
706        self.command_tasks.begin_shutdown();
707        self.cancel_active_subscriptions()?;
708        self.ib_client = None;
709        let tasks_result = self.finish_tasks().await;
710        self.is_connected.store(false, Ordering::Relaxed);
711        tracing::info!("Disconnected Interactive Brokers data client");
712        tasks_result
713    }
714
715    fn is_connected(&self) -> bool {
716        self.is_connected.load(Ordering::Relaxed)
717    }
718
719    fn is_disconnected(&self) -> bool {
720        !self.is_connected()
721    }
722
723    // Subscription handlers
724    fn subscribe_quotes(&mut self, cmd: SubscribeQuotes) -> anyhow::Result<()> {
725        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
726            self.subscribe_quotes_member(instrument_id, &cmd)
727        })
728    }
729
730    fn subscribe_index_prices(&mut self, cmd: SubscribeIndexPrices) -> anyhow::Result<()> {
731        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
732            tracing::debug!("Subscribing to index prices for {}", instrument_id);
733
734            if self.has_subscription(instrument_id, SubscriptionType::IndexPrices)? {
735                tracing::debug!(
736                    "Index price subscription already active for {}",
737                    instrument_id
738                );
739                return Ok(());
740            }
741
742            let client = self
743                .ib_client
744                .as_ref()
745                .context("IB client not connected. Call connect() first")?;
746
747            let instrument = self
748                .instrument_provider
749                .find(&instrument_id)
750                .context(format!("Instrument {instrument_id} not found in provider"))?;
751
752            let contract = self
753                .instrument_provider
754                .resolve_contract_for_instrument(instrument_id)
755                .context("Failed to convert instrument_id to IB contract")?;
756
757            if !matches!(contract.security_type, SecurityType::Index) {
758                tracing::warn!(
759                    "Index price subscription not supported for security type {:?} on {}",
760                    contract.security_type,
761                    instrument_id
762                );
763                return Ok(());
764            }
765
766            let prices = IbPriceConverter::for_instrument(
767                &instrument,
768                self.instrument_provider.get_price_magnifier(&instrument_id),
769            );
770            let data_sender = self.data_sender.clone();
771            let clock = self.clock;
772
773            let subscription_token = self.cancellation_token.child_token();
774
775            let subscription_id = next_subscription_id();
776            lock(&self.subscriptions)?.insert(
777                (instrument_id, SubscriptionType::IndexPrices),
778                SubscriptionInfo {
779                    cancellation_token: subscription_token.clone(),
780                    id: subscription_id,
781                },
782            );
783
784            let guard = SubscriptionGuard {
785                subscriptions: Arc::clone(&self.subscriptions),
786                key: (instrument_id, SubscriptionType::IndexPrices),
787                id: subscription_id,
788            };
789
790            let client_clone = client.as_arc().clone();
791            let data_farm_state = Arc::clone(&self.data_farm_state);
792            let stream_config = StreamConfig::new(self.client_id, &self.config);
793
794            let task = async move {
795                if let Err(e) = handle_index_price_subscription(
796                    client_clone,
797                    contract,
798                    instrument_id,
799                    prices,
800                    data_sender,
801                    clock,
802                    subscription_token,
803                    data_farm_state,
804                    stream_config,
805                )
806                .await
807                {
808                    tracing::error!(
809                        "Index price subscription error for {}: {:?}",
810                        instrument_id,
811                        e
812                    );
813                }
814
815                guard.release();
816            };
817
818            self.session_tasks
819                .spawn(task)
820                .context("failed to register IB data subscription task")?;
821
822            tracing::debug!("Index price subscription started for {}", instrument_id);
823            Ok(())
824        })
825    }
826
827    fn subscribe_option_greeks(&mut self, cmd: SubscribeOptionGreeks) -> anyhow::Result<()> {
828        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
829            tracing::debug!("Subscribing to option greeks for {}", instrument_id);
830
831            if lock(&self.option_greeks_subscriptions)?.contains_key(&instrument_id) {
832                tracing::debug!(
833                    "Option greeks subscription already active for {}",
834                    instrument_id
835                );
836                return Ok(());
837            }
838
839            let client = self
840                .ib_client
841                .as_ref()
842                .context("IB client not connected. Call connect() first")?;
843
844            let instrument = self
845                .instrument_provider
846                .find(&instrument_id)
847                .context(format!("Instrument {instrument_id} not found in provider"))?;
848
849            if !matches!(
850                instrument,
851                InstrumentAny::OptionContract(_)
852                    | InstrumentAny::FuturesContract(_)
853                    | InstrumentAny::CryptoOption(_)
854            ) && !matches!(
855                self.instrument_provider
856                    .resolve_contract_for_instrument(instrument_id)?
857                    .security_type,
858                SecurityType::Option | SecurityType::FuturesOption
859            ) {
860                tracing::warn!(
861                    "Option greeks subscription is only supported for option instruments: {}",
862                    instrument_id
863                );
864                return Ok(());
865            }
866
867            let contract = self
868                .instrument_provider
869                .resolve_contract_for_instrument(instrument_id)
870                .context("Failed to convert instrument_id to IB contract")?;
871
872            let data_sender = self.data_sender.clone();
873            let option_greeks_cache = Arc::clone(&self.option_greeks_cache);
874            let clock = self.clock;
875            let subscription_token = self.cancellation_token.child_token();
876            let subscription_token_clone = subscription_token.clone();
877            let client_clone = client.as_arc().clone();
878            let data_farm_state = Arc::clone(&self.data_farm_state);
879            let stream_config = StreamConfig::new(self.client_id, &self.config);
880
881            let task = async move {
882                if let Err(e) = handle_option_greeks_subscription(
883                    client_clone,
884                    contract,
885                    instrument_id,
886                    data_sender,
887                    option_greeks_cache,
888                    clock,
889                    subscription_token_clone,
890                    data_farm_state,
891                    stream_config,
892                )
893                .await
894                {
895                    tracing::error!(
896                        "Option greeks subscription error for {}: {:?}",
897                        instrument_id,
898                        e
899                    );
900                }
901            };
902
903            self.session_tasks
904                .spawn(task)
905                .context("failed to register IB data subscription task")?;
906
907            let mut subscriptions = lock(&self.option_greeks_subscriptions)?;
908            subscriptions.insert(instrument_id, subscription_token);
909
910            tracing::debug!("Option greeks subscription started for {}", instrument_id);
911            Ok(())
912        })
913    }
914
915    fn unsubscribe_quotes(&mut self, cmd: &UnsubscribeQuotes) -> anyhow::Result<()> {
916        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
917            tracing::debug!("Unsubscribing from quotes for {instrument_id}");
918
919            let mut subscriptions = lock(&self.subscriptions)?;
920            if let Some(sub_info) = subscriptions.remove(&(instrument_id, SubscriptionType::Quotes))
921            {
922                sub_info.cancellation_token.cancel();
923                tracing::debug!("Unsubscribed from quotes for {instrument_id}");
924            } else {
925                tracing::warn!("No active quote subscription found for {instrument_id}");
926            }
927
928            // Clear quote cache for this instrument
929            {
930                // Quote cache doesn't have per-instrument clear, but we can clear all
931                // In practice, the cache will naturally expire as new quotes arrive
932            }
933
934            Ok(())
935        })
936    }
937
938    fn unsubscribe_index_prices(&mut self, cmd: &UnsubscribeIndexPrices) -> anyhow::Result<()> {
939        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
940            tracing::debug!("Unsubscribing from index prices for {}", instrument_id);
941
942            let mut subscriptions = lock(&self.subscriptions)?;
943
944            if let Some(sub_info) =
945                subscriptions.remove(&(instrument_id, SubscriptionType::IndexPrices))
946            {
947                sub_info.cancellation_token.cancel();
948                tracing::debug!("Unsubscribed from index prices for {}", instrument_id);
949            } else {
950                tracing::warn!(
951                    "No active index price subscription found for {}",
952                    instrument_id
953                );
954            }
955
956            Ok(())
957        })
958    }
959
960    fn unsubscribe_option_greeks(&mut self, cmd: &UnsubscribeOptionGreeks) -> anyhow::Result<()> {
961        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
962            tracing::debug!("Unsubscribing from option greeks for {}", instrument_id);
963
964            let mut subscriptions = lock(&self.option_greeks_subscriptions)?;
965            if let Some(subscription_token) = subscriptions.remove(&instrument_id) {
966                subscription_token.cancel();
967                tracing::debug!("Unsubscribed from option greeks for {}", instrument_id);
968            } else {
969                tracing::warn!(
970                    "No active option greeks subscription found for {}",
971                    instrument_id
972                );
973            }
974
975            Ok(())
976        })
977    }
978
979    fn subscribe_trades(&mut self, cmd: SubscribeTrades) -> anyhow::Result<()> {
980        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
981            tracing::debug!("Subscribing to trades for {}", instrument_id);
982
983            if self.has_subscription(instrument_id, SubscriptionType::Trades)? {
984                tracing::debug!("Trade subscription already active for {}", instrument_id);
985                return Ok(());
986            }
987
988            let client = self
989                .ib_client
990                .as_ref()
991                .context("IB client not connected. Call connect() first")?;
992
993            // Get instrument from provider
994            let instrument = self
995                .instrument_provider
996                .find(&instrument_id)
997                .context(format!("Instrument {instrument_id} not found in provider"))?;
998
999            // Check if instrument is a CurrencyPair (IB doesn't support trades for CurrencyPair)
1000            if matches!(instrument, InstrumentAny::CurrencyPair(_)) {
1001                tracing::error!(
1002                    "Interactive Brokers does not support trades for CurrencyPair instruments: {}",
1003                    instrument_id
1004                );
1005                return Ok(());
1006            }
1007
1008            let prices = IbPriceConverter::for_instrument(
1009                &instrument,
1010                self.instrument_provider.get_price_magnifier(&instrument_id),
1011            );
1012            let size_precision = instrument.size_precision();
1013
1014            // Convert instrument_id to IB contract
1015            let contract = self
1016                .instrument_provider
1017                .resolve_contract_for_instrument(instrument_id)
1018                .context("Failed to convert instrument_id to IB contract")?;
1019
1020            let data_sender = self.data_sender.clone();
1021            let clock = self.clock;
1022
1023            // Create subscription-specific cancellation token
1024            let subscription_token = self.cancellation_token.child_token();
1025
1026            let subscription_id = next_subscription_id();
1027            lock(&self.subscriptions)?.insert(
1028                (instrument_id, SubscriptionType::Trades),
1029                SubscriptionInfo {
1030                    cancellation_token: subscription_token.clone(),
1031                    id: subscription_id,
1032                },
1033            );
1034
1035            let guard = SubscriptionGuard {
1036                subscriptions: Arc::clone(&self.subscriptions),
1037                key: (instrument_id, SubscriptionType::Trades),
1038                id: subscription_id,
1039            };
1040
1041            // Spawn subscription task
1042            let client_clone = client.as_arc().clone();
1043            let data_farm_state = Arc::clone(&self.data_farm_state);
1044            let stream_config = StreamConfig::new(self.client_id, &self.config);
1045
1046            let task = async move {
1047                if let Err(e) = handle_trade_subscription(
1048                    client_clone,
1049                    contract,
1050                    instrument_id,
1051                    prices,
1052                    size_precision,
1053                    data_sender,
1054                    clock,
1055                    subscription_token,
1056                    data_farm_state,
1057                    stream_config,
1058                )
1059                .await
1060                {
1061                    tracing::error!("Trade subscription error for {}: {:?}", instrument_id, e);
1062                }
1063
1064                guard.release();
1065            };
1066
1067            self.session_tasks
1068                .spawn(task)
1069                .context("failed to register IB data subscription task")?;
1070
1071            tracing::debug!("Trade subscription started for {}", instrument_id);
1072            Ok(())
1073        })
1074    }
1075
1076    fn unsubscribe_trades(&mut self, cmd: &UnsubscribeTrades) -> anyhow::Result<()> {
1077        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1078            tracing::debug!("Unsubscribing from trades for {}", instrument_id);
1079
1080            let mut subscriptions = lock(&self.subscriptions)?;
1081            if let Some(sub_info) = subscriptions.remove(&(instrument_id, SubscriptionType::Trades))
1082            {
1083                sub_info.cancellation_token.cancel();
1084                tracing::debug!("Unsubscribed from trades for {}", instrument_id);
1085            } else {
1086                tracing::warn!("No active trade subscription found for {}", instrument_id);
1087            }
1088
1089            Ok(())
1090        })
1091    }
1092
1093    fn subscribe_bars(&mut self, cmd: SubscribeBars) -> anyhow::Result<()> {
1094        execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
1095            let cmd = cmd.with_bar_types(bar_type);
1096            tracing::debug!("Subscribing to bars for {}", bar_type);
1097
1098            let client = self
1099                .ib_client
1100                .as_ref()
1101                .context("IB client not connected. Call connect() first")?;
1102
1103            // Get instrument from provider
1104            let instrument_id = bar_type.instrument_id();
1105            if self.has_subscription(instrument_id, SubscriptionType::Bars(bar_type))? {
1106                tracing::debug!("Bar subscription already active for {}", bar_type);
1107                return Ok(());
1108            }
1109
1110            let instrument = self
1111                .instrument_provider
1112                .find(&instrument_id)
1113                .context(format!("Instrument {instrument_id} not found in provider"))?;
1114
1115            let prices = IbPriceConverter::for_instrument(
1116                &instrument,
1117                self.instrument_provider.get_price_magnifier(&instrument_id),
1118            );
1119            let size_precision = instrument.size_precision();
1120
1121            // Convert instrument_id to IB contract
1122            let contract = self
1123                .instrument_provider
1124                .resolve_contract_for_instrument(instrument_id)
1125                .context("Failed to convert instrument_id to IB contract")?;
1126
1127            let bar_type_str = bar_type.to_string();
1128            let data_sender = self.data_sender.clone();
1129            let clock = self.clock;
1130            let last_bars = Arc::clone(&self.last_bars);
1131            let bar_timeout_tasks = Arc::clone(&self.bar_timeout_tasks);
1132            let handle_revised_bars = self.config.handle_revised_bars;
1133            let use_rth = self.config.use_regular_trading_hours;
1134            let start_ns = parse_start_ns(cmd.params.as_ref());
1135            // Crypto (ZEROHASH/PAXOS) trade-price bars must request AGGTRADES, not
1136            // TRADES (TWS rejects TRADES for crypto, error 10299) - on BOTH the
1137            // realtime (reqRealTimeBars) and historical (reqHistoricalData) paths, per
1138            // the Java engine's whatToShowFor rule. Capture the flag before `contract`
1139            // is moved into the subscription task below.
1140            let is_crypto = is_crypto_contract(&contract);
1141
1142            // Create subscription-specific cancellation token
1143            let subscription_token = self.cancellation_token.child_token();
1144
1145            let subscription_id = next_subscription_id();
1146            lock(&self.subscriptions)?.insert(
1147                (instrument_id, SubscriptionType::Bars(bar_type)),
1148                SubscriptionInfo {
1149                    cancellation_token: subscription_token.clone(),
1150                    id: subscription_id,
1151                },
1152            );
1153
1154            let guard = SubscriptionGuard {
1155                subscriptions: Arc::clone(&self.subscriptions),
1156                key: (instrument_id, SubscriptionType::Bars(bar_type)),
1157                id: subscription_id,
1158            };
1159
1160            // Spawn subscription task
1161            let client_clone = client.as_arc().clone();
1162            let data_farm_state = Arc::clone(&self.data_farm_state);
1163            let stream_config = StreamConfig::new(self.client_id, &self.config);
1164
1165            let task = async move {
1166                let result = if bar_type.spec().timedelta().as_secs() == 5 {
1167                    handle_realtime_bars_subscription(
1168                        client_clone,
1169                        contract,
1170                        bar_type,
1171                        bar_type_str,
1172                        instrument_id,
1173                        price_type_to_ib_realtime_what_to_show_for_security(
1174                            bar_type.spec().price_type,
1175                            is_crypto,
1176                        ),
1177                        prices,
1178                        size_precision,
1179                        data_sender,
1180                        clock,
1181                        last_bars,
1182                        bar_timeout_tasks,
1183                        handle_revised_bars,
1184                        use_rth,
1185                        subscription_token,
1186                        Arc::clone(&data_farm_state),
1187                        stream_config,
1188                    )
1189                    .await
1190                } else {
1191                    handle_historical_bars_subscription(
1192                        client_clone,
1193                        contract,
1194                        bar_type,
1195                        price_type_to_ib_what_to_show_for_security(
1196                            bar_type.spec().price_type,
1197                            is_crypto,
1198                        ),
1199                        prices,
1200                        size_precision,
1201                        use_rth,
1202                        start_ns,
1203                        data_sender,
1204                        handle_revised_bars,
1205                        clock,
1206                        subscription_token,
1207                        Arc::clone(&data_farm_state),
1208                        stream_config,
1209                    )
1210                    .await
1211                };
1212
1213                if let Err(e) = result {
1214                    tracing::error!("Bars subscription error for {}: {:?}", bar_type, e);
1215                }
1216
1217                guard.release();
1218            };
1219
1220            self.session_tasks
1221                .spawn(task)
1222                .context("failed to register IB data subscription task")?;
1223
1224            tracing::debug!("Real-time bars subscription started for {}", bar_type);
1225            Ok(())
1226        })
1227    }
1228
1229    fn unsubscribe_bars(&mut self, cmd: &UnsubscribeBars) -> anyhow::Result<()> {
1230        execute_bar_subscriptions(&cmd.bar_types, |bar_type| {
1231            tracing::debug!("Unsubscribing from bars for {}", bar_type);
1232
1233            let instrument_id = bar_type.instrument_id();
1234            let mut subscriptions = lock(&self.subscriptions)?;
1235
1236            if let Some(sub_info) =
1237                subscriptions.remove(&(instrument_id, SubscriptionType::Bars(bar_type)))
1238            {
1239                sub_info.cancellation_token.cancel();
1240                tracing::debug!("Unsubscribed from bars for {}", bar_type);
1241            } else {
1242                tracing::warn!("No active bar subscription found for {}", bar_type);
1243            }
1244
1245            Ok(())
1246        })
1247    }
1248
1249    fn subscribe_book_deltas(&mut self, cmd: SubscribeBookDeltas) -> anyhow::Result<()> {
1250        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1251            let cmd = cmd.with_instrument_ids(instrument_id);
1252            tracing::debug!("Subscribing to book deltas for {}", instrument_id);
1253
1254            if self.has_subscription(instrument_id, SubscriptionType::BookDeltas)? {
1255                tracing::debug!(
1256                    "Book delta subscription already active for {}",
1257                    instrument_id
1258                );
1259                return Ok(());
1260            }
1261
1262            // Validate book type (IB doesn't support L3_MBO)
1263            if cmd.book_type == BookType::L3_MBO {
1264                tracing::error!(
1265                    "Cannot subscribe to order book deltas: L3_MBO data is not published by Interactive Brokers. Valid book types are L1_MBP, L2_MBP"
1266                );
1267                return Ok(());
1268            }
1269
1270            let client = self
1271                .ib_client
1272                .as_ref()
1273                .context("IB client not connected. Call connect() first")?;
1274
1275            // Get instrument from provider
1276            let instrument = self
1277                .instrument_provider
1278                .find(&instrument_id)
1279                .context(format!("Instrument {instrument_id} not found in provider"))?;
1280
1281            // Book levels are not rounded to the increment: two IB levels could round to one price
1282            let prices = IbPriceConverter::new(instrument.price_precision())
1283                .with_price_magnifier(self.instrument_provider.get_price_magnifier(&instrument_id));
1284            let size_precision = instrument.size_precision();
1285
1286            // Convert instrument_id to IB contract
1287            let contract = self
1288                .instrument_provider
1289                .resolve_contract_for_instrument(instrument_id)
1290                .context("Failed to convert instrument_id to IB contract")?;
1291
1292            let data_sender = self.data_sender.clone();
1293            let clock = self.clock;
1294
1295            // Create subscription-specific cancellation token
1296            let subscription_token = self.cancellation_token.child_token();
1297
1298            // Get depth from command or default to 20 (Python default)
1299            let depth_rows = cmd.depth.map_or(20, |d| d.get() as i32);
1300
1301            // Get is_smart_depth from params or default to true
1302            let is_smart_depth = cmd
1303                .params
1304                .as_ref()
1305                .and_then(|params| params.get_str("is_smart_depth"))
1306                .is_none_or(parse_bool_param_value);
1307
1308            let subscription_id = next_subscription_id();
1309            lock(&self.subscriptions)?.insert(
1310                (instrument_id, SubscriptionType::BookDeltas),
1311                SubscriptionInfo {
1312                    cancellation_token: subscription_token.clone(),
1313                    id: subscription_id,
1314                },
1315            );
1316
1317            let guard = SubscriptionGuard {
1318                subscriptions: Arc::clone(&self.subscriptions),
1319                key: (instrument_id, SubscriptionType::BookDeltas),
1320                id: subscription_id,
1321            };
1322
1323            // Spawn subscription task
1324            let client_clone = client.as_arc().clone();
1325            let data_farm_state = Arc::clone(&self.data_farm_state);
1326            let stream_config = StreamConfig::new(self.client_id, &self.config);
1327
1328            let task = async move {
1329                if let Err(e) = handle_market_depth_subscription(
1330                    client_clone,
1331                    contract,
1332                    instrument_id,
1333                    prices,
1334                    size_precision,
1335                    depth_rows,
1336                    is_smart_depth,
1337                    data_sender,
1338                    clock,
1339                    subscription_token,
1340                    data_farm_state,
1341                    stream_config,
1342                )
1343                .await
1344                {
1345                    tracing::error!(
1346                        "Market depth subscription error for {}: {:?}",
1347                        instrument_id,
1348                        e
1349                    );
1350                }
1351
1352                guard.release();
1353            };
1354
1355            self.session_tasks
1356                .spawn(task)
1357                .context("failed to register IB data subscription task")?;
1358
1359            tracing::debug!("Market depth subscription started for {}", instrument_id);
1360            Ok(())
1361        })
1362    }
1363
1364    fn unsubscribe_book_deltas(&mut self, cmd: &UnsubscribeBookDeltas) -> anyhow::Result<()> {
1365        execute_instrument_subscriptions(&cmd.instrument_ids, |instrument_id| {
1366            tracing::debug!("Unsubscribing from book deltas for {}", instrument_id);
1367
1368            let mut subscriptions = lock(&self.subscriptions)?;
1369
1370            if let Some(sub_info) =
1371                subscriptions.remove(&(instrument_id, SubscriptionType::BookDeltas))
1372            {
1373                sub_info.cancellation_token.cancel();
1374                tracing::debug!("Unsubscribed from book deltas for {}", instrument_id);
1375            } else {
1376                tracing::warn!(
1377                    "No active book delta subscription found for {}",
1378                    instrument_id
1379                );
1380            }
1381
1382            Ok(())
1383        })
1384    }
1385
1386    // Request handlers
1387    fn request_instrument(&self, cmd: RequestInstrument) -> anyhow::Result<()> {
1388        let instrument_id = cmd.single_instrument_id()?;
1389
1390        tracing::debug!("Requesting instrument: {instrument_id}");
1391
1392        if cmd.start.is_some() {
1393            tracing::warn!(
1394                "Requesting instrument {instrument_id} with specified `start` which has no effect"
1395            );
1396        }
1397
1398        if cmd.end.is_some() {
1399            tracing::warn!(
1400                "Requesting instrument {instrument_id} with specified `end` which has no effect"
1401            );
1402        }
1403
1404        // Check if force_instrument_update is requested
1405        let force_update = cmd
1406            .params
1407            .as_ref()
1408            .and_then(|params| params.get_str("force_instrument_update"))
1409            .is_some_and(parse_bool_param_value);
1410
1411        // Get instrument from provider (or load if not found or force_update)
1412        let instrument = if force_update || self.instrument_provider.find(&instrument_id).is_none()
1413        {
1414            // Need to load instrument - spawn async task
1415            let client = self
1416                .ib_client
1417                .as_ref()
1418                .context("IB client not connected. Call connect() first")?;
1419            let instrument_provider = Arc::clone(&self.instrument_provider);
1420            let data_sender = self.data_sender.clone();
1421            let clock = self.clock;
1422            let request_id = cmd.request_id;
1423            let client_id = cmd.client_id.unwrap_or(self.client_id);
1424            let params = cmd.params.clone();
1425            let start_nanos = cmd.start.map(datetime_to_unix_nanos);
1426            let end_nanos = cmd.end.map(datetime_to_unix_nanos);
1427
1428            let client_clone = client.as_arc().clone();
1429
1430            self.spawn_command(async move {
1431                let filters = params_to_string_filters(params.as_ref());
1432
1433                if let Err(e) = load_requested_instrument(
1434                    &instrument_provider,
1435                    &client_clone,
1436                    instrument_id,
1437                    force_update,
1438                    filters,
1439                    &data_sender,
1440                )
1441                .await
1442                {
1443                    tracing::error!(
1444                        "Failed to fetch contract details for {}: {:?}",
1445                        instrument_id,
1446                        e
1447                    );
1448                    return;
1449                }
1450
1451                if let Some(instrument) = instrument_provider.find(&instrument_id) {
1452                    let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
1453                        request_id,
1454                        client_id,
1455                        instrument_id,
1456                        instrument,
1457                        start_nanos,
1458                        end_nanos,
1459                        clock.get_time_ns(),
1460                        params,
1461                    )));
1462
1463                    if let Err(e) = data_sender.send(DataEvent::Response(response)) {
1464                        tracing::error!("Failed to send instrument response: {e}");
1465                    }
1466                }
1467            });
1468
1469            // Return early, response will be sent async
1470            return Ok(());
1471        } else {
1472            // Instrument already in provider
1473            self.instrument_provider
1474                .find(&instrument_id)
1475                .context(format!("Instrument {instrument_id} not found in provider"))?
1476        };
1477
1478        let start_nanos = cmd.start.map(datetime_to_unix_nanos);
1479        let end_nanos = cmd.end.map(datetime_to_unix_nanos);
1480
1481        let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
1482            cmd.request_id,
1483            cmd.client_id.unwrap_or(self.client_id),
1484            instrument_id,
1485            instrument,
1486            start_nanos,
1487            end_nanos,
1488            self.clock.get_time_ns(),
1489            cmd.params,
1490        )));
1491
1492        if let Err(e) = self.data_sender.send(DataEvent::Response(response)) {
1493            tracing::error!("Failed to send instrument response: {e}");
1494        }
1495
1496        Ok(())
1497    }
1498
1499    fn request_instruments(&self, cmd: RequestInstruments) -> anyhow::Result<()> {
1500        tracing::debug!("Requesting all instruments for venue: {:?}", cmd.venue);
1501
1502        let client = self
1503            .ib_client
1504            .as_ref()
1505            .context("IB client not connected. Call connect() first")?;
1506
1507        // Check for force_instrument_update
1508        let force_update = cmd
1509            .params
1510            .as_ref()
1511            .and_then(|params| params.get_str("force_instrument_update"))
1512            .is_some_and(parse_bool_param_value);
1513
1514        // Check if ib_contracts parameter is provided for batch loading
1515        let mut contract_specs_to_load: Vec<serde_json::Value> = Vec::new();
1516
1517        if let Some(params) = &cmd.params
1518            && let Some(ib_contracts_value) = params.get("ib_contracts")
1519        {
1520            match ib_contracts_value {
1521                serde_json::Value::Array(contract_specs) => {
1522                    tracing::debug!(
1523                        "Parsed {} structured contract specs from ib_contracts",
1524                        contract_specs.len()
1525                    );
1526                    contract_specs_to_load = contract_specs.clone();
1527                }
1528                serde_json::Value::String(ib_contracts_json_str) => {
1529                    match serde_json::from_str::<serde_json::Value>(ib_contracts_json_str) {
1530                        Ok(serde_json::Value::Array(contract_specs)) => {
1531                            tracing::debug!(
1532                                "Parsed {} contract specs from ib_contracts JSON",
1533                                contract_specs.len()
1534                            );
1535                            log::debug!("Parsed ib_contracts payload: {ib_contracts_json_str}");
1536                            contract_specs_to_load = contract_specs;
1537                        }
1538                        Ok(value) => {
1539                            tracing::warn!(
1540                                "Expected ib_contracts JSON array, received {}. Continuing without contracts",
1541                                value
1542                            );
1543                        }
1544                        Err(e) => {
1545                            tracing::warn!(
1546                                "Failed to parse ib_contracts JSON: {}. Continuing without contracts",
1547                                e
1548                            );
1549                        }
1550                    }
1551                }
1552                value => {
1553                    tracing::warn!(
1554                        "Expected ib_contracts array or JSON string, received {}. Continuing without contracts",
1555                        value
1556                    );
1557                }
1558            }
1559        }
1560
1561        // If force_update is requested or we need to batch load, spawn async task
1562        let instrument_provider = Arc::clone(&self.instrument_provider);
1563        let client_clone = client.as_arc().clone();
1564        let data_sender = self.data_sender.clone();
1565        let clock = self.clock;
1566        let request_id = cmd.request_id;
1567        let client_id = cmd.client_id.unwrap_or(self.client_id);
1568        let venue = cmd.venue.unwrap_or(*IB_VENUE);
1569        let params = cmd.params.clone();
1570        let start_nanos = cmd.start.map(datetime_to_unix_nanos);
1571        let end_nanos = cmd.end.map(datetime_to_unix_nanos);
1572
1573        // Handle batch loading if contracts are provided or force_update is requested
1574        if !contract_specs_to_load.is_empty() || force_update {
1575            let contract_specs_to_load_clone = contract_specs_to_load;
1576            let return_loaded_only = !contract_specs_to_load_clone.is_empty();
1577
1578            self.spawn_command(async move {
1579                let mut loaded_instrument_ids = Vec::new();
1580
1581                // Load instruments from contracts if provided
1582                if !contract_specs_to_load_clone.is_empty() {
1583                    for contract_spec in contract_specs_to_load_clone {
1584                        let contract =
1585                            match parse_contract_from_json(&contract_spec)
1586                            {
1587                                Ok(contract) => contract,
1588                                Err(e) => {
1589                                    tracing::warn!(
1590                                        "Failed to parse IB contract spec {:?}: {}",
1591                                        contract_spec,
1592                                        e
1593                                    );
1594                                    continue;
1595                                }
1596                            };
1597
1598                        log::debug!(
1599                            "Loading instrument from IB contract spec (sec_type={:?}, symbol={}, local_symbol={}, exchange={}, expiry={})",
1600                            contract.security_type,
1601                            contract.symbol.as_str(),
1602                            contract.local_symbol.as_str(),
1603                            contract.exchange.as_str(),
1604                            contract.last_trade_date_or_contract_month.as_str()
1605                        );
1606
1607                        match instrument_provider
1608                            .load_contract_spec(&client_clone, &contract, Some(&contract_spec))
1609                            .await
1610                        {
1611                            Ok(mut instrument_ids) => {
1612                                loaded_instrument_ids.append(&mut instrument_ids);
1613                            }
1614                            Err(e) => {
1615                                tracing::warn!(
1616                                    "Failed to load IB contract spec {:?}: {}",
1617                                    contract_spec,
1618                                    e
1619                                );
1620                            }
1621                        }
1622                    }
1623                }
1624
1625                // If force_update, also reload all existing instruments
1626                if force_update && !return_loaded_only {
1627                    let all_instrument_ids: Vec<InstrumentId> = instrument_provider
1628                        .get_all()
1629                        .into_iter()
1630                        .map(|inst| inst.id())
1631                        .collect();
1632
1633                    if !all_instrument_ids.is_empty()
1634                        && let Ok(mut reloaded_ids) = instrument_provider
1635                            .batch_load(&client_clone, all_instrument_ids, None)
1636                            .await
1637                    {
1638                        loaded_instrument_ids.append(&mut reloaded_ids);
1639                    }
1640                }
1641
1642                let instruments = if return_loaded_only {
1643                    instrument_provider.find_all(&loaded_instrument_ids)
1644                } else {
1645                    Vec::new()
1646                };
1647
1648                let instruments_count = instruments.len();
1649
1650                let response = DataResponse::Instruments(InstrumentsResponse::new(
1651                    request_id,
1652                    client_id,
1653                    venue,
1654                    instruments,
1655                    start_nanos,
1656                    end_nanos,
1657                    clock.get_time_ns(),
1658                    params,
1659                ));
1660
1661                if let Err(e) = data_sender.send(DataEvent::Response(response)) {
1662                    tracing::error!("Failed to send instruments response: {e}");
1663                } else {
1664                    tracing::debug!(
1665                        "Successfully sent {} instruments response (loaded {} new instruments)",
1666                        instruments_count,
1667                        loaded_instrument_ids.len()
1668                    );
1669                }
1670            });
1671        } else {
1672            let response = DataResponse::Instruments(InstrumentsResponse::new(
1673                cmd.request_id,
1674                cmd.client_id.unwrap_or(self.client_id),
1675                venue,
1676                Vec::new(),
1677                start_nanos,
1678                end_nanos,
1679                self.clock.get_time_ns(),
1680                cmd.params,
1681            ));
1682
1683            if let Err(e) = self.data_sender.send(DataEvent::Response(response)) {
1684                tracing::error!("Failed to send instruments response: {e}");
1685            } else {
1686                tracing::debug!("Successfully sent empty instruments response");
1687            }
1688        }
1689
1690        Ok(())
1691    }
1692
1693    fn request_quotes(&self, cmd: RequestQuotes) -> anyhow::Result<()> {
1694        let instrument_id = cmd.single_instrument_id()?;
1695
1696        tracing::debug!("Requesting quotes for {instrument_id}");
1697
1698        let client = self
1699            .ib_client
1700            .as_ref()
1701            .context("IB client not connected. Call connect() first")?;
1702
1703        // Get instrument from provider
1704        let instrument = self
1705            .instrument_provider
1706            .find(&instrument_id)
1707            .context(format!("Instrument {instrument_id} not found in provider"))?;
1708
1709        let prices = IbPriceConverter::for_instrument(
1710            &instrument,
1711            self.instrument_provider.get_price_magnifier(&instrument_id),
1712        );
1713        let size_precision = instrument.size_precision();
1714
1715        // Convert instrument_id to IB contract
1716        let contract = self
1717            .instrument_provider
1718            .resolve_contract_for_instrument(instrument_id)
1719            .context("Failed to convert instrument_id to IB contract")?;
1720
1721        let limit = cmd
1722            .limit
1723            .map(|limit| limit.get())
1724            .or_else(|| cmd.start.is_none().then_some(HISTORICAL_TICK_DEFAULT_LIMIT));
1725        let number_of_ticks = limit.unwrap_or(1000).min(1000) as i32;
1726
1727        let data_sender = self.data_sender.clone();
1728        let clock = self.clock;
1729        let request_id = cmd.request_id;
1730        let client_id = cmd.client_id.unwrap_or(self.client_id);
1731        let params = cmd.params.clone();
1732        let start_nanos = cmd.start.map(datetime_to_unix_nanos);
1733        let end_nanos = cmd.end.map(datetime_to_unix_nanos);
1734
1735        // Spawn async task to handle the request with pagination
1736        let client_clone = client.as_arc().clone();
1737        let start_nanos_clone = start_nanos;
1738        let end_nanos_clone = end_nanos;
1739        let cmd_start = cmd.start;
1740        let cmd_end = cmd.end;
1741        let trading_hours = request_trading_hours(self.config.use_regular_trading_hours);
1742
1743        self.spawn_command(async move {
1744            let mut all_quotes = Vec::new();
1745            // Work backwards from end_date, updating end to the earliest tick received
1746            let mut current_end_date = Some(historical_tick_request_end(
1747                cmd_end.unwrap_or_else(jiff::Timestamp::now),
1748            ));
1749
1750            let current_start_date = cmd_start;
1751
1752            loop {
1753                if !should_continue_historical_tick_pagination(
1754                    current_start_date,
1755                    current_end_date,
1756                    all_quotes.len(),
1757                    limit,
1758                ) {
1759                    break;
1760                }
1761
1762                let current_end_ib = current_end_date.as_ref().map(jiff_to_ib_datetime);
1763
1764                // Make request for this batch
1765                let mut builder = client_clone
1766                    .historical_ticks(&contract, number_of_ticks)
1767                    .trading_hours(trading_hours);
1768
1769                // IB accepts start or end on reqHistoricalTicks, not both; backward
1770                // pagination anchors on the end and filters the start client-side.
1771                if let Some(end) = current_end_ib {
1772                    builder = builder.ending(end);
1773                }
1774
1775                match builder.bid_ask(IgnoreSize::No).await {
1776                    Ok(subscription) => {
1777                        let mut subscription = subscription.filter_data();
1778                        let mut batch_quotes = Vec::new();
1779
1780                        while let Some(tick_result) = subscription.next().await {
1781                            let tick = match tick_result {
1782                                Ok(tick) => tick,
1783                                Err(e) => {
1784                                    tracing::warn!("Historical quote ticks stream error: {e:?}");
1785                                    continue;
1786                                }
1787                            };
1788
1789                            let ts_event =
1790                                super::convert::ib_timestamp_to_unix_nanos(&tick.timestamp);
1791                            let ts_init = ts_event;
1792
1793                            match super::parse::parse_quote_tick(
1794                                instrument_id,
1795                                Some(tick.price_bid),
1796                                Some(tick.price_ask),
1797                                tick.size_bid,
1798                                tick.size_ask,
1799                                prices,
1800                                size_precision,
1801                                ts_event,
1802                                ts_init,
1803                            ) {
1804                                Ok(quote_tick) => batch_quotes.push(quote_tick),
1805                                Err(e) => {
1806                                    tracing::warn!("Failed to parse quote tick: {:?}", e);
1807                                }
1808                            }
1809                        }
1810
1811                        if !extend_historical_tick_batch(
1812                            &mut all_quotes,
1813                            batch_quotes,
1814                            current_start_date,
1815                            &mut current_end_date,
1816                            start_nanos_clone,
1817                            end_nanos_clone,
1818                            limit,
1819                            |quote| quote.ts_event,
1820                        ) {
1821                            break;
1822                        }
1823                    }
1824                    Err(e) => {
1825                        tracing::error!(
1826                            "Historical quotes request failed for {}: {:?}",
1827                            instrument_id,
1828                            e
1829                        );
1830                        break;
1831                    }
1832                }
1833            }
1834
1835            retain_historical_ticks_in_range(
1836                &mut all_quotes,
1837                start_nanos_clone,
1838                end_nanos_clone,
1839                |quote| quote.ts_event,
1840            );
1841
1842            all_quotes.sort_by_key(|q| q.ts_event);
1843            if let Some(limit) = limit
1844                && all_quotes.len() > limit
1845            {
1846                all_quotes = all_quotes.split_off(all_quotes.len() - limit);
1847            }
1848
1849            let quotes_count = all_quotes.len();
1850            let response = DataResponse::Quotes(QuotesResponse::new(
1851                request_id,
1852                client_id,
1853                instrument_id,
1854                all_quotes,
1855                start_nanos_clone,
1856                end_nanos_clone,
1857                clock.get_time_ns(),
1858                params,
1859            ));
1860
1861            if let Err(e) = data_sender.send(DataEvent::Response(response)) {
1862                tracing::error!("Failed to send quotes response: {e}");
1863            } else {
1864                tracing::debug!(
1865                    "Successfully sent {} quotes for {}",
1866                    quotes_count,
1867                    instrument_id
1868                );
1869            }
1870        });
1871
1872        Ok(())
1873    }
1874
1875    fn request_trades(&self, cmd: RequestTrades) -> anyhow::Result<()> {
1876        let instrument_id = cmd.single_instrument_id()?;
1877
1878        tracing::debug!("Requesting trades for {instrument_id}");
1879
1880        let client = self
1881            .ib_client
1882            .as_ref()
1883            .context("IB client not connected. Call connect() first")?;
1884
1885        // Get instrument from provider
1886        let instrument = self
1887            .instrument_provider
1888            .find(&instrument_id)
1889            .context(format!("Instrument {instrument_id} not found in provider"))?;
1890
1891        // Check if instrument is a CurrencyPair (IB doesn't support trades for CurrencyPair)
1892        if matches!(instrument, InstrumentAny::CurrencyPair(_)) {
1893            tracing::error!(
1894                "Interactive Brokers does not support trades for CurrencyPair instruments: {instrument_id}"
1895            );
1896            return Ok(());
1897        }
1898
1899        let prices = IbPriceConverter::for_instrument(
1900            &instrument,
1901            self.instrument_provider.get_price_magnifier(&instrument_id),
1902        );
1903        let size_precision = instrument.size_precision();
1904
1905        // Convert instrument_id to IB contract
1906        let contract = self
1907            .instrument_provider
1908            .resolve_contract_for_instrument(instrument_id)
1909            .context("Failed to convert instrument_id to IB contract")?;
1910
1911        let limit = cmd
1912            .limit
1913            .map(|limit| limit.get())
1914            .or_else(|| cmd.start.is_none().then_some(HISTORICAL_TICK_DEFAULT_LIMIT));
1915        let number_of_ticks = limit.unwrap_or(1000).min(1000) as i32;
1916
1917        let data_sender = self.data_sender.clone();
1918        let clock = self.clock;
1919        let request_id = cmd.request_id;
1920        let client_id = cmd.client_id.unwrap_or(self.client_id);
1921        let params = cmd.params.clone();
1922        let start_nanos = cmd.start.map(datetime_to_unix_nanos);
1923        let end_nanos = cmd.end.map(datetime_to_unix_nanos);
1924
1925        // Spawn async task to handle the request with pagination
1926        let client_clone = client.as_arc().clone();
1927        let start_nanos_clone = start_nanos;
1928        let end_nanos_clone = end_nanos;
1929        let cmd_start = cmd.start;
1930        let cmd_end = cmd.end;
1931        let trading_hours = request_trading_hours(self.config.use_regular_trading_hours);
1932
1933        self.spawn_command(async move {
1934            let mut all_trades = Vec::new();
1935            // Work backwards from end_date, updating end to the earliest tick received
1936            let mut current_end_date = Some(historical_tick_request_end(
1937                cmd_end.unwrap_or_else(jiff::Timestamp::now),
1938            ));
1939
1940            let current_start_date = cmd_start;
1941
1942            loop {
1943                if !should_continue_historical_tick_pagination(
1944                    current_start_date,
1945                    current_end_date,
1946                    all_trades.len(),
1947                    limit,
1948                ) {
1949                    break;
1950                }
1951
1952                let current_end_ib = current_end_date.as_ref().map(jiff_to_ib_datetime);
1953
1954                // Make request for this batch
1955                let mut builder = client_clone
1956                    .historical_ticks(&contract, number_of_ticks)
1957                    .trading_hours(trading_hours);
1958
1959                // IB accepts start or end on reqHistoricalTicks, not both; backward
1960                // pagination anchors on the end and filters the start client-side.
1961                if let Some(end) = current_end_ib {
1962                    builder = builder.ending(end);
1963                }
1964
1965                match builder.trade().await {
1966                    Ok(subscription) => {
1967                        let mut subscription = subscription.filter_data();
1968                        let mut batch_trades = Vec::new();
1969
1970                        while let Some(tick_result) = subscription.next().await {
1971                            let tick = match tick_result {
1972                                Ok(tick) => tick,
1973                                Err(e) => {
1974                                    tracing::warn!("Historical trade ticks stream error: {e:?}");
1975                                    continue;
1976                                }
1977                            };
1978
1979                            let ts_event =
1980                                super::convert::ib_timestamp_to_unix_nanos(&tick.timestamp);
1981                            let ts_init = ts_event;
1982
1983                            // Generate trade ID from exchange and special conditions if available
1984                            let trade_id = None;
1985
1986                            let Some(size) = tick.size else {
1987                                tracing::warn!(
1988                                    "Skipping historical trade tick with no size for {}",
1989                                    instrument_id
1990                                );
1991                                continue;
1992                            };
1993
1994                            match super::parse::parse_trade_tick(
1995                                instrument_id,
1996                                tick.price,
1997                                size,
1998                                prices,
1999                                size_precision,
2000                                ts_event,
2001                                ts_init,
2002                                trade_id,
2003                            ) {
2004                                Ok(trade_tick) => batch_trades.push(trade_tick),
2005                                Err(e) => {
2006                                    log_tick_parse_error("trade", instrument_id, &e);
2007                                }
2008                            }
2009                        }
2010
2011                        if !extend_historical_tick_batch(
2012                            &mut all_trades,
2013                            batch_trades,
2014                            current_start_date,
2015                            &mut current_end_date,
2016                            start_nanos_clone,
2017                            end_nanos_clone,
2018                            limit,
2019                            |trade| trade.ts_event,
2020                        ) {
2021                            break;
2022                        }
2023                    }
2024                    Err(e) => {
2025                        tracing::error!(
2026                            "Historical trades request failed for {}: {:?}",
2027                            instrument_id,
2028                            e
2029                        );
2030                        break;
2031                    }
2032                }
2033            }
2034
2035            retain_historical_ticks_in_range(
2036                &mut all_trades,
2037                start_nanos_clone,
2038                end_nanos_clone,
2039                |trade| trade.ts_event,
2040            );
2041
2042            all_trades.sort_by_key(|t| t.ts_event);
2043            if let Some(limit) = limit
2044                && all_trades.len() > limit
2045            {
2046                all_trades = all_trades.split_off(all_trades.len() - limit);
2047            }
2048
2049            let trades_count = all_trades.len();
2050            let response = DataResponse::Trades(TradesResponse::new(
2051                request_id,
2052                client_id,
2053                instrument_id,
2054                all_trades,
2055                start_nanos_clone,
2056                end_nanos_clone,
2057                clock.get_time_ns(),
2058                params,
2059            ));
2060
2061            if let Err(e) = data_sender.send(DataEvent::Response(response)) {
2062                tracing::error!("Failed to send trades response: {e}");
2063            } else {
2064                tracing::debug!(
2065                    "Successfully sent {} trades for {}",
2066                    trades_count,
2067                    instrument_id
2068                );
2069            }
2070        });
2071
2072        Ok(())
2073    }
2074
2075    fn request_bars(&self, cmd: RequestBars) -> anyhow::Result<()> {
2076        let bar_type = cmd.single_bar_type()?;
2077
2078        tracing::debug!("Requesting bars for {bar_type}");
2079
2080        // Validate bar spec (only time-aggregated bars are supported)
2081        if !bar_type.spec().is_time_aggregated() {
2082            tracing::error!(
2083                "Cannot request {bar_type} bars: only time bars are aggregated by Interactive Brokers"
2084            );
2085            return Ok(());
2086        }
2087
2088        let client = self
2089            .ib_client
2090            .as_ref()
2091            .context("IB client not connected. Call connect() first")?;
2092
2093        // Get instrument from provider
2094        let instrument_id = bar_type.instrument_id();
2095        let instrument = self
2096            .instrument_provider
2097            .find(&instrument_id)
2098            .context(format!("Instrument {instrument_id} not found in provider"))?;
2099
2100        let prices = IbPriceConverter::for_instrument(
2101            &instrument,
2102            self.instrument_provider.get_price_magnifier(&instrument_id),
2103        );
2104        let size_precision = instrument.size_precision();
2105
2106        // Convert instrument_id to IB contract
2107        let contract = self
2108            .instrument_provider
2109            .resolve_contract_for_instrument(instrument_id)
2110            .context("Failed to convert instrument_id to IB contract")?;
2111
2112        // Convert bar type to IB formats
2113        let ib_bar_size = bar_type_to_ib_bar_size(&bar_type)
2114            .context("Failed to convert bar type to IB bar size")?;
2115        // Crypto trade-price bars require AGGTRADES (TWS rejects TRADES for crypto,
2116        // error 10299); mirror the Java engine's whatToShowFor rule.
2117        let is_crypto = is_crypto_contract(&contract);
2118        let ib_what_to_show =
2119            price_type_to_ib_what_to_show_for_security(bar_type.spec().price_type, is_crypto);
2120
2121        // Omit the end date for continuous futures (IB error 10339).
2122        let is_continuous_future = contract.security_type == SecurityType::ContinuousFuture;
2123        let range = bar_request_range(cmd.start, cmd.end, None, is_continuous_future);
2124
2125        let data_sender = self.data_sender.clone();
2126        let clock = self.clock;
2127        let request_id = cmd.request_id;
2128        let client_id = cmd.client_id.unwrap_or(self.client_id);
2129        let params = cmd.params.clone();
2130        let start_nanos = cmd.start.map(datetime_to_unix_nanos);
2131        let end_nanos = cmd.end.map(datetime_to_unix_nanos);
2132        let limit = cmd.limit.map(|limit| limit.get());
2133
2134        let client_clone = client.as_arc().clone();
2135        let trading_hours = request_trading_hours(self.config.use_regular_trading_hours);
2136
2137        self.spawn_command(async move {
2138            let mut all_bars = Vec::new();
2139
2140            if let Some(range) = range {
2141                for bar_request in range.requests(&bar_type.spec()) {
2142                    let request = bar_request
2143                        .apply(client_clone.historical_data(&contract, ib_bar_size))
2144                        .what_to_show(ib_what_to_show)
2145                        .trading_hours(trading_hours);
2146
2147                    let historical_data = match request.fetch().await {
2148                        Ok(historical_data) => historical_data,
2149                        Err(e) => {
2150                            tracing::warn!(
2151                                "Historical data request failed for {bar_type} over {bar_request:?}: {e}"
2152                            );
2153                            all_bars.clear();
2154                            break;
2155                        }
2156                    };
2157
2158                    // Convert IB bars to Nautilus bars
2159                    for ib_bar in &historical_data.bars {
2160                        match ib_bar_to_nautilus_bar(ib_bar, bar_type, prices, size_precision) {
2161                            Ok(bar) => {
2162                                if range.contains(bar.ts_event) {
2163                                    all_bars.push(bar);
2164                                }
2165                            }
2166                            Err(e) => {
2167                                tracing::warn!(
2168                                    "Failed to convert IB bar to Nautilus bar: {:?}",
2169                                    e
2170                                );
2171                            }
2172                        }
2173                    }
2174                }
2175            }
2176
2177            // Return aggregated results
2178            if all_bars.is_empty() {
2179                tracing::warn!("No bar data received for {}", bar_type);
2180            }
2181
2182            all_bars.sort_by_key(|b| b.ts_event);
2183            all_bars.dedup();
2184            if let Some(limit) = limit
2185                && all_bars.len() > limit
2186            {
2187                all_bars = all_bars.split_off(all_bars.len() - limit);
2188            }
2189
2190            let bars_count = all_bars.len();
2191
2192            let response = DataResponse::Bars(BarsResponse::new(
2193                request_id,
2194                client_id,
2195                bar_type,
2196                all_bars,
2197                start_nanos,
2198                end_nanos,
2199                clock.get_time_ns(),
2200                params,
2201            ));
2202
2203            if let Err(e) = data_sender.send(DataEvent::Response(response)) {
2204                tracing::error!("Failed to send bars response: {e}");
2205            } else {
2206                tracing::debug!("Successfully sent {bars_count} bars for {bar_type}");
2207            }
2208        });
2209
2210        Ok(())
2211    }
2212}
2213
2214impl InteractiveBrokersDataClient {
2215    fn has_subscription(
2216        &self,
2217        instrument_id: InstrumentId,
2218        subscription_type: SubscriptionType,
2219    ) -> anyhow::Result<bool> {
2220        let subscriptions = lock(&self.subscriptions)?;
2221        Ok(subscriptions.contains_key(&(instrument_id, subscription_type)))
2222    }
2223
2224    fn clear_bar_tracking_state(&self) {
2225        if let Ok(mut tasks) = self.bar_timeout_tasks.try_lock() {
2226            for task in tasks.values_mut() {
2227                task.abort();
2228            }
2229        } else {
2230            tracing::warn!("Failed to lock IB bar timeout tasks for cleanup");
2231        }
2232
2233        if let Ok(mut last_bars) = self.last_bars.try_lock() {
2234            last_bars.clear();
2235        } else {
2236            tracing::warn!("Failed to lock IB last bars for cleanup");
2237        }
2238    }
2239}
2240
2241impl Drop for InteractiveBrokersDataClient {
2242    fn drop(&mut self) {
2243        let _ = self.stop();
2244    }
2245}
2246
2247impl InteractiveBrokersDataClient {
2248    fn subscribe_quotes_member(
2249        &self,
2250        instrument_id: InstrumentId,
2251        cmd: &SubscribeQuotes,
2252    ) -> anyhow::Result<()> {
2253        tracing::debug!("Subscribing to quotes for {instrument_id}");
2254
2255        if self.has_subscription(instrument_id, SubscriptionType::Quotes)? {
2256            tracing::debug!("Quote subscription already active for {instrument_id}");
2257            return Ok(());
2258        }
2259
2260        let client = self
2261            .ib_client
2262            .as_ref()
2263            .context("IB client not connected. Call connect() first")?;
2264
2265        // Get instrument from provider
2266        let instrument = self
2267            .instrument_provider
2268            .find(&instrument_id)
2269            .context(format!("Instrument {instrument_id} not found in provider"))?;
2270
2271        let prices = IbPriceConverter::for_instrument(
2272            &instrument,
2273            self.instrument_provider.get_price_magnifier(&instrument_id),
2274        );
2275        let size_precision = instrument.size_precision();
2276
2277        // Convert instrument_id to IB contract
2278        let contract = self
2279            .instrument_provider
2280            .resolve_contract_for_instrument(instrument_id)
2281            .context("Failed to convert instrument_id to IB contract")?;
2282
2283        // Check if contract is BAG (spread) or if batch_quotes parameter is set
2284        // BAG contracts have SecurityType::Spread or combo_legs populated
2285        let is_bag = matches!(
2286            contract.security_type,
2287            ibapi::contracts::SecurityType::Spread
2288        ) || !contract.combo_legs.is_empty();
2289        let batch_quotes = cmd
2290            .params
2291            .as_ref()
2292            .and_then(|params| params.get_str("batch_quotes"))
2293            .map_or(self.config.batch_quotes, parse_bool_param_value);
2294
2295        let use_market_data = is_bag || batch_quotes;
2296
2297        let data_sender = self.data_sender.clone();
2298        let quote_cache = Arc::clone(&self.quote_cache);
2299        let clock = self.clock;
2300
2301        let subscription_token = self.cancellation_token.child_token();
2302
2303        // Record the subscription before spawning so a fast-failing task cannot
2304        // finish before its registry entry exists.
2305        let subscription_id = next_subscription_id();
2306        lock(&self.subscriptions)?.insert(
2307            (instrument_id, SubscriptionType::Quotes),
2308            SubscriptionInfo {
2309                cancellation_token: subscription_token.clone(),
2310                id: subscription_id,
2311            },
2312        );
2313
2314        let guard = SubscriptionGuard {
2315            subscriptions: Arc::clone(&self.subscriptions),
2316            key: (instrument_id, SubscriptionType::Quotes),
2317            id: subscription_id,
2318        };
2319
2320        // Spawn subscription task
2321        let client_clone = client.as_arc().clone();
2322        let ignore_size_updates = self.config.ignore_quote_tick_size_updates;
2323        let data_farm_state = Arc::clone(&self.data_farm_state);
2324        let stream_config = StreamConfig::new(self.client_id, &self.config);
2325
2326        let task = async move {
2327            if use_market_data {
2328                // Use market_data (reqMktData) for BAG contracts or when batch_quotes is requested
2329                tracing::debug!(
2330                    "Using market_data subscription for {} (BAG: {}, batch_quotes: {})",
2331                    instrument_id,
2332                    is_bag,
2333                    batch_quotes
2334                );
2335
2336                if let Err(e) = handle_quote_subscription(
2337                    client_clone,
2338                    contract,
2339                    instrument_id,
2340                    prices,
2341                    size_precision,
2342                    data_sender,
2343                    quote_cache,
2344                    clock,
2345                    subscription_token,
2346                    ignore_size_updates,
2347                    Arc::clone(&data_farm_state),
2348                    stream_config,
2349                )
2350                .await
2351                {
2352                    tracing::error!("Quote subscription error for {}: {:?}", instrument_id, e);
2353                }
2354            } else {
2355                // Try tick_by_tick_bid_ask first for regular contracts (better performance)
2356                // Fallback to market_data if it fails (e.g., for BAG contracts not detected upfront)
2357                tracing::debug!(
2358                    "Attempting tick_by_tick_bid_ask subscription for {}",
2359                    instrument_id
2360                );
2361
2362                match handle_tick_by_tick_quote_subscription(
2363                    client_clone.clone(),
2364                    contract.clone(),
2365                    instrument_id,
2366                    prices,
2367                    size_precision,
2368                    data_sender.clone(),
2369                    clock,
2370                    subscription_token.clone(),
2371                    Arc::clone(&data_farm_state),
2372                    stream_config,
2373                )
2374                .await
2375                {
2376                    Ok(()) => {
2377                        // Success - subscription is active
2378                    }
2379                    Err(e) => {
2380                        tracing::warn!(
2381                            "tick_by_tick_bid_ask failed for {} (may be BAG contract), falling back to market_data: {:?}",
2382                            instrument_id,
2383                            e
2384                        );
2385
2386                        // Fallback to market_data (reqMktData) - works for BAG contracts
2387                        if let Err(fallback_err) = handle_quote_subscription(
2388                            client_clone,
2389                            contract,
2390                            instrument_id,
2391                            prices,
2392                            size_precision,
2393                            data_sender,
2394                            quote_cache,
2395                            clock,
2396                            subscription_token,
2397                            ignore_size_updates,
2398                            Arc::clone(&data_farm_state),
2399                            stream_config,
2400                        )
2401                        .await
2402                        {
2403                            tracing::error!(
2404                                "Quote subscription fallback also failed for {}: {:?}",
2405                                instrument_id,
2406                                fallback_err
2407                            );
2408                        } else {
2409                            tracing::debug!(
2410                                "Successfully subscribed to {} using market_data fallback",
2411                                instrument_id
2412                            );
2413                        }
2414                    }
2415                }
2416            }
2417
2418            guard.release();
2419        };
2420
2421        self.session_tasks
2422            .spawn(task)
2423            .context("failed to register IB data subscription task")?;
2424
2425        tracing::debug!(
2426            "Quote subscription started for {instrument_id} (method: {})",
2427            if use_market_data {
2428                "market_data"
2429            } else {
2430                "tick_by_tick_bid_ask"
2431            }
2432        );
2433        Ok(())
2434    }
2435}
2436
2437// Loads a requested instrument into the provider. A generic spread is built as an IB BAG from its
2438// qualified legs, as `load_ids` does, and its legs are published first because spread fills update
2439// the leg positions.
2440async fn load_requested_instrument(
2441    instrument_provider: &InteractiveBrokersInstrumentProvider,
2442    client: &ibapi::Client,
2443    instrument_id: InstrumentId,
2444    force_update: bool,
2445    filters: Option<HashMap<String, String>>,
2446    data_sender: &EventSender<DataEvent>,
2447) -> anyhow::Result<()> {
2448    if !is_spread_instrument_id(&instrument_id) {
2449        return instrument_provider
2450            .fetch_contract_details(client, instrument_id, force_update, filters)
2451            .await;
2452    }
2453
2454    instrument_provider
2455        .fetch_spread_instrument(client, instrument_id, force_update, filters)
2456        .await?;
2457    let leg_ids = parse_spread_instrument_id_to_legs(&instrument_id)?
2458        .into_iter()
2459        .map(|(leg_id, _)| leg_id)
2460        .collect::<Vec<_>>();
2461
2462    for leg in instrument_provider.find_all(&leg_ids) {
2463        if let Err(e) = data_sender.send(DataEvent::Data(leg.into())) {
2464            tracing::error!("Failed to send spread leg instrument: {e}");
2465        }
2466    }
2467
2468    Ok(())
2469}
2470
2471#[cfg(test)]
2472mod tests {
2473    use nautilus_common::live::get_runtime;
2474    use rstest::rstest;
2475
2476    use super::*;
2477    use crate::common::consts::IB_CLIENT_ID;
2478
2479    #[rstest]
2480    #[case(true, ibapi::market_data::TradingHours::Regular)]
2481    #[case(false, ibapi::market_data::TradingHours::Extended)]
2482    fn test_request_trading_hours_uses_config(
2483        #[case] use_regular_trading_hours: bool,
2484        #[case] expected: ibapi::market_data::TradingHours,
2485    ) {
2486        assert_eq!(request_trading_hours(use_regular_trading_hours), expected);
2487    }
2488
2489    #[rstest]
2490    #[case("true", true)]
2491    #[case("True", true)]
2492    #[case("1", true)]
2493    #[case("false", false)]
2494    #[case("False", false)]
2495    #[case("0", false)]
2496    fn test_parse_bool_param_value(#[case] value: &str, #[case] expected: bool) {
2497        assert_eq!(parse_bool_param_value(value), expected);
2498    }
2499
2500    #[rstest]
2501    fn test_datetime_to_unix_nanos() {
2502        let dt = jiff::Timestamp::new(1, 2).unwrap();
2503
2504        assert_eq!(datetime_to_unix_nanos(dt), UnixNanos::from(1_000_000_002));
2505    }
2506
2507    #[rstest]
2508    fn test_subscription_keys_keep_quote_and_trade_streams_independent() {
2509        let instrument_id = InstrumentId::from("AAPL.NASDAQ");
2510        let quote_token = CancellationToken::new();
2511        let trade_token = CancellationToken::new();
2512        let mut subscriptions = AHashMap::new();
2513        subscriptions.insert(
2514            (instrument_id, SubscriptionType::Quotes),
2515            SubscriptionInfo {
2516                cancellation_token: quote_token.clone(),
2517                id: next_subscription_id(),
2518            },
2519        );
2520
2521        subscriptions.insert(
2522            (instrument_id, SubscriptionType::Trades),
2523            SubscriptionInfo {
2524                cancellation_token: trade_token.clone(),
2525                id: next_subscription_id(),
2526            },
2527        );
2528
2529        let quote = subscriptions
2530            .remove(&(instrument_id, SubscriptionType::Quotes))
2531            .unwrap();
2532        quote.cancellation_token.cancel();
2533
2534        assert!(quote_token.is_cancelled());
2535        assert!(!trade_token.is_cancelled());
2536        assert_eq!(subscriptions.len(), 1);
2537        assert!(subscriptions.contains_key(&(instrument_id, SubscriptionType::Trades)));
2538    }
2539
2540    #[rstest]
2541    fn test_subscription_keys_keep_bar_types_of_one_instrument_independent() {
2542        let minute = BarType::from("AAPL.NASDAQ-1-MINUTE-LAST-EXTERNAL");
2543        let five_minute = BarType::from("AAPL.NASDAQ-5-MINUTE-LAST-EXTERNAL");
2544        let instrument_id = minute.instrument_id();
2545        let minute_token = CancellationToken::new();
2546        let five_minute_token = CancellationToken::new();
2547        let mut subscriptions = AHashMap::new();
2548        subscriptions.insert(
2549            (instrument_id, SubscriptionType::Bars(minute)),
2550            SubscriptionInfo {
2551                cancellation_token: minute_token.clone(),
2552                id: next_subscription_id(),
2553            },
2554        );
2555
2556        subscriptions.insert(
2557            (instrument_id, SubscriptionType::Bars(five_minute)),
2558            SubscriptionInfo {
2559                cancellation_token: five_minute_token.clone(),
2560                id: next_subscription_id(),
2561            },
2562        );
2563
2564        let five = subscriptions
2565            .remove(&(instrument_id, SubscriptionType::Bars(five_minute)))
2566            .unwrap();
2567        five.cancellation_token.cancel();
2568
2569        assert!(five_minute_token.is_cancelled());
2570        assert!(!minute_token.is_cancelled());
2571        assert_eq!(subscriptions.len(), 1);
2572        assert!(subscriptions.contains_key(&(instrument_id, SubscriptionType::Bars(minute))));
2573    }
2574
2575    #[rstest]
2576    fn test_subscription_guard_release_removes_only_matching_registration() {
2577        let subscriptions = Arc::new(Mutex::new(AHashMap::new()));
2578        let instrument_id = InstrumentId::from("AAPL.NASDAQ");
2579        let key = (instrument_id, SubscriptionType::Quotes);
2580
2581        let first_id = next_subscription_id();
2582        subscriptions.lock().unwrap().insert(
2583            key,
2584            SubscriptionInfo {
2585                cancellation_token: CancellationToken::new(),
2586                id: first_id,
2587            },
2588        );
2589
2590        let stale_guard = SubscriptionGuard {
2591            subscriptions: Arc::clone(&subscriptions),
2592            key,
2593            id: first_id,
2594        };
2595
2596        // A newer registration for the same key must survive the stale guard.
2597        let second_id = next_subscription_id();
2598        subscriptions.lock().unwrap().insert(
2599            key,
2600            SubscriptionInfo {
2601                cancellation_token: CancellationToken::new(),
2602                id: second_id,
2603            },
2604        );
2605
2606        stale_guard.release();
2607        assert!(subscriptions.lock().unwrap().contains_key(&key));
2608
2609        let current_guard = SubscriptionGuard {
2610            subscriptions: Arc::clone(&subscriptions),
2611            key,
2612            id: second_id,
2613        };
2614
2615        current_guard.release();
2616        assert!(!subscriptions.lock().unwrap().contains_key(&key));
2617    }
2618
2619    #[rstest]
2620    fn test_subscription_guard_drop_removes_registration_without_release() {
2621        let subscriptions = Arc::new(Mutex::new(AHashMap::new()));
2622        let instrument_id = InstrumentId::from("AAPL.NASDAQ");
2623        let key = (instrument_id, SubscriptionType::Quotes);
2624
2625        let id = next_subscription_id();
2626        subscriptions.lock().unwrap().insert(
2627            key,
2628            SubscriptionInfo {
2629                cancellation_token: CancellationToken::new(),
2630                id,
2631            },
2632        );
2633
2634        {
2635            let _guard = SubscriptionGuard {
2636                subscriptions: Arc::clone(&subscriptions),
2637                key,
2638                id,
2639            };
2640            // Dropped without release(), as when a stream task panics.
2641        }
2642
2643        assert!(!subscriptions.lock().unwrap().contains_key(&key));
2644    }
2645
2646    #[rstest]
2647    fn test_venue_is_none_for_default_routing() {
2648        // IB is a multi-venue adapter: `venue()` must be `None` so the data engine registers
2649        // the client for default routing. A `Some(venue)` here means venue-routed subscribe
2650        // commands (e.g. `BTC/USD.ZEROHASH` bars, `AAPL.NASDAQ` quotes) never reach the client.
2651        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel();
2652        nautilus_common::live::runner::replace_data_event_sender(sender);
2653
2654        let config = InteractiveBrokersDataClientConfig::default();
2655
2656        let provider = Arc::new(
2657            InteractiveBrokersInstrumentProvider::new(config.instrument_provider.clone()).unwrap(),
2658        );
2659        let client = InteractiveBrokersDataClient::new(*IB_CLIENT_ID, config, provider).unwrap();
2660
2661        assert_eq!(client.venue(), None);
2662    }
2663
2664    #[rstest]
2665    fn test_stop_closes_tasks_until_async_generation_preparation() {
2666        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel();
2667        nautilus_common::live::runner::replace_data_event_sender(sender);
2668
2669        let config = InteractiveBrokersDataClientConfig::default();
2670
2671        let provider = Arc::new(
2672            InteractiveBrokersInstrumentProvider::new(config.instrument_provider.clone()).unwrap(),
2673        );
2674        let mut client =
2675            InteractiveBrokersDataClient::new(*IB_CLIENT_ID, config, provider).unwrap();
2676
2677        client
2678            .session_tasks
2679            .spawn(async { std::future::pending::<()>().await })
2680            .unwrap();
2681
2682        client.stop().unwrap();
2683
2684        assert!(client.cancellation_token.is_cancelled());
2685        assert!(client.cancellation_token.child_token().is_cancelled());
2686        assert!(!client.session_tasks.is_open());
2687        assert!(!client.command_tasks.is_open());
2688
2689        get_runtime()
2690            .block_on(client.prepare_task_groups())
2691            .expect("prepare replacement task generation");
2692
2693        assert!(!client.cancellation_token.is_cancelled());
2694        assert!(!client.cancellation_token.child_token().is_cancelled());
2695        assert!(client.session_tasks.is_open());
2696        assert!(client.command_tasks.is_open());
2697    }
2698
2699    #[rstest]
2700    fn test_stop_retains_bar_timeout_handle_until_async_finish() {
2701        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel();
2702        nautilus_common::live::runner::replace_data_event_sender(sender);
2703
2704        let config = InteractiveBrokersDataClientConfig::default();
2705
2706        let provider = Arc::new(
2707            InteractiveBrokersInstrumentProvider::new(config.instrument_provider.clone()).unwrap(),
2708        );
2709        let mut client =
2710            InteractiveBrokersDataClient::new(*IB_CLIENT_ID, config, provider).unwrap();
2711        let bar_type = "AAPL.NASDAQ-1-MINUTE-LAST-EXTERNAL".to_string();
2712
2713        let task = get_runtime().spawn(async {
2714            std::future::pending::<()>().await;
2715        });
2716
2717        get_runtime().block_on(async {
2718            client
2719                .bar_timeout_tasks
2720                .lock()
2721                .await
2722                .insert(bar_type.clone(), TaskSlot::from_handle(task));
2723            client.last_bars.lock().await.insert(
2724                bar_type.clone(),
2725                ibapi::market_data::realtime::Bar {
2726                    date: time::OffsetDateTime::from_unix_timestamp(1).unwrap(),
2727                    open: 1.0,
2728                    high: 1.0,
2729                    low: 1.0,
2730                    close: 1.0,
2731                    volume: 1.0,
2732                    wap: 1.0,
2733                    count: 1,
2734                },
2735            );
2736        });
2737
2738        client.stop().unwrap();
2739
2740        get_runtime().block_on(async {
2741            assert_eq!(client.bar_timeout_tasks.lock().await.len(), 1);
2742            assert!(client.last_bars.lock().await.is_empty());
2743
2744            client
2745                .prepare_task_groups()
2746                .await
2747                .expect("finish stopped task generation");
2748
2749            assert!(client.bar_timeout_tasks.lock().await.is_empty());
2750        });
2751    }
2752}