Skip to main content

nautilus_coinbase/
execution.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Live execution client implementation for the Coinbase Advanced Trade adapter.
17
18use std::{
19    collections::hash_map::Entry,
20    future::Future,
21    sync::Arc,
22    time::{Duration, Instant},
23};
24
25use ahash::{AHashMap, AHashSet};
26use anyhow::Context;
27use async_trait::async_trait;
28use nautilus_common::{
29    clients::ExecutionClient,
30    factories::OrderEventFactory,
31    live::runner::get_exec_event_sender,
32    messages::{
33        ExecutionReport,
34        execution::{
35            BatchCancelOrders, CancelAllOrders, CancelOrder, GenerateFillReports,
36            GenerateFillReportsBuilder, GenerateOrderStatusReport, GenerateOrderStatusReports,
37            GenerateOrderStatusReportsBuilder, GeneratePositionStatusReports,
38            GeneratePositionStatusReportsBuilder, ModifyOrder, QueryAccount, QueryOrder,
39            SubmitOrder, SubmitOrderList,
40        },
41    },
42    msgbus::{self, TypedHandler},
43};
44use nautilus_core::{
45    DurationNanos, Params, UUID4, UnixNanos,
46    time::{AtomicTime, get_atomic_clock_realtime},
47};
48use nautilus_live::{
49    ExecutionClientCore, ExecutionEventEmitter, SocketControl,
50    execution::failure::CommandFailure,
51    task::{TaskGroup, TaskGroupGuard},
52};
53use nautilus_model::{
54    accounts::AccountAny,
55    enums::{AccountType, OmsType, OrderStatus, OrderType, TriggerType},
56    events::{OrderDeniedReason, OrderEventAny, OrderUpdated},
57    identifiers::{
58        AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, Symbol, TradeId, TraderId,
59        Venue, VenueOrderId,
60    },
61    instruments::{Instrument, InstrumentAny},
62    orders::{Order, OrderAny},
63    reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
64    types::{AccountBalance, MarginBalance, Price, Quantity},
65};
66use nautilus_network::retry::RetryConfig;
67use parking_lot::Mutex;
68use rust_decimal::Decimal;
69use ustr::Ustr;
70
71use crate::{
72    common::{
73        consts::COINBASE_VENUE,
74        credential::CoinbaseCredential,
75        enums::{CoinbaseProductType, CoinbaseWsChannel},
76    },
77    config::CoinbaseExecutionClientConfig,
78    http::{
79        client::CoinbaseHttpClient, error::Error as CoinbaseHttpError,
80        parse::parse_ws_cfm_account_state,
81    },
82    websocket::{
83        client::CoinbaseWebSocketClient,
84        handler::{NautilusWsMessage, UserOrderUpdate},
85        messages::WsOrderUpdate,
86    },
87};
88
89// Coinbase does not publish a formal max for batch_cancel; conservative chunk
90// size mirrors the 100 used by other adapters and keeps request bodies small.
91const BATCH_CANCEL_CHUNK: usize = 100;
92
93// Coinbase spot account is ready as soon as the REST account state lands, but
94// the engine registers it asynchronously; wait up to 30s for that to happen.
95const ACCOUNT_REGISTERED_TIMEOUT_SECS: f64 = 30.0;
96
97/// Live execution client for Coinbase Advanced Trade.
98#[derive(Debug)]
99pub struct CoinbaseExecutionClient {
100    core: ExecutionClientCore,
101    clock: &'static AtomicTime,
102    config: CoinbaseExecutionClientConfig,
103    emitter: ExecutionEventEmitter,
104    http_client: CoinbaseHttpClient,
105    ws_user: CoinbaseWebSocketClient,
106    session_tasks: TaskGroup,
107    pending_tasks: TaskGroup,
108    shutdown_errors: Vec<String>,
109    instruments_cache: Arc<AHashMap<String, InstrumentAny>>,
110    fills_observed: Arc<Mutex<AHashMap<VenueOrderId, AHashSet<TradeId>>>>,
111    fills_pending: Arc<Mutex<AHashMap<VenueOrderId, OrderStatusReport>>>,
112    fills_published: Arc<tokio::sync::Mutex<AHashMap<VenueOrderId, OrderStatusReport>>>,
113    fills_start: Option<UnixNanos>,
114    lifecycle: Arc<Mutex<RecoveryLifecycle>>,
115    order_event_handler: Option<TypedHandler<OrderEventAny>>,
116    order_contexts: Arc<Mutex<AHashMap<String, OrderContext>>>,
117    // Caches REST-derived metadata for orders this client did not submit
118    // (keyed by `venue_order_id`). Populated lazily when the user-channel
119    // handler encounters an unknown order whose `OrderStatusReport` would
120    // otherwise lack `price` / `trigger_price` / `trigger_type` and panic
121    // the engine's reconstruction path. Separate from `order_contexts`
122    // because external orders may carry a `client_order_id` we never set.
123    external_order_contexts: Arc<Mutex<AHashMap<String, OrderContext>>>,
124}
125
126impl CoinbaseExecutionClient {
127    /// Creates a new [`CoinbaseExecutionClient`].
128    ///
129    /// # Errors
130    ///
131    /// Returns an error if credentials cannot be resolved or the underlying
132    /// HTTP / WebSocket client cannot be constructed.
133    pub fn new(
134        core: ExecutionClientCore,
135        config: CoinbaseExecutionClientConfig,
136    ) -> anyhow::Result<Self> {
137        let credential = CoinbaseCredential::resolve(
138            config.api_key.as_ref().map(|value| value.expose_secret()),
139            config
140                .api_secret
141                .as_ref()
142                .map(|value| value.expose_secret()),
143        )
144        .ok_or_else(|| {
145                    anyhow::anyhow!(
146                        "Coinbase credentials not available; set COINBASE_API_KEY and COINBASE_API_SECRET or pass them in the config"
147                    )
148                })?;
149        let proxy_url = config
150            .proxy_url
151            .as_ref()
152            .map(|value| value.expose_secret().to_owned());
153
154        let retry_config = RetryConfig {
155            max_retries: config.max_retries,
156            initial_delay_ms: config.retry_delay_initial_ms,
157            max_delay_ms: config.retry_delay_max_ms,
158            backoff_factor: 2.0,
159            jitter_ms: 250,
160            operation_timeout_ms: Some(60_000),
161            immediate_first: false,
162            max_elapsed_ms: Some(180_000),
163        };
164
165        let http_client = CoinbaseHttpClient::with_credentials(
166            credential.clone(),
167            config.environment,
168            config.http_timeout_secs,
169            proxy_url.clone(),
170            Some(retry_config),
171        )
172        .map_err(|e| anyhow::anyhow!("Failed to create Coinbase HTTP client: {e}"))?;
173
174        if let Some(ref url) = config.base_url_rest {
175            http_client.set_base_url(url.clone());
176        }
177
178        let ws_url = config.ws_url();
179        let ws_user = CoinbaseWebSocketClient::with_credential(
180            &ws_url,
181            credential,
182            config.transport_backend,
183            proxy_url,
184        )
185        .with_socket_control(SocketControl::new(
186            core.client_id,
187            Some(*COINBASE_VENUE),
188            "coinbase-user-streams",
189        ));
190
191        let clock = get_atomic_clock_realtime();
192        let emitter = ExecutionEventEmitter::new(
193            clock,
194            core.trader_id,
195            core.account_id,
196            core.account_type,
197            None,
198        );
199
200        let session_tasks = TaskGroup::new();
201        let pending_tasks = TaskGroup::new();
202
203        let lifecycle = RecoveryLifecycle::new(core.trader_id, core.account_id, core.account_type);
204
205        Ok(Self {
206            core,
207            clock,
208            config,
209            emitter,
210            http_client,
211            ws_user,
212            session_tasks,
213            pending_tasks,
214            shutdown_errors: Vec::new(),
215            instruments_cache: Arc::new(AHashMap::new()),
216            fills_observed: Arc::new(Mutex::new(AHashMap::new())),
217            fills_pending: Arc::new(Mutex::new(AHashMap::new())),
218            fills_published: Arc::new(tokio::sync::Mutex::new(AHashMap::new())),
219            fills_start: None,
220            lifecycle: Arc::new(Mutex::new(lifecycle)),
221            order_event_handler: None,
222            order_contexts: Arc::new(Mutex::new(AHashMap::new())),
223            external_order_contexts: Arc::new(Mutex::new(AHashMap::new())),
224        })
225    }
226
227    fn order_recovery(&self) -> UserOrderRecovery {
228        UserOrderRecovery {
229            http_client: self.http_client.clone(),
230            emitter: self.emitter.clone(),
231            instruments: Arc::clone(&self.instruments_cache),
232            observed: Arc::clone(&self.fills_observed),
233            pending: Arc::clone(&self.fills_pending),
234            published: Arc::clone(&self.fills_published),
235            order_contexts: Arc::clone(&self.order_contexts),
236            external_order_contexts: Arc::clone(&self.external_order_contexts),
237            start: self.fills_start.unwrap_or_default(),
238            lifecycle: Arc::clone(&self.lifecycle),
239        }
240    }
241
242    fn subscribe_order_events(&mut self) {
243        if self.order_event_handler.is_some() {
244            return;
245        }
246
247        let core = self.core.clone();
248        let lifecycle = Arc::clone(&self.lifecycle);
249        let emitter = self.emitter.clone();
250
251        let handler = TypedHandler::from(move |event: &OrderEventAny| {
252            if event.instrument_id().venue != core.venue
253                || matches!(event, OrderEventAny::Initialized(_))
254            {
255                return;
256            }
257
258            let order = core
259                .cache()
260                .order(&event.client_order_id())
261                .map(|order| order.clone());
262
263            if let Some(order) = order
264                && order.account_id().is_none_or(|id| id == core.account_id)
265            {
266                let mut lifecycle = lifecycle.lock();
267                lifecycle.track(order);
268                if let Err(e) = lifecycle.dispatch(&emitter) {
269                    log::warn!("Retaining Coinbase lifecycle recovery: {e}");
270                }
271            }
272        });
273
274        msgbus::subscribe_order_events("events.order.*".into(), handler.clone(), Some(10));
275        self.order_event_handler = Some(handler);
276
277        let cache = self.core.cache();
278        let mut lifecycle = self.lifecycle.lock();
279
280        for order in cache.orders_refs(Some(&self.core.venue), None, None, None, None) {
281            if order
282                .account_id()
283                .is_none_or(|id| id == self.core.account_id)
284            {
285                lifecycle.track(order.clone());
286            }
287        }
288    }
289
290    fn spawn_task<F>(&self, description: &'static str, fut: F)
291    where
292        F: Future<Output = anyhow::Result<()>> + Send + 'static,
293    {
294        let future = async move {
295            if let Err(e) = fut.await {
296                log::warn!("{description} failed: {e:?}");
297            }
298        };
299
300        if let Err(e) = self.pending_tasks.spawn(future) {
301            log::warn!("Skipping Coinbase {description} after shutdown began: {e}");
302        }
303    }
304
305    fn abort_pending_tasks(&self) {
306        self.pending_tasks.begin_shutdown();
307    }
308
309    fn abort_session_tasks(&self) {
310        self.session_tasks.begin_shutdown();
311        self.ws_user.begin_shutdown();
312    }
313
314    async fn await_pending_tasks(&self) -> anyhow::Result<()> {
315        self.pending_tasks.begin_shutdown();
316        self.pending_tasks
317            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
318            .await
319            .map_err(|e| anyhow::anyhow!("Failed to terminate Coinbase execution tasks: {e}"))?;
320        Ok(())
321    }
322
323    async fn await_session_tasks(&self) -> anyhow::Result<()> {
324        self.session_tasks.begin_shutdown();
325        self.session_tasks
326            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
327            .await
328            .map_err(|e| anyhow::anyhow!("Failed to terminate Coinbase session tasks: {e}"))?;
329        Ok(())
330    }
331
332    async fn teardown_partial_connect(&mut self) -> anyhow::Result<()> {
333        self.abort_session_tasks();
334        self.abort_pending_tasks();
335
336        if let Err(e) = self.ws_user.disconnect().await {
337            self.shutdown_errors.push(e.to_string());
338        }
339        let (session_result, pending_result) =
340            tokio::join!(self.await_session_tasks(), self.await_pending_tasks());
341        self.core.set_disconnected();
342
343        if let Err(e) = session_result {
344            self.shutdown_errors.push(e.to_string());
345        }
346
347        if let Err(e) = pending_result {
348            self.shutdown_errors.push(e.to_string());
349        }
350
351        if !self.shutdown_errors.is_empty() {
352            anyhow::bail!(std::mem::take(&mut self.shutdown_errors).join("; "));
353        }
354        Ok(())
355    }
356
357    // Returns true when the exec client was created with a Margin account,
358    // indicating it should handle CFM-backed derivatives traffic.
359    fn is_margin(&self) -> bool {
360        self.core.account_type == AccountType::Margin
361    }
362
363    // Returns true when the instrument resides in the connect-time bootstrap
364    // cache. For the Cash (spot) factory this gates spot-only traffic; for the
365    // Margin factory the cache contains CFM perp + future products.
366    fn is_instrument_cached(&self, instrument_id: &InstrumentId) -> bool {
367        self.instruments_cache
368            .contains_key(instrument_id.symbol.as_str())
369    }
370
371    async fn await_account_registered(&self, timeout_secs: f64) -> anyhow::Result<()> {
372        let account_id = self.core.account_id;
373
374        if self.core.cache().account(&account_id).is_some() {
375            log::info!("Account {account_id} registered");
376            return Ok(());
377        }
378
379        let start = Instant::now();
380        let timeout = Duration::from_secs_f64(timeout_secs);
381        let interval = Duration::from_millis(10);
382
383        loop {
384            tokio::time::sleep(interval).await;
385
386            if self.core.cache().account(&account_id).is_some() {
387                log::info!("Account {account_id} registered");
388                return Ok(());
389            }
390
391            if start.elapsed() >= timeout {
392                anyhow::bail!(
393                    "Timeout waiting for account {account_id} to be registered after {timeout_secs}s"
394                );
395            }
396        }
397    }
398}
399
400#[async_trait(?Send)]
401impl ExecutionClient for CoinbaseExecutionClient {
402    fn is_connected(&self) -> bool {
403        self.core.is_connected()
404    }
405
406    fn client_id(&self) -> ClientId {
407        self.core.client_id
408    }
409
410    fn account_id(&self) -> AccountId {
411        self.core.account_id
412    }
413
414    fn venue(&self) -> Venue {
415        *COINBASE_VENUE
416    }
417
418    fn oms_type(&self) -> OmsType {
419        self.core.oms_type
420    }
421
422    fn provides_bulk_position_coverage(&self, instrument_id: InstrumentId) -> bool {
423        self.is_margin()
424            && self
425                .instruments_cache
426                .get(instrument_id.symbol.as_str())
427                .is_some_and(|instrument| instrument.id() == instrument_id)
428    }
429
430    fn get_account(&self) -> Option<AccountAny> {
431        self.core.cache().account_owned(&self.core.account_id)
432    }
433
434    async fn connect(&mut self) -> anyhow::Result<()> {
435        if self.core.is_connected() && self.pending_tasks.is_open() && self.session_tasks.is_open()
436        {
437            return Ok(());
438        }
439
440        if !self.pending_tasks.is_open() {
441            self.await_pending_tasks().await?;
442            self.pending_tasks
443                .start_generation()
444                .map_err(|e| anyhow::anyhow!("Failed to start Coinbase task generation: {e}"))?;
445        }
446
447        if !self.session_tasks.is_open() || !self.session_tasks.is_empty() {
448            self.abort_session_tasks();
449            self.ws_user
450                .disconnect()
451                .await
452                .context("failed to close stale Coinbase user WebSocket")?;
453            self.await_session_tasks().await?;
454            self.session_tasks
455                .start_generation()
456                .map_err(|e| anyhow::anyhow!("Failed to start Coinbase session generation: {e}"))?;
457        }
458        let ws_user = self.ws_user.clone();
459        let setup_guard =
460            TaskGroupGuard::new(&[&self.session_tasks, &self.pending_tasks], move || {
461                ws_user.begin_shutdown();
462            });
463
464        // If the underlying WS is still alive from a prior stop() that did not
465        // explicitly disconnect, tear it down before reconnecting. The
466        // in-handler signal path can race with the Disconnect command, leaving
467        // the inner connection_mode stale even after disconnect().await, so
468        // we rebuild the client outright to guarantee clean cmd_tx/out_rx
469        // pairs and a fresh signal.
470        if self.ws_user.is_active() || self.ws_user.is_reconnecting() {
471            log::debug!("Tearing down stale user WS before reconnect");
472            self.ws_user
473                .disconnect()
474                .await
475                .context("failed to close stale Coinbase user WebSocket")?;
476            let credential = CoinbaseCredential::resolve(
477                self.config
478                    .api_key
479                    .as_ref()
480                    .map(|value| value.expose_secret()),
481                self.config
482                    .api_secret
483                    .as_ref()
484                    .map(|value| value.expose_secret()),
485            )
486            .ok_or_else(|| anyhow::anyhow!("Coinbase credentials unavailable for WS reset"))?;
487            self.ws_user = CoinbaseWebSocketClient::with_credential(
488                &self.config.ws_url(),
489                credential,
490                self.config.transport_backend,
491                self.config
492                    .proxy_url
493                    .as_ref()
494                    .map(|value| value.expose_secret().to_owned()),
495            );
496        }
497
498        if self.core.instruments_initialized() {
499            // Instruments were loaded externally; still propagate the cached
500            // set to the WS client on reconnect scenarios.
501            let cached: Vec<InstrumentAny> = self.instruments_cache.values().cloned().collect();
502            if !cached.is_empty() {
503                self.ws_user.initialize_instruments(cached).await;
504            }
505        } else {
506            // The Cash (spot) factory loads only spot products; the Margin
507            // (derivatives) factory loads the futures universe so CFM perps
508            // and dated futures can be reconciled. Mixing the two through a
509            // single client is intentionally unsupported, so each factory
510            // picks one branch.
511            let instruments = if self.is_margin() {
512                self.http_client
513                    .request_instruments(Some(CoinbaseProductType::Future))
514                    .await
515                    .context("failed to load Coinbase futures instruments")?
516            } else {
517                self.http_client
518                    .request_instruments(Some(CoinbaseProductType::Spot))
519                    .await
520                    .context("failed to load Coinbase instruments")?
521            };
522
523            let product_kind = if self.is_margin() { "futures" } else { "spot" };
524
525            if instruments.is_empty() {
526                log::warn!("Coinbase instrument bootstrap returned no {product_kind} instruments");
527            } else {
528                log::debug!(
529                    "Coinbase exec client loaded {} {product_kind} instruments",
530                    instruments.len()
531                );
532            }
533
534            let mut map: AHashMap<String, InstrumentAny> =
535                AHashMap::with_capacity(instruments.len());
536            for inst in &instruments {
537                map.insert(inst.id().symbol.as_str().to_string(), inst.clone());
538            }
539            self.instruments_cache = Arc::new(map);
540
541            // Propagate to the WS client so the feed handler can resolve
542            // user-channel product IDs to cached instruments.
543            self.ws_user.initialize_instruments(instruments).await;
544
545            self.core.set_instruments_initialized();
546        }
547
548        if self.fills_start.is_none() {
549            let time = self.http_client.request_server_time().await?;
550            self.fills_start = Some(time.saturating_sub(DurationNanos::from_millis(1)));
551        }
552
553        let session_result = async {
554            self.ws_user.set_account_id(self.core.account_id).await;
555            self.ws_user.connect().await?;
556
557            // Subscribe to the user channel (product-agnostic). User channel with
558            // an empty product list returns events for all products.
559            self.ws_user
560                .subscribe(CoinbaseWsChannel::User, &[])
561                .await
562                .context("failed to subscribe to Coinbase user channel")?;
563
564            if self.is_margin() {
565                self.ws_user
566                    .subscribe(CoinbaseWsChannel::FuturesBalanceSummary, &[])
567                    .await
568                    .context("failed to subscribe to Coinbase futures_balance_summary channel")?;
569            }
570
571            if let Some(rx) = self.ws_user.take_out_rx() {
572                let recovery = self.order_recovery();
573                let retry_interval = Duration::from_millis(self.config.retry_delay_max_ms.max(1));
574
575                let clock = self.clock;
576                let is_margin = self.is_margin();
577                self.session_tasks.spawn(async move {
578                    recovery.run(rx, retry_interval, clock, is_margin).await;
579                })?;
580            }
581
582            let account_state = if self.is_margin() {
583                self.http_client
584                    .request_cfm_account_state(self.core.account_id)
585                    .await
586                    .context("failed to request Coinbase CFM account state")?
587            } else {
588                self.http_client
589                    .request_account_state(self.core.account_id)
590                    .await
591                    .context("failed to request Coinbase account state")?
592            };
593
594            if !account_state.balances.is_empty() {
595                log::debug!(
596                    "Received account state with {} balance(s)",
597                    account_state.balances.len()
598                );
599            }
600            self.emitter.send_account_state(account_state);
601
602            self.await_account_registered(ACCOUNT_REGISTERED_TIMEOUT_SECS)
603                .await?;
604
605            Ok::<(), anyhow::Error>(())
606        }
607        .await;
608
609        if let Err(e) = session_result {
610            if let Err(teardown_error) = self.teardown_partial_connect().await {
611                return Err(e.context(format!(
612                    "Coinbase execution startup teardown failed: {teardown_error}"
613                )));
614            }
615            return Err(e);
616        }
617
618        self.core.set_connected();
619        setup_guard.disarm();
620        log::info!("Connected: client_id={}", self.core.client_id);
621        Ok(())
622    }
623
624    async fn disconnect(&mut self) -> anyhow::Result<()> {
625        self.teardown_partial_connect().await?;
626        log::info!("Disconnected: client_id={}", self.core.client_id);
627        Ok(())
628    }
629
630    fn start(&mut self) -> anyhow::Result<()> {
631        if self.core.is_started() {
632            return Ok(());
633        }
634
635        let sender = get_exec_event_sender();
636        self.emitter.set_sender(sender);
637        self.subscribe_order_events();
638        self.core.set_started();
639
640        log::info!(
641            "Started: client_id={}, account_id={}, account_type={:?}, environment={:?}",
642            self.core.client_id,
643            self.core.account_id,
644            self.core.account_type,
645            self.config.environment,
646        );
647        Ok(())
648    }
649
650    fn stop(&mut self) -> anyhow::Result<()> {
651        if self.core.is_stopped() {
652            return Ok(());
653        }
654
655        if let Some(handler) = self.order_event_handler.take() {
656            msgbus::unsubscribe_order_events("events.order.*".into(), &handler);
657        }
658
659        self.core.set_stopped();
660        self.core.set_disconnected();
661
662        self.abort_session_tasks();
663        self.abort_pending_tasks();
664        log::info!("Stopped: client_id={}", self.core.client_id);
665        Ok(())
666    }
667
668    fn query_account(&self, _cmd: QueryAccount) -> anyhow::Result<()> {
669        let http_client = self.http_client.clone();
670        let account_id = self.core.account_id;
671        let emitter = self.emitter.clone();
672        let is_margin = self.is_margin();
673
674        self.spawn_task("query_account", async move {
675            let account_state = if is_margin {
676                http_client
677                    .request_cfm_account_state(account_id)
678                    .await
679                    .context("failed to request Coinbase CFM account state")?
680            } else {
681                http_client
682                    .request_account_state(account_id)
683                    .await
684                    .context("failed to request Coinbase account state")?
685            };
686            emitter.send_account_state(account_state);
687            Ok(())
688        });
689        Ok(())
690    }
691
692    fn query_order(&self, cmd: QueryOrder) -> anyhow::Result<()> {
693        if let Some(order) = self.core.cache().order(&cmd.client_order_id) {
694            self.lifecycle.lock().track(order.clone());
695        }
696
697        let recovery = self.order_recovery();
698        self.spawn_task("query_order", async move {
699            let report = recovery
700                .http_client
701                .request_order_status_report(
702                    recovery.emitter.account_id(),
703                    Some(cmd.client_order_id),
704                    cmd.venue_order_id,
705                )
706                .await?;
707            recovery.queue(report.clone());
708            recovery
709                .lifecycle
710                .lock()
711                .accept(&report, &recovery.emitter)?;
712            recovery.retry().await;
713            Ok(())
714        });
715
716        Ok(())
717    }
718
719    fn generate_account_state(
720        &self,
721        balances: Vec<AccountBalance>,
722        margins: Vec<MarginBalance>,
723        reported: bool,
724        ts_event: UnixNanos,
725        info: Option<Params>,
726    ) -> anyhow::Result<()> {
727        self.emitter
728            .emit_account_state(balances, margins, reported, ts_event, info);
729        Ok(())
730    }
731
732    async fn generate_order_status_report(
733        &self,
734        cmd: &GenerateOrderStatusReport,
735    ) -> anyhow::Result<Option<OrderStatusReport>> {
736        let report = self
737            .http_client
738            .request_order_status_report(
739                self.core.account_id,
740                cmd.client_order_id,
741                cmd.venue_order_id,
742            )
743            .await
744            .ok();
745
746        // Filter reports to instruments this client bootstrapped. A Cash
747        // client drops derivatives reports (and vice-versa) so mixed activity
748        // on the same venue account does not poison the engine state
749        // associated with either exec client.
750        Ok(report.filter(|r| self.is_instrument_cached(&r.instrument_id)))
751    }
752
753    async fn generate_order_status_reports(
754        &self,
755        cmd: &GenerateOrderStatusReports,
756    ) -> anyhow::Result<Vec<OrderStatusReport>> {
757        let start = cmd.start.map(|ts| ts.to_datetime_utc());
758        let end = cmd.end.map(|ts| ts.to_datetime_utc());
759
760        let mut reports = self
761            .http_client
762            .request_order_status_reports(
763                self.core.account_id,
764                cmd.instrument_id,
765                cmd.open_only,
766                start,
767                end,
768                None,
769            )
770            .await?;
771
772        let before = reports.len();
773        reports.retain(|r| self.is_instrument_cached(&r.instrument_id));
774        if reports.len() != before {
775            let scope = if self.is_margin() {
776                "non-futures"
777            } else {
778                "non-spot"
779            };
780            log::debug!("Filtered {} {scope} order reports", before - reports.len());
781        }
782        Ok(reports)
783    }
784
785    async fn generate_fill_reports(
786        &self,
787        cmd: GenerateFillReports,
788    ) -> anyhow::Result<Vec<FillReport>> {
789        let start = cmd.start.map(|ts| ts.to_datetime_utc());
790        let end = cmd.end.map(|ts| ts.to_datetime_utc());
791
792        let mut reports = self
793            .http_client
794            .request_fill_reports(
795                self.core.account_id,
796                cmd.instrument_id,
797                cmd.venue_order_id,
798                start,
799                end,
800                None,
801            )
802            .await?;
803
804        let before = reports.len();
805        reports.retain(|r| self.is_instrument_cached(&r.instrument_id));
806        if reports.len() != before {
807            let scope = if self.is_margin() {
808                "non-futures"
809            } else {
810                "non-spot"
811            };
812            log::debug!("Filtered {} {scope} fill reports", before - reports.len());
813        }
814        Ok(reports)
815    }
816
817    async fn generate_position_status_reports(
818        &self,
819        cmd: &GeneratePositionStatusReports,
820    ) -> anyhow::Result<Vec<PositionStatusReport>> {
821        // Coinbase spot has no positions.
822        if !self.is_margin() {
823            return Ok(Vec::new());
824        }
825
826        // Errors propagate (matching `generate_order_status_reports` /
827        // `generate_fill_reports`) so `generate_mass_status` and the live
828        // manager's reconciliation path see venue failures rather than
829        // receive a silently-empty report set.
830        if let Some(instrument_id) = cmd.instrument_id {
831            let report = self
832                .http_client
833                .request_position_status_report(self.core.account_id, instrument_id)
834                .await
835                .with_context(|| format!("failed to request CFM position for {instrument_id}"))?;
836            Ok(report.map(|r| vec![r]).unwrap_or_default())
837        } else {
838            self.http_client
839                .request_position_status_reports(self.core.account_id)
840                .await
841                .context("failed to request CFM positions")
842        }
843    }
844
845    async fn generate_mass_status(
846        &self,
847        lookback_mins: Option<u64>,
848    ) -> anyhow::Result<Option<ExecutionMassStatus>> {
849        log::info!("Generating ExecutionMassStatus (lookback_mins={lookback_mins:?})");
850
851        let ts_now = self.clock.get_time_ns();
852        let start = lookback_mins
853            .map(DurationNanos::try_from_mins)
854            .transpose()?
855            .map(|lookback| ts_now.saturating_sub(lookback));
856
857        let order_cmd = GenerateOrderStatusReportsBuilder::default()
858            .ts_init(ts_now)
859            .open_only(false)
860            .start(start)
861            .build()
862            .map_err(|e| anyhow::anyhow!("{e}"))?;
863        let fill_cmd = GenerateFillReportsBuilder::default()
864            .ts_init(ts_now)
865            .start(start)
866            .build()
867            .map_err(|e| anyhow::anyhow!("{e}"))?;
868        let position_cmd = GeneratePositionStatusReportsBuilder::default()
869            .ts_init(ts_now)
870            .build()
871            .map_err(|e| anyhow::anyhow!("{e}"))?;
872
873        let (order_reports, fill_reports, position_reports) = tokio::try_join!(
874            self.generate_order_status_reports(&order_cmd),
875            self.generate_fill_reports(fill_cmd),
876            self.generate_position_status_reports(&position_cmd),
877        )?;
878
879        log::info!("Received {} OrderStatusReports", order_reports.len());
880        log::info!("Received {} FillReports", fill_reports.len());
881        log::info!("Received {} PositionReports", position_reports.len());
882
883        let mut mass_status = ExecutionMassStatus::new(
884            self.core.client_id,
885            self.core.account_id,
886            *COINBASE_VENUE,
887            ts_now,
888            None,
889        );
890
891        mass_status.add_order_reports(order_reports);
892        mass_status.add_fill_reports(fill_reports);
893        mass_status.add_position_reports(position_reports);
894
895        Ok(Some(mass_status))
896    }
897
898    fn submit_order(&self, cmd: SubmitOrder) -> anyhow::Result<()> {
899        let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
900        if order.is_closed() {
901            log::warn!("Cannot submit closed order {}", order.client_order_id());
902            return Ok(());
903        }
904
905        if let Err(reason) = validate_order(&order, &self.instruments_cache) {
906            self.emitter.emit_order_denied(&order, &reason.to_string());
907            return Ok(());
908        }
909
910        self.lifecycle.lock().track(order.clone());
911        let instrument_id = order.instrument_id();
912
913        // The user channel does not need a product-wide alias registration:
914        // `order_contexts` (keyed by `client_order_id`) records the
915        // submitted `product_id` to restore on working user-channel reports. A product-wide map would
916        // misroute external or canonical-side orders that share the same
917        // wire `product_id`.
918
919        log::debug!("OrderSubmitted client_order_id={}", order.client_order_id());
920        self.emitter.emit_order_submitted(&order);
921
922        let http_client = self.http_client.clone();
923        let emitter = self.emitter.clone();
924        let order_contexts = Arc::clone(&self.order_contexts);
925        let clock = self.clock;
926        let strategy_id = order.strategy_id();
927        let client_order_id = order.client_order_id();
928        let side = order.order_side();
929        let order_type = order.order_type();
930        let quantity = order.quantity();
931        let time_in_force = order.time_in_force();
932        let price = order.price();
933        let trigger_price = order.trigger_price();
934        let trigger_type = order.trigger_type();
935        let expire_time = order.expire_time();
936        let post_only = order.is_post_only();
937        let is_quote_quantity = order.is_quote_quantity();
938        let reduce_only = order.is_reduce_only();
939
940        // Cache limit/trigger metadata under `client_order_id` synchronously
941        // before the spawn so user-channel updates that race the REST submit
942        // response can still patch their reports. Coinbase's user channel does
943        // not echo `price`, `stop_price`, `trigger_type`, or whether the order
944        // is `post_only`, so without this the engine reconciler would clear
945        // the local price or lose maker-only semantics.
946        {
947            let mut map = self.order_contexts.lock();
948            map.insert(
949                client_order_id.to_string(),
950                OrderContext {
951                    price,
952                    trigger_price,
953                    trigger_type,
954                    post_only,
955                    submitted_product_id: Some(instrument_id.symbol.inner()),
956                },
957            );
958        }
959        let (leverage, margin_type) = if self.core.account_type == AccountType::Margin {
960            (
961                self.config.default_leverage,
962                self.config.default_margin_type,
963            )
964        } else {
965            (None, None)
966        };
967        let retail_portfolio_id = self.config.retail_portfolio_id.clone();
968
969        self.spawn_task("submit_order", async move {
970            let result = http_client
971                .submit_order(
972                    client_order_id,
973                    instrument_id,
974                    side,
975                    order_type,
976                    quantity,
977                    time_in_force,
978                    price,
979                    trigger_price,
980                    expire_time,
981                    post_only,
982                    is_quote_quantity,
983                    leverage,
984                    margin_type,
985                    reduce_only,
986                    retail_portfolio_id,
987                )
988                .await;
989
990            match result {
991                Ok(response) => {
992                    if response.success {
993                        let venue_id = response
994                            .success_response
995                            .as_ref()
996                            .map(|s| s.order_id.clone())
997                            .unwrap_or(response.order_id);
998
999                        if venue_id.is_empty() {
1000                            log::warn!(
1001                                "Submit succeeded but no order_id returned for {client_order_id}"
1002                            );
1003                        } else {
1004                            let venue_order_id = VenueOrderId::new(&venue_id);
1005                            let ts_event = clock.get_time_ns();
1006                            emitter.emit_order_accepted(&order, venue_order_id, ts_event);
1007                        }
1008                    } else {
1009                        let reason = response.error_response.as_ref().map_or_else(
1010                            || response.failure_reason.clone(),
1011                            |e| format!("{}: {}", e.error, e.message),
1012                        );
1013                        // `INVALID_LIMIT_PRICE_POST_ONLY` is Coinbase's reject
1014                        // code when a `post_only` order would have crossed
1015                        // the spread by the time it reached the matching
1016                        // engine. Mark the rejection so strategies can react
1017                        // (typically: re-quote at the new TOB).
1018                        let due_post_only = reason.contains("INVALID_LIMIT_PRICE_POST_ONLY")
1019                            || response.error_response.as_ref().is_some_and(|e| {
1020                                e.preview_failure_reason == "PREVIEW_INVALID_LIMIT_PRICE_POSTONLY"
1021                                    || e.new_order_failure_reason == "INVALID_LIMIT_PRICE_POST_ONLY"
1022                            });
1023                        // Order never made it to the venue: drop the cached
1024                        // metadata so the map does not grow unbounded with
1025                        // dead entries.
1026                        order_contexts.lock().remove(client_order_id.as_str());
1027                        let ts_event = clock.get_time_ns();
1028                        emitter.emit_order_rejected_event(
1029                            strategy_id,
1030                            instrument_id,
1031                            client_order_id,
1032                            &format!("submit-order-rejected: {reason}"),
1033                            ts_event,
1034                            due_post_only,
1035                        );
1036                    }
1037                }
1038                Err(e) => {
1039                    handle_coinbase_submit_failure(
1040                        &e,
1041                        &order_contexts,
1042                        &emitter,
1043                        strategy_id,
1044                        instrument_id,
1045                        client_order_id,
1046                        clock.get_time_ns(),
1047                    );
1048                    return Err(e.context("submit order failed"));
1049                }
1050            }
1051            Ok(())
1052        });
1053
1054        Ok(())
1055    }
1056
1057    fn submit_order_list(&self, cmd: SubmitOrderList) -> anyhow::Result<()> {
1058        let orders = self.core.get_orders_for_list(&cmd.order_list)?;
1059        let denied = OrderDeniedReason::UnsupportedOrderList {
1060            detail: "order lists are not supported by Coinbase Advanced Trade".to_string(),
1061        }
1062        .to_string();
1063
1064        for order in &orders {
1065            self.emitter.emit_order_denied(order, &denied);
1066        }
1067
1068        Ok(())
1069    }
1070
1071    fn modify_order(&self, cmd: ModifyOrder) -> anyhow::Result<()> {
1072        let ts_event = self.clock.get_time_ns();
1073
1074        let Some(venue_order_id) = cmd.venue_order_id else {
1075            self.emitter.emit_order_modify_denied_event(
1076                cmd.strategy_id,
1077                cmd.instrument_id,
1078                cmd.client_order_id,
1079                None,
1080                "modify-order requires venue_order_id",
1081                ts_event,
1082                Some(cmd.command_id),
1083            );
1084            return Ok(());
1085        };
1086
1087        if cmd.price.is_none() && cmd.quantity.is_none() && cmd.trigger_price.is_none() {
1088            self.emitter.emit_order_modify_denied_event(
1089                cmd.strategy_id,
1090                cmd.instrument_id,
1091                cmd.client_order_id,
1092                Some(venue_order_id),
1093                "modify-order requires price, quantity, or trigger_price",
1094                ts_event,
1095                Some(cmd.command_id),
1096            );
1097            return Ok(());
1098        }
1099
1100        // Coinbase's `/orders/edit` requires both `price` and `size` to be
1101        // present in the request even when only one is changing; omitting
1102        // `size` is interpreted as 0 and rejected with `INVALID_EDITED_SIZE` /
1103        // `CANNOT_EDIT_TO_BELOW_FILLED_SIZE`. Auto-fill missing fields from
1104        // the cached order so strategies can call `modify_order(price=...)`
1105        // without having to look up the current quantity themselves.
1106        let (auto_price, auto_quantity) = {
1107            let cache = self.core.cache();
1108            let cached = cache.order(&cmd.client_order_id);
1109            let cached_price = cached.as_ref().and_then(|o| o.price());
1110            let cached_qty = cached.as_ref().map(|o| o.quantity());
1111            (cmd.price.or(cached_price), cmd.quantity.or(cached_qty))
1112        };
1113
1114        let http_client = self.http_client.clone();
1115        let emitter = self.emitter.clone();
1116        let order_contexts = Arc::clone(&self.order_contexts);
1117        let clock = self.clock;
1118        let strategy_id = cmd.strategy_id;
1119        let instrument_id = cmd.instrument_id;
1120        let client_order_id = cmd.client_order_id;
1121        let price = auto_price;
1122        let quantity = auto_quantity;
1123        let trigger_price = cmd.trigger_price;
1124
1125        self.spawn_task("modify_order", async move {
1126            let result = http_client
1127                .modify_order(venue_order_id, price, quantity, trigger_price)
1128                .await;
1129
1130            match result {
1131                Ok(resp) => {
1132                    if resp.success {
1133                        // Refresh the submit-time metadata cache so subsequent
1134                        // user-channel updates patch with the new price /
1135                        // trigger_price (Coinbase user channel does not echo
1136                        // these fields, so a stale cache would let the
1137                        // reconciler revert the local order to the pre-edit
1138                        // values).
1139                        let mut map = order_contexts.lock();
1140                        if let Some(meta) = map.get_mut(client_order_id.as_str()) {
1141                            if price.is_some() {
1142                                meta.price = price;
1143                            }
1144
1145                            if trigger_price.is_some() {
1146                                meta.trigger_price = trigger_price;
1147                            }
1148                        }
1149                    } else {
1150                        let reason = resp
1151                            .errors
1152                            .iter()
1153                            .map(|e| {
1154                                if e.edit_failure_reason.is_empty() {
1155                                    e.preview_failure_reason.clone()
1156                                } else {
1157                                    e.edit_failure_reason.clone()
1158                                }
1159                            })
1160                            .collect::<Vec<_>>()
1161                            .join(",");
1162                        let ts_event = clock.get_time_ns();
1163                        emitter.emit_order_modify_rejected_event(
1164                            strategy_id,
1165                            instrument_id,
1166                            client_order_id,
1167                            Some(venue_order_id),
1168                            &format!("modify-order-rejected: {reason}"),
1169                            ts_event,
1170                            Some(cmd.command_id),
1171                        );
1172                    }
1173                }
1174                Err(e) => {
1175                    match classify_coinbase_command_failure(&e) {
1176                        Some(CommandFailure::NotSent(reason)) => emitter
1177                            .emit_order_modify_denied_event(
1178                                strategy_id,
1179                                instrument_id,
1180                                client_order_id,
1181                                Some(venue_order_id),
1182                                &reason,
1183                                clock.get_time_ns(),
1184                                Some(cmd.command_id),
1185                            ),
1186                        Some(CommandFailure::VenueRejected(reason)) => emitter
1187                            .emit_order_modify_rejected_event(
1188                                strategy_id,
1189                                instrument_id,
1190                                client_order_id,
1191                                Some(venue_order_id),
1192                                &reason,
1193                                clock.get_time_ns(),
1194                                Some(cmd.command_id),
1195                            ),
1196                        Some(CommandFailure::Ambiguous(_)) => log::warn!(
1197                            "Ambiguous modify failure for {client_order_id}, awaiting reconciliation: {e}"
1198                        ),
1199                        None => log::warn!(
1200                            "Modify command failed without venue-declared outcome for {client_order_id}: {e}"
1201                        ),
1202                    }
1203                    return Err(e.context("modify order failed"));
1204                }
1205            }
1206
1207            Ok(())
1208        });
1209
1210        Ok(())
1211    }
1212
1213    fn cancel_order(&self, cmd: CancelOrder) -> anyhow::Result<()> {
1214        let Some(venue_order_id) = cmd.venue_order_id else {
1215            self.emitter.emit_order_cancel_denied_event(
1216                cmd.strategy_id,
1217                cmd.instrument_id,
1218                cmd.client_order_id,
1219                None,
1220                "cancel-order requires venue_order_id",
1221                self.clock.get_time_ns(),
1222                Some(cmd.command_id),
1223            );
1224            return Ok(());
1225        };
1226
1227        let http_client = self.http_client.clone();
1228        let emitter = self.emitter.clone();
1229        let clock = self.clock;
1230        let strategy_id = cmd.strategy_id;
1231        let instrument_id = cmd.instrument_id;
1232        let client_order_id = cmd.client_order_id;
1233
1234        self.spawn_task("cancel_order", async move {
1235            match http_client.cancel_orders(&[venue_order_id]).await {
1236                Ok(resp) => {
1237                    if let Some(result) = resp.results.first()
1238                        && !result.success
1239                    {
1240                        let ts_event = clock.get_time_ns();
1241                        emitter.emit_order_cancel_rejected_event(
1242                            strategy_id,
1243                            instrument_id,
1244                            client_order_id,
1245                            Some(venue_order_id),
1246                            &format!("cancel-order-rejected: {}", result.failure_reason),
1247                            ts_event,
1248                            Some(cmd.command_id),
1249                        );
1250                    }
1251                }
1252                Err(e) => {
1253                    if is_coinbase_local_command_failure(&e) {
1254                        emitter.emit_order_cancel_denied_event(
1255                            strategy_id,
1256                            instrument_id,
1257                            client_order_id,
1258                            Some(venue_order_id),
1259                            &e.to_string(),
1260                            clock.get_time_ns(),
1261                            Some(cmd.command_id),
1262                        );
1263                    } else if is_coinbase_explicit_command_rejection(&e) {
1264                        emitter.emit_order_cancel_rejected_event(
1265                            strategy_id,
1266                            instrument_id,
1267                            client_order_id,
1268                            Some(venue_order_id),
1269                            &e.to_string(),
1270                            clock.get_time_ns(),
1271                            Some(cmd.command_id),
1272                        );
1273                    } else if is_coinbase_ambiguous_command_failure(&e) {
1274                        log::warn!(
1275                            "Ambiguous cancel failure for {client_order_id}, awaiting reconciliation: {e}"
1276                        );
1277                    } else {
1278                        log::warn!(
1279                            "Cancel command failed without venue-declared outcome for {client_order_id}: {e}"
1280                        );
1281                    }
1282                    return Err(e.context("cancel order failed"));
1283                }
1284            }
1285            Ok(())
1286        });
1287
1288        Ok(())
1289    }
1290
1291    fn cancel_all_orders(&self, cmd: CancelAllOrders) -> anyhow::Result<()> {
1292        let http_client = self.http_client.clone();
1293        let account_id = self.core.account_id;
1294        let instrument_id = cmd.instrument_id;
1295        let side_filter = cmd.order_side;
1296        let emitter = self.emitter.clone();
1297        let clock = self.clock;
1298        let strategy_id = cmd.strategy_id;
1299
1300        self.spawn_task("cancel_all_orders", async move {
1301            // Coinbase's `order_status=OPEN` filter excludes PENDING / QUEUED
1302            // orders that were submitted very recently and are still cancelable.
1303            // Fetch all reports and filter to any open status locally so a cancel-
1304            // all issued right after submission does not leave working orders behind.
1305            let reports = http_client
1306                .request_order_status_reports(
1307                    account_id,
1308                    Some(instrument_id),
1309                    false,
1310                    None,
1311                    None,
1312                    None,
1313                )
1314                .await
1315                .context("failed to list orders for cancel_all")?;
1316
1317            // Filter to statuses that are safe to cancel and to the requested
1318            // side since Coinbase's batch-cancel endpoint has no side parameter.
1319            //
1320            // Coinbase's `PENDING` / `QUEUED` / `OPEN` all map to `Accepted`
1321            // and are cancelable. We can't use `OrderStatus::is_open()` because
1322            // it includes `PendingCancel`, and re-cancelling a `CANCEL_QUEUED`
1323            // order risks `CancelRejected` flipping the order back to its prior
1324            // working status.
1325            let filtered: Vec<CancelRequest> = reports
1326                .into_iter()
1327                .filter(|r| {
1328                    matches!(
1329                        r.order_status,
1330                        OrderStatus::Accepted
1331                            | OrderStatus::Triggered
1332                            | OrderStatus::PendingUpdate
1333                            | OrderStatus::PartiallyFilled
1334                    )
1335                })
1336                .filter(|r| side_filter.is_none_or(|side| r.order_side == side.into()))
1337                .map(|r| CancelRequest {
1338                    strategy_id,
1339                    instrument_id,
1340                    client_order_id: r.client_order_id,
1341                    venue_order_id: r.venue_order_id,
1342                    command_id: cmd.command_id,
1343                })
1344                .collect();
1345
1346            cancel_order_batches(&http_client, &emitter, clock, &filtered, "cancel-all").await;
1347            Ok(())
1348        });
1349
1350        Ok(())
1351    }
1352
1353    fn batch_cancel_orders(&self, cmd: BatchCancelOrders) -> anyhow::Result<()> {
1354        if cmd.cancels.is_empty() {
1355            return Ok(());
1356        }
1357
1358        let http_client = self.http_client.clone();
1359        let emitter = self.emitter.clone();
1360        let clock = self.clock;
1361
1362        let cancels = cmd.cancels;
1363
1364        self.spawn_task("batch_cancel_orders", async move {
1365            let mut requests = Vec::new();
1366
1367            for cancel in cancels {
1368                let Some(venue_order_id) = cancel.venue_order_id else {
1369                    emitter.emit_order_cancel_denied_event(
1370                        cancel.strategy_id,
1371                        cancel.instrument_id,
1372                        cancel.client_order_id,
1373                        None,
1374                        "cancel-order requires venue_order_id",
1375                        clock.get_time_ns(),
1376                        Some(cancel.command_id),
1377                    );
1378                    continue;
1379                };
1380
1381                requests.push(CancelRequest {
1382                    strategy_id: cancel.strategy_id,
1383                    instrument_id: cancel.instrument_id,
1384                    client_order_id: Some(cancel.client_order_id),
1385                    venue_order_id,
1386                    command_id: cancel.command_id,
1387                });
1388            }
1389
1390            cancel_order_batches(&http_client, &emitter, clock, &requests, "batch-cancel").await;
1391            Ok(())
1392        });
1393
1394        Ok(())
1395    }
1396}
1397
1398struct CancelRequest {
1399    strategy_id: StrategyId,
1400    instrument_id: InstrumentId,
1401    client_order_id: Option<ClientOrderId>,
1402    venue_order_id: VenueOrderId,
1403    command_id: UUID4,
1404}
1405
1406async fn cancel_order_batches(
1407    http: &CoinbaseHttpClient,
1408    emitter: &ExecutionEventEmitter,
1409    clock: &AtomicTime,
1410    requests: &[CancelRequest],
1411    task_name: &str,
1412) {
1413    let mut offset = 0;
1414    while offset < requests.len() {
1415        let end = offset + cancel_batch_len(&requests[offset..]);
1416        let chunk = &requests[offset..end];
1417        offset = end;
1418        let venue_ids: Vec<_> = chunk.iter().map(|request| request.venue_order_id).collect();
1419        match http.cancel_orders(&venue_ids).await {
1420            Ok(response) => {
1421                for result in response.results {
1422                    if result.success {
1423                        continue;
1424                    }
1425
1426                    let Some(request) = chunk
1427                        .iter()
1428                        .find(|request| request.venue_order_id.as_str() == result.order_id)
1429                    else {
1430                        continue;
1431                    };
1432
1433                    let Some(client_order_id) = request.client_order_id else {
1434                        continue;
1435                    };
1436
1437                    emitter.emit_order_cancel_rejected_event(
1438                        request.strategy_id,
1439                        request.instrument_id,
1440                        client_order_id,
1441                        Some(request.venue_order_id),
1442                        &format!("{task_name}-rejected: {}", result.failure_reason),
1443                        clock.get_time_ns(),
1444                        Some(request.command_id),
1445                    );
1446                }
1447            }
1448            Err(e) => emit_batch_cancel_failure(emitter, clock, chunk, task_name, &e),
1449        }
1450    }
1451}
1452
1453fn cancel_batch_len(requests: &[CancelRequest]) -> usize {
1454    let mut targets = AHashSet::new();
1455    requests
1456        .iter()
1457        .take(BATCH_CANCEL_CHUNK)
1458        .take_while(|request| targets.insert(request.venue_order_id))
1459        .count()
1460}
1461
1462fn emit_batch_cancel_failure(
1463    emitter: &ExecutionEventEmitter,
1464    clock: &AtomicTime,
1465    requests: &[CancelRequest],
1466    task_name: &str,
1467    error: &anyhow::Error,
1468) {
1469    match classify_coinbase_command_failure(error) {
1470        Some(CommandFailure::NotSent(reason)) => {
1471            for request in requests {
1472                let Some(client_order_id) = request.client_order_id else {
1473                    continue;
1474                };
1475
1476                emitter.emit_order_cancel_denied_event(
1477                    request.strategy_id,
1478                    request.instrument_id,
1479                    client_order_id,
1480                    Some(request.venue_order_id),
1481                    &reason,
1482                    clock.get_time_ns(),
1483                    Some(request.command_id),
1484                );
1485            }
1486        }
1487        Some(CommandFailure::VenueRejected(reason)) => {
1488            for request in requests {
1489                let Some(client_order_id) = request.client_order_id else {
1490                    continue;
1491                };
1492
1493                emitter.emit_order_cancel_rejected_event(
1494                    request.strategy_id,
1495                    request.instrument_id,
1496                    client_order_id,
1497                    Some(request.venue_order_id),
1498                    &reason,
1499                    clock.get_time_ns(),
1500                    Some(request.command_id),
1501                );
1502            }
1503        }
1504        Some(CommandFailure::Ambiguous(_)) => log::warn!(
1505            "Ambiguous {task_name} failure for {} orders, awaiting reconciliation: {error}",
1506            requests.len()
1507        ),
1508        None => log::warn!(
1509            "{task_name} command failed without venue-declared outcome for {} orders: {error}",
1510            requests.len()
1511        ),
1512    }
1513}
1514
1515fn classify_coinbase_command_failure(error: &anyhow::Error) -> Option<CommandFailure> {
1516    if is_coinbase_local_command_failure(error) {
1517        Some(CommandFailure::not_sent(error.to_string()))
1518    } else if is_coinbase_explicit_command_rejection(error) {
1519        Some(CommandFailure::venue_rejected(error.to_string()))
1520    } else if is_coinbase_ambiguous_command_failure(error) {
1521        Some(CommandFailure::ambiguous(error.to_string()))
1522    } else {
1523        None
1524    }
1525}
1526
1527fn validate_order(
1528    order: &impl Order,
1529    instruments_cache: &AHashMap<String, InstrumentAny>,
1530) -> Result<(), OrderDeniedReason> {
1531    if order.is_reduce_only() {
1532        return Err(OrderDeniedReason::UnsupportedReduceOnly);
1533    }
1534
1535    let instrument_id = order.instrument_id();
1536    if !instruments_cache.contains_key(instrument_id.symbol.as_str()) {
1537        return Err(OrderDeniedReason::InstrumentNotFound { instrument_id });
1538    }
1539
1540    Ok(())
1541}
1542
1543fn handle_coinbase_submit_failure(
1544    err: &anyhow::Error,
1545    order_contexts: &Mutex<AHashMap<String, OrderContext>>,
1546    emitter: &ExecutionEventEmitter,
1547    strategy_id: StrategyId,
1548    instrument_id: InstrumentId,
1549    client_order_id: ClientOrderId,
1550    ts_event: UnixNanos,
1551) {
1552    if is_coinbase_local_command_failure(err) {
1553        order_contexts.lock().remove(client_order_id.as_str());
1554        emitter.emit_order_rejected_event(
1555            strategy_id,
1556            instrument_id,
1557            client_order_id,
1558            &format!("submit-order-error: {err}"),
1559            ts_event,
1560            false,
1561        );
1562    } else if is_coinbase_explicit_command_rejection(err) {
1563        order_contexts.lock().remove(client_order_id.as_str());
1564        emitter.emit_order_rejected_event(
1565            strategy_id,
1566            instrument_id,
1567            client_order_id,
1568            &format!("submit-order-rejected: {err}"),
1569            ts_event,
1570            false,
1571        );
1572    } else if is_coinbase_ambiguous_submit_failure(err) {
1573        log::warn!(
1574            "Ambiguous submit failure for {client_order_id}, awaiting reconciliation: {err}"
1575        );
1576    } else {
1577        order_contexts.lock().remove(client_order_id.as_str());
1578        log::warn!(
1579            "Submit command failed without venue-declared outcome for {client_order_id}: {err}"
1580        );
1581    }
1582}
1583
1584fn is_coinbase_local_command_failure(err: &anyhow::Error) -> bool {
1585    match coinbase_http_error(err) {
1586        None => true,
1587        Some(CoinbaseHttpError::Auth(message)) => !message.starts_with("HTTP "),
1588        _ => false,
1589    }
1590}
1591
1592fn is_coinbase_explicit_command_rejection(err: &anyhow::Error) -> bool {
1593    match coinbase_http_error(err) {
1594        Some(CoinbaseHttpError::Auth(message) | CoinbaseHttpError::BadRequest(message)) => {
1595            message.starts_with("HTTP ")
1596        }
1597        _ => false,
1598    }
1599}
1600
1601fn is_coinbase_ambiguous_submit_failure(err: &anyhow::Error) -> bool {
1602    matches!(
1603        coinbase_http_error(err),
1604        Some(CoinbaseHttpError::RateLimit { .. })
1605    ) || is_coinbase_ambiguous_command_failure(err)
1606}
1607
1608fn is_coinbase_ambiguous_command_failure(err: &anyhow::Error) -> bool {
1609    matches!(
1610        coinbase_http_error(err),
1611        Some(
1612            CoinbaseHttpError::Transport(_)
1613                | CoinbaseHttpError::Serde(_)
1614                | CoinbaseHttpError::Exchange(_)
1615                | CoinbaseHttpError::Timeout
1616                | CoinbaseHttpError::Decode(_)
1617        )
1618    ) || matches!(
1619        coinbase_http_error(err),
1620        Some(CoinbaseHttpError::Http { status, .. }) if *status >= 500
1621    )
1622}
1623
1624fn coinbase_http_error(err: &anyhow::Error) -> Option<&CoinbaseHttpError> {
1625    err.chain()
1626        .find_map(|cause| cause.downcast_ref::<CoinbaseHttpError>())
1627}
1628
1629#[derive(Debug)]
1630struct UserOrderRecovery {
1631    http_client: CoinbaseHttpClient,
1632    emitter: ExecutionEventEmitter,
1633    instruments: Arc<AHashMap<String, InstrumentAny>>,
1634    observed: Arc<Mutex<AHashMap<VenueOrderId, AHashSet<TradeId>>>>,
1635    pending: Arc<Mutex<AHashMap<VenueOrderId, OrderStatusReport>>>,
1636    published: Arc<tokio::sync::Mutex<AHashMap<VenueOrderId, OrderStatusReport>>>,
1637    order_contexts: Arc<Mutex<AHashMap<String, OrderContext>>>,
1638    external_order_contexts: Arc<Mutex<AHashMap<String, OrderContext>>>,
1639    start: UnixNanos,
1640    lifecycle: Arc<Mutex<RecoveryLifecycle>>,
1641}
1642
1643impl UserOrderRecovery {
1644    async fn run(
1645        &self,
1646        mut rx: tokio::sync::mpsc::UnboundedReceiver<NautilusWsMessage>,
1647        retry_interval: Duration,
1648        clock: &'static AtomicTime,
1649        is_margin: bool,
1650    ) {
1651        let mut interval = tokio::time::interval(retry_interval);
1652        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1653
1654        loop {
1655            tokio::select! {
1656                message = rx.recv() => {
1657                    let Some(message) = message else { break };
1658                    match message {
1659                        NautilusWsMessage::UserOrder(carrier) => self.handle_update(*carrier).await,
1660                        NautilusWsMessage::FuturesBalanceSummary(summary) => {
1661                            let ts = clock.get_time_ns();
1662                            let account_id = self.emitter.account_id();
1663                            match parse_ws_cfm_account_state(&summary, account_id, ts, ts) {
1664                                Ok(state) => self.emitter.send_account_state(state),
1665                                Err(e) => log::warn!("Failed to parse futures_balance_summary into AccountState: {e}"),
1666                            }
1667                        }
1668                        NautilusWsMessage::Reconnected => {
1669                            log::info!("Coinbase user WebSocket reconnected");
1670                            let account_id = self.emitter.account_id();
1671                            let refresh = if is_margin {
1672                                self.http_client.request_cfm_account_state(account_id).await
1673                            } else {
1674                                self.http_client.request_account_state(account_id).await
1675                            };
1676
1677                            match refresh {
1678                                Ok(state) => self.emitter.send_account_state(state),
1679                                Err(e) => log::warn!("Failed to refresh account state on reconnect: {e}"),
1680                            }
1681                        }
1682                        NautilusWsMessage::Error(e) => log::warn!("Coinbase user WebSocket error: {e}"),
1683                        _ => {}
1684                    }
1685                }
1686                _ = interval.tick() => {
1687                    self.retry().await;
1688                }
1689            }
1690        }
1691    }
1692
1693    async fn handle_update(&self, carrier: UserOrderUpdate) {
1694        let needs_fills =
1695            carrier.report.filled_qty.is_positive() || carrier.update.status.is_terminal();
1696        let is_snapshot = carrier.is_snapshot;
1697        let own_context = self
1698            .order_contexts
1699            .lock()
1700            .contains_key(&carrier.update.client_order_id);
1701        if needs_fills {
1702            self.queue((*carrier.report).clone());
1703            if let Err(e) = self.lifecycle.lock().accept(&carrier.report, &self.emitter) {
1704                log::warn!("Retaining Coinbase acceptance recovery: {e}");
1705            }
1706        }
1707
1708        if needs_fills && !own_context {
1709            self.retry().await;
1710            return;
1711        }
1712
1713        let context = resolve_order_context(
1714            &carrier.update,
1715            carrier.report.order_type,
1716            carrier.report.price.is_none(),
1717            &self.order_contexts,
1718            &self.external_order_contexts,
1719            &self.http_client,
1720            self.emitter.account_id(),
1721        )
1722        .await;
1723
1724        if let Some(report) = prepare_user_order_report(carrier, context) {
1725            if !is_snapshot
1726                && own_context
1727                && let Err(e) = self.lifecycle.lock().update(&report, &self.emitter)
1728            {
1729                log::warn!("Failed to publish Coinbase amendment: {e}");
1730            }
1731
1732            if !needs_fills && let Err(e) = self.publish(&[(report, Vec::new())]) {
1733                log::warn!("Failed to publish Coinbase acceptance: {e}");
1734            }
1735        }
1736
1737        if needs_fills {
1738            self.retry().await;
1739        }
1740    }
1741
1742    async fn retry(&self) {
1743        if let Err(e) = self.discover().await {
1744            log::warn!("Coinbase fill recovery scan remains pending: {e}");
1745            return;
1746        }
1747
1748        if let Err(e) = self.recover().await {
1749            log::warn!("Retaining pending Coinbase recovery: {e}");
1750        }
1751    }
1752
1753    async fn discover(&self) -> anyhow::Result<()> {
1754        let fills = self.http_client.request_fill_ids(self.start).await?;
1755        {
1756            let mut observed = self.observed.lock();
1757
1758            for (venue_order_id, instrument_id, trade_id) in fills {
1759                if self.instruments.contains_key(instrument_id.symbol.as_str()) {
1760                    observed.entry(venue_order_id).or_default().insert(trade_id);
1761                }
1762            }
1763        }
1764
1765        let mut reports: AHashMap<_, _> = self
1766            .http_client
1767            .request_order_status_reports(self.emitter.account_id(), None, false, None, None, None)
1768            .await?
1769            .into_iter()
1770            .filter(|report| {
1771                self.instruments
1772                    .contains_key(report.instrument_id.symbol.as_str())
1773                    && report.filled_qty.is_positive()
1774                    && report.ts_last >= self.start
1775            })
1776            .map(|report| (report.venue_order_id, report))
1777            .collect();
1778
1779        let orders: Vec<_> = self.observed.lock().keys().copied().collect();
1780
1781        for venue_order_id in orders {
1782            if let Entry::Vacant(entry) = reports.entry(venue_order_id) {
1783                let report = self
1784                    .http_client
1785                    .request_order_status_report(
1786                        self.emitter.account_id(),
1787                        None,
1788                        Some(venue_order_id),
1789                    )
1790                    .await?;
1791                entry.insert(report);
1792            }
1793        }
1794
1795        for report in reports.into_values() {
1796            self.queue(report);
1797        }
1798
1799        Ok(())
1800    }
1801
1802    fn queue(&self, report: OrderStatusReport) {
1803        let mut pending = self.pending.lock();
1804        let entry = pending
1805            .entry(report.venue_order_id)
1806            .or_insert_with(|| report.clone());
1807        if report.filled_qty > entry.filled_qty
1808            || (report.filled_qty == entry.filled_qty && !entry.order_status.is_closed())
1809        {
1810            *entry = report;
1811        }
1812    }
1813
1814    async fn recover(&self) -> anyhow::Result<()> {
1815        let mut published = self.published.lock().await;
1816        let targets = self.pending.lock().clone();
1817        let mut reports = Vec::with_capacity(targets.len());
1818
1819        for (venue_order_id, expected) in &targets {
1820            let report = self
1821                .http_client
1822                .request_order_status_report(self.emitter.account_id(), None, Some(*venue_order_id))
1823                .await?;
1824            anyhow::ensure!(
1825                self.instruments
1826                    .contains_key(report.instrument_id.symbol.as_str()),
1827                "order instrument is outside the configured product family"
1828            );
1829            anyhow::ensure!(
1830                report.filled_qty >= expected.filled_qty
1831                    && (!expected.order_status.is_closed() || report.order_status.is_closed()),
1832                "REST order state has not reached the user-channel update"
1833            );
1834
1835            if let Some(prior) = published.get(venue_order_id) {
1836                anyhow::ensure!(
1837                    report.filled_qty >= prior.filled_qty
1838                        && (!prior.order_status.is_closed() || report.order_status.is_closed()),
1839                    "REST order state precedes published recovery"
1840                );
1841            }
1842
1843            let fills = if report.filled_qty.is_positive()
1844                || self.observed.lock().contains_key(venue_order_id)
1845            {
1846                self.http_client
1847                    .request_fill_reports(
1848                        self.emitter.account_id(),
1849                        None,
1850                        Some(*venue_order_id),
1851                        None,
1852                        None,
1853                        None,
1854                    )
1855                    .await?
1856            } else {
1857                Vec::new()
1858            };
1859
1860            let fills = prepare_order_fills(&report, fills)?;
1861            reports.push((report, fills));
1862        }
1863
1864        let mut pending = self.pending.lock();
1865        let mut observed = self.observed.lock();
1866        anyhow::ensure!(
1867            pending.len() == targets.len()
1868                && targets
1869                    .iter()
1870                    .all(|(id, target)| pending.get(id) == Some(target)),
1871            "recovery target changed while reading REST"
1872        );
1873
1874        let trades: AHashMap<_, AHashSet<_>> = reports
1875            .iter()
1876            .map(|(report, fills)| {
1877                (
1878                    report.venue_order_id,
1879                    fills.iter().map(|fill| fill.trade_id).collect(),
1880                )
1881            })
1882            .collect();
1883
1884        anyhow::ensure!(
1885            observed
1886                .iter()
1887                .all(|(id, ids)| trades.get(id).is_some_and(|trades| ids.is_subset(trades))),
1888            "REST fills have not reached discovered trades"
1889        );
1890        self.publish(&reports)?;
1891        observed.clear();
1892
1893        for (report, _) in reports {
1894            let venue_order_id = report.venue_order_id;
1895            if pending.get(&venue_order_id) == targets.get(&venue_order_id) {
1896                pending.remove(&venue_order_id);
1897            }
1898
1899            if report.order_status.is_closed() {
1900                if let Some(client_order_id) = report.client_order_id {
1901                    self.order_contexts.lock().remove(client_order_id.as_str());
1902                }
1903
1904                self.external_order_contexts
1905                    .lock()
1906                    .remove(venue_order_id.as_str());
1907            }
1908
1909            published.insert(venue_order_id, report);
1910        }
1911
1912        Ok(())
1913    }
1914
1915    fn publish(&self, reports: &[(OrderStatusReport, Vec<FillReport>)]) -> anyhow::Result<()> {
1916        let mut external = Vec::new();
1917        {
1918            let mut lifecycle = self.lifecycle.lock();
1919            for (report, _) in reports {
1920                if lifecycle.order(report).is_some() {
1921                    lifecycle
1922                        .pending
1923                        .insert(report.venue_order_id, report.clone());
1924                } else {
1925                    // Establish order context without inferring fills before the real trades
1926                    let mut accepted = report.clone();
1927                    if accepted.order_status != OrderStatus::Rejected {
1928                        accepted.order_status = OrderStatus::Accepted;
1929                    }
1930
1931                    accepted.filled_qty = Quantity::zero(report.filled_qty.precision);
1932                    accepted.avg_px = None;
1933                    accepted.ts_last = accepted.ts_accepted;
1934                    self.emitter
1935                        .try_send_execution_report(ExecutionReport::Order(Box::new(accepted)))?;
1936                    external.push(report);
1937                }
1938            }
1939
1940            lifecycle.dispatch(&self.emitter)?;
1941        }
1942
1943        let mut fills: Vec<_> = reports.iter().flat_map(|(_, fills)| fills).collect();
1944        fills.sort_by_key(|fill| (fill.ts_event, fill.trade_id, fill.venue_order_id));
1945        for fill in fills {
1946            self.emitter
1947                .try_send_execution_report(ExecutionReport::Fill(Box::new(fill.clone())))?;
1948        }
1949
1950        for report in external {
1951            self.emitter
1952                .try_send_execution_report(ExecutionReport::Order(Box::new(report.clone())))?;
1953        }
1954
1955        Ok(())
1956    }
1957}
1958
1959#[derive(Debug)]
1960struct RecoveryLifecycle {
1961    factory: OrderEventFactory,
1962    orders: AHashMap<ClientOrderId, OrderAny>,
1963    venue_ids: AHashMap<VenueOrderId, ClientOrderId>,
1964    pending: AHashMap<VenueOrderId, OrderStatusReport>,
1965    accepted: AHashSet<VenueOrderId>,
1966}
1967
1968impl RecoveryLifecycle {
1969    fn new(trader_id: TraderId, account_id: AccountId, account_type: AccountType) -> Self {
1970        Self {
1971            factory: OrderEventFactory::new(trader_id, account_id, account_type, None),
1972            orders: AHashMap::new(),
1973            venue_ids: AHashMap::new(),
1974            pending: AHashMap::new(),
1975            accepted: AHashSet::new(),
1976        }
1977    }
1978
1979    fn track(&mut self, order: OrderAny) {
1980        if let Some(venue_order_id) = order.venue_order_id() {
1981            if !matches!(
1982                order.status(),
1983                OrderStatus::Initialized
1984                    | OrderStatus::Submitted
1985                    | OrderStatus::PendingUpdate
1986                    | OrderStatus::PendingCancel
1987            ) {
1988                self.accepted.remove(&venue_order_id);
1989            }
1990
1991            self.venue_ids
1992                .insert(venue_order_id, order.client_order_id());
1993        }
1994
1995        self.orders.insert(order.client_order_id(), order);
1996    }
1997
1998    fn dispatch(&mut self, emitter: &ExecutionEventEmitter) -> anyhow::Result<()> {
1999        let reports: Vec<_> = self.pending.values().cloned().collect();
2000        for report in reports {
2001            if self.dispatch_order(&report, emitter)? {
2002                self.pending.remove(&report.venue_order_id);
2003            }
2004        }
2005
2006        Ok(())
2007    }
2008
2009    fn dispatch_order(
2010        &mut self,
2011        report: &OrderStatusReport,
2012        emitter: &ExecutionEventEmitter,
2013    ) -> anyhow::Result<bool> {
2014        self.accept(report, emitter)?;
2015
2016        let Some(order) = self.order(report).cloned() else {
2017            return Ok(false);
2018        };
2019
2020        if order.is_closed() {
2021            return Ok(true);
2022        }
2023
2024        let factory = &self.factory;
2025        let now = get_atomic_clock_realtime().get_time_ns();
2026
2027        if (order.order_type() == OrderType::Market
2028            && order.is_quote_quantity()
2029            && report.order_status.is_closed()
2030            && report.filled_qty.is_positive())
2031            || (!order.is_quote_quantity() && report.filled_qty > order.quantity())
2032            || (report.order_status == OrderStatus::Filled && report.quantity != order.quantity())
2033        {
2034            let updated = OrderUpdated::new(
2035                order.trader_id(),
2036                order.strategy_id(),
2037                order.instrument_id(),
2038                order.client_order_id(),
2039                report.quantity,
2040                UUID4::new(),
2041                report.ts_last,
2042                now,
2043                true,
2044                Some(report.venue_order_id),
2045                Some(emitter.account_id()),
2046                None,
2047                None,
2048                None,
2049                false,
2050            );
2051            emitter.try_send_order_event(OrderEventAny::Updated(updated))?;
2052        }
2053
2054        if order.filled_qty() < report.filled_qty {
2055            return Ok(false);
2056        }
2057
2058        let event = match report.order_status {
2059            OrderStatus::Canceled => factory.generate_order_canceled(
2060                &order,
2061                Some(report.venue_order_id),
2062                report.ts_last,
2063                now,
2064            ),
2065            OrderStatus::Expired => factory.generate_order_expired(
2066                &order,
2067                Some(report.venue_order_id),
2068                report.ts_last,
2069                now,
2070            ),
2071            OrderStatus::Rejected => factory.generate_order_rejected(
2072                &order,
2073                report.cancel_reason.as_deref().unwrap_or("rejected"),
2074                report.ts_last,
2075                now,
2076                false,
2077            ),
2078            _ => return Ok(true),
2079        };
2080
2081        emitter.try_send_order_event(event)?;
2082        Ok(true)
2083    }
2084
2085    fn accept(
2086        &mut self,
2087        report: &OrderStatusReport,
2088        emitter: &ExecutionEventEmitter,
2089    ) -> anyhow::Result<()> {
2090        let Some(order) = self.order(report) else {
2091            return Ok(());
2092        };
2093
2094        let factory = &self.factory;
2095        let now = get_atomic_clock_realtime().get_time_ns();
2096        let awaiting_acceptance = matches!(
2097            order.status(),
2098            OrderStatus::Initialized | OrderStatus::Submitted
2099        );
2100
2101        if awaiting_acceptance
2102            && report.order_status != OrderStatus::Rejected
2103            && !self.accepted.contains(&report.venue_order_id)
2104        {
2105            let event = factory.generate_order_accepted(
2106                order,
2107                report.venue_order_id,
2108                report.ts_accepted,
2109                now,
2110            );
2111            emitter.try_send_order_event(event)?;
2112            self.accepted.insert(report.venue_order_id);
2113        }
2114
2115        Ok(())
2116    }
2117
2118    fn update(
2119        &self,
2120        report: &OrderStatusReport,
2121        emitter: &ExecutionEventEmitter,
2122    ) -> anyhow::Result<()> {
2123        let Some(order) = self.order(report) else {
2124            return Ok(());
2125        };
2126
2127        if order.is_closed()
2128            || (order.order_type() == OrderType::Market && order.is_quote_quantity())
2129            || (order.quantity() == report.quantity
2130                && order.price() == report.price
2131                && order.trigger_price() == report.trigger_price)
2132        {
2133            return Ok(());
2134        }
2135
2136        let factory = &self.factory;
2137        let now = get_atomic_clock_realtime().get_time_ns();
2138        let event = factory.generate_order_updated(
2139            order,
2140            report.venue_order_id,
2141            report.quantity,
2142            report.price,
2143            report.trigger_price,
2144            None,
2145            report.ts_last,
2146            now,
2147        );
2148        emitter.try_send_order_event(event)
2149    }
2150
2151    fn order(&self, report: &OrderStatusReport) -> Option<&OrderAny> {
2152        report
2153            .client_order_id
2154            .and_then(|id| self.orders.get(&id))
2155            .or_else(|| {
2156                self.venue_ids
2157                    .get(&report.venue_order_id)
2158                    .and_then(|id| self.orders.get(id))
2159            })
2160    }
2161}
2162
2163fn prepare_order_fills(
2164    report: &OrderStatusReport,
2165    fills: Vec<FillReport>,
2166) -> anyhow::Result<Vec<FillReport>> {
2167    // Quote-sized market orders have no final base quantity before terminal REST state
2168    anyhow::ensure!(
2169        report.order_type != OrderType::Market
2170            || report.order_status.is_closed()
2171            || report.quantity > report.filled_qty,
2172        "market order base quantity is not final"
2173    );
2174    let mut unique = AHashMap::new();
2175
2176    for mut fill in fills {
2177        anyhow::ensure!(
2178            fill.account_id == report.account_id
2179                && fill.venue_order_id == report.venue_order_id
2180                && fill.instrument_id == report.instrument_id
2181                && report.order_side == Some(fill.order_side),
2182            "fill identity does not match the order report"
2183        );
2184        fill.client_order_id = report.client_order_id;
2185        unique.entry(fill.trade_id).or_insert(fill);
2186    }
2187
2188    let mut fills: Vec<_> = unique.into_values().collect();
2189    fills.sort_by_key(|fill| (fill.ts_event, fill.trade_id));
2190
2191    let quantity = fills.iter().try_fold(Decimal::ZERO, |quantity, fill| {
2192        quantity
2193            .checked_add(fill.last_qty.as_decimal())
2194            .context("fill quantity overflow")
2195    })?;
2196
2197    anyhow::ensure!(
2198        quantity == report.filled_qty.as_decimal(),
2199        "REST fills do not cover the order filled quantity"
2200    );
2201    Ok(fills)
2202}
2203
2204fn prepare_user_order_report(
2205    carrier: UserOrderUpdate,
2206    context: Option<OrderContext>,
2207) -> Option<OrderStatusReport> {
2208    let UserOrderUpdate {
2209        mut report,
2210        update,
2211        is_snapshot,
2212        ..
2213    } = carrier;
2214    let have_context = context.is_some();
2215    if let Some(meta) = context {
2216        if report.price.is_none() {
2217            report.price = meta.price;
2218        }
2219
2220        if report.trigger_price.is_none() {
2221            report.trigger_price = meta.trigger_price;
2222        }
2223
2224        if report.trigger_type.is_none() {
2225            report.trigger_type = meta.trigger_type;
2226        }
2227
2228        report.post_only = meta.post_only;
2229        if let Some(submitted) = meta.submitted_product_id {
2230            report.instrument_id = InstrumentId::new(Symbol::new(submitted), *COINBASE_VENUE);
2231        }
2232    }
2233
2234    let safe = match report.order_type {
2235        OrderType::Limit | OrderType::LimitIfTouched => report.price.is_some(),
2236        OrderType::StopLimit => report.price.is_some() && report.trigger_price.is_some(),
2237        OrderType::StopMarket | OrderType::MarketIfTouched => report.trigger_price.is_some(),
2238        _ => true,
2239    };
2240
2241    if !safe {
2242        log::warn!(
2243            "Suppressed unsafe OrderStatusReport for {} {}: missing price/trigger after enrichment",
2244            report.order_type,
2245            update.order_id
2246        );
2247        return None;
2248    }
2249
2250    (!is_snapshot || have_context).then_some(*report)
2251}
2252
2253// Returns the submit-time / enriched metadata for `update`, fetching from
2254// REST and populating the enrichment cache the first time an external order
2255// is seen. `order_contexts` (keyed by `client_order_id`) covers orders this
2256// client placed; `external_order_contexts` (keyed by venue `order_id`) covers
2257// external orders whose `OrderStatusReport` would otherwise be unsafe to
2258// reconstruct (LIMIT / STOP_LIMIT with `price = None`).
2259async fn resolve_order_context(
2260    update: &WsOrderUpdate,
2261    order_type: OrderType,
2262    report_price_missing: bool,
2263    order_contexts: &Arc<Mutex<AHashMap<String, OrderContext>>>,
2264    external_order_contexts: &Arc<Mutex<AHashMap<String, OrderContext>>>,
2265    http_client: &CoinbaseHttpClient,
2266    account_id: AccountId,
2267) -> Option<OrderContext> {
2268    if !update.client_order_id.is_empty() {
2269        let map = order_contexts.lock();
2270        if let Some(meta) = map.get(&update.client_order_id) {
2271            return Some(meta.clone());
2272        }
2273    }
2274
2275    if let Some(meta) = external_order_contexts.lock().get(&update.order_id) {
2276        return Some(meta.clone());
2277    }
2278
2279    let needs_enrichment = report_price_missing
2280        && matches!(
2281            order_type,
2282            OrderType::Limit
2283                | OrderType::StopLimit
2284                | OrderType::LimitIfTouched
2285                | OrderType::StopMarket
2286                | OrderType::MarketIfTouched
2287        );
2288
2289    if !needs_enrichment {
2290        return None;
2291    }
2292
2293    let venue_order_id = VenueOrderId::new(update.order_id.as_str());
2294
2295    match http_client
2296        .request_order_status_report(account_id, None, Some(venue_order_id))
2297        .await
2298    {
2299        Ok(rest_report) => {
2300            let post_only_from_rest = matches!(order_type, OrderType::Limit | OrderType::StopLimit)
2301                && rest_report.post_only;
2302
2303            let meta = OrderContext {
2304                price: rest_report.price,
2305                trigger_price: rest_report.trigger_price,
2306                trigger_type: rest_report.trigger_type,
2307                post_only: post_only_from_rest,
2308                submitted_product_id: None,
2309            };
2310            external_order_contexts
2311                .lock()
2312                .insert(update.order_id.clone(), meta.clone());
2313            Some(meta)
2314        }
2315        Err(e) => {
2316            log::warn!(
2317                "Failed to enrich external order {} via REST: {e}",
2318                update.order_id
2319            );
2320            None
2321        }
2322    }
2323}
2324
2325// Restores price and trigger fields omitted from unfilled user-channel reports
2326// using submit-time metadata keyed by client_order_id.
2327#[derive(Debug, Default, Clone)]
2328struct OrderContext {
2329    price: Option<Price>,
2330    trigger_price: Option<Price>,
2331    trigger_type: Option<TriggerType>,
2332    post_only: bool,
2333    // The `product_id` the order was submitted with. Coinbase rewrites
2334    // aliased products to the canonical id on the user channel, so
2335    // `update.product_id` always reads as the canonical (e.g. `BTC-USD`)
2336    // even for an order placed on the alias side (`BTC-USDC`). Looking the
2337    // submitted id up by `client_order_id` lets us re-key user-channel
2338    // echoes back to the caller's id without rewriting *every* canonical
2339    // event globally.
2340    submitted_product_id: Option<Ustr>,
2341}
2342
2343#[cfg(test)]
2344mod tests {
2345    use std::{cell::RefCell, rc::Rc};
2346
2347    use nautilus_common::{
2348        cache::Cache,
2349        enums::Environment,
2350        messages::{ExecutionEvent, ExecutionReport},
2351    };
2352    use nautilus_live::{manager::ExecutionManager, node::LiveNode};
2353    use nautilus_model::{
2354        accounts::CashAccount,
2355        enums::{AccountType, OrderSide, TimeInForce},
2356        events::{AccountState, OrderEventAny, OrderPendingCancel, OrderPendingUpdate},
2357        identifiers::{Symbol, TraderId, Venue},
2358        instruments::{
2359            CurrencyPair,
2360            stubs::{crypto_future_btcusdt, crypto_perpetual_ethusdt},
2361        },
2362        orders::OrderTestBuilder,
2363        types::{Currency, Money, Quantity},
2364    };
2365    use rstest::rstest;
2366    use ustr::Ustr;
2367
2368    use super::*;
2369    use crate::{
2370        common::{
2371            consts::COINBASE_VENUE,
2372            enums::{
2373                CoinbaseContractExpiryType, CoinbaseOrderSide as CbSide,
2374                CoinbaseOrderStatus as CbStatus, CoinbaseOrderType as CbType,
2375                CoinbaseProductType as CbProductType, CoinbaseRiskManagedBy,
2376                CoinbaseTimeInForce as CbTif, CoinbaseTriggerStatus,
2377            },
2378        },
2379        websocket::messages::WsOrderUpdate,
2380    };
2381
2382    #[rstest]
2383    #[case::limit(101, false, 100)]
2384    #[case::repeated(3, true, 1)]
2385    fn test_cancel_batch_len(
2386        #[case] count: usize,
2387        #[case] repeated: bool,
2388        #[case] expected: usize,
2389    ) {
2390        let requests: Vec<_> = (0..count)
2391            .map(|index| CancelRequest {
2392                strategy_id: StrategyId::from("S-BATCH"),
2393                instrument_id: InstrumentId::from("BTC-USD.COINBASE"),
2394                client_order_id: Some(ClientOrderId::from(format!("C-{index}"))),
2395                venue_order_id: VenueOrderId::from(format!(
2396                    "V-{}",
2397                    if repeated { 0 } else { index }
2398                )),
2399                command_id: UUID4::new(),
2400            })
2401            .collect();
2402
2403        assert_eq!(cancel_batch_len(&requests), expected);
2404    }
2405
2406    #[rstest]
2407    #[case::not_sent(CoinbaseHttpError::auth("No credentials configured"), Some(false))]
2408    #[case::venue_rejected(
2409        CoinbaseHttpError::bad_request("HTTP 400: invalid cancellation"),
2410        Some(true)
2411    )]
2412    #[case::ambiguous(CoinbaseHttpError::Timeout, None)]
2413    #[case::unclassified(CoinbaseHttpError::rate_limit(None), None)]
2414    fn test_batch_cancel_failure_preserves_child_identity(
2415        #[case] error: CoinbaseHttpError,
2416        #[case] rejected: Option<bool>,
2417    ) {
2418        let (emitter, mut receiver) = make_emitter();
2419        let clock = AtomicTime::new(false, UnixNanos::from(53));
2420
2421        let requests = [0, 1, 2].map(|index| CancelRequest {
2422            strategy_id: StrategyId::from(format!("S-BATCH-{index}")),
2423            instrument_id: InstrumentId::from(if index == 0 {
2424                "BTC-USD.COINBASE"
2425            } else {
2426                "ETH-USD.COINBASE"
2427            }),
2428            client_order_id: (index != 1).then(|| ClientOrderId::from(format!("O-BATCH-{index}"))),
2429            venue_order_id: VenueOrderId::from(format!("V-BATCH-{index}")),
2430            command_id: UUID4::new(),
2431        });
2432
2433        let error = anyhow::Error::new(error);
2434        emit_batch_cancel_failure(&emitter, &clock, &requests, "batch_cancel_orders", &error);
2435        if let Some(rejected) = rejected {
2436            for request in requests
2437                .iter()
2438                .filter(|request| request.client_order_id.is_some())
2439            {
2440                let ExecutionEvent::Order(event) = receiver.try_recv().unwrap() else {
2441                    panic!("expected cancellation outcome");
2442                };
2443
2444                let (strategy, instrument, client, venue, cause, reason, ts_event) = match event {
2445                    OrderEventAny::CancelDenied(event) if !rejected => (
2446                        event.strategy_id,
2447                        event.instrument_id,
2448                        event.client_order_id,
2449                        event.venue_order_id,
2450                        event.causation_id,
2451                        event.reason,
2452                        event.ts_event,
2453                    ),
2454                    OrderEventAny::CancelRejected(event) if rejected => (
2455                        event.strategy_id,
2456                        event.instrument_id,
2457                        event.client_order_id,
2458                        event.venue_order_id,
2459                        event.causation_id,
2460                        event.reason,
2461                        event.ts_event,
2462                    ),
2463                    other => panic!("unexpected cancellation outcome {other:?}"),
2464                };
2465
2466                assert_eq!(
2467                    (
2468                        strategy,
2469                        instrument,
2470                        client,
2471                        venue,
2472                        cause,
2473                        reason.as_str(),
2474                        ts_event
2475                    ),
2476                    (
2477                        request.strategy_id,
2478                        request.instrument_id,
2479                        request.client_order_id.unwrap(),
2480                        Some(request.venue_order_id),
2481                        Some(request.command_id),
2482                        error.to_string().as_str(),
2483                        UnixNanos::from(53)
2484                    )
2485                );
2486            }
2487        }
2488
2489        assert!(matches!(
2490            receiver.try_recv(),
2491            Err(tokio::sync::mpsc::error::TryRecvError::Empty)
2492        ));
2493    }
2494
2495    #[rstest]
2496    #[case::cash(AccountType::Cash, true, false)]
2497    #[case::margin_cached(AccountType::Margin, true, true)]
2498    #[case::margin_uncached(AccountType::Margin, false, false)]
2499    fn test_bulk_position_coverage(
2500        #[case] account_type: AccountType,
2501        #[case] cached: bool,
2502        #[case] expected: bool,
2503        #[values(false, true)] perpetual: bool,
2504    ) {
2505        let cache = Rc::new(RefCell::new(Cache::default()));
2506
2507        let core = ExecutionClientCore::new(
2508            TraderId::from("TRADER-001"),
2509            ClientId::from("COINBASE-TEST"),
2510            *COINBASE_VENUE,
2511            OmsType::Netting,
2512            AccountId::from("COINBASE-001"),
2513            account_type,
2514            None,
2515            cache,
2516        );
2517
2518        let config = CoinbaseExecutionClientConfig {
2519            api_key: Some("test-key".into()),
2520            api_secret: Some("test-secret".into()),
2521            account_type,
2522            ..Default::default()
2523        };
2524
2525        let mut client = CoinbaseExecutionClient::new(core, config).unwrap();
2526
2527        let instrument = if perpetual {
2528            let mut instrument = crypto_perpetual_ethusdt();
2529            instrument.id = InstrumentId::from("ETH-PERP.COINBASE");
2530            InstrumentAny::CryptoPerpetual(instrument)
2531        } else {
2532            let mut instrument =
2533                crypto_future_btcusdt(2, 6, Price::from("0.01"), Quantity::from("0.000001"));
2534            instrument.id = InstrumentId::from("BIP-20DEC30-CDE.COINBASE");
2535            InstrumentAny::CryptoFuture(instrument)
2536        };
2537
2538        let instrument_id = instrument.id();
2539
2540        if cached {
2541            client.instruments_cache = Arc::new(AHashMap::from([(
2542                instrument_id.symbol.as_str().to_owned(),
2543                instrument,
2544            )]));
2545        }
2546
2547        assert_eq!(
2548            client.provides_bulk_position_coverage(instrument_id),
2549            expected
2550        );
2551        let foreign_id = InstrumentId::new(instrument_id.symbol, Venue::from("OTHER"));
2552        assert!(!client.provides_bulk_position_coverage(foreign_id));
2553        assert!(!client.provides_bulk_position_coverage(InstrumentId::from("BTC-USD.COINBASE")));
2554        assert!(!client.provides_bulk_position_coverage(InstrumentId::from("UNKNOWN.COINBASE")));
2555    }
2556
2557    #[rstest]
2558    fn test_submit_local_command_failure_classification() {
2559        let err = anyhow::anyhow!("Unsupported Coinbase order configuration");
2560
2561        assert!(is_coinbase_local_command_failure(&err));
2562        assert!(!is_coinbase_explicit_command_rejection(&err));
2563        assert!(!is_coinbase_ambiguous_command_failure(&err));
2564    }
2565
2566    #[rstest]
2567    fn test_submit_http_exchange_failure_classification() {
2568        let err = anyhow::Error::new(CoinbaseHttpError::exchange("HTTP 500: unavailable"))
2569            .context("failed to submit order");
2570
2571        assert!(!is_coinbase_local_command_failure(&err));
2572        assert!(!is_coinbase_explicit_command_rejection(&err));
2573        assert!(is_coinbase_ambiguous_command_failure(&err));
2574    }
2575
2576    #[rstest]
2577    fn test_submit_http_bad_request_failure_classification() {
2578        let err = anyhow::Error::new(CoinbaseHttpError::bad_request("HTTP 400: bad request"))
2579            .context("failed to submit order");
2580
2581        assert!(!is_coinbase_local_command_failure(&err));
2582        assert!(is_coinbase_explicit_command_rejection(&err));
2583        assert!(!is_coinbase_ambiguous_command_failure(&err));
2584    }
2585
2586    #[rstest]
2587    #[case(401)]
2588    #[case(403)]
2589    fn test_submit_http_auth_failure_classification(#[case] status: u16) {
2590        let err = anyhow::Error::new(CoinbaseHttpError::auth(format!(
2591            "HTTP {status}: authentication failed"
2592        )))
2593        .context("failed to submit order");
2594
2595        assert!(!is_coinbase_local_command_failure(&err));
2596        assert!(is_coinbase_explicit_command_rejection(&err));
2597        assert!(!is_coinbase_ambiguous_command_failure(&err));
2598    }
2599
2600    #[rstest]
2601    fn test_submit_http_rate_limit_failure_classification() {
2602        let err = anyhow::Error::new(CoinbaseHttpError::rate_limit(None))
2603            .context("failed to submit order");
2604
2605        assert!(!is_coinbase_local_command_failure(&err));
2606        assert!(!is_coinbase_explicit_command_rejection(&err));
2607        assert!(is_coinbase_ambiguous_submit_failure(&err));
2608        assert!(!is_coinbase_ambiguous_command_failure(&err));
2609    }
2610
2611    #[tokio::test]
2612    async fn test_rate_limited_submit_retains_context() {
2613        let (emitter, mut rx) = make_emitter();
2614        let order_contexts = Arc::new(Mutex::new(AHashMap::new()));
2615
2616        let context = OrderContext {
2617            price: Some(Price::from("100.00")),
2618            trigger_price: Some(Price::from("99.00")),
2619            trigger_type: Some(TriggerType::LastPrice),
2620            post_only: true,
2621            submitted_product_id: Some(Ustr::from("BTC-USDC")),
2622        };
2623        order_contexts
2624            .lock()
2625            .insert("client-1".to_string(), context.clone());
2626        let err = anyhow::Error::new(CoinbaseHttpError::rate_limit(Some(1_000)))
2627            .context("failed to submit order");
2628
2629        handle_coinbase_submit_failure(
2630            &err,
2631            &order_contexts,
2632            &emitter,
2633            StrategyId::from("S-001"),
2634            InstrumentId::from("BTC-USD.COINBASE"),
2635            ClientOrderId::from("client-1"),
2636            UnixNanos::default(),
2637        );
2638
2639        assert!(rx.try_recv().is_err());
2640        {
2641            let map = order_contexts.lock();
2642            let retained = map.get("client-1").expect("submit context retained");
2643            assert_eq!(retained.price, context.price);
2644            assert_eq!(retained.trigger_price, context.trigger_price);
2645            assert_eq!(retained.trigger_type, context.trigger_type);
2646            assert_eq!(retained.post_only, context.post_only);
2647            assert_eq!(retained.submitted_product_id, context.submitted_product_id);
2648        }
2649    }
2650
2651    #[rstest]
2652    #[case(None)]
2653    #[case(Some(400))]
2654    #[case(Some(401))]
2655    #[case(Some(403))]
2656    fn test_definitive_submit_failure_rejects_and_removes_context(#[case] status: Option<u16>) {
2657        let (emitter, mut rx) = make_emitter();
2658        let order_contexts = Mutex::new(AHashMap::from_iter([(
2659            "client-rejected".to_string(),
2660            make_limit_context(),
2661        )]));
2662        let (err, reason) = match status {
2663            None => (
2664                anyhow::anyhow!("Unsupported Coinbase order configuration"),
2665                "submit-order-error: Unsupported Coinbase order configuration".to_string(),
2666            ),
2667            Some(400) => (
2668                anyhow::Error::new(CoinbaseHttpError::bad_request("HTTP 400: refused")),
2669                "submit-order-rejected: bad request: HTTP 400: refused".to_string(),
2670            ),
2671            Some(status @ (401 | 403)) => (
2672                anyhow::Error::new(CoinbaseHttpError::auth(format!("HTTP {status}: refused"))),
2673                format!("submit-order-rejected: auth error: HTTP {status}: refused"),
2674            ),
2675            Some(status) => panic!("unsupported status {status}"),
2676        };
2677
2678        handle_coinbase_submit_failure(
2679            &err,
2680            &order_contexts,
2681            &emitter,
2682            StrategyId::from("S-REJECT"),
2683            InstrumentId::from("BTC-USD.COINBASE"),
2684            ClientOrderId::from("client-rejected"),
2685            UnixNanos::from(42_u64),
2686        );
2687
2688        assert!(order_contexts.lock().is_empty());
2689        let event = rx.try_recv().expect("order rejection emitted");
2690        let ExecutionEvent::Order(OrderEventAny::Rejected(rejected)) = event else {
2691            panic!("expected OrderRejected event, was {event:?}");
2692        };
2693        assert_eq!(rejected.trader_id, TraderId::from("TRADER-001"));
2694        assert_eq!(rejected.strategy_id, StrategyId::from("S-REJECT"));
2695        assert_eq!(
2696            rejected.instrument_id,
2697            InstrumentId::from("BTC-USD.COINBASE")
2698        );
2699        assert_eq!(
2700            rejected.client_order_id,
2701            ClientOrderId::from("client-rejected")
2702        );
2703        assert_eq!(rejected.account_id, AccountId::from("COINBASE-001"));
2704        assert_eq!(rejected.reason, reason);
2705        assert_eq!(rejected.ts_event, UnixNanos::from(42_u64));
2706        assert!(!rejected.reconciliation);
2707        assert!(!rejected.due_post_only);
2708        assert!(rx.try_recv().is_err());
2709    }
2710
2711    #[rstest]
2712    fn test_submit_unmapped_http_4xx_failure_classification() {
2713        let err = anyhow::Error::new(CoinbaseHttpError::http(409, "conflict"))
2714            .context("failed to submit order");
2715
2716        assert!(!is_coinbase_local_command_failure(&err));
2717        assert!(!is_coinbase_explicit_command_rejection(&err));
2718        assert!(!is_coinbase_ambiguous_command_failure(&err));
2719    }
2720
2721    #[rstest]
2722    fn test_submit_local_auth_failure_classification() {
2723        let err = anyhow::Error::new(CoinbaseHttpError::auth("No credentials configured"))
2724            .context("failed to submit order");
2725
2726        assert!(is_coinbase_local_command_failure(&err));
2727        assert!(!is_coinbase_explicit_command_rejection(&err));
2728        assert!(!is_coinbase_ambiguous_command_failure(&err));
2729    }
2730
2731    fn test_instrument() -> InstrumentAny {
2732        let instrument_id = InstrumentId::new(Symbol::new("BTC-USD"), *COINBASE_VENUE);
2733        InstrumentAny::CurrencyPair(
2734            CurrencyPair::builder()
2735                .instrument_id(instrument_id)
2736                .raw_symbol(Symbol::new("BTC-USD"))
2737                .base_currency(Currency::get_or_create_crypto("BTC"))
2738                .quote_currency(Currency::get_or_create_crypto("USD"))
2739                .price_precision(2)
2740                .size_precision(8)
2741                .price_increment(Price::from("0.01"))
2742                .size_increment(Quantity::from("0.00000001"))
2743                .min_quantity(Quantity::from("0.00000001"))
2744                .ts_event(UnixNanos::default())
2745                .ts_init(UnixNanos::default())
2746                .build()
2747                .unwrap(),
2748        )
2749    }
2750
2751    fn make_emitter() -> (
2752        ExecutionEventEmitter,
2753        tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
2754    ) {
2755        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
2756        let mut emitter = ExecutionEventEmitter::new(
2757            get_atomic_clock_realtime(),
2758            TraderId::from("TRADER-001"),
2759            AccountId::new("COINBASE-001"),
2760            AccountType::Cash,
2761            None,
2762        );
2763        emitter.set_sender(tx);
2764        (emitter, rx)
2765    }
2766
2767    fn make_user_order_update(
2768        cumulative: &str,
2769        leaves: &str,
2770        avg_price: &str,
2771        total_fees: &str,
2772        status: CbStatus,
2773    ) -> WsOrderUpdate {
2774        WsOrderUpdate {
2775            order_id: "venue-1".to_string(),
2776            client_order_id: "client-1".to_string(),
2777            contract_expiry_type: CoinbaseContractExpiryType::Unknown,
2778            cumulative_quantity: cumulative.to_string(),
2779            leaves_quantity: leaves.to_string(),
2780            avg_price: avg_price.to_string(),
2781            total_fees: total_fees.to_string(),
2782            status,
2783            product_id: Ustr::from("BTC-USD"),
2784            product_type: CbProductType::Spot,
2785            creation_time: String::new(),
2786            order_side: CbSide::Buy,
2787            order_type: CbType::Limit,
2788            risk_managed_by: CoinbaseRiskManagedBy::Unknown,
2789            time_in_force: CbTif::GoodUntilCancelled,
2790            trigger_status: CoinbaseTriggerStatus::InvalidOrderType,
2791            cancel_reason: String::new(),
2792            reject_reason: String::new(),
2793            total_value_after_fees: String::new(),
2794        }
2795    }
2796
2797    fn make_carrier(update: WsOrderUpdate) -> UserOrderUpdate {
2798        make_carrier_with_kind(update, false)
2799    }
2800
2801    fn make_limit_context() -> OrderContext {
2802        OrderContext {
2803            price: Some(Price::from("100.00")),
2804            ..OrderContext::default()
2805        }
2806    }
2807
2808    fn make_carrier_with_kind(update: WsOrderUpdate, is_snapshot: bool) -> UserOrderUpdate {
2809        let instrument = test_instrument();
2810        let report = crate::websocket::parse::parse_ws_user_event_to_order_status_report(
2811            &update,
2812            &instrument,
2813            AccountId::new("COINBASE-001"),
2814            UnixNanos::default(),
2815            UnixNanos::default(),
2816        )
2817        .unwrap();
2818        UserOrderUpdate {
2819            report: Box::new(report),
2820            update: Box::new(update),
2821            instrument,
2822            is_snapshot,
2823            ts_event: UnixNanos::default(),
2824            ts_init: UnixNanos::default(),
2825        }
2826    }
2827
2828    struct RecoverySpec {
2829        recovery: UserOrderRecovery,
2830        rx: tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
2831        state: Arc<Mutex<RecoveryResponses>>,
2832        server: tokio::task::JoinHandle<()>,
2833        client: Option<CoinbaseExecutionClient>,
2834    }
2835
2836    struct RecoveryResponses {
2837        order: serde_json::Value,
2838        fills: serde_json::Value,
2839        fills_targeted: Option<serde_json::Value>,
2840        fail_fills: bool,
2841        orders: AHashMap<String, serde_json::Value>,
2842        discover_orders: bool,
2843        on_discovery: Option<Box<dyn FnOnce() + Send>>,
2844        requests: Vec<String>,
2845        on_order: Option<Box<dyn FnOnce() + Send>>,
2846    }
2847
2848    impl RecoverySpec {
2849        async fn new() -> Self {
2850            let mut order: serde_json::Value =
2851                serde_json::from_str(include_str!("../test_data/http_order.json")).unwrap();
2852            order["order"]["order_id"] = "venue-1".into();
2853            order["order"]["client_order_id"] = "client-1".into();
2854            order["order"]["order_configuration"]["limit_limit_gtc"]["base_size"] = "1.0".into();
2855            order["order"]["status"] = "FILLED".into();
2856            order["order"]["filled_size"] = "1.0".into();
2857            order["order"]["average_filled_price"] = "112.00".into();
2858            order["order"]["created_time"] = "2024-01-15T09:59:00Z".into();
2859            order["order"]["last_fill_time"] = "2024-01-15T10:31:00Z".into();
2860            let fixture: serde_json::Value =
2861                serde_json::from_str(include_str!("../test_data/http_fills.json")).unwrap();
2862            let mut first = fixture["fills"][0].clone();
2863            first["order_id"] = "venue-1".into();
2864            first["trade_id"] = "trade-alpha".into();
2865            first["size"] = "0.4".into();
2866            first["price"] = "100.00".into();
2867            first["commission"] = "0.04".into();
2868            let mut second = first.clone();
2869            second["trade_id"] = "trade-beta".into();
2870            second["size"] = "0.6".into();
2871            second["price"] = "120.00".into();
2872            second["commission"] = "0.06".into();
2873            second["trade_time"] = "2024-01-15T10:31:00Z".into();
2874            second["liquidity_indicator"] = "TAKER".into();
2875
2876            let state = Arc::new(Mutex::new(RecoveryResponses {
2877                order,
2878                fills: serde_json::json!({"fills": [second, first], "cursor": ""}),
2879                fills_targeted: None,
2880                fail_fills: false,
2881                orders: AHashMap::new(),
2882                discover_orders: true,
2883                on_discovery: None,
2884                requests: Vec::new(),
2885                on_order: None,
2886            }));
2887
2888            let router = axum::Router::new().fallback(axum::routing::get({
2889                let state = Arc::clone(&state);
2890                move |uri: axum::http::Uri| {
2891                    let state = Arc::clone(&state);
2892                    async move {
2893                        let (status, value) = state.lock().response(&uri);
2894                        (status, axum::Json(value))
2895                    }
2896                }
2897            }));
2898
2899            let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2900            let addr = listener.local_addr().unwrap();
2901            let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
2902            let rng = aws_lc_rs::rand::SystemRandom::new();
2903            let key = aws_lc_rs::signature::EcdsaKeyPair::generate_pkcs8(
2904                &aws_lc_rs::signature::ECDSA_P256_SHA256_FIXED_SIGNING,
2905                &rng,
2906            )
2907            .unwrap();
2908            let pem = pem::encode(&pem::Pem::new("PRIVATE KEY", key.as_ref().to_vec()));
2909            let credential = CoinbaseCredential::new("test-key".into(), pem);
2910
2911            let http_client = CoinbaseHttpClient::with_credentials(
2912                credential,
2913                crate::common::enums::CoinbaseEnvironment::Live,
2914                5,
2915                None,
2916                Some(RetryConfig {
2917                    max_retries: 0,
2918                    ..Default::default()
2919                }),
2920            )
2921            .unwrap();
2922
2923            http_client.set_base_url(format!("http://{addr}"));
2924            let instrument = test_instrument();
2925            http_client.cache_instrument(&instrument);
2926            let (emitter, rx) = make_emitter();
2927
2928            let recovery = UserOrderRecovery {
2929                http_client,
2930                emitter,
2931                instruments: Arc::new(AHashMap::from([("BTC-USD".into(), instrument)])),
2932                observed: Arc::new(Mutex::new(AHashMap::new())),
2933                pending: Arc::new(Mutex::new(AHashMap::new())),
2934                published: Arc::new(tokio::sync::Mutex::new(AHashMap::new())),
2935                order_contexts: Arc::new(Mutex::new(AHashMap::new())),
2936                external_order_contexts: Arc::new(Mutex::new(AHashMap::new())),
2937                lifecycle: Arc::new(Mutex::new(RecoveryLifecycle::new(
2938                    TraderId::from("TRADER-001"),
2939                    AccountId::from("COINBASE-001"),
2940                    AccountType::Cash,
2941                ))),
2942                start: crate::http::parse::parse_rfc3339_timestamp("2024-01-01T00:00:00Z").unwrap(),
2943            };
2944
2945            Self {
2946                recovery,
2947                rx,
2948                state,
2949                server,
2950                client: None,
2951            }
2952        }
2953
2954        fn attach(&mut self, node: &LiveNode) {
2955            let core = ExecutionClientCore::new(
2956                TraderId::from("TRADER-001"),
2957                ClientId::from("COINBASE-TEST"),
2958                *COINBASE_VENUE,
2959                OmsType::Netting,
2960                AccountId::from("COINBASE-001"),
2961                AccountType::Cash,
2962                None,
2963                node.kernel().cache.clone(),
2964            );
2965
2966            let config = CoinbaseExecutionClientConfig {
2967                api_key: Some("test-key".into()),
2968                api_secret: Some("test-secret".into()),
2969                ..Default::default()
2970            };
2971
2972            let mut client = CoinbaseExecutionClient::new(core, config).unwrap();
2973            client.emitter = self.recovery.emitter.clone();
2974            client.lifecycle = Arc::clone(&self.recovery.lifecycle);
2975            client.subscribe_order_events();
2976            self.client = Some(client);
2977        }
2978
2979        fn apply(&mut self, node: &LiveNode) {
2980            while let Ok(event) = self.rx.try_recv() {
2981                match event {
2982                    ExecutionEvent::Report(report) => node
2983                        .kernel()
2984                        .exec_engine
2985                        .borrow_mut()
2986                        .reconcile_execution_report(&report),
2987                    ExecutionEvent::Order(event) => {
2988                        node.kernel().exec_engine.borrow_mut().process(&event);
2989                    }
2990                    other => panic!("unexpected recovery event: {other:?}"),
2991                }
2992            }
2993        }
2994
2995        fn reports(&mut self) -> Vec<ExecutionReport> {
2996            let mut reports = Vec::new();
2997
2998            while let Ok(event) = self.rx.try_recv() {
2999                let ExecutionEvent::Report(report) = event else {
3000                    panic!("expected execution report, was {event:?}");
3001                };
3002
3003                reports.push(report);
3004            }
3005
3006            reports
3007        }
3008
3009        fn bundle(&mut self) -> (OrderStatusReport, Vec<FillReport>) {
3010            let mut order = None;
3011            let mut fills = Vec::new();
3012
3013            for report in self.reports() {
3014                match report {
3015                    ExecutionReport::Order(report) => order = Some(*report),
3016                    ExecutionReport::Fill(fill) => fills.push(*fill),
3017                    other => panic!("unexpected recovery report: {other:?}"),
3018                }
3019            }
3020
3021            (order.expect("final order report emitted"), fills)
3022        }
3023    }
3024
3025    impl RecoveryResponses {
3026        fn response(
3027            &mut self,
3028            uri: &axum::http::Uri,
3029        ) -> (axum::http::StatusCode, serde_json::Value) {
3030            self.requests.push(uri.to_string());
3031            let value = if uri.path().ends_with("/fills") {
3032                return self.response_fills(uri);
3033            } else if uri.path().ends_with("/orders/historical/batch") {
3034                self.response_orders()
3035            } else {
3036                if let Some(action) = self.on_order.take() {
3037                    action();
3038                }
3039
3040                let order_id = uri.path().rsplit('/').next().unwrap();
3041                self.orders.get(order_id).unwrap_or(&self.order).clone()
3042            };
3043
3044            (axum::http::StatusCode::OK, value)
3045        }
3046
3047        fn response_fills(
3048            &mut self,
3049            uri: &axum::http::Uri,
3050        ) -> (axum::http::StatusCode, serde_json::Value) {
3051            if self.fail_fills {
3052                return (
3053                    axum::http::StatusCode::SERVICE_UNAVAILABLE,
3054                    serde_json::Value::Null,
3055                );
3056            }
3057
3058            let mut fills = self.fills.clone();
3059            let order_id = url::form_urlencoded::parse(uri.query().unwrap_or_default().as_bytes())
3060                .find(|(key, _)| key == "order_ids")
3061                .map(|(_, value)| value.into_owned());
3062
3063            if let Some(order_id) = order_id {
3064                fills = self.fills_targeted.as_ref().unwrap_or(&fills).clone();
3065                fills["fills"]
3066                    .as_array_mut()
3067                    .unwrap()
3068                    .retain(|fill| fill["order_id"] == order_id);
3069            } else if let Some(action) = self.on_discovery.take() {
3070                action();
3071            }
3072
3073            (axum::http::StatusCode::OK, fills)
3074        }
3075
3076        fn response_orders(&self) -> serde_json::Value {
3077            let orders: Vec<_> = if self.discover_orders {
3078                std::iter::once(self.order["order"].clone())
3079                    .chain(self.orders.values().map(|order| order["order"].clone()))
3080                    .collect()
3081            } else {
3082                Vec::new()
3083            };
3084
3085            serde_json::json!({"orders": orders, "has_next": false, "cursor": ""})
3086        }
3087    }
3088
3089    impl Drop for RecoverySpec {
3090        fn drop(&mut self) {
3091            if let Some(client) = &mut self.client
3092                && let Some(handler) = client.order_event_handler.take()
3093            {
3094                msgbus::unsubscribe_order_events("events.order.*".into(), &handler);
3095            }
3096
3097            self.server.abort();
3098        }
3099    }
3100
3101    #[tokio::test]
3102    async fn test_user_fill_update_preserves_distinct_rest_trades() {
3103        let mut spec = RecoverySpec::new().await;
3104        let update = make_user_order_update("1.0", "0", "112.00", "0.10", CbStatus::Filled);
3105
3106        spec.recovery.handle_update(make_carrier(update)).await;
3107        let (report, fills) = spec.bundle();
3108        let rest = spec
3109            .recovery
3110            .http_client
3111            .request_fill_reports(
3112                report.account_id,
3113                None,
3114                Some(report.venue_order_id),
3115                None,
3116                None,
3117                None,
3118            )
3119            .await
3120            .unwrap();
3121        let mut expected = prepare_order_fills(&report, rest).unwrap();
3122        for (expected, actual) in expected.iter_mut().zip(&fills) {
3123            expected.report_id = actual.report_id;
3124            expected.ts_init = actual.ts_init;
3125        }
3126
3127        assert_eq!(fills, expected);
3128        assert_eq!(report.quantity, Quantity::from("1.00000000"));
3129        assert_eq!(report.filled_qty, Quantity::from("1.00000000"));
3130        assert_eq!(report.order_status, OrderStatus::Filled);
3131        assert_eq!(fills.len(), 2);
3132        assert_eq!(fills[0].trade_id.as_str(), "trade-alpha");
3133        assert_eq!(fills[0].last_qty, Quantity::from("0.40000000"));
3134        assert_eq!(fills[0].last_px, Price::from("100.00"));
3135        assert_eq!(fills[0].commission.to_string(), "0.04 USD");
3136        assert_eq!(fills[1].trade_id.as_str(), "trade-beta");
3137        assert_eq!(fills[1].last_qty, Quantity::from("0.60000000"));
3138        assert_eq!(fills[1].last_px, Price::from("120.00"));
3139        assert_eq!(fills[1].commission.to_string(), "0.06 USD");
3140        assert!(spec.recovery.pending.lock().is_empty());
3141    }
3142
3143    #[rstest]
3144    #[case::missing_fill(false)]
3145    #[case::request_failure(true)]
3146    #[tokio::test]
3147    async fn test_terminal_fill_retries_without_another_user_update(#[case] request_failure: bool) {
3148        let mut spec = RecoverySpec::new().await;
3149        spec.recovery
3150            .order_contexts
3151            .lock()
3152            .insert("client-1".into(), make_limit_context());
3153        let complete = spec.state.lock().fills.clone();
3154        {
3155            let mut state = spec.state.lock();
3156            state.fail_fills = request_failure;
3157            state.fills["fills"].as_array_mut().unwrap().pop();
3158        }
3159
3160        let update = make_user_order_update("1.0", "0", "112.00", "0.10", CbStatus::Filled);
3161        spec.recovery.handle_update(make_carrier(update)).await;
3162
3163        assert!(spec.rx.try_recv().is_err());
3164        assert_eq!(spec.recovery.pending.lock().len(), 1);
3165        assert_eq!(spec.recovery.order_contexts.lock().len(), 1);
3166
3167        {
3168            let mut state = spec.state.lock();
3169            state.fail_fills = false;
3170            state.fills = complete;
3171        }
3172
3173        spec.recovery.retry().await;
3174        let (report, fills) = spec.bundle();
3175
3176        assert_eq!(report.order_status, OrderStatus::Filled);
3177        assert_eq!(fills.len(), 2);
3178        assert!(spec.recovery.pending.lock().is_empty());
3179        assert!(spec.recovery.order_contexts.lock().is_empty());
3180    }
3181
3182    #[tokio::test]
3183    async fn test_reconnect_discovers_closed_order_and_fetches_full_history() {
3184        let mut spec = RecoverySpec::new().await;
3185        let all = spec.state.lock().fills.clone();
3186        // Discovery sees only the later fill; the targeted request must recover the earlier partial too
3187        spec.state.lock().fills["fills"]
3188            .as_array_mut()
3189            .unwrap()
3190            .pop();
3191        spec.recovery.discover().await.unwrap();
3192        spec.state.lock().fills = all;
3193        spec.recovery.retry().await;
3194        let (report, fills) = spec.bundle();
3195        let requests = &spec.state.lock().requests;
3196        let discovery = url::Url::parse(&format!("http://localhost{}", requests[0])).unwrap();
3197        let target =
3198            url::Url::parse(&format!("http://localhost{}", requests.last().unwrap())).unwrap();
3199
3200        assert_eq!(report.order_status, OrderStatus::Filled);
3201        assert_eq!(fills.len(), 2);
3202        assert!(
3203            discovery
3204                .query_pairs()
3205                .any(|(key, value)| key == "start_sequence_timestamp"
3206                    && value == "2024-01-01T00:00:00+00:00")
3207        );
3208        assert!(
3209            target
3210                .query_pairs()
3211                .any(|(key, value)| key == "order_ids" && value == "venue-1")
3212        );
3213        assert!(
3214            !target
3215                .query_pairs()
3216                .any(|(key, _)| key == "start_sequence_timestamp")
3217        );
3218    }
3219
3220    #[tokio::test]
3221    async fn test_snapshot_and_replay_keep_rest_identity() {
3222        let mut spec = RecoverySpec::new().await;
3223        let update = make_user_order_update("1.0", "0", "112.00", "0.10", CbStatus::Filled);
3224        spec.recovery
3225            .handle_update(make_carrier_with_kind(update.clone(), true))
3226            .await;
3227        let (_, first) = spec.bundle();
3228        spec.recovery.handle_update(make_carrier(update)).await;
3229        let (_, replay) = spec.bundle();
3230
3231        assert_eq!(first.len(), replay.len());
3232
3233        for (first, mut replay) in first.iter().zip(replay) {
3234            replay.report_id = first.report_id;
3235            replay.ts_init = first.ts_init;
3236            assert_eq!(*first, replay);
3237        }
3238
3239        assert_eq!(
3240            first
3241                .iter()
3242                .map(|fill| fill.trade_id.as_str())
3243                .collect::<Vec<_>>(),
3244            vec!["trade-alpha", "trade-beta"]
3245        );
3246    }
3247
3248    #[rstest]
3249    #[case(false)]
3250    #[case(true)]
3251    fn test_prepare_user_order_report_requires_snapshot_context(#[case] snapshot: bool) {
3252        let update = make_user_order_update("0", "1.0", "0", "0", CbStatus::Open);
3253        let carrier = make_carrier_with_kind(update, snapshot);
3254        let mut context = make_limit_context();
3255        context.post_only = true;
3256        context.submitted_product_id = Some(Ustr::from("BTC-USDC"));
3257
3258        let report = prepare_user_order_report(carrier.clone(), Some(context)).unwrap();
3259        let unknown = prepare_user_order_report(carrier, None);
3260
3261        assert_eq!(report.price, Some(Price::from("100.00")));
3262        assert_eq!(
3263            report.instrument_id,
3264            InstrumentId::from("BTC-USDC.COINBASE")
3265        );
3266        assert_eq!(report.quantity, Quantity::from("1.00000000"));
3267        assert!(report.post_only);
3268        assert!(unknown.is_none());
3269    }
3270
3271    #[rstest]
3272    #[case(CbType::Limit, false)]
3273    #[case(CbType::StopLimit, false)]
3274    #[case(CbType::Market, true)]
3275    fn test_prepare_user_order_report_preserves_safety_gate(
3276        #[case] order_type: CbType,
3277        #[case] safe: bool,
3278    ) {
3279        let mut update = make_user_order_update("0", "1.0", "0", "0", CbStatus::Open);
3280        update.order_type = order_type;
3281        let report = prepare_user_order_report(make_carrier(update), None);
3282        assert_eq!(report.is_some(), safe);
3283    }
3284
3285    #[rstest]
3286    #[case::startup_first(true)]
3287    #[case::live_first(false)]
3288    #[tokio::test]
3289    async fn test_startup_and_reconnect_overlap_apply_each_fill_once(#[case] startup_first: bool) {
3290        let mut spec = RecoverySpec::new().await;
3291        let node = recovery_node();
3292        let kernel = node.kernel();
3293        let account_id = AccountId::from("COINBASE-001");
3294        let client_id = ClientId::from("COINBASE-TEST");
3295
3296        let core = ExecutionClientCore::new(
3297            TraderId::from("TRADER-001"),
3298            client_id,
3299            *COINBASE_VENUE,
3300            OmsType::Netting,
3301            account_id,
3302            AccountType::Cash,
3303            None,
3304            kernel.cache.clone(),
3305        );
3306
3307        let config = CoinbaseExecutionClientConfig {
3308            api_key: Some("test-key".into()),
3309            api_secret: Some("test-secret".into()),
3310            ..Default::default()
3311        };
3312
3313        let mut client = CoinbaseExecutionClient::new(core, config).unwrap();
3314        client.instruments_cache = Arc::clone(&spec.recovery.instruments);
3315        kernel
3316            .exec_engine
3317            .borrow_mut()
3318            .register_client(Box::new(client))
3319            .unwrap();
3320        let mut manager = ExecutionManager::new(
3321            kernel.clock.clone(),
3322            kernel.cache.clone(),
3323            Default::default(),
3324        )
3325        .unwrap();
3326        let full_order = spec.state.lock().order.clone();
3327        let full_fills = spec.state.lock().fills.clone();
3328        {
3329            let mut state = spec.state.lock();
3330            state.order["order"]["status"] = "OPEN".into();
3331            state.order["order"]["filled_size"] = "0.4".into();
3332            state.order["order"]["average_filled_price"] = "100.00".into();
3333            state.fills["fills"].as_array_mut().unwrap().remove(0);
3334        }
3335
3336        let report = spec
3337            .recovery
3338            .http_client
3339            .request_order_status_report(account_id, None, Some(VenueOrderId::from("venue-1")))
3340            .await
3341            .unwrap();
3342        let fills = spec
3343            .recovery
3344            .http_client
3345            .request_fill_reports(
3346                account_id,
3347                None,
3348                Some(report.venue_order_id),
3349                None,
3350                None,
3351                None,
3352            )
3353            .await
3354            .unwrap();
3355
3356        let mut mass = ExecutionMassStatus::new(
3357            client_id,
3358            account_id,
3359            *COINBASE_VENUE,
3360            UnixNanos::default(),
3361            None,
3362        );
3363        mass.add_order_reports(vec![report.clone()]);
3364        mass.add_fill_reports(fills.clone());
3365        if startup_first {
3366            manager.reconcile_execution_mass_status(&mass, &kernel.exec_engine);
3367        }
3368
3369        let update = make_user_order_update("0.4", "0.6", "100.00", "0.04", CbStatus::Open);
3370        spec.recovery.handle_update(make_carrier(update)).await;
3371        let live_reports = spec.reports();
3372        apply_recovery_reports(&node, &live_reports);
3373
3374        if !startup_first {
3375            manager.reconcile_execution_mass_status(&mass, &kernel.exec_engine);
3376        }
3377
3378        {
3379            let mut state = spec.state.lock();
3380            state.order = full_order;
3381            state.fills = full_fills;
3382        }
3383
3384        spec.recovery.retry().await;
3385        let reports = spec.reports();
3386        apply_recovery_reports(&node, &reports);
3387        let event_count = kernel
3388            .cache
3389            .borrow()
3390            .order(&ClientOrderId::from("client-1"))
3391            .unwrap()
3392            .event_count();
3393        apply_recovery_reports(&node, &reports);
3394
3395        let cache = kernel.cache.borrow();
3396        let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
3397        let positions = cache.positions_refs(None, None, None, None, None);
3398        assert_eq!(order.status(), OrderStatus::Filled);
3399        assert_eq!(order.filled_qty(), Quantity::from("1.00000000"));
3400        assert_eq!(order.event_count(), event_count);
3401        assert_eq!(
3402            order
3403                .trade_ids()
3404                .iter()
3405                .map(|id| id.as_str())
3406                .collect::<Vec<_>>(),
3407            vec!["trade-alpha", "trade-beta"]
3408        );
3409        assert_eq!(positions.len(), 1);
3410        assert_eq!(positions[0].quantity, Quantity::from("1.00000000"));
3411        assert_eq!(positions[0].avg_px_open, rust_decimal_macros::dec!(112));
3412        assert_eq!(positions[0].commissions(), vec![Money::from("0.10 USD")]);
3413        assert_eq!(positions[0].event_count(), 2);
3414    }
3415
3416    #[rstest]
3417    fn test_prepare_user_order_report_retains_trigger_metadata() {
3418        let mut update = make_user_order_update("0", "1", "0", "0", CbStatus::Open);
3419        update.order_type = CbType::StopLimit;
3420
3421        let context = OrderContext {
3422            price: Some(Price::from("100.50")),
3423            trigger_price: Some(Price::from("99.00")),
3424            trigger_type: Some(TriggerType::LastPrice),
3425            post_only: true,
3426            submitted_product_id: Some(Ustr::from("BTC-USDC")),
3427        };
3428
3429        let report = prepare_user_order_report(make_carrier(update), Some(context)).unwrap();
3430        assert_eq!(report.price, Some(Price::from("100.50")));
3431        assert_eq!(report.trigger_price, Some(Price::from("99.00")));
3432        assert_eq!(report.trigger_type, Some(TriggerType::LastPrice));
3433        assert_eq!(
3434            report.instrument_id,
3435            InstrumentId::from("BTC-USDC.COINBASE")
3436        );
3437        assert!(report.post_only);
3438    }
3439
3440    #[rstest]
3441    #[case::partial_cancel("0.4", CbStatus::Cancelled, "CANCELLED")]
3442    #[case::empty_cancel("0", CbStatus::Cancelled, "CANCELLED")]
3443    #[case::empty_expire("0", CbStatus::Expired, "EXPIRED")]
3444    #[tokio::test]
3445    async fn test_terminal_recovery_retains_original_qty(
3446        #[case] cumulative: &str,
3447        #[case] status: CbStatus,
3448        #[case] rest_status: &str,
3449    ) {
3450        let mut spec = RecoverySpec::new().await;
3451        {
3452            let mut state = spec.state.lock();
3453            state.order["order"]["status"] = rest_status.into();
3454            state.order["order"]["filled_size"] = cumulative.into();
3455            let fills = state.fills["fills"].as_array_mut().unwrap();
3456            if cumulative == "0" {
3457                fills.clear();
3458            } else {
3459                fills.remove(0);
3460            }
3461        }
3462
3463        let update = make_user_order_update(cumulative, "0", "100", "0.04", status);
3464        spec.recovery.handle_update(make_carrier(update)).await;
3465        let (report, fills) = spec.bundle();
3466        assert_eq!(report.quantity, Quantity::from("1.00000000"));
3467        assert_eq!(
3468            report.filled_qty.as_decimal(),
3469            cumulative.parse::<Decimal>().unwrap()
3470        );
3471        assert_eq!(fills.len(), usize::from(cumulative != "0"));
3472    }
3473
3474    #[tokio::test]
3475    async fn test_recovery_uses_rest_alias_and_fee_currency_after_context_removal() {
3476        let mut spec = RecoverySpec::new().await;
3477
3478        let InstrumentAny::CurrencyPair(mut pair) = test_instrument() else {
3479            unreachable!()
3480        };
3481
3482        pair.id = InstrumentId::from("BTC-USDC.COINBASE");
3483        pair.raw_symbol = Symbol::from("BTC-USDC");
3484        pair.quote_currency = Currency::USDC();
3485        let instrument = InstrumentAny::CurrencyPair(pair);
3486        spec.recovery.http_client.cache_instrument(&instrument);
3487        spec.recovery.instruments = Arc::new(AHashMap::from([("BTC-USDC".into(), instrument)]));
3488        {
3489            let mut state = spec.state.lock();
3490            state.order["order"]["product_id"] = "BTC-USDC".into();
3491            for fill in state.fills["fills"].as_array_mut().unwrap() {
3492                fill["product_id"] = "BTC-USDC".into();
3493            }
3494        }
3495
3496        let update = make_user_order_update("1", "0", "112", "0.10", CbStatus::Filled);
3497        spec.recovery
3498            .handle_update(make_carrier(update.clone()))
3499            .await;
3500        let (first_order, first_fills) = spec.bundle();
3501        spec.recovery.handle_update(make_carrier(update)).await;
3502        let (replay_order, replay_fills) = spec.bundle();
3503
3504        assert_eq!(
3505            first_order.instrument_id,
3506            InstrumentId::from("BTC-USDC.COINBASE")
3507        );
3508        assert_eq!(replay_order.instrument_id, first_order.instrument_id);
3509        assert_eq!(first_fills.len(), 2);
3510        assert_eq!(replay_fills.len(), 2);
3511
3512        for (first, replay) in first_fills.iter().zip(&replay_fills) {
3513            assert_eq!(first.instrument_id, first_order.instrument_id);
3514            assert_eq!(replay.instrument_id, first.instrument_id);
3515            assert_eq!(first.commission.currency, Currency::USDC());
3516            assert_eq!(replay.commission, first.commission);
3517            assert_eq!(replay.trade_id, first.trade_id);
3518        }
3519    }
3520
3521    #[tokio::test]
3522    async fn test_recovery_deduplicates_rest_rows_before_coverage() {
3523        let mut spec = RecoverySpec::new().await;
3524        {
3525            let mut state = spec.state.lock();
3526            let duplicate = state.fills["fills"][1].clone();
3527            state.fills["fills"].as_array_mut().unwrap().push(duplicate);
3528        }
3529
3530        let update = make_user_order_update("1", "0", "112", "0.10", CbStatus::Filled);
3531        spec.recovery.handle_update(make_carrier(update)).await;
3532        let (_, fills) = spec.bundle();
3533        assert_eq!(fills.len(), 2);
3534        assert_eq!(
3535            fills
3536                .iter()
3537                .map(|fill| fill.last_qty.as_decimal())
3538                .sum::<Decimal>(),
3539            rust_decimal_macros::dec!(1)
3540        );
3541    }
3542
3543    #[tokio::test]
3544    async fn test_discovery_does_not_hide_an_unparsable_fill() {
3545        let mut spec = RecoverySpec::new().await;
3546        spec.state.lock().fills["fills"][0]["size"] = "invalid".into();
3547        spec.recovery.retry().await;
3548        assert_eq!(spec.recovery.pending.lock().len(), 1);
3549        assert!(spec.rx.try_recv().is_err());
3550
3551        spec.state.lock().fills["fills"][0]["size"] = "0.6".into();
3552        spec.recovery.retry().await;
3553        let (_, fills) = spec.bundle();
3554        assert_eq!(fills.len(), 2);
3555        assert!(spec.recovery.pending.lock().is_empty());
3556    }
3557
3558    #[tokio::test]
3559    async fn test_pending_terminal_is_not_replaced_by_stale_update() {
3560        let spec = RecoverySpec::new().await;
3561        let terminal = make_carrier(make_user_order_update(
3562            "1",
3563            "0",
3564            "112",
3565            "0.10",
3566            CbStatus::Filled,
3567        ));
3568        let stale = make_carrier(make_user_order_update(
3569            "0.4",
3570            "0.6",
3571            "100",
3572            "0.04",
3573            CbStatus::Open,
3574        ));
3575        spec.recovery.queue(*terminal.report.clone());
3576        spec.recovery.queue(*stale.report);
3577        let pending = spec.recovery.pending.lock();
3578        assert_eq!(pending[&VenueOrderId::from("venue-1")], *terminal.report);
3579    }
3580
3581    #[tokio::test]
3582    async fn test_recovery_timer_publishes_pending_terminal_without_user_message() {
3583        let mut spec = RecoverySpec::new().await;
3584        let carrier = make_carrier(make_user_order_update(
3585            "1",
3586            "0",
3587            "112",
3588            "0.10",
3589            CbStatus::Filled,
3590        ));
3591        spec.recovery.queue(*carrier.report);
3592        let (_tx, rx) = tokio::sync::mpsc::unbounded_channel();
3593
3594        let reports = {
3595            let run = spec.recovery.run(
3596                rx,
3597                Duration::from_millis(1),
3598                get_atomic_clock_realtime(),
3599                false,
3600            );
3601            tokio::pin!(run);
3602
3603            let collect = async {
3604                let mut reports = Vec::new();
3605
3606                loop {
3607                    let ExecutionEvent::Report(report) = spec.rx.recv().await.unwrap() else {
3608                        panic!("expected execution report");
3609                    };
3610
3611                    let complete = matches!(&report, ExecutionReport::Fill(fill) if fill.trade_id.as_str() == "trade-beta");
3612                    reports.push(report);
3613                    if complete {
3614                        return reports;
3615                    }
3616                }
3617            };
3618
3619            tokio::select! {
3620                () = &mut run => panic!("recovery loop stopped while the input channel is open"),
3621                result = tokio::time::timeout(Duration::from_secs(2), collect) => result.unwrap(),
3622            }
3623        };
3624
3625        assert_eq!(reports.len(), 3);
3626
3627        let trade_ids: Vec<_> = reports
3628            .iter()
3629            .filter_map(|report| match report {
3630                ExecutionReport::Fill(fill) => Some(fill.trade_id.as_str()),
3631                _ => None,
3632            })
3633            .collect();
3634
3635        assert_eq!(trade_ids, vec!["trade-alpha", "trade-beta"]);
3636        assert!(spec.recovery.pending.lock().is_empty());
3637    }
3638
3639    #[tokio::test]
3640    async fn test_recovery_preserves_new_target_queued_during_request() {
3641        let mut spec = RecoverySpec::new().await;
3642        let full_order = spec.state.lock().order.clone();
3643        let full_fills = spec.state.lock().fills.clone();
3644        let partial = make_carrier(make_user_order_update(
3645            "0.4",
3646            "0.6",
3647            "100",
3648            "0.04",
3649            CbStatus::Open,
3650        ));
3651        let terminal = make_carrier(make_user_order_update(
3652            "1",
3653            "0",
3654            "112",
3655            "0.10",
3656            CbStatus::Filled,
3657        ));
3658        spec.recovery.queue(*partial.report);
3659        let pending = Arc::clone(&spec.recovery.pending);
3660        {
3661            let mut state = spec.state.lock();
3662            state.order["order"]["status"] = "OPEN".into();
3663            state.order["order"]["filled_size"] = "0.4".into();
3664            state.fills["fills"].as_array_mut().unwrap().remove(0);
3665            state.on_order = Some(Box::new(move || {
3666                pending
3667                    .lock()
3668                    .insert(terminal.report.venue_order_id, *terminal.report);
3669            }));
3670        }
3671
3672        let error = spec.recovery.recover().await.unwrap_err();
3673        assert_eq!(
3674            error.to_string(),
3675            "recovery target changed while reading REST"
3676        );
3677        assert!(spec.rx.try_recv().is_err());
3678        assert_eq!(
3679            spec.recovery.pending.lock()[&VenueOrderId::from("venue-1")].filled_qty,
3680            Quantity::from("1.00000000")
3681        );
3682
3683        {
3684            let mut state = spec.state.lock();
3685            state.order = full_order;
3686            state.fills = full_fills;
3687        }
3688
3689        spec.recovery.retry().await;
3690        let (terminal, fills) = spec.bundle();
3691        assert_eq!(terminal.order_status, OrderStatus::Filled);
3692        assert_eq!(fills.len(), 2);
3693        assert!(spec.recovery.pending.lock().is_empty());
3694    }
3695
3696    #[tokio::test]
3697    async fn test_recovery_retains_pending_on_closed_event_receiver() {
3698        let mut spec = RecoverySpec::new().await;
3699        spec.rx.close();
3700        let carrier = make_carrier(make_user_order_update(
3701            "1",
3702            "0",
3703            "112",
3704            "0.10",
3705            CbStatus::Filled,
3706        ));
3707        spec.recovery.handle_update(carrier).await;
3708        assert_eq!(spec.recovery.pending.lock().len(), 1);
3709
3710        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
3711        spec.recovery.emitter.set_sender(tx);
3712        spec.rx = rx;
3713        spec.recovery.retry().await;
3714        let (terminal, fills) = spec.bundle();
3715        assert_eq!(terminal.order_status, OrderStatus::Filled);
3716        assert_eq!(fills.len(), 2);
3717        assert!(spec.recovery.pending.lock().is_empty());
3718    }
3719
3720    #[tokio::test]
3721    async fn test_server_time_uses_venue_timestamp_and_rejects_missing_field() {
3722        let spec = RecoverySpec::new().await;
3723        spec.state.lock().order["iso"] = "2024-01-02T03:04:05.123Z".into();
3724        let time = spec
3725            .recovery
3726            .http_client
3727            .request_server_time()
3728            .await
3729            .unwrap();
3730        assert_eq!(
3731            time,
3732            crate::http::parse::parse_rfc3339_timestamp("2024-01-02T03:04:05.123Z").unwrap()
3733        );
3734
3735        spec.state.lock().order["iso"] = serde_json::Value::Null;
3736        let error = spec
3737            .recovery
3738            .http_client
3739            .request_server_time()
3740            .await
3741            .unwrap_err();
3742        assert_eq!(error.to_string(), "server time response is missing iso");
3743    }
3744
3745    #[rstest]
3746    #[case::older_partial("1", CbStatus::Filled)]
3747    #[case::active_before_cancel("0.4", CbStatus::Cancelled)]
3748    #[tokio::test]
3749    async fn test_recovery_waits_for_rest_to_reach_user_state(
3750        #[case] expected_qty: &str,
3751        #[case] status: CbStatus,
3752    ) {
3753        let mut spec = RecoverySpec::new().await;
3754        {
3755            let mut state = spec.state.lock();
3756            state.order["order"]["status"] = "OPEN".into();
3757            state.order["order"]["filled_size"] = "0.4".into();
3758            state.fills["fills"].as_array_mut().unwrap().remove(0);
3759        }
3760
3761        let carrier = make_carrier(make_user_order_update(
3762            expected_qty,
3763            "0",
3764            "112",
3765            "0.10",
3766            status,
3767        ));
3768        spec.recovery.queue(*carrier.report);
3769        let error = spec.recovery.recover().await.unwrap_err();
3770        assert_eq!(
3771            error.to_string(),
3772            "REST order state has not reached the user-channel update"
3773        );
3774        assert_eq!(spec.recovery.pending.lock().len(), 1);
3775        assert!(spec.rx.try_recv().is_err());
3776    }
3777
3778    #[tokio::test]
3779    async fn test_query_order_uses_ordered_rest_recovery() {
3780        let mut spec = RecoverySpec::new().await;
3781        let cache = Rc::new(RefCell::new(Cache::default()));
3782
3783        let core = ExecutionClientCore::new(
3784            TraderId::from("TRADER-001"),
3785            ClientId::from("COINBASE-TEST"),
3786            *COINBASE_VENUE,
3787            OmsType::Netting,
3788            AccountId::from("COINBASE-001"),
3789            AccountType::Cash,
3790            None,
3791            cache,
3792        );
3793
3794        let config = CoinbaseExecutionClientConfig {
3795            api_key: Some("test-key".into()),
3796            api_secret: Some("test-secret".into()),
3797            ..Default::default()
3798        };
3799
3800        let mut client = CoinbaseExecutionClient::new(core, config).unwrap();
3801        client.http_client = spec.recovery.http_client.clone();
3802        client.emitter = spec.recovery.emitter.clone();
3803        client.instruments_cache = Arc::clone(&spec.recovery.instruments);
3804
3805        let command = QueryOrder::new(
3806            TraderId::from("TRADER-001"),
3807            None,
3808            StrategyId::from("S-001"),
3809            InstrumentId::from("BTC-USD.COINBASE"),
3810            ClientOrderId::from("client-1"),
3811            Some(VenueOrderId::from("venue-1")),
3812            UUID4::new(),
3813            UnixNanos::default(),
3814            None,
3815            None,
3816        );
3817        client.query_order(command).unwrap();
3818        tokio::time::timeout(Duration::from_secs(2), client.await_pending_tasks())
3819            .await
3820            .unwrap()
3821            .unwrap();
3822        let (report, fills) = spec.bundle();
3823
3824        assert_eq!(report.order_status, OrderStatus::Filled);
3825        assert_eq!(
3826            fills
3827                .iter()
3828                .map(|fill| fill.trade_id.as_str())
3829                .collect::<Vec<_>>(),
3830            vec!["trade-alpha", "trade-beta"]
3831        );
3832        client.await_pending_tasks().await.unwrap();
3833    }
3834
3835    #[rstest]
3836    #[case::visible(false)]
3837    #[case::delayed_closing_fill(true)]
3838    #[tokio::test]
3839    async fn test_recovery_orders_interleaved_fills_across_orders(#[case] delayed: bool) {
3840        let mut spec = RecoverySpec::new().await;
3841        {
3842            let mut state = spec.state.lock();
3843            let mut closing = state.order.clone();
3844            closing["order"]["order_id"] = "venue-2".into();
3845            closing["order"]["client_order_id"] = "client-2".into();
3846            closing["order"]["side"] = "SELL".into();
3847            closing["order"]["filled_size"] = "0.4".into();
3848            closing["order"]["average_filled_price"] = "110.00".into();
3849            closing["order"]["last_fill_time"] = "2024-01-15T10:01:00Z".into();
3850            closing["order"]["order_configuration"]["limit_limit_gtc"]["base_size"] = "0.4".into();
3851            state.orders.insert("venue-2".into(), closing);
3852            state.order["order"]["last_fill_time"] = "2024-01-15T10:02:00Z".into();
3853            state.fills["fills"][1]["trade_time"] = "2024-01-15T10:00:00Z".into();
3854            state.fills["fills"][0]["trade_time"] = "2024-01-15T10:02:00Z".into();
3855            let mut fill = state.fills["fills"][1].clone();
3856            fill["order_id"] = "venue-2".into();
3857            fill["trade_id"] = "trade-gamma".into();
3858            fill["side"] = "SELL".into();
3859            fill["price"] = "110.00".into();
3860            fill["commission"] = "0.08".into();
3861            fill["trade_time"] = "2024-01-15T10:01:00Z".into();
3862            state.fills["fills"].as_array_mut().unwrap().insert(0, fill);
3863        }
3864
3865        let node = recovery_node();
3866
3867        if delayed {
3868            let closing = spec.state.lock().fills["fills"]
3869                .as_array_mut()
3870                .unwrap()
3871                .remove(0);
3872            spec.recovery.retry().await;
3873            assert!(spec.rx.try_recv().is_err());
3874            assert_eq!(spec.recovery.pending.lock().len(), 2);
3875            spec.state.lock().fills["fills"]
3876                .as_array_mut()
3877                .unwrap()
3878                .push(closing);
3879        }
3880
3881        spec.recovery.retry().await;
3882        let reports = spec.reports();
3883
3884        let trades: Vec<_> = reports
3885            .iter()
3886            .filter_map(|report| match report {
3887                ExecutionReport::Fill(fill) => Some(fill.trade_id.as_str()),
3888                _ => None,
3889            })
3890            .collect();
3891
3892        assert_eq!(trades, vec!["trade-alpha", "trade-gamma", "trade-beta"]);
3893        apply_recovery_reports(&node, &reports);
3894        let events_before: Vec<_> = ["client-1", "client-2"]
3895            .iter()
3896            .map(|id| {
3897                node.kernel()
3898                    .cache
3899                    .borrow()
3900                    .order(&ClientOrderId::from(*id))
3901                    .unwrap()
3902                    .event_count()
3903            })
3904            .collect();
3905
3906        spec.recovery.retry().await;
3907        apply_recovery_reports(&node, &spec.reports());
3908        let cache = node.kernel().cache.borrow();
3909        let positions = cache.positions_open_refs(None, None, None, None, None);
3910        assert_eq!(positions.len(), 1);
3911        assert_eq!(positions[0].quantity, Quantity::from("0.600000"));
3912        assert_eq!(positions[0].avg_px_open, rust_decimal_macros::dec!(120));
3913
3914        for (id, events) in ["client-1", "client-2"].iter().zip(events_before) {
3915            let order = cache.order(&ClientOrderId::from(*id)).unwrap();
3916            assert_eq!(order.status(), OrderStatus::Filled);
3917            assert_eq!(order.event_count(), events);
3918            assert_eq!(
3919                order
3920                    .events()
3921                    .iter()
3922                    .filter(|event| matches!(event, OrderEventAny::FillVoided(_)))
3923                    .count(),
3924                0
3925            );
3926        }
3927    }
3928
3929    #[rstest]
3930    #[case::already_published(true)]
3931    #[case::startup_only(false)]
3932    #[tokio::test]
3933    async fn test_stale_recovery_cannot_void_newer_fills(#[case] published: bool) {
3934        let mut spec = RecoverySpec::new().await;
3935        let node = recovery_node();
3936        spec.recovery.retry().await;
3937        apply_recovery_reports(&node, &spec.reports());
3938        if !published {
3939            spec.recovery.published.lock().await.clear();
3940        }
3941
3942        let before = node
3943            .kernel()
3944            .cache
3945            .borrow()
3946            .order(&ClientOrderId::from("client-1"))
3947            .unwrap()
3948            .event_count();
3949        {
3950            let mut state = spec.state.lock();
3951            state.order["order"]["status"] = "OPEN".into();
3952            state.order["order"]["filled_size"] = "0.4".into();
3953            state.order["order"]["average_filled_price"] = "100.00".into();
3954            state.fills["fills"].as_array_mut().unwrap().remove(0);
3955        }
3956
3957        let partial = make_carrier(make_user_order_update(
3958            "0.4",
3959            "0.6",
3960            "100",
3961            "0.04",
3962            CbStatus::Open,
3963        ));
3964        spec.recovery.queue(*partial.report);
3965        let result = spec.recovery.recover().await;
3966
3967        if published {
3968            assert_eq!(
3969                result.unwrap_err().to_string(),
3970                "REST order state precedes published recovery"
3971            );
3972            assert!(spec.rx.try_recv().is_err());
3973        } else {
3974            result.unwrap();
3975            apply_recovery_reports(&node, &spec.reports());
3976        }
3977
3978        let cache = node.kernel().cache.borrow();
3979        let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
3980        let positions = cache.positions_open_refs(None, None, None, None, None);
3981        assert_eq!(order.filled_qty(), Quantity::from("1.000000"));
3982        assert_eq!(order.status(), OrderStatus::Filled);
3983        assert_eq!(order.event_count(), before);
3984        assert_eq!(
3985            order
3986                .events()
3987                .iter()
3988                .filter(|event| matches!(event, OrderEventAny::FillVoided(_)))
3989                .count(),
3990            0
3991        );
3992        assert_eq!(positions.len(), 1);
3993        assert_eq!(positions[0].quantity, Quantity::from("1.000000"));
3994        assert_eq!(positions[0].avg_px_open, rust_decimal_macros::dec!(112));
3995    }
3996
3997    #[rstest]
3998    #[case::closed_order(false)]
3999    #[case::query_pending(true)]
4000    #[tokio::test]
4001    async fn test_idle_recovery_discovers_fills_visible_after_empty_scan(
4002        #[case] query_pending: bool,
4003    ) {
4004        let mut spec = RecoverySpec::new().await;
4005        let complete = spec.state.lock().fills.clone();
4006        let scanned = Arc::new(tokio::sync::Notify::new());
4007        {
4008            let scanned = Arc::clone(&scanned);
4009            let mut state = spec.state.lock();
4010            state.fills["fills"] = serde_json::Value::Array(Vec::new());
4011            state.discover_orders = false;
4012            state.on_discovery = Some(Box::new(move || scanned.notify_one()));
4013        }
4014
4015        let state = Arc::clone(&spec.state);
4016        let pending = Arc::clone(&spec.recovery.pending);
4017        let target = make_carrier(make_user_order_update(
4018            "1",
4019            "0",
4020            "112",
4021            "0.10",
4022            CbStatus::Filled,
4023        ));
4024
4025        let make_visible = async move {
4026            scanned.notified().await;
4027            state.lock().fills = complete;
4028
4029            if query_pending {
4030                pending
4031                    .lock()
4032                    .insert(target.report.venue_order_id, *target.report);
4033            }
4034        };
4035
4036        let (_tx, rx) = tokio::sync::mpsc::unbounded_channel();
4037        let run = spec.recovery.run(
4038            rx,
4039            Duration::from_millis(1),
4040            get_atomic_clock_realtime(),
4041            false,
4042        );
4043
4044        let collect = async {
4045            make_visible.await;
4046            let mut trades = Vec::new();
4047
4048            loop {
4049                let ExecutionEvent::Report(report) = spec.rx.recv().await.unwrap() else {
4050                    panic!("expected execution report");
4051                };
4052
4053                match report {
4054                    ExecutionReport::Fill(fill) => {
4055                        trades.push(fill.trade_id);
4056                        if trades.len() == 2 {
4057                            return trades;
4058                        }
4059                    }
4060                    ExecutionReport::Order(_) => {}
4061                    other => panic!("unexpected recovery report: {other:?}"),
4062                }
4063            }
4064        };
4065
4066        tokio::pin!(run);
4067        let trades = tokio::select! {
4068            () = &mut run => panic!("recovery loop stopped"),
4069            result = tokio::time::timeout(Duration::from_secs(2), collect) => result.unwrap(),
4070        };
4071        assert_eq!(
4072            trades.iter().map(|id| id.as_str()).collect::<Vec<_>>(),
4073            vec!["trade-alpha", "trade-beta"]
4074        );
4075    }
4076
4077    #[rstest]
4078    #[case::zero_total("0", false)]
4079    #[case::partial_total("0.4", true)]
4080    #[case::empty_targeted_history("0", true)]
4081    #[tokio::test]
4082    async fn test_discovered_trades_survive_stale_totals_and_later_empty_scans(
4083        #[case] filled_qty: &str,
4084        #[case] targeted: bool,
4085    ) {
4086        let mut spec = RecoverySpec::new().await;
4087        let complete = spec.state.lock().fills.clone();
4088        let order = spec.state.lock().order.clone();
4089        {
4090            let mut state = spec.state.lock();
4091            state.order["order"]["status"] = "OPEN".into();
4092            state.order["order"]["filled_size"] = filled_qty.into();
4093
4094            if targeted {
4095                let mut fills = complete.clone();
4096                fills["fills"]
4097                    .as_array_mut()
4098                    .unwrap()
4099                    .retain(|fill| filled_qty == "0.4" && fill["trade_id"] == "trade-alpha");
4100                state.fills_targeted = Some(fills);
4101            }
4102        }
4103
4104        spec.recovery.retry().await;
4105        assert!(spec.rx.try_recv().is_err());
4106        assert_eq!(
4107            spec.recovery.observed.lock()[&VenueOrderId::from("venue-1")].len(),
4108            2
4109        );
4110        spec.state.lock().fills["fills"] = serde_json::Value::Array(Vec::new());
4111        spec.recovery.retry().await;
4112        assert!(spec.rx.try_recv().is_err());
4113        assert_eq!(
4114            spec.recovery.observed.lock()[&VenueOrderId::from("venue-1")].len(),
4115            2
4116        );
4117        {
4118            let mut state = spec.state.lock();
4119            state.order = order;
4120            state.fills = complete;
4121            state.fills_targeted = None;
4122        }
4123
4124        spec.recovery.retry().await;
4125        let (report, fills) = spec.bundle();
4126        assert_eq!(report.filled_qty, Quantity::from("1.00000000"));
4127        assert_eq!(
4128            fills
4129                .iter()
4130                .map(|fill| fill.trade_id.as_str())
4131                .collect::<Vec<_>>(),
4132            vec!["trade-alpha", "trade-beta"]
4133        );
4134        assert!(spec.recovery.observed.lock().is_empty());
4135        assert!(spec.recovery.pending.lock().is_empty());
4136    }
4137
4138    #[tokio::test]
4139    async fn test_recovery_preserves_newer_cached_amendment() {
4140        let mut spec = RecoverySpec::new().await;
4141        {
4142            let mut state = spec.state.lock();
4143            state.order["order"]["status"] = "OPEN".into();
4144            state.order["order"]["filled_size"] = "0.4".into();
4145            state.order["order"]["average_filled_price"] = "100.00".into();
4146            state.fills["fills"].as_array_mut().unwrap().remove(0);
4147        }
4148
4149        let node = recovery_node();
4150        spec.attach(&node);
4151        spec.recovery.retry().await;
4152        apply_recovery_reports(&node, &spec.reports());
4153        let mut amended = spec
4154            .recovery
4155            .http_client
4156            .request_order_status_report(
4157                AccountId::from("COINBASE-001"),
4158                None,
4159                Some(VenueOrderId::from("venue-1")),
4160            )
4161            .await
4162            .unwrap();
4163        amended.quantity = Quantity::from("2.00000000");
4164        amended.price = Some(Price::from("10100.00"));
4165        amended.ts_last = UnixNanos::from(amended.ts_last.as_u64() + 1);
4166        node.kernel()
4167            .exec_engine
4168            .borrow_mut()
4169            .reconcile_order_status_report(&amended);
4170        {
4171            let cache = node.kernel().cache.borrow();
4172            let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4173            assert_eq!(order.quantity(), amended.quantity);
4174            assert_eq!(order.price(), amended.price);
4175        }
4176
4177        spec.recovery.retry().await;
4178        apply_recovery_reports(&node, &spec.reports());
4179        let cache = node.kernel().cache.borrow();
4180        let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4181        assert_eq!(order.quantity(), amended.quantity);
4182        assert_eq!(order.price(), amended.price);
4183        assert_eq!(order.filled_qty(), Quantity::from("0.40000000"));
4184    }
4185
4186    #[tokio::test]
4187    async fn test_external_recovery_emits_order_status_before_real_fills() {
4188        let mut spec = RecoverySpec::new().await;
4189        spec.recovery.retry().await;
4190        let reports = spec.reports();
4191
4192        let ExecutionReport::Order(accepted) = &reports[0] else {
4193            panic!("order status must precede external fills")
4194        };
4195
4196        let ExecutionReport::Order(terminal) = &reports[3] else {
4197            panic!("final order status must follow external fills")
4198        };
4199
4200        let fills: Vec<_> = reports[1..3]
4201            .iter()
4202            .map(|report| {
4203                let ExecutionReport::Fill(fill) = report else {
4204                    panic!("expected canonical fill report")
4205                };
4206
4207                assert_eq!(fill.account_id, accepted.account_id);
4208                assert_eq!(fill.instrument_id, accepted.instrument_id);
4209                assert_eq!(fill.venue_order_id, accepted.venue_order_id);
4210                fill.trade_id
4211            })
4212            .collect();
4213
4214        assert_eq!(reports.len(), 4);
4215        assert_eq!(accepted.order_status, OrderStatus::Accepted);
4216        assert_eq!(accepted.quantity, terminal.quantity);
4217        assert_eq!(accepted.price, terminal.price);
4218        assert_eq!(accepted.filled_qty, Quantity::from("0.00000000"));
4219        assert_eq!(accepted.avg_px, None);
4220        assert_eq!(accepted.ts_last, accepted.ts_accepted);
4221        assert_eq!(terminal.order_status, OrderStatus::Filled);
4222        assert_eq!(terminal.filled_qty, Quantity::from("1.00000000"));
4223        assert_eq!(
4224            fills,
4225            vec![TradeId::from("trade-alpha"), TradeId::from("trade-beta")]
4226        );
4227        let node = recovery_node();
4228        apply_recovery_reports(&node, &reports);
4229        let cache = node.kernel().cache.borrow();
4230        let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4231        assert_eq!(order.status(), OrderStatus::Filled);
4232        assert_eq!(order.filled_qty(), terminal.filled_qty);
4233        assert_eq!(
4234            order.trade_ids().into_iter().copied().collect::<Vec<_>>(),
4235            fills
4236        );
4237    }
4238
4239    #[rstest]
4240    #[case::partial_cancel("CANCELLED", OrderStatus::Canceled, "0.4")]
4241    #[case::partial_expire("EXPIRED", OrderStatus::Expired, "0.4")]
4242    #[case::empty_cancel("CANCELLED", OrderStatus::Canceled, "0")]
4243    #[case::empty_expire("EXPIRED", OrderStatus::Expired, "0")]
4244    #[tokio::test]
4245    async fn test_unknown_recovery_applies_native_terminal_after_fills(
4246        #[case] status: &str,
4247        #[case] expected: OrderStatus,
4248        #[case] filled: &str,
4249        #[values(false, true)] known: bool,
4250        #[values(false, true)] missing_client_id: bool,
4251    ) {
4252        let mut spec = RecoverySpec::new().await;
4253
4254        let client_order_id = ClientOrderId::from(if missing_client_id {
4255            "venue-1"
4256        } else {
4257            "client-1"
4258        });
4259
4260        if missing_client_id {
4261            spec.state.lock().order["order"]["client_order_id"] = "".into();
4262        }
4263
4264        {
4265            let mut state = spec.state.lock();
4266            state.order["order"]["status"] = status.into();
4267            state.order["order"]["filled_size"] = filled.into();
4268            state.order["order"]["average_filled_price"] = "100".into();
4269            let fills = state.fills["fills"].as_array_mut().unwrap();
4270            fills.remove(0);
4271            if filled == "0" {
4272                fills.clear();
4273            }
4274        }
4275
4276        let node = recovery_node();
4277
4278        if known {
4279            let mut report = spec
4280                .recovery
4281                .http_client
4282                .request_order_status_report(
4283                    AccountId::from("COINBASE-001"),
4284                    None,
4285                    Some(VenueOrderId::from("venue-1")),
4286                )
4287                .await
4288                .unwrap();
4289            report.quantity = Quantity::from("2.00000000");
4290            report.price = Some(Price::from("10100.00"));
4291            report.filled_qty = Quantity::from("0.00000000");
4292            report.avg_px = None;
4293            report.order_status = OrderStatus::Accepted;
4294            node.kernel()
4295                .exec_engine
4296                .borrow_mut()
4297                .reconcile_order_status_report(&report);
4298        }
4299
4300        spec.attach(&node);
4301
4302        let status = if expected == OrderStatus::Canceled {
4303            CbStatus::Cancelled
4304        } else {
4305            CbStatus::Expired
4306        };
4307
4308        let mut update = make_user_order_update(filled, "0", "100", "0.04", status);
4309        if missing_client_id {
4310            update.client_order_id.clear();
4311        }
4312
4313        let carrier = make_carrier(update);
4314        spec.recovery.handle_update(carrier).await;
4315        spec.apply(&node);
4316        let before = node
4317            .kernel()
4318            .cache
4319            .borrow()
4320            .order(&client_order_id)
4321            .unwrap()
4322            .event_count();
4323        spec.recovery.retry().await;
4324        spec.apply(&node);
4325        let cache = node.kernel().cache.borrow();
4326        let order = cache.order(&client_order_id).unwrap();
4327        assert_eq!(order.status(), expected);
4328        assert_eq!(
4329            order.quantity(),
4330            Quantity::from(if known { "2.00000000" } else { "1.00000000" })
4331        );
4332        assert_eq!(
4333            order.price(),
4334            Some(Price::from(if known { "10100.00" } else { "10000.00" }))
4335        );
4336        assert_eq!(
4337            order.filled_qty().as_decimal(),
4338            filled.parse::<Decimal>().unwrap()
4339        );
4340        assert_eq!(order.event_count(), before);
4341        assert_eq!(order.trade_ids().len(), usize::from(filled != "0"));
4342        assert!(spec.recovery.lifecycle.lock().pending.is_empty());
4343    }
4344
4345    #[rstest]
4346    #[tokio::test]
4347    async fn test_recovery_accepts_cached_submitted_order_without_replacing_terms(
4348        #[values(false, true)] fills_lag: bool,
4349    ) {
4350        let mut spec = RecoverySpec::new().await;
4351        {
4352            let mut state = spec.state.lock();
4353            state.order["order"]["status"] = "OPEN".into();
4354            state.order["order"]["filled_size"] = if fills_lag { "0.4" } else { "0" }.into();
4355            if fills_lag {
4356                state.fills["fills"].as_array_mut().unwrap().remove(0);
4357                state.fail_fills = true;
4358            } else {
4359                state.fills["fills"].as_array_mut().unwrap().clear();
4360            }
4361        }
4362
4363        let mut order = OrderTestBuilder::new(OrderType::Limit)
4364            .trader_id(TraderId::from("TRADER-001"))
4365            .strategy_id(StrategyId::from("STRATEGY-007"))
4366            .instrument_id(InstrumentId::from("BTC-USD.COINBASE"))
4367            .client_order_id(ClientOrderId::from("client-1"))
4368            .side(OrderSide::Buy)
4369            .quantity(Quantity::from("2.00000000"))
4370            .price(Price::from("10100.00"))
4371            .build();
4372
4373        let factory = OrderEventFactory::new(
4374            TraderId::from("TRADER-001"),
4375            AccountId::from("COINBASE-001"),
4376            AccountType::Cash,
4377            None,
4378        );
4379        let submitted = factory.generate_order_submitted(&order, UnixNanos::from(10));
4380        order.apply(submitted).unwrap();
4381        let node = recovery_node();
4382        node.kernel()
4383            .cache
4384            .borrow_mut()
4385            .add_order(order, None, None, false)
4386            .unwrap();
4387        spec.attach(&node);
4388        let carrier = make_carrier(make_user_order_update(
4389            if fills_lag { "0.4" } else { "0" },
4390            if fills_lag { "0.6" } else { "1" },
4391            "100",
4392            "0.04",
4393            CbStatus::Open,
4394        ));
4395        spec.recovery.handle_update(carrier).await;
4396        spec.apply(&node);
4397        {
4398            let cache = node.kernel().cache.borrow();
4399            let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4400            assert_eq!(order.status(), OrderStatus::Accepted);
4401            assert_eq!(order.quantity(), Quantity::from("2.00000000"));
4402            assert_eq!(order.price(), Some(Price::from("10100.00")));
4403            assert_eq!(order.filled_qty(), Quantity::from("0.00000000"));
4404        }
4405
4406        if fills_lag {
4407            assert_eq!(spec.recovery.pending.lock().len(), 1);
4408        } else {
4409            assert!(spec.recovery.pending.lock().is_empty());
4410        }
4411    }
4412
4413    #[rstest]
4414    #[case::cancel(OrderStatus::PendingCancel)]
4415    #[case::update(OrderStatus::PendingUpdate)]
4416    #[tokio::test]
4417    async fn test_recovery_preserves_pending_command_on_unchanged_open_report(
4418        #[case] expected_status: OrderStatus,
4419    ) {
4420        let mut spec = RecoverySpec::new().await;
4421        {
4422            let mut state = spec.state.lock();
4423            state.order["order"]["status"] = "OPEN".into();
4424            state.order["order"]["filled_size"] = "0".into();
4425            state.fills["fills"].as_array_mut().unwrap().clear();
4426        }
4427
4428        let report = spec
4429            .recovery
4430            .http_client
4431            .request_order_status_report(
4432                AccountId::from("COINBASE-001"),
4433                None,
4434                Some(VenueOrderId::from("venue-1")),
4435            )
4436            .await
4437            .unwrap();
4438        let node = recovery_node();
4439        node.kernel()
4440            .exec_engine
4441            .borrow_mut()
4442            .reconcile_order_status_report(&report);
4443        let order = node
4444            .kernel()
4445            .cache
4446            .borrow()
4447            .order(&ClientOrderId::from("client-1"))
4448            .unwrap()
4449            .clone();
4450        let ts = UnixNanos::from(report.ts_last.as_u64() + 1);
4451
4452        let pending = match expected_status {
4453            OrderStatus::PendingCancel => OrderEventAny::PendingCancel(OrderPendingCancel::new(
4454                order.trader_id(),
4455                order.strategy_id(),
4456                order.instrument_id(),
4457                order.client_order_id(),
4458                order.account_id(),
4459                UUID4::new(),
4460                ts,
4461                ts,
4462                false,
4463                order.venue_order_id(),
4464            )),
4465            OrderStatus::PendingUpdate => OrderEventAny::PendingUpdate(OrderPendingUpdate::new(
4466                order.trader_id(),
4467                order.strategy_id(),
4468                order.instrument_id(),
4469                order.client_order_id(),
4470                order.account_id(),
4471                UUID4::new(),
4472                ts,
4473                ts,
4474                false,
4475                order.venue_order_id(),
4476            )),
4477            _ => unreachable!(),
4478        };
4479
4480        node.kernel().exec_engine.borrow_mut().process(&pending);
4481        let before = node
4482            .kernel()
4483            .cache
4484            .borrow()
4485            .order(&order.client_order_id())
4486            .unwrap()
4487            .event_count();
4488        node.kernel()
4489            .exec_engine
4490            .borrow_mut()
4491            .reconcile_order_status_report(&report);
4492        {
4493            let cache = node.kernel().cache.borrow();
4494            let order = cache.order(&order.client_order_id()).unwrap();
4495            assert_eq!(order.status(), expected_status);
4496            assert_eq!(order.event_count(), before);
4497        }
4498
4499        spec.attach(&node);
4500        spec.recovery.queue(report);
4501        spec.recovery.retry().await;
4502        spec.apply(&node);
4503        let cache = node.kernel().cache.borrow();
4504        let order = cache.order(&order.client_order_id()).unwrap();
4505        assert_eq!(order.status(), expected_status);
4506        assert_eq!(order.event_count(), before);
4507        assert_eq!(order.quantity(), Quantity::from("1.00000000"));
4508        assert_eq!(order.price(), Some(Price::from("10000.00")));
4509        assert_eq!(order.filled_qty(), Quantity::from("0.00000000"));
4510        assert!(order.trade_ids().is_empty());
4511        assert!(
4512            cache
4513                .positions_refs(None, None, None, None, None)
4514                .is_empty()
4515        );
4516    }
4517
4518    #[tokio::test]
4519    async fn test_partial_user_amendment_survives_snapshot_and_rest_replay() {
4520        let mut spec = RecoverySpec::new().await;
4521        {
4522            let mut state = spec.state.lock();
4523            state.order["order"]["status"] = "OPEN".into();
4524            state.order["order"]["filled_size"] = "0.4".into();
4525            state.order["order"]["average_filled_price"] = "100".into();
4526            state.fills["fills"].as_array_mut().unwrap().remove(0);
4527        }
4528
4529        let node = recovery_node();
4530        spec.attach(&node);
4531        spec.recovery.retry().await;
4532        spec.apply(&node);
4533
4534        let context = OrderContext {
4535            price: Some(Price::from("10100.00")),
4536            ..Default::default()
4537        };
4538
4539        spec.recovery
4540            .order_contexts
4541            .lock()
4542            .insert("client-1".into(), context);
4543        let update = make_user_order_update("0.4", "1.6", "100", "0.04", CbStatus::Open);
4544        spec.recovery.handle_update(make_carrier(update)).await;
4545        spec.apply(&node);
4546        {
4547            let cache = node.kernel().cache.borrow();
4548            let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4549            assert_eq!(order.quantity(), Quantity::from("2.00000000"));
4550            assert_eq!(order.price(), Some(Price::from("10100.00")));
4551        }
4552
4553        let stale = make_user_order_update("0.4", "0.6", "100", "0.04", CbStatus::Open);
4554        spec.recovery
4555            .handle_update(make_carrier_with_kind(stale, true))
4556            .await;
4557        spec.apply(&node);
4558        let cache = node.kernel().cache.borrow();
4559        let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4560        assert_eq!(order.quantity(), Quantity::from("2.00000000"));
4561        assert_eq!(order.price(), Some(Price::from("10100.00")));
4562        assert_eq!(order.filled_qty(), Quantity::from("0.40000000"));
4563        assert_eq!(
4564            order.trade_ids().into_iter().copied().collect::<Vec<_>>(),
4565            vec![TradeId::from("trade-alpha")]
4566        );
4567    }
4568
4569    #[rstest]
4570    #[case::partial("OPEN", "2", "0.5", OrderStatus::PartiallyFilled)]
4571    #[case::filled_increase("FILLED", "1", "0.5", OrderStatus::Filled)]
4572    #[case::filled_decrease("FILLED", "1", "2", OrderStatus::Filled)]
4573    #[tokio::test]
4574    async fn test_recovery_updates_missed_quantity_amendment_before_fills(
4575        #[case] status: &str,
4576        #[case] quantity: &str,
4577        #[case] cached_quantity: &str,
4578        #[case] expected_status: OrderStatus,
4579    ) {
4580        let mut spec = RecoverySpec::new().await;
4581        {
4582            let mut state = spec.state.lock();
4583            state.order["order"]["status"] = status.into();
4584            state.order["order"]["order_configuration"]["limit_limit_gtc"]["base_size"] =
4585                quantity.into();
4586        }
4587
4588        let mut order = OrderTestBuilder::new(OrderType::Limit)
4589            .trader_id(TraderId::from("TRADER-001"))
4590            .strategy_id(StrategyId::from("STRATEGY-007"))
4591            .instrument_id(InstrumentId::from("BTC-USD.COINBASE"))
4592            .client_order_id(ClientOrderId::from("client-1"))
4593            .side(OrderSide::Buy)
4594            .quantity(cached_quantity.parse().unwrap())
4595            .price(Price::from("10100.00"))
4596            .build();
4597
4598        let factory = OrderEventFactory::new(
4599            TraderId::from("TRADER-001"),
4600            AccountId::from("COINBASE-001"),
4601            AccountType::Cash,
4602            None,
4603        );
4604        let accepted = factory.generate_order_accepted(
4605            &order,
4606            VenueOrderId::from("venue-1"),
4607            UnixNanos::from(1),
4608            UnixNanos::from(1),
4609        );
4610        order.apply(accepted).unwrap();
4611        let before = order.event_count();
4612        let node = recovery_node();
4613        node.kernel()
4614            .cache
4615            .borrow_mut()
4616            .add_order(order, None, None, false)
4617            .unwrap();
4618        spec.attach(&node);
4619        spec.recovery.retry().await;
4620        spec.apply(&node);
4621        let cache = node.kernel().cache.borrow();
4622        let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4623        let events = order.events();
4624        let update_index = events
4625            .iter()
4626            .position(|event| matches!(event, OrderEventAny::Updated(_)))
4627            .unwrap();
4628        let fill_index = events
4629            .iter()
4630            .position(|event| matches!(event, OrderEventAny::Filled(_)))
4631            .unwrap();
4632        let positions = cache.positions_open_refs(None, None, None, None, None);
4633        assert!(update_index < fill_index);
4634        assert_eq!(order.status(), expected_status);
4635        assert_eq!(
4636            order.quantity().as_decimal(),
4637            quantity.parse::<Decimal>().unwrap()
4638        );
4639        assert_eq!(order.price(), Some(Price::from("10100.00")));
4640        assert_eq!(order.filled_qty(), Quantity::from("1.00000000"));
4641        assert_eq!(order.event_count(), before + 3);
4642        assert_eq!(
4643            order.trade_ids().into_iter().copied().collect::<Vec<_>>(),
4644            vec![TradeId::from("trade-alpha"), TradeId::from("trade-beta")]
4645        );
4646        assert_eq!(positions.len(), 1);
4647        assert_eq!(positions[0].quantity, Quantity::from("1.00000000"));
4648        assert_eq!(positions[0].avg_px_open, Decimal::from(112));
4649        assert_eq!(positions[0].commissions(), vec![Money::from("0.10 USD")]);
4650    }
4651
4652    #[rstest]
4653    #[tokio::test]
4654    async fn test_quote_market_recovery_normalizes_base_units_after_terminal_totals(
4655        #[values(false, true)] known: bool,
4656        #[values(false, true)] snapshot: bool,
4657        #[values(false, true)] partial: bool,
4658    ) {
4659        let mut spec = RecoverySpec::new().await;
4660        {
4661            let mut state = spec.state.lock();
4662            state.order["order"]["order_type"] = "MARKET".into();
4663            state.order["order"]["time_in_force"] = "IMMEDIATE_OR_CANCEL".into();
4664            state.order["order"]["order_configuration"] =
4665                serde_json::json!({"market_market_ioc": {"quote_size": "112"}});
4666            state.order["order"]["size_in_quote"] = true.into();
4667            state.fills["fills"][0]["size_in_quote"] = true.into();
4668            state.fills["fills"][0]["size"] = "72".into();
4669            state.fills["fills"][1]["size_in_quote"] = true.into();
4670            state.fills["fills"][1]["size"] = "40".into();
4671        }
4672
4673        let node = recovery_node();
4674
4675        if known {
4676            let mut order = OrderTestBuilder::new(OrderType::Market)
4677                .trader_id(TraderId::from("TRADER-001"))
4678                .strategy_id(StrategyId::from("STRATEGY-007"))
4679                .instrument_id(InstrumentId::from("BTC-USD.COINBASE"))
4680                .client_order_id(ClientOrderId::from("client-1"))
4681                .side(OrderSide::Buy)
4682                .quantity(Quantity::from("112.00"))
4683                .quote_quantity(true)
4684                .time_in_force(TimeInForce::Ioc)
4685                .build();
4686
4687            let factory = OrderEventFactory::new(
4688                TraderId::from("TRADER-001"),
4689                AccountId::from("COINBASE-001"),
4690                AccountType::Cash,
4691                None,
4692            );
4693            let accepted = factory.generate_order_accepted(
4694                &order,
4695                VenueOrderId::from("venue-1"),
4696                UnixNanos::from(1),
4697                UnixNanos::from(1),
4698            );
4699            order.apply(accepted).unwrap();
4700            node.kernel()
4701                .cache
4702                .borrow_mut()
4703                .add_order(order, None, None, false)
4704                .unwrap();
4705            spec.recovery
4706                .order_contexts
4707                .lock()
4708                .insert("client-1".into(), OrderContext::default());
4709        }
4710
4711        spec.attach(&node);
4712        let final_order = spec.state.lock().order.clone();
4713        let final_fills = spec.state.lock().fills.clone();
4714
4715        if partial {
4716            {
4717                let mut state = spec.state.lock();
4718                state.order["order"]["status"] = "OPEN".into();
4719                state.order["order"]["filled_size"] = "0.4".into();
4720                state.order["order"]["average_filled_price"] = "100".into();
4721                state.fills["fills"].as_array_mut().unwrap().remove(0);
4722            }
4723
4724            let mut update = make_user_order_update("0.4", "0", "100", "0.04", CbStatus::Open);
4725            update.order_type = CbType::Market;
4726            spec.recovery.handle_update(make_carrier(update)).await;
4727            assert!(spec.rx.try_recv().is_err());
4728            assert_eq!(spec.recovery.pending.lock().len(), 1);
4729        }
4730
4731        {
4732            let mut state = spec.state.lock();
4733            state.order = final_order;
4734            state.fills = final_fills;
4735        }
4736
4737        let mut update = make_user_order_update("1", "0", "112", "0.10", CbStatus::Filled);
4738        update.order_type = CbType::Market;
4739        spec.recovery
4740            .handle_update(make_carrier_with_kind(update, snapshot))
4741            .await;
4742        spec.apply(&node);
4743        let before = node
4744            .kernel()
4745            .cache
4746            .borrow()
4747            .order(&ClientOrderId::from("client-1"))
4748            .unwrap()
4749            .event_count();
4750        spec.recovery.retry().await;
4751        spec.apply(&node);
4752        let cache = node.kernel().cache.borrow();
4753        let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4754        let positions = cache.positions_refs(None, None, None, None, None);
4755        assert_eq!(order.status(), OrderStatus::Filled);
4756        assert_eq!(order.quantity(), Quantity::from("1.00000000"));
4757        assert_eq!(order.filled_qty(), Quantity::from("1.00000000"));
4758        assert!(!order.is_quote_quantity());
4759        assert_eq!(
4760            order.trade_ids().into_iter().copied().collect::<Vec<_>>(),
4761            vec![TradeId::from("trade-alpha"), TradeId::from("trade-beta")]
4762        );
4763        assert_eq!(order.event_count(), before);
4764        assert_eq!(positions.len(), 1);
4765        assert_eq!(positions[0].quantity, Quantity::from("1.00000000"));
4766        assert_eq!(positions[0].avg_px_open, Decimal::from(112));
4767        assert_eq!(positions[0].commissions(), vec![Money::from("0.10 USD")]);
4768    }
4769
4770    fn apply_recovery_reports(node: &LiveNode, reports: &[ExecutionReport]) {
4771        for report in reports {
4772            node.kernel()
4773                .exec_engine
4774                .borrow_mut()
4775                .reconcile_execution_report(report);
4776        }
4777    }
4778
4779    fn recovery_node() -> LiveNode {
4780        let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
4781            .unwrap()
4782            .build()
4783            .unwrap();
4784        let kernel = node.kernel();
4785        kernel
4786            .cache
4787            .borrow_mut()
4788            .add_instrument(test_instrument())
4789            .unwrap();
4790        let account_id = AccountId::from("COINBASE-001");
4791
4792        let state = AccountState::new(
4793            account_id,
4794            AccountType::Cash,
4795            vec![AccountBalance::new(
4796                Money::from("100000 USD"),
4797                Money::from("0 USD"),
4798                Money::from("100000 USD"),
4799            )],
4800            vec![],
4801            true,
4802            UUID4::new(),
4803            UnixNanos::default(),
4804            UnixNanos::default(),
4805            None,
4806        );
4807        kernel
4808            .cache
4809            .borrow_mut()
4810            .add_account(AccountAny::Cash(CashAccount::new(state, true, false)))
4811            .unwrap();
4812        node
4813    }
4814}