1use std::{
19 collections::hash_map::Entry,
20 future::Future,
21 sync::Arc,
22 time::{Duration, Instant},
23};
24
25use ahash::{AHashMap, AHashSet};
26use anyhow::Context;
27use async_trait::async_trait;
28use nautilus_common::{
29 clients::ExecutionClient,
30 factories::OrderEventFactory,
31 live::runner::get_exec_event_sender,
32 messages::{
33 ExecutionReport,
34 execution::{
35 BatchCancelOrders, CancelAllOrders, CancelOrder, GenerateFillReports,
36 GenerateFillReportsBuilder, GenerateOrderStatusReport, GenerateOrderStatusReports,
37 GenerateOrderStatusReportsBuilder, GeneratePositionStatusReports,
38 GeneratePositionStatusReportsBuilder, ModifyOrder, QueryAccount, QueryOrder,
39 SubmitOrder, SubmitOrderList,
40 },
41 },
42 msgbus::{self, TypedHandler},
43};
44use nautilus_core::{
45 DurationNanos, Params, UUID4, UnixNanos,
46 time::{AtomicTime, get_atomic_clock_realtime},
47};
48use nautilus_live::{
49 ExecutionClientCore, ExecutionEventEmitter, SocketControl,
50 execution::failure::CommandFailure,
51 task::{TaskGroup, TaskGroupGuard},
52};
53use nautilus_model::{
54 accounts::AccountAny,
55 enums::{AccountType, OmsType, OrderStatus, OrderType, TriggerType},
56 events::{OrderDeniedReason, OrderEventAny, OrderUpdated},
57 identifiers::{
58 AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, Symbol, TradeId, TraderId,
59 Venue, VenueOrderId,
60 },
61 instruments::{Instrument, InstrumentAny},
62 orders::{Order, OrderAny},
63 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
64 types::{AccountBalance, MarginBalance, Price, Quantity},
65};
66use nautilus_network::retry::RetryConfig;
67use parking_lot::Mutex;
68use rust_decimal::Decimal;
69use ustr::Ustr;
70
71use crate::{
72 common::{
73 consts::COINBASE_VENUE,
74 credential::CoinbaseCredential,
75 enums::{CoinbaseProductType, CoinbaseWsChannel},
76 },
77 config::CoinbaseExecutionClientConfig,
78 http::{
79 client::CoinbaseHttpClient, error::Error as CoinbaseHttpError,
80 parse::parse_ws_cfm_account_state,
81 },
82 websocket::{
83 client::CoinbaseWebSocketClient,
84 handler::{NautilusWsMessage, UserOrderUpdate},
85 messages::WsOrderUpdate,
86 },
87};
88
89const BATCH_CANCEL_CHUNK: usize = 100;
92
93const ACCOUNT_REGISTERED_TIMEOUT_SECS: f64 = 30.0;
96
97#[derive(Debug)]
99pub struct CoinbaseExecutionClient {
100 core: ExecutionClientCore,
101 clock: &'static AtomicTime,
102 config: CoinbaseExecutionClientConfig,
103 emitter: ExecutionEventEmitter,
104 http_client: CoinbaseHttpClient,
105 ws_user: CoinbaseWebSocketClient,
106 session_tasks: TaskGroup,
107 pending_tasks: TaskGroup,
108 shutdown_errors: Vec<String>,
109 instruments_cache: Arc<AHashMap<String, InstrumentAny>>,
110 fills_observed: Arc<Mutex<AHashMap<VenueOrderId, AHashSet<TradeId>>>>,
111 fills_pending: Arc<Mutex<AHashMap<VenueOrderId, OrderStatusReport>>>,
112 fills_published: Arc<tokio::sync::Mutex<AHashMap<VenueOrderId, OrderStatusReport>>>,
113 fills_start: Option<UnixNanos>,
114 lifecycle: Arc<Mutex<RecoveryLifecycle>>,
115 order_event_handler: Option<TypedHandler<OrderEventAny>>,
116 order_contexts: Arc<Mutex<AHashMap<String, OrderContext>>>,
117 external_order_contexts: Arc<Mutex<AHashMap<String, OrderContext>>>,
124}
125
126impl CoinbaseExecutionClient {
127 pub fn new(
134 core: ExecutionClientCore,
135 config: CoinbaseExecutionClientConfig,
136 ) -> anyhow::Result<Self> {
137 let credential = CoinbaseCredential::resolve(
138 config.api_key.as_ref().map(|value| value.expose_secret()),
139 config
140 .api_secret
141 .as_ref()
142 .map(|value| value.expose_secret()),
143 )
144 .ok_or_else(|| {
145 anyhow::anyhow!(
146 "Coinbase credentials not available; set COINBASE_API_KEY and COINBASE_API_SECRET or pass them in the config"
147 )
148 })?;
149 let proxy_url = config
150 .proxy_url
151 .as_ref()
152 .map(|value| value.expose_secret().to_owned());
153
154 let retry_config = RetryConfig {
155 max_retries: config.max_retries,
156 initial_delay_ms: config.retry_delay_initial_ms,
157 max_delay_ms: config.retry_delay_max_ms,
158 backoff_factor: 2.0,
159 jitter_ms: 250,
160 operation_timeout_ms: Some(60_000),
161 immediate_first: false,
162 max_elapsed_ms: Some(180_000),
163 };
164
165 let http_client = CoinbaseHttpClient::with_credentials(
166 credential.clone(),
167 config.environment,
168 config.http_timeout_secs,
169 proxy_url.clone(),
170 Some(retry_config),
171 )
172 .map_err(|e| anyhow::anyhow!("Failed to create Coinbase HTTP client: {e}"))?;
173
174 if let Some(ref url) = config.base_url_rest {
175 http_client.set_base_url(url.clone());
176 }
177
178 let ws_url = config.ws_url();
179 let ws_user = CoinbaseWebSocketClient::with_credential(
180 &ws_url,
181 credential,
182 config.transport_backend,
183 proxy_url,
184 )
185 .with_socket_control(SocketControl::new(
186 core.client_id,
187 Some(*COINBASE_VENUE),
188 "coinbase-user-streams",
189 ));
190
191 let clock = get_atomic_clock_realtime();
192 let emitter = ExecutionEventEmitter::new(
193 clock,
194 core.trader_id,
195 core.account_id,
196 core.account_type,
197 None,
198 );
199
200 let session_tasks = TaskGroup::new();
201 let pending_tasks = TaskGroup::new();
202
203 let lifecycle = RecoveryLifecycle::new(core.trader_id, core.account_id, core.account_type);
204
205 Ok(Self {
206 core,
207 clock,
208 config,
209 emitter,
210 http_client,
211 ws_user,
212 session_tasks,
213 pending_tasks,
214 shutdown_errors: Vec::new(),
215 instruments_cache: Arc::new(AHashMap::new()),
216 fills_observed: Arc::new(Mutex::new(AHashMap::new())),
217 fills_pending: Arc::new(Mutex::new(AHashMap::new())),
218 fills_published: Arc::new(tokio::sync::Mutex::new(AHashMap::new())),
219 fills_start: None,
220 lifecycle: Arc::new(Mutex::new(lifecycle)),
221 order_event_handler: None,
222 order_contexts: Arc::new(Mutex::new(AHashMap::new())),
223 external_order_contexts: Arc::new(Mutex::new(AHashMap::new())),
224 })
225 }
226
227 fn order_recovery(&self) -> UserOrderRecovery {
228 UserOrderRecovery {
229 http_client: self.http_client.clone(),
230 emitter: self.emitter.clone(),
231 instruments: Arc::clone(&self.instruments_cache),
232 observed: Arc::clone(&self.fills_observed),
233 pending: Arc::clone(&self.fills_pending),
234 published: Arc::clone(&self.fills_published),
235 order_contexts: Arc::clone(&self.order_contexts),
236 external_order_contexts: Arc::clone(&self.external_order_contexts),
237 start: self.fills_start.unwrap_or_default(),
238 lifecycle: Arc::clone(&self.lifecycle),
239 }
240 }
241
242 fn subscribe_order_events(&mut self) {
243 if self.order_event_handler.is_some() {
244 return;
245 }
246
247 let core = self.core.clone();
248 let lifecycle = Arc::clone(&self.lifecycle);
249 let emitter = self.emitter.clone();
250
251 let handler = TypedHandler::from(move |event: &OrderEventAny| {
252 if event.instrument_id().venue != core.venue
253 || matches!(event, OrderEventAny::Initialized(_))
254 {
255 return;
256 }
257
258 let order = core
259 .cache()
260 .order(&event.client_order_id())
261 .map(|order| order.clone());
262
263 if let Some(order) = order
264 && order.account_id().is_none_or(|id| id == core.account_id)
265 {
266 let mut lifecycle = lifecycle.lock();
267 lifecycle.track(order);
268 if let Err(e) = lifecycle.dispatch(&emitter) {
269 log::warn!("Retaining Coinbase lifecycle recovery: {e}");
270 }
271 }
272 });
273
274 msgbus::subscribe_order_events("events.order.*".into(), handler.clone(), Some(10));
275 self.order_event_handler = Some(handler);
276
277 let cache = self.core.cache();
278 let mut lifecycle = self.lifecycle.lock();
279
280 for order in cache.orders_refs(Some(&self.core.venue), None, None, None, None) {
281 if order
282 .account_id()
283 .is_none_or(|id| id == self.core.account_id)
284 {
285 lifecycle.track(order.clone());
286 }
287 }
288 }
289
290 fn spawn_task<F>(&self, description: &'static str, fut: F)
291 where
292 F: Future<Output = anyhow::Result<()>> + Send + 'static,
293 {
294 let future = async move {
295 if let Err(e) = fut.await {
296 log::warn!("{description} failed: {e:?}");
297 }
298 };
299
300 if let Err(e) = self.pending_tasks.spawn(future) {
301 log::warn!("Skipping Coinbase {description} after shutdown began: {e}");
302 }
303 }
304
305 fn abort_pending_tasks(&self) {
306 self.pending_tasks.begin_shutdown();
307 }
308
309 fn abort_session_tasks(&self) {
310 self.session_tasks.begin_shutdown();
311 self.ws_user.begin_shutdown();
312 }
313
314 async fn await_pending_tasks(&self) -> anyhow::Result<()> {
315 self.pending_tasks.begin_shutdown();
316 self.pending_tasks
317 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
318 .await
319 .map_err(|e| anyhow::anyhow!("Failed to terminate Coinbase execution tasks: {e}"))?;
320 Ok(())
321 }
322
323 async fn await_session_tasks(&self) -> anyhow::Result<()> {
324 self.session_tasks.begin_shutdown();
325 self.session_tasks
326 .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
327 .await
328 .map_err(|e| anyhow::anyhow!("Failed to terminate Coinbase session tasks: {e}"))?;
329 Ok(())
330 }
331
332 async fn teardown_partial_connect(&mut self) -> anyhow::Result<()> {
333 self.abort_session_tasks();
334 self.abort_pending_tasks();
335
336 if let Err(e) = self.ws_user.disconnect().await {
337 self.shutdown_errors.push(e.to_string());
338 }
339 let (session_result, pending_result) =
340 tokio::join!(self.await_session_tasks(), self.await_pending_tasks());
341 self.core.set_disconnected();
342
343 if let Err(e) = session_result {
344 self.shutdown_errors.push(e.to_string());
345 }
346
347 if let Err(e) = pending_result {
348 self.shutdown_errors.push(e.to_string());
349 }
350
351 if !self.shutdown_errors.is_empty() {
352 anyhow::bail!(std::mem::take(&mut self.shutdown_errors).join("; "));
353 }
354 Ok(())
355 }
356
357 fn is_margin(&self) -> bool {
360 self.core.account_type == AccountType::Margin
361 }
362
363 fn is_instrument_cached(&self, instrument_id: &InstrumentId) -> bool {
367 self.instruments_cache
368 .contains_key(instrument_id.symbol.as_str())
369 }
370
371 async fn await_account_registered(&self, timeout_secs: f64) -> anyhow::Result<()> {
372 let account_id = self.core.account_id;
373
374 if self.core.cache().account(&account_id).is_some() {
375 log::info!("Account {account_id} registered");
376 return Ok(());
377 }
378
379 let start = Instant::now();
380 let timeout = Duration::from_secs_f64(timeout_secs);
381 let interval = Duration::from_millis(10);
382
383 loop {
384 tokio::time::sleep(interval).await;
385
386 if self.core.cache().account(&account_id).is_some() {
387 log::info!("Account {account_id} registered");
388 return Ok(());
389 }
390
391 if start.elapsed() >= timeout {
392 anyhow::bail!(
393 "Timeout waiting for account {account_id} to be registered after {timeout_secs}s"
394 );
395 }
396 }
397 }
398}
399
400#[async_trait(?Send)]
401impl ExecutionClient for CoinbaseExecutionClient {
402 fn is_connected(&self) -> bool {
403 self.core.is_connected()
404 }
405
406 fn client_id(&self) -> ClientId {
407 self.core.client_id
408 }
409
410 fn account_id(&self) -> AccountId {
411 self.core.account_id
412 }
413
414 fn venue(&self) -> Venue {
415 *COINBASE_VENUE
416 }
417
418 fn oms_type(&self) -> OmsType {
419 self.core.oms_type
420 }
421
422 fn provides_bulk_position_coverage(&self, instrument_id: InstrumentId) -> bool {
423 self.is_margin()
424 && self
425 .instruments_cache
426 .get(instrument_id.symbol.as_str())
427 .is_some_and(|instrument| instrument.id() == instrument_id)
428 }
429
430 fn get_account(&self) -> Option<AccountAny> {
431 self.core.cache().account_owned(&self.core.account_id)
432 }
433
434 async fn connect(&mut self) -> anyhow::Result<()> {
435 if self.core.is_connected() && self.pending_tasks.is_open() && self.session_tasks.is_open()
436 {
437 return Ok(());
438 }
439
440 if !self.pending_tasks.is_open() {
441 self.await_pending_tasks().await?;
442 self.pending_tasks
443 .start_generation()
444 .map_err(|e| anyhow::anyhow!("Failed to start Coinbase task generation: {e}"))?;
445 }
446
447 if !self.session_tasks.is_open() || !self.session_tasks.is_empty() {
448 self.abort_session_tasks();
449 self.ws_user
450 .disconnect()
451 .await
452 .context("failed to close stale Coinbase user WebSocket")?;
453 self.await_session_tasks().await?;
454 self.session_tasks
455 .start_generation()
456 .map_err(|e| anyhow::anyhow!("Failed to start Coinbase session generation: {e}"))?;
457 }
458 let ws_user = self.ws_user.clone();
459 let setup_guard =
460 TaskGroupGuard::new(&[&self.session_tasks, &self.pending_tasks], move || {
461 ws_user.begin_shutdown();
462 });
463
464 if self.ws_user.is_active() || self.ws_user.is_reconnecting() {
471 log::debug!("Tearing down stale user WS before reconnect");
472 self.ws_user
473 .disconnect()
474 .await
475 .context("failed to close stale Coinbase user WebSocket")?;
476 let credential = CoinbaseCredential::resolve(
477 self.config
478 .api_key
479 .as_ref()
480 .map(|value| value.expose_secret()),
481 self.config
482 .api_secret
483 .as_ref()
484 .map(|value| value.expose_secret()),
485 )
486 .ok_or_else(|| anyhow::anyhow!("Coinbase credentials unavailable for WS reset"))?;
487 self.ws_user = CoinbaseWebSocketClient::with_credential(
488 &self.config.ws_url(),
489 credential,
490 self.config.transport_backend,
491 self.config
492 .proxy_url
493 .as_ref()
494 .map(|value| value.expose_secret().to_owned()),
495 );
496 }
497
498 if self.core.instruments_initialized() {
499 let cached: Vec<InstrumentAny> = self.instruments_cache.values().cloned().collect();
502 if !cached.is_empty() {
503 self.ws_user.initialize_instruments(cached).await;
504 }
505 } else {
506 let instruments = if self.is_margin() {
512 self.http_client
513 .request_instruments(Some(CoinbaseProductType::Future))
514 .await
515 .context("failed to load Coinbase futures instruments")?
516 } else {
517 self.http_client
518 .request_instruments(Some(CoinbaseProductType::Spot))
519 .await
520 .context("failed to load Coinbase instruments")?
521 };
522
523 let product_kind = if self.is_margin() { "futures" } else { "spot" };
524
525 if instruments.is_empty() {
526 log::warn!("Coinbase instrument bootstrap returned no {product_kind} instruments");
527 } else {
528 log::debug!(
529 "Coinbase exec client loaded {} {product_kind} instruments",
530 instruments.len()
531 );
532 }
533
534 let mut map: AHashMap<String, InstrumentAny> =
535 AHashMap::with_capacity(instruments.len());
536 for inst in &instruments {
537 map.insert(inst.id().symbol.as_str().to_string(), inst.clone());
538 }
539 self.instruments_cache = Arc::new(map);
540
541 self.ws_user.initialize_instruments(instruments).await;
544
545 self.core.set_instruments_initialized();
546 }
547
548 if self.fills_start.is_none() {
549 let time = self.http_client.request_server_time().await?;
550 self.fills_start = Some(time.saturating_sub(DurationNanos::from_millis(1)));
551 }
552
553 let session_result = async {
554 self.ws_user.set_account_id(self.core.account_id).await;
555 self.ws_user.connect().await?;
556
557 self.ws_user
560 .subscribe(CoinbaseWsChannel::User, &[])
561 .await
562 .context("failed to subscribe to Coinbase user channel")?;
563
564 if self.is_margin() {
565 self.ws_user
566 .subscribe(CoinbaseWsChannel::FuturesBalanceSummary, &[])
567 .await
568 .context("failed to subscribe to Coinbase futures_balance_summary channel")?;
569 }
570
571 if let Some(rx) = self.ws_user.take_out_rx() {
572 let recovery = self.order_recovery();
573 let retry_interval = Duration::from_millis(self.config.retry_delay_max_ms.max(1));
574
575 let clock = self.clock;
576 let is_margin = self.is_margin();
577 self.session_tasks.spawn(async move {
578 recovery.run(rx, retry_interval, clock, is_margin).await;
579 })?;
580 }
581
582 let account_state = if self.is_margin() {
583 self.http_client
584 .request_cfm_account_state(self.core.account_id)
585 .await
586 .context("failed to request Coinbase CFM account state")?
587 } else {
588 self.http_client
589 .request_account_state(self.core.account_id)
590 .await
591 .context("failed to request Coinbase account state")?
592 };
593
594 if !account_state.balances.is_empty() {
595 log::debug!(
596 "Received account state with {} balance(s)",
597 account_state.balances.len()
598 );
599 }
600 self.emitter.send_account_state(account_state);
601
602 self.await_account_registered(ACCOUNT_REGISTERED_TIMEOUT_SECS)
603 .await?;
604
605 Ok::<(), anyhow::Error>(())
606 }
607 .await;
608
609 if let Err(e) = session_result {
610 if let Err(teardown_error) = self.teardown_partial_connect().await {
611 return Err(e.context(format!(
612 "Coinbase execution startup teardown failed: {teardown_error}"
613 )));
614 }
615 return Err(e);
616 }
617
618 self.core.set_connected();
619 setup_guard.disarm();
620 log::info!("Connected: client_id={}", self.core.client_id);
621 Ok(())
622 }
623
624 async fn disconnect(&mut self) -> anyhow::Result<()> {
625 self.teardown_partial_connect().await?;
626 log::info!("Disconnected: client_id={}", self.core.client_id);
627 Ok(())
628 }
629
630 fn start(&mut self) -> anyhow::Result<()> {
631 if self.core.is_started() {
632 return Ok(());
633 }
634
635 let sender = get_exec_event_sender();
636 self.emitter.set_sender(sender);
637 self.subscribe_order_events();
638 self.core.set_started();
639
640 log::info!(
641 "Started: client_id={}, account_id={}, account_type={:?}, environment={:?}",
642 self.core.client_id,
643 self.core.account_id,
644 self.core.account_type,
645 self.config.environment,
646 );
647 Ok(())
648 }
649
650 fn stop(&mut self) -> anyhow::Result<()> {
651 if self.core.is_stopped() {
652 return Ok(());
653 }
654
655 if let Some(handler) = self.order_event_handler.take() {
656 msgbus::unsubscribe_order_events("events.order.*".into(), &handler);
657 }
658
659 self.core.set_stopped();
660 self.core.set_disconnected();
661
662 self.abort_session_tasks();
663 self.abort_pending_tasks();
664 log::info!("Stopped: client_id={}", self.core.client_id);
665 Ok(())
666 }
667
668 fn query_account(&self, _cmd: QueryAccount) -> anyhow::Result<()> {
669 let http_client = self.http_client.clone();
670 let account_id = self.core.account_id;
671 let emitter = self.emitter.clone();
672 let is_margin = self.is_margin();
673
674 self.spawn_task("query_account", async move {
675 let account_state = if is_margin {
676 http_client
677 .request_cfm_account_state(account_id)
678 .await
679 .context("failed to request Coinbase CFM account state")?
680 } else {
681 http_client
682 .request_account_state(account_id)
683 .await
684 .context("failed to request Coinbase account state")?
685 };
686 emitter.send_account_state(account_state);
687 Ok(())
688 });
689 Ok(())
690 }
691
692 fn query_order(&self, cmd: QueryOrder) -> anyhow::Result<()> {
693 if let Some(order) = self.core.cache().order(&cmd.client_order_id) {
694 self.lifecycle.lock().track(order.clone());
695 }
696
697 let recovery = self.order_recovery();
698 self.spawn_task("query_order", async move {
699 let report = recovery
700 .http_client
701 .request_order_status_report(
702 recovery.emitter.account_id(),
703 Some(cmd.client_order_id),
704 cmd.venue_order_id,
705 )
706 .await?;
707 recovery.queue(report.clone());
708 recovery
709 .lifecycle
710 .lock()
711 .accept(&report, &recovery.emitter)?;
712 recovery.retry().await;
713 Ok(())
714 });
715
716 Ok(())
717 }
718
719 fn generate_account_state(
720 &self,
721 balances: Vec<AccountBalance>,
722 margins: Vec<MarginBalance>,
723 reported: bool,
724 ts_event: UnixNanos,
725 info: Option<Params>,
726 ) -> anyhow::Result<()> {
727 self.emitter
728 .emit_account_state(balances, margins, reported, ts_event, info);
729 Ok(())
730 }
731
732 async fn generate_order_status_report(
733 &self,
734 cmd: &GenerateOrderStatusReport,
735 ) -> anyhow::Result<Option<OrderStatusReport>> {
736 let report = self
737 .http_client
738 .request_order_status_report(
739 self.core.account_id,
740 cmd.client_order_id,
741 cmd.venue_order_id,
742 )
743 .await
744 .ok();
745
746 Ok(report.filter(|r| self.is_instrument_cached(&r.instrument_id)))
751 }
752
753 async fn generate_order_status_reports(
754 &self,
755 cmd: &GenerateOrderStatusReports,
756 ) -> anyhow::Result<Vec<OrderStatusReport>> {
757 let start = cmd.start.map(|ts| ts.to_datetime_utc());
758 let end = cmd.end.map(|ts| ts.to_datetime_utc());
759
760 let mut reports = self
761 .http_client
762 .request_order_status_reports(
763 self.core.account_id,
764 cmd.instrument_id,
765 cmd.open_only,
766 start,
767 end,
768 None,
769 )
770 .await?;
771
772 let before = reports.len();
773 reports.retain(|r| self.is_instrument_cached(&r.instrument_id));
774 if reports.len() != before {
775 let scope = if self.is_margin() {
776 "non-futures"
777 } else {
778 "non-spot"
779 };
780 log::debug!("Filtered {} {scope} order reports", before - reports.len());
781 }
782 Ok(reports)
783 }
784
785 async fn generate_fill_reports(
786 &self,
787 cmd: GenerateFillReports,
788 ) -> anyhow::Result<Vec<FillReport>> {
789 let start = cmd.start.map(|ts| ts.to_datetime_utc());
790 let end = cmd.end.map(|ts| ts.to_datetime_utc());
791
792 let mut reports = self
793 .http_client
794 .request_fill_reports(
795 self.core.account_id,
796 cmd.instrument_id,
797 cmd.venue_order_id,
798 start,
799 end,
800 None,
801 )
802 .await?;
803
804 let before = reports.len();
805 reports.retain(|r| self.is_instrument_cached(&r.instrument_id));
806 if reports.len() != before {
807 let scope = if self.is_margin() {
808 "non-futures"
809 } else {
810 "non-spot"
811 };
812 log::debug!("Filtered {} {scope} fill reports", before - reports.len());
813 }
814 Ok(reports)
815 }
816
817 async fn generate_position_status_reports(
818 &self,
819 cmd: &GeneratePositionStatusReports,
820 ) -> anyhow::Result<Vec<PositionStatusReport>> {
821 if !self.is_margin() {
823 return Ok(Vec::new());
824 }
825
826 if let Some(instrument_id) = cmd.instrument_id {
831 let report = self
832 .http_client
833 .request_position_status_report(self.core.account_id, instrument_id)
834 .await
835 .with_context(|| format!("failed to request CFM position for {instrument_id}"))?;
836 Ok(report.map(|r| vec![r]).unwrap_or_default())
837 } else {
838 self.http_client
839 .request_position_status_reports(self.core.account_id)
840 .await
841 .context("failed to request CFM positions")
842 }
843 }
844
845 async fn generate_mass_status(
846 &self,
847 lookback_mins: Option<u64>,
848 ) -> anyhow::Result<Option<ExecutionMassStatus>> {
849 log::info!("Generating ExecutionMassStatus (lookback_mins={lookback_mins:?})");
850
851 let ts_now = self.clock.get_time_ns();
852 let start = lookback_mins
853 .map(DurationNanos::try_from_mins)
854 .transpose()?
855 .map(|lookback| ts_now.saturating_sub(lookback));
856
857 let order_cmd = GenerateOrderStatusReportsBuilder::default()
858 .ts_init(ts_now)
859 .open_only(false)
860 .start(start)
861 .build()
862 .map_err(|e| anyhow::anyhow!("{e}"))?;
863 let fill_cmd = GenerateFillReportsBuilder::default()
864 .ts_init(ts_now)
865 .start(start)
866 .build()
867 .map_err(|e| anyhow::anyhow!("{e}"))?;
868 let position_cmd = GeneratePositionStatusReportsBuilder::default()
869 .ts_init(ts_now)
870 .build()
871 .map_err(|e| anyhow::anyhow!("{e}"))?;
872
873 let (order_reports, fill_reports, position_reports) = tokio::try_join!(
874 self.generate_order_status_reports(&order_cmd),
875 self.generate_fill_reports(fill_cmd),
876 self.generate_position_status_reports(&position_cmd),
877 )?;
878
879 log::info!("Received {} OrderStatusReports", order_reports.len());
880 log::info!("Received {} FillReports", fill_reports.len());
881 log::info!("Received {} PositionReports", position_reports.len());
882
883 let mut mass_status = ExecutionMassStatus::new(
884 self.core.client_id,
885 self.core.account_id,
886 *COINBASE_VENUE,
887 ts_now,
888 None,
889 );
890
891 mass_status.add_order_reports(order_reports);
892 mass_status.add_fill_reports(fill_reports);
893 mass_status.add_position_reports(position_reports);
894
895 Ok(Some(mass_status))
896 }
897
898 fn submit_order(&self, cmd: SubmitOrder) -> anyhow::Result<()> {
899 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
900 if order.is_closed() {
901 log::warn!("Cannot submit closed order {}", order.client_order_id());
902 return Ok(());
903 }
904
905 if let Err(reason) = validate_order(&order, &self.instruments_cache) {
906 self.emitter.emit_order_denied(&order, &reason.to_string());
907 return Ok(());
908 }
909
910 self.lifecycle.lock().track(order.clone());
911 let instrument_id = order.instrument_id();
912
913 log::debug!("OrderSubmitted client_order_id={}", order.client_order_id());
920 self.emitter.emit_order_submitted(&order);
921
922 let http_client = self.http_client.clone();
923 let emitter = self.emitter.clone();
924 let order_contexts = Arc::clone(&self.order_contexts);
925 let clock = self.clock;
926 let strategy_id = order.strategy_id();
927 let client_order_id = order.client_order_id();
928 let side = order.order_side();
929 let order_type = order.order_type();
930 let quantity = order.quantity();
931 let time_in_force = order.time_in_force();
932 let price = order.price();
933 let trigger_price = order.trigger_price();
934 let trigger_type = order.trigger_type();
935 let expire_time = order.expire_time();
936 let post_only = order.is_post_only();
937 let is_quote_quantity = order.is_quote_quantity();
938 let reduce_only = order.is_reduce_only();
939
940 {
947 let mut map = self.order_contexts.lock();
948 map.insert(
949 client_order_id.to_string(),
950 OrderContext {
951 price,
952 trigger_price,
953 trigger_type,
954 post_only,
955 submitted_product_id: Some(instrument_id.symbol.inner()),
956 },
957 );
958 }
959 let (leverage, margin_type) = if self.core.account_type == AccountType::Margin {
960 (
961 self.config.default_leverage,
962 self.config.default_margin_type,
963 )
964 } else {
965 (None, None)
966 };
967 let retail_portfolio_id = self.config.retail_portfolio_id.clone();
968
969 self.spawn_task("submit_order", async move {
970 let result = http_client
971 .submit_order(
972 client_order_id,
973 instrument_id,
974 side,
975 order_type,
976 quantity,
977 time_in_force,
978 price,
979 trigger_price,
980 expire_time,
981 post_only,
982 is_quote_quantity,
983 leverage,
984 margin_type,
985 reduce_only,
986 retail_portfolio_id,
987 )
988 .await;
989
990 match result {
991 Ok(response) => {
992 if response.success {
993 let venue_id = response
994 .success_response
995 .as_ref()
996 .map(|s| s.order_id.clone())
997 .unwrap_or(response.order_id);
998
999 if venue_id.is_empty() {
1000 log::warn!(
1001 "Submit succeeded but no order_id returned for {client_order_id}"
1002 );
1003 } else {
1004 let venue_order_id = VenueOrderId::new(&venue_id);
1005 let ts_event = clock.get_time_ns();
1006 emitter.emit_order_accepted(&order, venue_order_id, ts_event);
1007 }
1008 } else {
1009 let reason = response.error_response.as_ref().map_or_else(
1010 || response.failure_reason.clone(),
1011 |e| format!("{}: {}", e.error, e.message),
1012 );
1013 let due_post_only = reason.contains("INVALID_LIMIT_PRICE_POST_ONLY")
1019 || response.error_response.as_ref().is_some_and(|e| {
1020 e.preview_failure_reason == "PREVIEW_INVALID_LIMIT_PRICE_POSTONLY"
1021 || e.new_order_failure_reason == "INVALID_LIMIT_PRICE_POST_ONLY"
1022 });
1023 order_contexts.lock().remove(client_order_id.as_str());
1027 let ts_event = clock.get_time_ns();
1028 emitter.emit_order_rejected_event(
1029 strategy_id,
1030 instrument_id,
1031 client_order_id,
1032 &format!("submit-order-rejected: {reason}"),
1033 ts_event,
1034 due_post_only,
1035 );
1036 }
1037 }
1038 Err(e) => {
1039 handle_coinbase_submit_failure(
1040 &e,
1041 &order_contexts,
1042 &emitter,
1043 strategy_id,
1044 instrument_id,
1045 client_order_id,
1046 clock.get_time_ns(),
1047 );
1048 return Err(e.context("submit order failed"));
1049 }
1050 }
1051 Ok(())
1052 });
1053
1054 Ok(())
1055 }
1056
1057 fn submit_order_list(&self, cmd: SubmitOrderList) -> anyhow::Result<()> {
1058 let orders = self.core.get_orders_for_list(&cmd.order_list)?;
1059 let denied = OrderDeniedReason::UnsupportedOrderList {
1060 detail: "order lists are not supported by Coinbase Advanced Trade".to_string(),
1061 }
1062 .to_string();
1063
1064 for order in &orders {
1065 self.emitter.emit_order_denied(order, &denied);
1066 }
1067
1068 Ok(())
1069 }
1070
1071 fn modify_order(&self, cmd: ModifyOrder) -> anyhow::Result<()> {
1072 let ts_event = self.clock.get_time_ns();
1073
1074 let Some(venue_order_id) = cmd.venue_order_id else {
1075 self.emitter.emit_order_modify_denied_event(
1076 cmd.strategy_id,
1077 cmd.instrument_id,
1078 cmd.client_order_id,
1079 None,
1080 "modify-order requires venue_order_id",
1081 ts_event,
1082 Some(cmd.command_id),
1083 );
1084 return Ok(());
1085 };
1086
1087 if cmd.price.is_none() && cmd.quantity.is_none() && cmd.trigger_price.is_none() {
1088 self.emitter.emit_order_modify_denied_event(
1089 cmd.strategy_id,
1090 cmd.instrument_id,
1091 cmd.client_order_id,
1092 Some(venue_order_id),
1093 "modify-order requires price, quantity, or trigger_price",
1094 ts_event,
1095 Some(cmd.command_id),
1096 );
1097 return Ok(());
1098 }
1099
1100 let (auto_price, auto_quantity) = {
1107 let cache = self.core.cache();
1108 let cached = cache.order(&cmd.client_order_id);
1109 let cached_price = cached.as_ref().and_then(|o| o.price());
1110 let cached_qty = cached.as_ref().map(|o| o.quantity());
1111 (cmd.price.or(cached_price), cmd.quantity.or(cached_qty))
1112 };
1113
1114 let http_client = self.http_client.clone();
1115 let emitter = self.emitter.clone();
1116 let order_contexts = Arc::clone(&self.order_contexts);
1117 let clock = self.clock;
1118 let strategy_id = cmd.strategy_id;
1119 let instrument_id = cmd.instrument_id;
1120 let client_order_id = cmd.client_order_id;
1121 let price = auto_price;
1122 let quantity = auto_quantity;
1123 let trigger_price = cmd.trigger_price;
1124
1125 self.spawn_task("modify_order", async move {
1126 let result = http_client
1127 .modify_order(venue_order_id, price, quantity, trigger_price)
1128 .await;
1129
1130 match result {
1131 Ok(resp) => {
1132 if resp.success {
1133 let mut map = order_contexts.lock();
1140 if let Some(meta) = map.get_mut(client_order_id.as_str()) {
1141 if price.is_some() {
1142 meta.price = price;
1143 }
1144
1145 if trigger_price.is_some() {
1146 meta.trigger_price = trigger_price;
1147 }
1148 }
1149 } else {
1150 let reason = resp
1151 .errors
1152 .iter()
1153 .map(|e| {
1154 if e.edit_failure_reason.is_empty() {
1155 e.preview_failure_reason.clone()
1156 } else {
1157 e.edit_failure_reason.clone()
1158 }
1159 })
1160 .collect::<Vec<_>>()
1161 .join(",");
1162 let ts_event = clock.get_time_ns();
1163 emitter.emit_order_modify_rejected_event(
1164 strategy_id,
1165 instrument_id,
1166 client_order_id,
1167 Some(venue_order_id),
1168 &format!("modify-order-rejected: {reason}"),
1169 ts_event,
1170 Some(cmd.command_id),
1171 );
1172 }
1173 }
1174 Err(e) => {
1175 match classify_coinbase_command_failure(&e) {
1176 Some(CommandFailure::NotSent(reason)) => emitter
1177 .emit_order_modify_denied_event(
1178 strategy_id,
1179 instrument_id,
1180 client_order_id,
1181 Some(venue_order_id),
1182 &reason,
1183 clock.get_time_ns(),
1184 Some(cmd.command_id),
1185 ),
1186 Some(CommandFailure::VenueRejected(reason)) => emitter
1187 .emit_order_modify_rejected_event(
1188 strategy_id,
1189 instrument_id,
1190 client_order_id,
1191 Some(venue_order_id),
1192 &reason,
1193 clock.get_time_ns(),
1194 Some(cmd.command_id),
1195 ),
1196 Some(CommandFailure::Ambiguous(_)) => log::warn!(
1197 "Ambiguous modify failure for {client_order_id}, awaiting reconciliation: {e}"
1198 ),
1199 None => log::warn!(
1200 "Modify command failed without venue-declared outcome for {client_order_id}: {e}"
1201 ),
1202 }
1203 return Err(e.context("modify order failed"));
1204 }
1205 }
1206
1207 Ok(())
1208 });
1209
1210 Ok(())
1211 }
1212
1213 fn cancel_order(&self, cmd: CancelOrder) -> anyhow::Result<()> {
1214 let Some(venue_order_id) = cmd.venue_order_id else {
1215 self.emitter.emit_order_cancel_denied_event(
1216 cmd.strategy_id,
1217 cmd.instrument_id,
1218 cmd.client_order_id,
1219 None,
1220 "cancel-order requires venue_order_id",
1221 self.clock.get_time_ns(),
1222 Some(cmd.command_id),
1223 );
1224 return Ok(());
1225 };
1226
1227 let http_client = self.http_client.clone();
1228 let emitter = self.emitter.clone();
1229 let clock = self.clock;
1230 let strategy_id = cmd.strategy_id;
1231 let instrument_id = cmd.instrument_id;
1232 let client_order_id = cmd.client_order_id;
1233
1234 self.spawn_task("cancel_order", async move {
1235 match http_client.cancel_orders(&[venue_order_id]).await {
1236 Ok(resp) => {
1237 if let Some(result) = resp.results.first()
1238 && !result.success
1239 {
1240 let ts_event = clock.get_time_ns();
1241 emitter.emit_order_cancel_rejected_event(
1242 strategy_id,
1243 instrument_id,
1244 client_order_id,
1245 Some(venue_order_id),
1246 &format!("cancel-order-rejected: {}", result.failure_reason),
1247 ts_event,
1248 Some(cmd.command_id),
1249 );
1250 }
1251 }
1252 Err(e) => {
1253 if is_coinbase_local_command_failure(&e) {
1254 emitter.emit_order_cancel_denied_event(
1255 strategy_id,
1256 instrument_id,
1257 client_order_id,
1258 Some(venue_order_id),
1259 &e.to_string(),
1260 clock.get_time_ns(),
1261 Some(cmd.command_id),
1262 );
1263 } else if is_coinbase_explicit_command_rejection(&e) {
1264 emitter.emit_order_cancel_rejected_event(
1265 strategy_id,
1266 instrument_id,
1267 client_order_id,
1268 Some(venue_order_id),
1269 &e.to_string(),
1270 clock.get_time_ns(),
1271 Some(cmd.command_id),
1272 );
1273 } else if is_coinbase_ambiguous_command_failure(&e) {
1274 log::warn!(
1275 "Ambiguous cancel failure for {client_order_id}, awaiting reconciliation: {e}"
1276 );
1277 } else {
1278 log::warn!(
1279 "Cancel command failed without venue-declared outcome for {client_order_id}: {e}"
1280 );
1281 }
1282 return Err(e.context("cancel order failed"));
1283 }
1284 }
1285 Ok(())
1286 });
1287
1288 Ok(())
1289 }
1290
1291 fn cancel_all_orders(&self, cmd: CancelAllOrders) -> anyhow::Result<()> {
1292 let http_client = self.http_client.clone();
1293 let account_id = self.core.account_id;
1294 let instrument_id = cmd.instrument_id;
1295 let side_filter = cmd.order_side;
1296 let emitter = self.emitter.clone();
1297 let clock = self.clock;
1298 let strategy_id = cmd.strategy_id;
1299
1300 self.spawn_task("cancel_all_orders", async move {
1301 let reports = http_client
1306 .request_order_status_reports(
1307 account_id,
1308 Some(instrument_id),
1309 false,
1310 None,
1311 None,
1312 None,
1313 )
1314 .await
1315 .context("failed to list orders for cancel_all")?;
1316
1317 let filtered: Vec<CancelRequest> = reports
1326 .into_iter()
1327 .filter(|r| {
1328 matches!(
1329 r.order_status,
1330 OrderStatus::Accepted
1331 | OrderStatus::Triggered
1332 | OrderStatus::PendingUpdate
1333 | OrderStatus::PartiallyFilled
1334 )
1335 })
1336 .filter(|r| side_filter.is_none_or(|side| r.order_side == side.into()))
1337 .map(|r| CancelRequest {
1338 strategy_id,
1339 instrument_id,
1340 client_order_id: r.client_order_id,
1341 venue_order_id: r.venue_order_id,
1342 command_id: cmd.command_id,
1343 })
1344 .collect();
1345
1346 cancel_order_batches(&http_client, &emitter, clock, &filtered, "cancel-all").await;
1347 Ok(())
1348 });
1349
1350 Ok(())
1351 }
1352
1353 fn batch_cancel_orders(&self, cmd: BatchCancelOrders) -> anyhow::Result<()> {
1354 if cmd.cancels.is_empty() {
1355 return Ok(());
1356 }
1357
1358 let http_client = self.http_client.clone();
1359 let emitter = self.emitter.clone();
1360 let clock = self.clock;
1361
1362 let cancels = cmd.cancels;
1363
1364 self.spawn_task("batch_cancel_orders", async move {
1365 let mut requests = Vec::new();
1366
1367 for cancel in cancels {
1368 let Some(venue_order_id) = cancel.venue_order_id else {
1369 emitter.emit_order_cancel_denied_event(
1370 cancel.strategy_id,
1371 cancel.instrument_id,
1372 cancel.client_order_id,
1373 None,
1374 "cancel-order requires venue_order_id",
1375 clock.get_time_ns(),
1376 Some(cancel.command_id),
1377 );
1378 continue;
1379 };
1380
1381 requests.push(CancelRequest {
1382 strategy_id: cancel.strategy_id,
1383 instrument_id: cancel.instrument_id,
1384 client_order_id: Some(cancel.client_order_id),
1385 venue_order_id,
1386 command_id: cancel.command_id,
1387 });
1388 }
1389
1390 cancel_order_batches(&http_client, &emitter, clock, &requests, "batch-cancel").await;
1391 Ok(())
1392 });
1393
1394 Ok(())
1395 }
1396}
1397
1398struct CancelRequest {
1399 strategy_id: StrategyId,
1400 instrument_id: InstrumentId,
1401 client_order_id: Option<ClientOrderId>,
1402 venue_order_id: VenueOrderId,
1403 command_id: UUID4,
1404}
1405
1406async fn cancel_order_batches(
1407 http: &CoinbaseHttpClient,
1408 emitter: &ExecutionEventEmitter,
1409 clock: &AtomicTime,
1410 requests: &[CancelRequest],
1411 task_name: &str,
1412) {
1413 let mut offset = 0;
1414 while offset < requests.len() {
1415 let end = offset + cancel_batch_len(&requests[offset..]);
1416 let chunk = &requests[offset..end];
1417 offset = end;
1418 let venue_ids: Vec<_> = chunk.iter().map(|request| request.venue_order_id).collect();
1419 match http.cancel_orders(&venue_ids).await {
1420 Ok(response) => {
1421 for result in response.results {
1422 if result.success {
1423 continue;
1424 }
1425
1426 let Some(request) = chunk
1427 .iter()
1428 .find(|request| request.venue_order_id.as_str() == result.order_id)
1429 else {
1430 continue;
1431 };
1432
1433 let Some(client_order_id) = request.client_order_id else {
1434 continue;
1435 };
1436
1437 emitter.emit_order_cancel_rejected_event(
1438 request.strategy_id,
1439 request.instrument_id,
1440 client_order_id,
1441 Some(request.venue_order_id),
1442 &format!("{task_name}-rejected: {}", result.failure_reason),
1443 clock.get_time_ns(),
1444 Some(request.command_id),
1445 );
1446 }
1447 }
1448 Err(e) => emit_batch_cancel_failure(emitter, clock, chunk, task_name, &e),
1449 }
1450 }
1451}
1452
1453fn cancel_batch_len(requests: &[CancelRequest]) -> usize {
1454 let mut targets = AHashSet::new();
1455 requests
1456 .iter()
1457 .take(BATCH_CANCEL_CHUNK)
1458 .take_while(|request| targets.insert(request.venue_order_id))
1459 .count()
1460}
1461
1462fn emit_batch_cancel_failure(
1463 emitter: &ExecutionEventEmitter,
1464 clock: &AtomicTime,
1465 requests: &[CancelRequest],
1466 task_name: &str,
1467 error: &anyhow::Error,
1468) {
1469 match classify_coinbase_command_failure(error) {
1470 Some(CommandFailure::NotSent(reason)) => {
1471 for request in requests {
1472 let Some(client_order_id) = request.client_order_id else {
1473 continue;
1474 };
1475
1476 emitter.emit_order_cancel_denied_event(
1477 request.strategy_id,
1478 request.instrument_id,
1479 client_order_id,
1480 Some(request.venue_order_id),
1481 &reason,
1482 clock.get_time_ns(),
1483 Some(request.command_id),
1484 );
1485 }
1486 }
1487 Some(CommandFailure::VenueRejected(reason)) => {
1488 for request in requests {
1489 let Some(client_order_id) = request.client_order_id else {
1490 continue;
1491 };
1492
1493 emitter.emit_order_cancel_rejected_event(
1494 request.strategy_id,
1495 request.instrument_id,
1496 client_order_id,
1497 Some(request.venue_order_id),
1498 &reason,
1499 clock.get_time_ns(),
1500 Some(request.command_id),
1501 );
1502 }
1503 }
1504 Some(CommandFailure::Ambiguous(_)) => log::warn!(
1505 "Ambiguous {task_name} failure for {} orders, awaiting reconciliation: {error}",
1506 requests.len()
1507 ),
1508 None => log::warn!(
1509 "{task_name} command failed without venue-declared outcome for {} orders: {error}",
1510 requests.len()
1511 ),
1512 }
1513}
1514
1515fn classify_coinbase_command_failure(error: &anyhow::Error) -> Option<CommandFailure> {
1516 if is_coinbase_local_command_failure(error) {
1517 Some(CommandFailure::not_sent(error.to_string()))
1518 } else if is_coinbase_explicit_command_rejection(error) {
1519 Some(CommandFailure::venue_rejected(error.to_string()))
1520 } else if is_coinbase_ambiguous_command_failure(error) {
1521 Some(CommandFailure::ambiguous(error.to_string()))
1522 } else {
1523 None
1524 }
1525}
1526
1527fn validate_order(
1528 order: &impl Order,
1529 instruments_cache: &AHashMap<String, InstrumentAny>,
1530) -> Result<(), OrderDeniedReason> {
1531 if order.is_reduce_only() {
1532 return Err(OrderDeniedReason::UnsupportedReduceOnly);
1533 }
1534
1535 let instrument_id = order.instrument_id();
1536 if !instruments_cache.contains_key(instrument_id.symbol.as_str()) {
1537 return Err(OrderDeniedReason::InstrumentNotFound { instrument_id });
1538 }
1539
1540 Ok(())
1541}
1542
1543fn handle_coinbase_submit_failure(
1544 err: &anyhow::Error,
1545 order_contexts: &Mutex<AHashMap<String, OrderContext>>,
1546 emitter: &ExecutionEventEmitter,
1547 strategy_id: StrategyId,
1548 instrument_id: InstrumentId,
1549 client_order_id: ClientOrderId,
1550 ts_event: UnixNanos,
1551) {
1552 if is_coinbase_local_command_failure(err) {
1553 order_contexts.lock().remove(client_order_id.as_str());
1554 emitter.emit_order_rejected_event(
1555 strategy_id,
1556 instrument_id,
1557 client_order_id,
1558 &format!("submit-order-error: {err}"),
1559 ts_event,
1560 false,
1561 );
1562 } else if is_coinbase_explicit_command_rejection(err) {
1563 order_contexts.lock().remove(client_order_id.as_str());
1564 emitter.emit_order_rejected_event(
1565 strategy_id,
1566 instrument_id,
1567 client_order_id,
1568 &format!("submit-order-rejected: {err}"),
1569 ts_event,
1570 false,
1571 );
1572 } else if is_coinbase_ambiguous_submit_failure(err) {
1573 log::warn!(
1574 "Ambiguous submit failure for {client_order_id}, awaiting reconciliation: {err}"
1575 );
1576 } else {
1577 order_contexts.lock().remove(client_order_id.as_str());
1578 log::warn!(
1579 "Submit command failed without venue-declared outcome for {client_order_id}: {err}"
1580 );
1581 }
1582}
1583
1584fn is_coinbase_local_command_failure(err: &anyhow::Error) -> bool {
1585 match coinbase_http_error(err) {
1586 None => true,
1587 Some(CoinbaseHttpError::Auth(message)) => !message.starts_with("HTTP "),
1588 _ => false,
1589 }
1590}
1591
1592fn is_coinbase_explicit_command_rejection(err: &anyhow::Error) -> bool {
1593 match coinbase_http_error(err) {
1594 Some(CoinbaseHttpError::Auth(message) | CoinbaseHttpError::BadRequest(message)) => {
1595 message.starts_with("HTTP ")
1596 }
1597 _ => false,
1598 }
1599}
1600
1601fn is_coinbase_ambiguous_submit_failure(err: &anyhow::Error) -> bool {
1602 matches!(
1603 coinbase_http_error(err),
1604 Some(CoinbaseHttpError::RateLimit { .. })
1605 ) || is_coinbase_ambiguous_command_failure(err)
1606}
1607
1608fn is_coinbase_ambiguous_command_failure(err: &anyhow::Error) -> bool {
1609 matches!(
1610 coinbase_http_error(err),
1611 Some(
1612 CoinbaseHttpError::Transport(_)
1613 | CoinbaseHttpError::Serde(_)
1614 | CoinbaseHttpError::Exchange(_)
1615 | CoinbaseHttpError::Timeout
1616 | CoinbaseHttpError::Decode(_)
1617 )
1618 ) || matches!(
1619 coinbase_http_error(err),
1620 Some(CoinbaseHttpError::Http { status, .. }) if *status >= 500
1621 )
1622}
1623
1624fn coinbase_http_error(err: &anyhow::Error) -> Option<&CoinbaseHttpError> {
1625 err.chain()
1626 .find_map(|cause| cause.downcast_ref::<CoinbaseHttpError>())
1627}
1628
1629#[derive(Debug)]
1630struct UserOrderRecovery {
1631 http_client: CoinbaseHttpClient,
1632 emitter: ExecutionEventEmitter,
1633 instruments: Arc<AHashMap<String, InstrumentAny>>,
1634 observed: Arc<Mutex<AHashMap<VenueOrderId, AHashSet<TradeId>>>>,
1635 pending: Arc<Mutex<AHashMap<VenueOrderId, OrderStatusReport>>>,
1636 published: Arc<tokio::sync::Mutex<AHashMap<VenueOrderId, OrderStatusReport>>>,
1637 order_contexts: Arc<Mutex<AHashMap<String, OrderContext>>>,
1638 external_order_contexts: Arc<Mutex<AHashMap<String, OrderContext>>>,
1639 start: UnixNanos,
1640 lifecycle: Arc<Mutex<RecoveryLifecycle>>,
1641}
1642
1643impl UserOrderRecovery {
1644 async fn run(
1645 &self,
1646 mut rx: tokio::sync::mpsc::UnboundedReceiver<NautilusWsMessage>,
1647 retry_interval: Duration,
1648 clock: &'static AtomicTime,
1649 is_margin: bool,
1650 ) {
1651 let mut interval = tokio::time::interval(retry_interval);
1652 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1653
1654 loop {
1655 tokio::select! {
1656 message = rx.recv() => {
1657 let Some(message) = message else { break };
1658 match message {
1659 NautilusWsMessage::UserOrder(carrier) => self.handle_update(*carrier).await,
1660 NautilusWsMessage::FuturesBalanceSummary(summary) => {
1661 let ts = clock.get_time_ns();
1662 let account_id = self.emitter.account_id();
1663 match parse_ws_cfm_account_state(&summary, account_id, ts, ts) {
1664 Ok(state) => self.emitter.send_account_state(state),
1665 Err(e) => log::warn!("Failed to parse futures_balance_summary into AccountState: {e}"),
1666 }
1667 }
1668 NautilusWsMessage::Reconnected => {
1669 log::info!("Coinbase user WebSocket reconnected");
1670 let account_id = self.emitter.account_id();
1671 let refresh = if is_margin {
1672 self.http_client.request_cfm_account_state(account_id).await
1673 } else {
1674 self.http_client.request_account_state(account_id).await
1675 };
1676
1677 match refresh {
1678 Ok(state) => self.emitter.send_account_state(state),
1679 Err(e) => log::warn!("Failed to refresh account state on reconnect: {e}"),
1680 }
1681 }
1682 NautilusWsMessage::Error(e) => log::warn!("Coinbase user WebSocket error: {e}"),
1683 _ => {}
1684 }
1685 }
1686 _ = interval.tick() => {
1687 self.retry().await;
1688 }
1689 }
1690 }
1691 }
1692
1693 async fn handle_update(&self, carrier: UserOrderUpdate) {
1694 let needs_fills =
1695 carrier.report.filled_qty.is_positive() || carrier.update.status.is_terminal();
1696 let is_snapshot = carrier.is_snapshot;
1697 let own_context = self
1698 .order_contexts
1699 .lock()
1700 .contains_key(&carrier.update.client_order_id);
1701 if needs_fills {
1702 self.queue((*carrier.report).clone());
1703 if let Err(e) = self.lifecycle.lock().accept(&carrier.report, &self.emitter) {
1704 log::warn!("Retaining Coinbase acceptance recovery: {e}");
1705 }
1706 }
1707
1708 if needs_fills && !own_context {
1709 self.retry().await;
1710 return;
1711 }
1712
1713 let context = resolve_order_context(
1714 &carrier.update,
1715 carrier.report.order_type,
1716 carrier.report.price.is_none(),
1717 &self.order_contexts,
1718 &self.external_order_contexts,
1719 &self.http_client,
1720 self.emitter.account_id(),
1721 )
1722 .await;
1723
1724 if let Some(report) = prepare_user_order_report(carrier, context) {
1725 if !is_snapshot
1726 && own_context
1727 && let Err(e) = self.lifecycle.lock().update(&report, &self.emitter)
1728 {
1729 log::warn!("Failed to publish Coinbase amendment: {e}");
1730 }
1731
1732 if !needs_fills && let Err(e) = self.publish(&[(report, Vec::new())]) {
1733 log::warn!("Failed to publish Coinbase acceptance: {e}");
1734 }
1735 }
1736
1737 if needs_fills {
1738 self.retry().await;
1739 }
1740 }
1741
1742 async fn retry(&self) {
1743 if let Err(e) = self.discover().await {
1744 log::warn!("Coinbase fill recovery scan remains pending: {e}");
1745 return;
1746 }
1747
1748 if let Err(e) = self.recover().await {
1749 log::warn!("Retaining pending Coinbase recovery: {e}");
1750 }
1751 }
1752
1753 async fn discover(&self) -> anyhow::Result<()> {
1754 let fills = self.http_client.request_fill_ids(self.start).await?;
1755 {
1756 let mut observed = self.observed.lock();
1757
1758 for (venue_order_id, instrument_id, trade_id) in fills {
1759 if self.instruments.contains_key(instrument_id.symbol.as_str()) {
1760 observed.entry(venue_order_id).or_default().insert(trade_id);
1761 }
1762 }
1763 }
1764
1765 let mut reports: AHashMap<_, _> = self
1766 .http_client
1767 .request_order_status_reports(self.emitter.account_id(), None, false, None, None, None)
1768 .await?
1769 .into_iter()
1770 .filter(|report| {
1771 self.instruments
1772 .contains_key(report.instrument_id.symbol.as_str())
1773 && report.filled_qty.is_positive()
1774 && report.ts_last >= self.start
1775 })
1776 .map(|report| (report.venue_order_id, report))
1777 .collect();
1778
1779 let orders: Vec<_> = self.observed.lock().keys().copied().collect();
1780
1781 for venue_order_id in orders {
1782 if let Entry::Vacant(entry) = reports.entry(venue_order_id) {
1783 let report = self
1784 .http_client
1785 .request_order_status_report(
1786 self.emitter.account_id(),
1787 None,
1788 Some(venue_order_id),
1789 )
1790 .await?;
1791 entry.insert(report);
1792 }
1793 }
1794
1795 for report in reports.into_values() {
1796 self.queue(report);
1797 }
1798
1799 Ok(())
1800 }
1801
1802 fn queue(&self, report: OrderStatusReport) {
1803 let mut pending = self.pending.lock();
1804 let entry = pending
1805 .entry(report.venue_order_id)
1806 .or_insert_with(|| report.clone());
1807 if report.filled_qty > entry.filled_qty
1808 || (report.filled_qty == entry.filled_qty && !entry.order_status.is_closed())
1809 {
1810 *entry = report;
1811 }
1812 }
1813
1814 async fn recover(&self) -> anyhow::Result<()> {
1815 let mut published = self.published.lock().await;
1816 let targets = self.pending.lock().clone();
1817 let mut reports = Vec::with_capacity(targets.len());
1818
1819 for (venue_order_id, expected) in &targets {
1820 let report = self
1821 .http_client
1822 .request_order_status_report(self.emitter.account_id(), None, Some(*venue_order_id))
1823 .await?;
1824 anyhow::ensure!(
1825 self.instruments
1826 .contains_key(report.instrument_id.symbol.as_str()),
1827 "order instrument is outside the configured product family"
1828 );
1829 anyhow::ensure!(
1830 report.filled_qty >= expected.filled_qty
1831 && (!expected.order_status.is_closed() || report.order_status.is_closed()),
1832 "REST order state has not reached the user-channel update"
1833 );
1834
1835 if let Some(prior) = published.get(venue_order_id) {
1836 anyhow::ensure!(
1837 report.filled_qty >= prior.filled_qty
1838 && (!prior.order_status.is_closed() || report.order_status.is_closed()),
1839 "REST order state precedes published recovery"
1840 );
1841 }
1842
1843 let fills = if report.filled_qty.is_positive()
1844 || self.observed.lock().contains_key(venue_order_id)
1845 {
1846 self.http_client
1847 .request_fill_reports(
1848 self.emitter.account_id(),
1849 None,
1850 Some(*venue_order_id),
1851 None,
1852 None,
1853 None,
1854 )
1855 .await?
1856 } else {
1857 Vec::new()
1858 };
1859
1860 let fills = prepare_order_fills(&report, fills)?;
1861 reports.push((report, fills));
1862 }
1863
1864 let mut pending = self.pending.lock();
1865 let mut observed = self.observed.lock();
1866 anyhow::ensure!(
1867 pending.len() == targets.len()
1868 && targets
1869 .iter()
1870 .all(|(id, target)| pending.get(id) == Some(target)),
1871 "recovery target changed while reading REST"
1872 );
1873
1874 let trades: AHashMap<_, AHashSet<_>> = reports
1875 .iter()
1876 .map(|(report, fills)| {
1877 (
1878 report.venue_order_id,
1879 fills.iter().map(|fill| fill.trade_id).collect(),
1880 )
1881 })
1882 .collect();
1883
1884 anyhow::ensure!(
1885 observed
1886 .iter()
1887 .all(|(id, ids)| trades.get(id).is_some_and(|trades| ids.is_subset(trades))),
1888 "REST fills have not reached discovered trades"
1889 );
1890 self.publish(&reports)?;
1891 observed.clear();
1892
1893 for (report, _) in reports {
1894 let venue_order_id = report.venue_order_id;
1895 if pending.get(&venue_order_id) == targets.get(&venue_order_id) {
1896 pending.remove(&venue_order_id);
1897 }
1898
1899 if report.order_status.is_closed() {
1900 if let Some(client_order_id) = report.client_order_id {
1901 self.order_contexts.lock().remove(client_order_id.as_str());
1902 }
1903
1904 self.external_order_contexts
1905 .lock()
1906 .remove(venue_order_id.as_str());
1907 }
1908
1909 published.insert(venue_order_id, report);
1910 }
1911
1912 Ok(())
1913 }
1914
1915 fn publish(&self, reports: &[(OrderStatusReport, Vec<FillReport>)]) -> anyhow::Result<()> {
1916 let mut external = Vec::new();
1917 {
1918 let mut lifecycle = self.lifecycle.lock();
1919 for (report, _) in reports {
1920 if lifecycle.order(report).is_some() {
1921 lifecycle
1922 .pending
1923 .insert(report.venue_order_id, report.clone());
1924 } else {
1925 let mut accepted = report.clone();
1927 if accepted.order_status != OrderStatus::Rejected {
1928 accepted.order_status = OrderStatus::Accepted;
1929 }
1930
1931 accepted.filled_qty = Quantity::zero(report.filled_qty.precision);
1932 accepted.avg_px = None;
1933 accepted.ts_last = accepted.ts_accepted;
1934 self.emitter
1935 .try_send_execution_report(ExecutionReport::Order(Box::new(accepted)))?;
1936 external.push(report);
1937 }
1938 }
1939
1940 lifecycle.dispatch(&self.emitter)?;
1941 }
1942
1943 let mut fills: Vec<_> = reports.iter().flat_map(|(_, fills)| fills).collect();
1944 fills.sort_by_key(|fill| (fill.ts_event, fill.trade_id, fill.venue_order_id));
1945 for fill in fills {
1946 self.emitter
1947 .try_send_execution_report(ExecutionReport::Fill(Box::new(fill.clone())))?;
1948 }
1949
1950 for report in external {
1951 self.emitter
1952 .try_send_execution_report(ExecutionReport::Order(Box::new(report.clone())))?;
1953 }
1954
1955 Ok(())
1956 }
1957}
1958
1959#[derive(Debug)]
1960struct RecoveryLifecycle {
1961 factory: OrderEventFactory,
1962 orders: AHashMap<ClientOrderId, OrderAny>,
1963 venue_ids: AHashMap<VenueOrderId, ClientOrderId>,
1964 pending: AHashMap<VenueOrderId, OrderStatusReport>,
1965 accepted: AHashSet<VenueOrderId>,
1966}
1967
1968impl RecoveryLifecycle {
1969 fn new(trader_id: TraderId, account_id: AccountId, account_type: AccountType) -> Self {
1970 Self {
1971 factory: OrderEventFactory::new(trader_id, account_id, account_type, None),
1972 orders: AHashMap::new(),
1973 venue_ids: AHashMap::new(),
1974 pending: AHashMap::new(),
1975 accepted: AHashSet::new(),
1976 }
1977 }
1978
1979 fn track(&mut self, order: OrderAny) {
1980 if let Some(venue_order_id) = order.venue_order_id() {
1981 if !matches!(
1982 order.status(),
1983 OrderStatus::Initialized
1984 | OrderStatus::Submitted
1985 | OrderStatus::PendingUpdate
1986 | OrderStatus::PendingCancel
1987 ) {
1988 self.accepted.remove(&venue_order_id);
1989 }
1990
1991 self.venue_ids
1992 .insert(venue_order_id, order.client_order_id());
1993 }
1994
1995 self.orders.insert(order.client_order_id(), order);
1996 }
1997
1998 fn dispatch(&mut self, emitter: &ExecutionEventEmitter) -> anyhow::Result<()> {
1999 let reports: Vec<_> = self.pending.values().cloned().collect();
2000 for report in reports {
2001 if self.dispatch_order(&report, emitter)? {
2002 self.pending.remove(&report.venue_order_id);
2003 }
2004 }
2005
2006 Ok(())
2007 }
2008
2009 fn dispatch_order(
2010 &mut self,
2011 report: &OrderStatusReport,
2012 emitter: &ExecutionEventEmitter,
2013 ) -> anyhow::Result<bool> {
2014 self.accept(report, emitter)?;
2015
2016 let Some(order) = self.order(report).cloned() else {
2017 return Ok(false);
2018 };
2019
2020 if order.is_closed() {
2021 return Ok(true);
2022 }
2023
2024 let factory = &self.factory;
2025 let now = get_atomic_clock_realtime().get_time_ns();
2026
2027 if (order.order_type() == OrderType::Market
2028 && order.is_quote_quantity()
2029 && report.order_status.is_closed()
2030 && report.filled_qty.is_positive())
2031 || (!order.is_quote_quantity() && report.filled_qty > order.quantity())
2032 || (report.order_status == OrderStatus::Filled && report.quantity != order.quantity())
2033 {
2034 let updated = OrderUpdated::new(
2035 order.trader_id(),
2036 order.strategy_id(),
2037 order.instrument_id(),
2038 order.client_order_id(),
2039 report.quantity,
2040 UUID4::new(),
2041 report.ts_last,
2042 now,
2043 true,
2044 Some(report.venue_order_id),
2045 Some(emitter.account_id()),
2046 None,
2047 None,
2048 None,
2049 false,
2050 );
2051 emitter.try_send_order_event(OrderEventAny::Updated(updated))?;
2052 }
2053
2054 if order.filled_qty() < report.filled_qty {
2055 return Ok(false);
2056 }
2057
2058 let event = match report.order_status {
2059 OrderStatus::Canceled => factory.generate_order_canceled(
2060 &order,
2061 Some(report.venue_order_id),
2062 report.ts_last,
2063 now,
2064 ),
2065 OrderStatus::Expired => factory.generate_order_expired(
2066 &order,
2067 Some(report.venue_order_id),
2068 report.ts_last,
2069 now,
2070 ),
2071 OrderStatus::Rejected => factory.generate_order_rejected(
2072 &order,
2073 report.cancel_reason.as_deref().unwrap_or("rejected"),
2074 report.ts_last,
2075 now,
2076 false,
2077 ),
2078 _ => return Ok(true),
2079 };
2080
2081 emitter.try_send_order_event(event)?;
2082 Ok(true)
2083 }
2084
2085 fn accept(
2086 &mut self,
2087 report: &OrderStatusReport,
2088 emitter: &ExecutionEventEmitter,
2089 ) -> anyhow::Result<()> {
2090 let Some(order) = self.order(report) else {
2091 return Ok(());
2092 };
2093
2094 let factory = &self.factory;
2095 let now = get_atomic_clock_realtime().get_time_ns();
2096 let awaiting_acceptance = matches!(
2097 order.status(),
2098 OrderStatus::Initialized | OrderStatus::Submitted
2099 );
2100
2101 if awaiting_acceptance
2102 && report.order_status != OrderStatus::Rejected
2103 && !self.accepted.contains(&report.venue_order_id)
2104 {
2105 let event = factory.generate_order_accepted(
2106 order,
2107 report.venue_order_id,
2108 report.ts_accepted,
2109 now,
2110 );
2111 emitter.try_send_order_event(event)?;
2112 self.accepted.insert(report.venue_order_id);
2113 }
2114
2115 Ok(())
2116 }
2117
2118 fn update(
2119 &self,
2120 report: &OrderStatusReport,
2121 emitter: &ExecutionEventEmitter,
2122 ) -> anyhow::Result<()> {
2123 let Some(order) = self.order(report) else {
2124 return Ok(());
2125 };
2126
2127 if order.is_closed()
2128 || (order.order_type() == OrderType::Market && order.is_quote_quantity())
2129 || (order.quantity() == report.quantity
2130 && order.price() == report.price
2131 && order.trigger_price() == report.trigger_price)
2132 {
2133 return Ok(());
2134 }
2135
2136 let factory = &self.factory;
2137 let now = get_atomic_clock_realtime().get_time_ns();
2138 let event = factory.generate_order_updated(
2139 order,
2140 report.venue_order_id,
2141 report.quantity,
2142 report.price,
2143 report.trigger_price,
2144 None,
2145 report.ts_last,
2146 now,
2147 );
2148 emitter.try_send_order_event(event)
2149 }
2150
2151 fn order(&self, report: &OrderStatusReport) -> Option<&OrderAny> {
2152 report
2153 .client_order_id
2154 .and_then(|id| self.orders.get(&id))
2155 .or_else(|| {
2156 self.venue_ids
2157 .get(&report.venue_order_id)
2158 .and_then(|id| self.orders.get(id))
2159 })
2160 }
2161}
2162
2163fn prepare_order_fills(
2164 report: &OrderStatusReport,
2165 fills: Vec<FillReport>,
2166) -> anyhow::Result<Vec<FillReport>> {
2167 anyhow::ensure!(
2169 report.order_type != OrderType::Market
2170 || report.order_status.is_closed()
2171 || report.quantity > report.filled_qty,
2172 "market order base quantity is not final"
2173 );
2174 let mut unique = AHashMap::new();
2175
2176 for mut fill in fills {
2177 anyhow::ensure!(
2178 fill.account_id == report.account_id
2179 && fill.venue_order_id == report.venue_order_id
2180 && fill.instrument_id == report.instrument_id
2181 && report.order_side == Some(fill.order_side),
2182 "fill identity does not match the order report"
2183 );
2184 fill.client_order_id = report.client_order_id;
2185 unique.entry(fill.trade_id).or_insert(fill);
2186 }
2187
2188 let mut fills: Vec<_> = unique.into_values().collect();
2189 fills.sort_by_key(|fill| (fill.ts_event, fill.trade_id));
2190
2191 let quantity = fills.iter().try_fold(Decimal::ZERO, |quantity, fill| {
2192 quantity
2193 .checked_add(fill.last_qty.as_decimal())
2194 .context("fill quantity overflow")
2195 })?;
2196
2197 anyhow::ensure!(
2198 quantity == report.filled_qty.as_decimal(),
2199 "REST fills do not cover the order filled quantity"
2200 );
2201 Ok(fills)
2202}
2203
2204fn prepare_user_order_report(
2205 carrier: UserOrderUpdate,
2206 context: Option<OrderContext>,
2207) -> Option<OrderStatusReport> {
2208 let UserOrderUpdate {
2209 mut report,
2210 update,
2211 is_snapshot,
2212 ..
2213 } = carrier;
2214 let have_context = context.is_some();
2215 if let Some(meta) = context {
2216 if report.price.is_none() {
2217 report.price = meta.price;
2218 }
2219
2220 if report.trigger_price.is_none() {
2221 report.trigger_price = meta.trigger_price;
2222 }
2223
2224 if report.trigger_type.is_none() {
2225 report.trigger_type = meta.trigger_type;
2226 }
2227
2228 report.post_only = meta.post_only;
2229 if let Some(submitted) = meta.submitted_product_id {
2230 report.instrument_id = InstrumentId::new(Symbol::new(submitted), *COINBASE_VENUE);
2231 }
2232 }
2233
2234 let safe = match report.order_type {
2235 OrderType::Limit | OrderType::LimitIfTouched => report.price.is_some(),
2236 OrderType::StopLimit => report.price.is_some() && report.trigger_price.is_some(),
2237 OrderType::StopMarket | OrderType::MarketIfTouched => report.trigger_price.is_some(),
2238 _ => true,
2239 };
2240
2241 if !safe {
2242 log::warn!(
2243 "Suppressed unsafe OrderStatusReport for {} {}: missing price/trigger after enrichment",
2244 report.order_type,
2245 update.order_id
2246 );
2247 return None;
2248 }
2249
2250 (!is_snapshot || have_context).then_some(*report)
2251}
2252
2253async fn resolve_order_context(
2260 update: &WsOrderUpdate,
2261 order_type: OrderType,
2262 report_price_missing: bool,
2263 order_contexts: &Arc<Mutex<AHashMap<String, OrderContext>>>,
2264 external_order_contexts: &Arc<Mutex<AHashMap<String, OrderContext>>>,
2265 http_client: &CoinbaseHttpClient,
2266 account_id: AccountId,
2267) -> Option<OrderContext> {
2268 if !update.client_order_id.is_empty() {
2269 let map = order_contexts.lock();
2270 if let Some(meta) = map.get(&update.client_order_id) {
2271 return Some(meta.clone());
2272 }
2273 }
2274
2275 if let Some(meta) = external_order_contexts.lock().get(&update.order_id) {
2276 return Some(meta.clone());
2277 }
2278
2279 let needs_enrichment = report_price_missing
2280 && matches!(
2281 order_type,
2282 OrderType::Limit
2283 | OrderType::StopLimit
2284 | OrderType::LimitIfTouched
2285 | OrderType::StopMarket
2286 | OrderType::MarketIfTouched
2287 );
2288
2289 if !needs_enrichment {
2290 return None;
2291 }
2292
2293 let venue_order_id = VenueOrderId::new(update.order_id.as_str());
2294
2295 match http_client
2296 .request_order_status_report(account_id, None, Some(venue_order_id))
2297 .await
2298 {
2299 Ok(rest_report) => {
2300 let post_only_from_rest = matches!(order_type, OrderType::Limit | OrderType::StopLimit)
2301 && rest_report.post_only;
2302
2303 let meta = OrderContext {
2304 price: rest_report.price,
2305 trigger_price: rest_report.trigger_price,
2306 trigger_type: rest_report.trigger_type,
2307 post_only: post_only_from_rest,
2308 submitted_product_id: None,
2309 };
2310 external_order_contexts
2311 .lock()
2312 .insert(update.order_id.clone(), meta.clone());
2313 Some(meta)
2314 }
2315 Err(e) => {
2316 log::warn!(
2317 "Failed to enrich external order {} via REST: {e}",
2318 update.order_id
2319 );
2320 None
2321 }
2322 }
2323}
2324
2325#[derive(Debug, Default, Clone)]
2328struct OrderContext {
2329 price: Option<Price>,
2330 trigger_price: Option<Price>,
2331 trigger_type: Option<TriggerType>,
2332 post_only: bool,
2333 submitted_product_id: Option<Ustr>,
2341}
2342
2343#[cfg(test)]
2344mod tests {
2345 use std::{cell::RefCell, rc::Rc};
2346
2347 use nautilus_common::{
2348 cache::Cache,
2349 enums::Environment,
2350 messages::{ExecutionEvent, ExecutionReport},
2351 };
2352 use nautilus_live::{manager::ExecutionManager, node::LiveNode};
2353 use nautilus_model::{
2354 accounts::CashAccount,
2355 enums::{AccountType, OrderSide, TimeInForce},
2356 events::{AccountState, OrderEventAny, OrderPendingCancel, OrderPendingUpdate},
2357 identifiers::{Symbol, TraderId, Venue},
2358 instruments::{
2359 CurrencyPair,
2360 stubs::{crypto_future_btcusdt, crypto_perpetual_ethusdt},
2361 },
2362 orders::OrderTestBuilder,
2363 types::{Currency, Money, Quantity},
2364 };
2365 use rstest::rstest;
2366 use ustr::Ustr;
2367
2368 use super::*;
2369 use crate::{
2370 common::{
2371 consts::COINBASE_VENUE,
2372 enums::{
2373 CoinbaseContractExpiryType, CoinbaseOrderSide as CbSide,
2374 CoinbaseOrderStatus as CbStatus, CoinbaseOrderType as CbType,
2375 CoinbaseProductType as CbProductType, CoinbaseRiskManagedBy,
2376 CoinbaseTimeInForce as CbTif, CoinbaseTriggerStatus,
2377 },
2378 },
2379 websocket::messages::WsOrderUpdate,
2380 };
2381
2382 #[rstest]
2383 #[case::limit(101, false, 100)]
2384 #[case::repeated(3, true, 1)]
2385 fn test_cancel_batch_len(
2386 #[case] count: usize,
2387 #[case] repeated: bool,
2388 #[case] expected: usize,
2389 ) {
2390 let requests: Vec<_> = (0..count)
2391 .map(|index| CancelRequest {
2392 strategy_id: StrategyId::from("S-BATCH"),
2393 instrument_id: InstrumentId::from("BTC-USD.COINBASE"),
2394 client_order_id: Some(ClientOrderId::from(format!("C-{index}"))),
2395 venue_order_id: VenueOrderId::from(format!(
2396 "V-{}",
2397 if repeated { 0 } else { index }
2398 )),
2399 command_id: UUID4::new(),
2400 })
2401 .collect();
2402
2403 assert_eq!(cancel_batch_len(&requests), expected);
2404 }
2405
2406 #[rstest]
2407 #[case::not_sent(CoinbaseHttpError::auth("No credentials configured"), Some(false))]
2408 #[case::venue_rejected(
2409 CoinbaseHttpError::bad_request("HTTP 400: invalid cancellation"),
2410 Some(true)
2411 )]
2412 #[case::ambiguous(CoinbaseHttpError::Timeout, None)]
2413 #[case::unclassified(CoinbaseHttpError::rate_limit(None), None)]
2414 fn test_batch_cancel_failure_preserves_child_identity(
2415 #[case] error: CoinbaseHttpError,
2416 #[case] rejected: Option<bool>,
2417 ) {
2418 let (emitter, mut receiver) = make_emitter();
2419 let clock = AtomicTime::new(false, UnixNanos::from(53));
2420
2421 let requests = [0, 1, 2].map(|index| CancelRequest {
2422 strategy_id: StrategyId::from(format!("S-BATCH-{index}")),
2423 instrument_id: InstrumentId::from(if index == 0 {
2424 "BTC-USD.COINBASE"
2425 } else {
2426 "ETH-USD.COINBASE"
2427 }),
2428 client_order_id: (index != 1).then(|| ClientOrderId::from(format!("O-BATCH-{index}"))),
2429 venue_order_id: VenueOrderId::from(format!("V-BATCH-{index}")),
2430 command_id: UUID4::new(),
2431 });
2432
2433 let error = anyhow::Error::new(error);
2434 emit_batch_cancel_failure(&emitter, &clock, &requests, "batch_cancel_orders", &error);
2435 if let Some(rejected) = rejected {
2436 for request in requests
2437 .iter()
2438 .filter(|request| request.client_order_id.is_some())
2439 {
2440 let ExecutionEvent::Order(event) = receiver.try_recv().unwrap() else {
2441 panic!("expected cancellation outcome");
2442 };
2443
2444 let (strategy, instrument, client, venue, cause, reason, ts_event) = match event {
2445 OrderEventAny::CancelDenied(event) if !rejected => (
2446 event.strategy_id,
2447 event.instrument_id,
2448 event.client_order_id,
2449 event.venue_order_id,
2450 event.causation_id,
2451 event.reason,
2452 event.ts_event,
2453 ),
2454 OrderEventAny::CancelRejected(event) if rejected => (
2455 event.strategy_id,
2456 event.instrument_id,
2457 event.client_order_id,
2458 event.venue_order_id,
2459 event.causation_id,
2460 event.reason,
2461 event.ts_event,
2462 ),
2463 other => panic!("unexpected cancellation outcome {other:?}"),
2464 };
2465
2466 assert_eq!(
2467 (
2468 strategy,
2469 instrument,
2470 client,
2471 venue,
2472 cause,
2473 reason.as_str(),
2474 ts_event
2475 ),
2476 (
2477 request.strategy_id,
2478 request.instrument_id,
2479 request.client_order_id.unwrap(),
2480 Some(request.venue_order_id),
2481 Some(request.command_id),
2482 error.to_string().as_str(),
2483 UnixNanos::from(53)
2484 )
2485 );
2486 }
2487 }
2488
2489 assert!(matches!(
2490 receiver.try_recv(),
2491 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
2492 ));
2493 }
2494
2495 #[rstest]
2496 #[case::cash(AccountType::Cash, true, false)]
2497 #[case::margin_cached(AccountType::Margin, true, true)]
2498 #[case::margin_uncached(AccountType::Margin, false, false)]
2499 fn test_bulk_position_coverage(
2500 #[case] account_type: AccountType,
2501 #[case] cached: bool,
2502 #[case] expected: bool,
2503 #[values(false, true)] perpetual: bool,
2504 ) {
2505 let cache = Rc::new(RefCell::new(Cache::default()));
2506
2507 let core = ExecutionClientCore::new(
2508 TraderId::from("TRADER-001"),
2509 ClientId::from("COINBASE-TEST"),
2510 *COINBASE_VENUE,
2511 OmsType::Netting,
2512 AccountId::from("COINBASE-001"),
2513 account_type,
2514 None,
2515 cache,
2516 );
2517
2518 let config = CoinbaseExecutionClientConfig {
2519 api_key: Some("test-key".into()),
2520 api_secret: Some("test-secret".into()),
2521 account_type,
2522 ..Default::default()
2523 };
2524
2525 let mut client = CoinbaseExecutionClient::new(core, config).unwrap();
2526
2527 let instrument = if perpetual {
2528 let mut instrument = crypto_perpetual_ethusdt();
2529 instrument.id = InstrumentId::from("ETH-PERP.COINBASE");
2530 InstrumentAny::CryptoPerpetual(instrument)
2531 } else {
2532 let mut instrument =
2533 crypto_future_btcusdt(2, 6, Price::from("0.01"), Quantity::from("0.000001"));
2534 instrument.id = InstrumentId::from("BIP-20DEC30-CDE.COINBASE");
2535 InstrumentAny::CryptoFuture(instrument)
2536 };
2537
2538 let instrument_id = instrument.id();
2539
2540 if cached {
2541 client.instruments_cache = Arc::new(AHashMap::from([(
2542 instrument_id.symbol.as_str().to_owned(),
2543 instrument,
2544 )]));
2545 }
2546
2547 assert_eq!(
2548 client.provides_bulk_position_coverage(instrument_id),
2549 expected
2550 );
2551 let foreign_id = InstrumentId::new(instrument_id.symbol, Venue::from("OTHER"));
2552 assert!(!client.provides_bulk_position_coverage(foreign_id));
2553 assert!(!client.provides_bulk_position_coverage(InstrumentId::from("BTC-USD.COINBASE")));
2554 assert!(!client.provides_bulk_position_coverage(InstrumentId::from("UNKNOWN.COINBASE")));
2555 }
2556
2557 #[rstest]
2558 fn test_submit_local_command_failure_classification() {
2559 let err = anyhow::anyhow!("Unsupported Coinbase order configuration");
2560
2561 assert!(is_coinbase_local_command_failure(&err));
2562 assert!(!is_coinbase_explicit_command_rejection(&err));
2563 assert!(!is_coinbase_ambiguous_command_failure(&err));
2564 }
2565
2566 #[rstest]
2567 fn test_submit_http_exchange_failure_classification() {
2568 let err = anyhow::Error::new(CoinbaseHttpError::exchange("HTTP 500: unavailable"))
2569 .context("failed to submit order");
2570
2571 assert!(!is_coinbase_local_command_failure(&err));
2572 assert!(!is_coinbase_explicit_command_rejection(&err));
2573 assert!(is_coinbase_ambiguous_command_failure(&err));
2574 }
2575
2576 #[rstest]
2577 fn test_submit_http_bad_request_failure_classification() {
2578 let err = anyhow::Error::new(CoinbaseHttpError::bad_request("HTTP 400: bad request"))
2579 .context("failed to submit order");
2580
2581 assert!(!is_coinbase_local_command_failure(&err));
2582 assert!(is_coinbase_explicit_command_rejection(&err));
2583 assert!(!is_coinbase_ambiguous_command_failure(&err));
2584 }
2585
2586 #[rstest]
2587 #[case(401)]
2588 #[case(403)]
2589 fn test_submit_http_auth_failure_classification(#[case] status: u16) {
2590 let err = anyhow::Error::new(CoinbaseHttpError::auth(format!(
2591 "HTTP {status}: authentication failed"
2592 )))
2593 .context("failed to submit order");
2594
2595 assert!(!is_coinbase_local_command_failure(&err));
2596 assert!(is_coinbase_explicit_command_rejection(&err));
2597 assert!(!is_coinbase_ambiguous_command_failure(&err));
2598 }
2599
2600 #[rstest]
2601 fn test_submit_http_rate_limit_failure_classification() {
2602 let err = anyhow::Error::new(CoinbaseHttpError::rate_limit(None))
2603 .context("failed to submit order");
2604
2605 assert!(!is_coinbase_local_command_failure(&err));
2606 assert!(!is_coinbase_explicit_command_rejection(&err));
2607 assert!(is_coinbase_ambiguous_submit_failure(&err));
2608 assert!(!is_coinbase_ambiguous_command_failure(&err));
2609 }
2610
2611 #[tokio::test]
2612 async fn test_rate_limited_submit_retains_context() {
2613 let (emitter, mut rx) = make_emitter();
2614 let order_contexts = Arc::new(Mutex::new(AHashMap::new()));
2615
2616 let context = OrderContext {
2617 price: Some(Price::from("100.00")),
2618 trigger_price: Some(Price::from("99.00")),
2619 trigger_type: Some(TriggerType::LastPrice),
2620 post_only: true,
2621 submitted_product_id: Some(Ustr::from("BTC-USDC")),
2622 };
2623 order_contexts
2624 .lock()
2625 .insert("client-1".to_string(), context.clone());
2626 let err = anyhow::Error::new(CoinbaseHttpError::rate_limit(Some(1_000)))
2627 .context("failed to submit order");
2628
2629 handle_coinbase_submit_failure(
2630 &err,
2631 &order_contexts,
2632 &emitter,
2633 StrategyId::from("S-001"),
2634 InstrumentId::from("BTC-USD.COINBASE"),
2635 ClientOrderId::from("client-1"),
2636 UnixNanos::default(),
2637 );
2638
2639 assert!(rx.try_recv().is_err());
2640 {
2641 let map = order_contexts.lock();
2642 let retained = map.get("client-1").expect("submit context retained");
2643 assert_eq!(retained.price, context.price);
2644 assert_eq!(retained.trigger_price, context.trigger_price);
2645 assert_eq!(retained.trigger_type, context.trigger_type);
2646 assert_eq!(retained.post_only, context.post_only);
2647 assert_eq!(retained.submitted_product_id, context.submitted_product_id);
2648 }
2649 }
2650
2651 #[rstest]
2652 #[case(None)]
2653 #[case(Some(400))]
2654 #[case(Some(401))]
2655 #[case(Some(403))]
2656 fn test_definitive_submit_failure_rejects_and_removes_context(#[case] status: Option<u16>) {
2657 let (emitter, mut rx) = make_emitter();
2658 let order_contexts = Mutex::new(AHashMap::from_iter([(
2659 "client-rejected".to_string(),
2660 make_limit_context(),
2661 )]));
2662 let (err, reason) = match status {
2663 None => (
2664 anyhow::anyhow!("Unsupported Coinbase order configuration"),
2665 "submit-order-error: Unsupported Coinbase order configuration".to_string(),
2666 ),
2667 Some(400) => (
2668 anyhow::Error::new(CoinbaseHttpError::bad_request("HTTP 400: refused")),
2669 "submit-order-rejected: bad request: HTTP 400: refused".to_string(),
2670 ),
2671 Some(status @ (401 | 403)) => (
2672 anyhow::Error::new(CoinbaseHttpError::auth(format!("HTTP {status}: refused"))),
2673 format!("submit-order-rejected: auth error: HTTP {status}: refused"),
2674 ),
2675 Some(status) => panic!("unsupported status {status}"),
2676 };
2677
2678 handle_coinbase_submit_failure(
2679 &err,
2680 &order_contexts,
2681 &emitter,
2682 StrategyId::from("S-REJECT"),
2683 InstrumentId::from("BTC-USD.COINBASE"),
2684 ClientOrderId::from("client-rejected"),
2685 UnixNanos::from(42_u64),
2686 );
2687
2688 assert!(order_contexts.lock().is_empty());
2689 let event = rx.try_recv().expect("order rejection emitted");
2690 let ExecutionEvent::Order(OrderEventAny::Rejected(rejected)) = event else {
2691 panic!("expected OrderRejected event, was {event:?}");
2692 };
2693 assert_eq!(rejected.trader_id, TraderId::from("TRADER-001"));
2694 assert_eq!(rejected.strategy_id, StrategyId::from("S-REJECT"));
2695 assert_eq!(
2696 rejected.instrument_id,
2697 InstrumentId::from("BTC-USD.COINBASE")
2698 );
2699 assert_eq!(
2700 rejected.client_order_id,
2701 ClientOrderId::from("client-rejected")
2702 );
2703 assert_eq!(rejected.account_id, AccountId::from("COINBASE-001"));
2704 assert_eq!(rejected.reason, reason);
2705 assert_eq!(rejected.ts_event, UnixNanos::from(42_u64));
2706 assert!(!rejected.reconciliation);
2707 assert!(!rejected.due_post_only);
2708 assert!(rx.try_recv().is_err());
2709 }
2710
2711 #[rstest]
2712 fn test_submit_unmapped_http_4xx_failure_classification() {
2713 let err = anyhow::Error::new(CoinbaseHttpError::http(409, "conflict"))
2714 .context("failed to submit order");
2715
2716 assert!(!is_coinbase_local_command_failure(&err));
2717 assert!(!is_coinbase_explicit_command_rejection(&err));
2718 assert!(!is_coinbase_ambiguous_command_failure(&err));
2719 }
2720
2721 #[rstest]
2722 fn test_submit_local_auth_failure_classification() {
2723 let err = anyhow::Error::new(CoinbaseHttpError::auth("No credentials configured"))
2724 .context("failed to submit order");
2725
2726 assert!(is_coinbase_local_command_failure(&err));
2727 assert!(!is_coinbase_explicit_command_rejection(&err));
2728 assert!(!is_coinbase_ambiguous_command_failure(&err));
2729 }
2730
2731 fn test_instrument() -> InstrumentAny {
2732 let instrument_id = InstrumentId::new(Symbol::new("BTC-USD"), *COINBASE_VENUE);
2733 InstrumentAny::CurrencyPair(
2734 CurrencyPair::builder()
2735 .instrument_id(instrument_id)
2736 .raw_symbol(Symbol::new("BTC-USD"))
2737 .base_currency(Currency::get_or_create_crypto("BTC"))
2738 .quote_currency(Currency::get_or_create_crypto("USD"))
2739 .price_precision(2)
2740 .size_precision(8)
2741 .price_increment(Price::from("0.01"))
2742 .size_increment(Quantity::from("0.00000001"))
2743 .min_quantity(Quantity::from("0.00000001"))
2744 .ts_event(UnixNanos::default())
2745 .ts_init(UnixNanos::default())
2746 .build()
2747 .unwrap(),
2748 )
2749 }
2750
2751 fn make_emitter() -> (
2752 ExecutionEventEmitter,
2753 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
2754 ) {
2755 let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
2756 let mut emitter = ExecutionEventEmitter::new(
2757 get_atomic_clock_realtime(),
2758 TraderId::from("TRADER-001"),
2759 AccountId::new("COINBASE-001"),
2760 AccountType::Cash,
2761 None,
2762 );
2763 emitter.set_sender(tx);
2764 (emitter, rx)
2765 }
2766
2767 fn make_user_order_update(
2768 cumulative: &str,
2769 leaves: &str,
2770 avg_price: &str,
2771 total_fees: &str,
2772 status: CbStatus,
2773 ) -> WsOrderUpdate {
2774 WsOrderUpdate {
2775 order_id: "venue-1".to_string(),
2776 client_order_id: "client-1".to_string(),
2777 contract_expiry_type: CoinbaseContractExpiryType::Unknown,
2778 cumulative_quantity: cumulative.to_string(),
2779 leaves_quantity: leaves.to_string(),
2780 avg_price: avg_price.to_string(),
2781 total_fees: total_fees.to_string(),
2782 status,
2783 product_id: Ustr::from("BTC-USD"),
2784 product_type: CbProductType::Spot,
2785 creation_time: String::new(),
2786 order_side: CbSide::Buy,
2787 order_type: CbType::Limit,
2788 risk_managed_by: CoinbaseRiskManagedBy::Unknown,
2789 time_in_force: CbTif::GoodUntilCancelled,
2790 trigger_status: CoinbaseTriggerStatus::InvalidOrderType,
2791 cancel_reason: String::new(),
2792 reject_reason: String::new(),
2793 total_value_after_fees: String::new(),
2794 }
2795 }
2796
2797 fn make_carrier(update: WsOrderUpdate) -> UserOrderUpdate {
2798 make_carrier_with_kind(update, false)
2799 }
2800
2801 fn make_limit_context() -> OrderContext {
2802 OrderContext {
2803 price: Some(Price::from("100.00")),
2804 ..OrderContext::default()
2805 }
2806 }
2807
2808 fn make_carrier_with_kind(update: WsOrderUpdate, is_snapshot: bool) -> UserOrderUpdate {
2809 let instrument = test_instrument();
2810 let report = crate::websocket::parse::parse_ws_user_event_to_order_status_report(
2811 &update,
2812 &instrument,
2813 AccountId::new("COINBASE-001"),
2814 UnixNanos::default(),
2815 UnixNanos::default(),
2816 )
2817 .unwrap();
2818 UserOrderUpdate {
2819 report: Box::new(report),
2820 update: Box::new(update),
2821 instrument,
2822 is_snapshot,
2823 ts_event: UnixNanos::default(),
2824 ts_init: UnixNanos::default(),
2825 }
2826 }
2827
2828 struct RecoverySpec {
2829 recovery: UserOrderRecovery,
2830 rx: tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
2831 state: Arc<Mutex<RecoveryResponses>>,
2832 server: tokio::task::JoinHandle<()>,
2833 client: Option<CoinbaseExecutionClient>,
2834 }
2835
2836 struct RecoveryResponses {
2837 order: serde_json::Value,
2838 fills: serde_json::Value,
2839 fills_targeted: Option<serde_json::Value>,
2840 fail_fills: bool,
2841 orders: AHashMap<String, serde_json::Value>,
2842 discover_orders: bool,
2843 on_discovery: Option<Box<dyn FnOnce() + Send>>,
2844 requests: Vec<String>,
2845 on_order: Option<Box<dyn FnOnce() + Send>>,
2846 }
2847
2848 impl RecoverySpec {
2849 async fn new() -> Self {
2850 let mut order: serde_json::Value =
2851 serde_json::from_str(include_str!("../test_data/http_order.json")).unwrap();
2852 order["order"]["order_id"] = "venue-1".into();
2853 order["order"]["client_order_id"] = "client-1".into();
2854 order["order"]["order_configuration"]["limit_limit_gtc"]["base_size"] = "1.0".into();
2855 order["order"]["status"] = "FILLED".into();
2856 order["order"]["filled_size"] = "1.0".into();
2857 order["order"]["average_filled_price"] = "112.00".into();
2858 order["order"]["created_time"] = "2024-01-15T09:59:00Z".into();
2859 order["order"]["last_fill_time"] = "2024-01-15T10:31:00Z".into();
2860 let fixture: serde_json::Value =
2861 serde_json::from_str(include_str!("../test_data/http_fills.json")).unwrap();
2862 let mut first = fixture["fills"][0].clone();
2863 first["order_id"] = "venue-1".into();
2864 first["trade_id"] = "trade-alpha".into();
2865 first["size"] = "0.4".into();
2866 first["price"] = "100.00".into();
2867 first["commission"] = "0.04".into();
2868 let mut second = first.clone();
2869 second["trade_id"] = "trade-beta".into();
2870 second["size"] = "0.6".into();
2871 second["price"] = "120.00".into();
2872 second["commission"] = "0.06".into();
2873 second["trade_time"] = "2024-01-15T10:31:00Z".into();
2874 second["liquidity_indicator"] = "TAKER".into();
2875
2876 let state = Arc::new(Mutex::new(RecoveryResponses {
2877 order,
2878 fills: serde_json::json!({"fills": [second, first], "cursor": ""}),
2879 fills_targeted: None,
2880 fail_fills: false,
2881 orders: AHashMap::new(),
2882 discover_orders: true,
2883 on_discovery: None,
2884 requests: Vec::new(),
2885 on_order: None,
2886 }));
2887
2888 let router = axum::Router::new().fallback(axum::routing::get({
2889 let state = Arc::clone(&state);
2890 move |uri: axum::http::Uri| {
2891 let state = Arc::clone(&state);
2892 async move {
2893 let (status, value) = state.lock().response(&uri);
2894 (status, axum::Json(value))
2895 }
2896 }
2897 }));
2898
2899 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2900 let addr = listener.local_addr().unwrap();
2901 let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
2902 let rng = aws_lc_rs::rand::SystemRandom::new();
2903 let key = aws_lc_rs::signature::EcdsaKeyPair::generate_pkcs8(
2904 &aws_lc_rs::signature::ECDSA_P256_SHA256_FIXED_SIGNING,
2905 &rng,
2906 )
2907 .unwrap();
2908 let pem = pem::encode(&pem::Pem::new("PRIVATE KEY", key.as_ref().to_vec()));
2909 let credential = CoinbaseCredential::new("test-key".into(), pem);
2910
2911 let http_client = CoinbaseHttpClient::with_credentials(
2912 credential,
2913 crate::common::enums::CoinbaseEnvironment::Live,
2914 5,
2915 None,
2916 Some(RetryConfig {
2917 max_retries: 0,
2918 ..Default::default()
2919 }),
2920 )
2921 .unwrap();
2922
2923 http_client.set_base_url(format!("http://{addr}"));
2924 let instrument = test_instrument();
2925 http_client.cache_instrument(&instrument);
2926 let (emitter, rx) = make_emitter();
2927
2928 let recovery = UserOrderRecovery {
2929 http_client,
2930 emitter,
2931 instruments: Arc::new(AHashMap::from([("BTC-USD".into(), instrument)])),
2932 observed: Arc::new(Mutex::new(AHashMap::new())),
2933 pending: Arc::new(Mutex::new(AHashMap::new())),
2934 published: Arc::new(tokio::sync::Mutex::new(AHashMap::new())),
2935 order_contexts: Arc::new(Mutex::new(AHashMap::new())),
2936 external_order_contexts: Arc::new(Mutex::new(AHashMap::new())),
2937 lifecycle: Arc::new(Mutex::new(RecoveryLifecycle::new(
2938 TraderId::from("TRADER-001"),
2939 AccountId::from("COINBASE-001"),
2940 AccountType::Cash,
2941 ))),
2942 start: crate::http::parse::parse_rfc3339_timestamp("2024-01-01T00:00:00Z").unwrap(),
2943 };
2944
2945 Self {
2946 recovery,
2947 rx,
2948 state,
2949 server,
2950 client: None,
2951 }
2952 }
2953
2954 fn attach(&mut self, node: &LiveNode) {
2955 let core = ExecutionClientCore::new(
2956 TraderId::from("TRADER-001"),
2957 ClientId::from("COINBASE-TEST"),
2958 *COINBASE_VENUE,
2959 OmsType::Netting,
2960 AccountId::from("COINBASE-001"),
2961 AccountType::Cash,
2962 None,
2963 node.kernel().cache.clone(),
2964 );
2965
2966 let config = CoinbaseExecutionClientConfig {
2967 api_key: Some("test-key".into()),
2968 api_secret: Some("test-secret".into()),
2969 ..Default::default()
2970 };
2971
2972 let mut client = CoinbaseExecutionClient::new(core, config).unwrap();
2973 client.emitter = self.recovery.emitter.clone();
2974 client.lifecycle = Arc::clone(&self.recovery.lifecycle);
2975 client.subscribe_order_events();
2976 self.client = Some(client);
2977 }
2978
2979 fn apply(&mut self, node: &LiveNode) {
2980 while let Ok(event) = self.rx.try_recv() {
2981 match event {
2982 ExecutionEvent::Report(report) => node
2983 .kernel()
2984 .exec_engine
2985 .borrow_mut()
2986 .reconcile_execution_report(&report),
2987 ExecutionEvent::Order(event) => {
2988 node.kernel().exec_engine.borrow_mut().process(&event);
2989 }
2990 other => panic!("unexpected recovery event: {other:?}"),
2991 }
2992 }
2993 }
2994
2995 fn reports(&mut self) -> Vec<ExecutionReport> {
2996 let mut reports = Vec::new();
2997
2998 while let Ok(event) = self.rx.try_recv() {
2999 let ExecutionEvent::Report(report) = event else {
3000 panic!("expected execution report, was {event:?}");
3001 };
3002
3003 reports.push(report);
3004 }
3005
3006 reports
3007 }
3008
3009 fn bundle(&mut self) -> (OrderStatusReport, Vec<FillReport>) {
3010 let mut order = None;
3011 let mut fills = Vec::new();
3012
3013 for report in self.reports() {
3014 match report {
3015 ExecutionReport::Order(report) => order = Some(*report),
3016 ExecutionReport::Fill(fill) => fills.push(*fill),
3017 other => panic!("unexpected recovery report: {other:?}"),
3018 }
3019 }
3020
3021 (order.expect("final order report emitted"), fills)
3022 }
3023 }
3024
3025 impl RecoveryResponses {
3026 fn response(
3027 &mut self,
3028 uri: &axum::http::Uri,
3029 ) -> (axum::http::StatusCode, serde_json::Value) {
3030 self.requests.push(uri.to_string());
3031 let value = if uri.path().ends_with("/fills") {
3032 return self.response_fills(uri);
3033 } else if uri.path().ends_with("/orders/historical/batch") {
3034 self.response_orders()
3035 } else {
3036 if let Some(action) = self.on_order.take() {
3037 action();
3038 }
3039
3040 let order_id = uri.path().rsplit('/').next().unwrap();
3041 self.orders.get(order_id).unwrap_or(&self.order).clone()
3042 };
3043
3044 (axum::http::StatusCode::OK, value)
3045 }
3046
3047 fn response_fills(
3048 &mut self,
3049 uri: &axum::http::Uri,
3050 ) -> (axum::http::StatusCode, serde_json::Value) {
3051 if self.fail_fills {
3052 return (
3053 axum::http::StatusCode::SERVICE_UNAVAILABLE,
3054 serde_json::Value::Null,
3055 );
3056 }
3057
3058 let mut fills = self.fills.clone();
3059 let order_id = url::form_urlencoded::parse(uri.query().unwrap_or_default().as_bytes())
3060 .find(|(key, _)| key == "order_ids")
3061 .map(|(_, value)| value.into_owned());
3062
3063 if let Some(order_id) = order_id {
3064 fills = self.fills_targeted.as_ref().unwrap_or(&fills).clone();
3065 fills["fills"]
3066 .as_array_mut()
3067 .unwrap()
3068 .retain(|fill| fill["order_id"] == order_id);
3069 } else if let Some(action) = self.on_discovery.take() {
3070 action();
3071 }
3072
3073 (axum::http::StatusCode::OK, fills)
3074 }
3075
3076 fn response_orders(&self) -> serde_json::Value {
3077 let orders: Vec<_> = if self.discover_orders {
3078 std::iter::once(self.order["order"].clone())
3079 .chain(self.orders.values().map(|order| order["order"].clone()))
3080 .collect()
3081 } else {
3082 Vec::new()
3083 };
3084
3085 serde_json::json!({"orders": orders, "has_next": false, "cursor": ""})
3086 }
3087 }
3088
3089 impl Drop for RecoverySpec {
3090 fn drop(&mut self) {
3091 if let Some(client) = &mut self.client
3092 && let Some(handler) = client.order_event_handler.take()
3093 {
3094 msgbus::unsubscribe_order_events("events.order.*".into(), &handler);
3095 }
3096
3097 self.server.abort();
3098 }
3099 }
3100
3101 #[tokio::test]
3102 async fn test_user_fill_update_preserves_distinct_rest_trades() {
3103 let mut spec = RecoverySpec::new().await;
3104 let update = make_user_order_update("1.0", "0", "112.00", "0.10", CbStatus::Filled);
3105
3106 spec.recovery.handle_update(make_carrier(update)).await;
3107 let (report, fills) = spec.bundle();
3108 let rest = spec
3109 .recovery
3110 .http_client
3111 .request_fill_reports(
3112 report.account_id,
3113 None,
3114 Some(report.venue_order_id),
3115 None,
3116 None,
3117 None,
3118 )
3119 .await
3120 .unwrap();
3121 let mut expected = prepare_order_fills(&report, rest).unwrap();
3122 for (expected, actual) in expected.iter_mut().zip(&fills) {
3123 expected.report_id = actual.report_id;
3124 expected.ts_init = actual.ts_init;
3125 }
3126
3127 assert_eq!(fills, expected);
3128 assert_eq!(report.quantity, Quantity::from("1.00000000"));
3129 assert_eq!(report.filled_qty, Quantity::from("1.00000000"));
3130 assert_eq!(report.order_status, OrderStatus::Filled);
3131 assert_eq!(fills.len(), 2);
3132 assert_eq!(fills[0].trade_id.as_str(), "trade-alpha");
3133 assert_eq!(fills[0].last_qty, Quantity::from("0.40000000"));
3134 assert_eq!(fills[0].last_px, Price::from("100.00"));
3135 assert_eq!(fills[0].commission.to_string(), "0.04 USD");
3136 assert_eq!(fills[1].trade_id.as_str(), "trade-beta");
3137 assert_eq!(fills[1].last_qty, Quantity::from("0.60000000"));
3138 assert_eq!(fills[1].last_px, Price::from("120.00"));
3139 assert_eq!(fills[1].commission.to_string(), "0.06 USD");
3140 assert!(spec.recovery.pending.lock().is_empty());
3141 }
3142
3143 #[rstest]
3144 #[case::missing_fill(false)]
3145 #[case::request_failure(true)]
3146 #[tokio::test]
3147 async fn test_terminal_fill_retries_without_another_user_update(#[case] request_failure: bool) {
3148 let mut spec = RecoverySpec::new().await;
3149 spec.recovery
3150 .order_contexts
3151 .lock()
3152 .insert("client-1".into(), make_limit_context());
3153 let complete = spec.state.lock().fills.clone();
3154 {
3155 let mut state = spec.state.lock();
3156 state.fail_fills = request_failure;
3157 state.fills["fills"].as_array_mut().unwrap().pop();
3158 }
3159
3160 let update = make_user_order_update("1.0", "0", "112.00", "0.10", CbStatus::Filled);
3161 spec.recovery.handle_update(make_carrier(update)).await;
3162
3163 assert!(spec.rx.try_recv().is_err());
3164 assert_eq!(spec.recovery.pending.lock().len(), 1);
3165 assert_eq!(spec.recovery.order_contexts.lock().len(), 1);
3166
3167 {
3168 let mut state = spec.state.lock();
3169 state.fail_fills = false;
3170 state.fills = complete;
3171 }
3172
3173 spec.recovery.retry().await;
3174 let (report, fills) = spec.bundle();
3175
3176 assert_eq!(report.order_status, OrderStatus::Filled);
3177 assert_eq!(fills.len(), 2);
3178 assert!(spec.recovery.pending.lock().is_empty());
3179 assert!(spec.recovery.order_contexts.lock().is_empty());
3180 }
3181
3182 #[tokio::test]
3183 async fn test_reconnect_discovers_closed_order_and_fetches_full_history() {
3184 let mut spec = RecoverySpec::new().await;
3185 let all = spec.state.lock().fills.clone();
3186 spec.state.lock().fills["fills"]
3188 .as_array_mut()
3189 .unwrap()
3190 .pop();
3191 spec.recovery.discover().await.unwrap();
3192 spec.state.lock().fills = all;
3193 spec.recovery.retry().await;
3194 let (report, fills) = spec.bundle();
3195 let requests = &spec.state.lock().requests;
3196 let discovery = url::Url::parse(&format!("http://localhost{}", requests[0])).unwrap();
3197 let target =
3198 url::Url::parse(&format!("http://localhost{}", requests.last().unwrap())).unwrap();
3199
3200 assert_eq!(report.order_status, OrderStatus::Filled);
3201 assert_eq!(fills.len(), 2);
3202 assert!(
3203 discovery
3204 .query_pairs()
3205 .any(|(key, value)| key == "start_sequence_timestamp"
3206 && value == "2024-01-01T00:00:00+00:00")
3207 );
3208 assert!(
3209 target
3210 .query_pairs()
3211 .any(|(key, value)| key == "order_ids" && value == "venue-1")
3212 );
3213 assert!(
3214 !target
3215 .query_pairs()
3216 .any(|(key, _)| key == "start_sequence_timestamp")
3217 );
3218 }
3219
3220 #[tokio::test]
3221 async fn test_snapshot_and_replay_keep_rest_identity() {
3222 let mut spec = RecoverySpec::new().await;
3223 let update = make_user_order_update("1.0", "0", "112.00", "0.10", CbStatus::Filled);
3224 spec.recovery
3225 .handle_update(make_carrier_with_kind(update.clone(), true))
3226 .await;
3227 let (_, first) = spec.bundle();
3228 spec.recovery.handle_update(make_carrier(update)).await;
3229 let (_, replay) = spec.bundle();
3230
3231 assert_eq!(first.len(), replay.len());
3232
3233 for (first, mut replay) in first.iter().zip(replay) {
3234 replay.report_id = first.report_id;
3235 replay.ts_init = first.ts_init;
3236 assert_eq!(*first, replay);
3237 }
3238
3239 assert_eq!(
3240 first
3241 .iter()
3242 .map(|fill| fill.trade_id.as_str())
3243 .collect::<Vec<_>>(),
3244 vec!["trade-alpha", "trade-beta"]
3245 );
3246 }
3247
3248 #[rstest]
3249 #[case(false)]
3250 #[case(true)]
3251 fn test_prepare_user_order_report_requires_snapshot_context(#[case] snapshot: bool) {
3252 let update = make_user_order_update("0", "1.0", "0", "0", CbStatus::Open);
3253 let carrier = make_carrier_with_kind(update, snapshot);
3254 let mut context = make_limit_context();
3255 context.post_only = true;
3256 context.submitted_product_id = Some(Ustr::from("BTC-USDC"));
3257
3258 let report = prepare_user_order_report(carrier.clone(), Some(context)).unwrap();
3259 let unknown = prepare_user_order_report(carrier, None);
3260
3261 assert_eq!(report.price, Some(Price::from("100.00")));
3262 assert_eq!(
3263 report.instrument_id,
3264 InstrumentId::from("BTC-USDC.COINBASE")
3265 );
3266 assert_eq!(report.quantity, Quantity::from("1.00000000"));
3267 assert!(report.post_only);
3268 assert!(unknown.is_none());
3269 }
3270
3271 #[rstest]
3272 #[case(CbType::Limit, false)]
3273 #[case(CbType::StopLimit, false)]
3274 #[case(CbType::Market, true)]
3275 fn test_prepare_user_order_report_preserves_safety_gate(
3276 #[case] order_type: CbType,
3277 #[case] safe: bool,
3278 ) {
3279 let mut update = make_user_order_update("0", "1.0", "0", "0", CbStatus::Open);
3280 update.order_type = order_type;
3281 let report = prepare_user_order_report(make_carrier(update), None);
3282 assert_eq!(report.is_some(), safe);
3283 }
3284
3285 #[rstest]
3286 #[case::startup_first(true)]
3287 #[case::live_first(false)]
3288 #[tokio::test]
3289 async fn test_startup_and_reconnect_overlap_apply_each_fill_once(#[case] startup_first: bool) {
3290 let mut spec = RecoverySpec::new().await;
3291 let node = recovery_node();
3292 let kernel = node.kernel();
3293 let account_id = AccountId::from("COINBASE-001");
3294 let client_id = ClientId::from("COINBASE-TEST");
3295
3296 let core = ExecutionClientCore::new(
3297 TraderId::from("TRADER-001"),
3298 client_id,
3299 *COINBASE_VENUE,
3300 OmsType::Netting,
3301 account_id,
3302 AccountType::Cash,
3303 None,
3304 kernel.cache.clone(),
3305 );
3306
3307 let config = CoinbaseExecutionClientConfig {
3308 api_key: Some("test-key".into()),
3309 api_secret: Some("test-secret".into()),
3310 ..Default::default()
3311 };
3312
3313 let mut client = CoinbaseExecutionClient::new(core, config).unwrap();
3314 client.instruments_cache = Arc::clone(&spec.recovery.instruments);
3315 kernel
3316 .exec_engine
3317 .borrow_mut()
3318 .register_client(Box::new(client))
3319 .unwrap();
3320 let mut manager = ExecutionManager::new(
3321 kernel.clock.clone(),
3322 kernel.cache.clone(),
3323 Default::default(),
3324 )
3325 .unwrap();
3326 let full_order = spec.state.lock().order.clone();
3327 let full_fills = spec.state.lock().fills.clone();
3328 {
3329 let mut state = spec.state.lock();
3330 state.order["order"]["status"] = "OPEN".into();
3331 state.order["order"]["filled_size"] = "0.4".into();
3332 state.order["order"]["average_filled_price"] = "100.00".into();
3333 state.fills["fills"].as_array_mut().unwrap().remove(0);
3334 }
3335
3336 let report = spec
3337 .recovery
3338 .http_client
3339 .request_order_status_report(account_id, None, Some(VenueOrderId::from("venue-1")))
3340 .await
3341 .unwrap();
3342 let fills = spec
3343 .recovery
3344 .http_client
3345 .request_fill_reports(
3346 account_id,
3347 None,
3348 Some(report.venue_order_id),
3349 None,
3350 None,
3351 None,
3352 )
3353 .await
3354 .unwrap();
3355
3356 let mut mass = ExecutionMassStatus::new(
3357 client_id,
3358 account_id,
3359 *COINBASE_VENUE,
3360 UnixNanos::default(),
3361 None,
3362 );
3363 mass.add_order_reports(vec![report.clone()]);
3364 mass.add_fill_reports(fills.clone());
3365 if startup_first {
3366 manager.reconcile_execution_mass_status(&mass, &kernel.exec_engine);
3367 }
3368
3369 let update = make_user_order_update("0.4", "0.6", "100.00", "0.04", CbStatus::Open);
3370 spec.recovery.handle_update(make_carrier(update)).await;
3371 let live_reports = spec.reports();
3372 apply_recovery_reports(&node, &live_reports);
3373
3374 if !startup_first {
3375 manager.reconcile_execution_mass_status(&mass, &kernel.exec_engine);
3376 }
3377
3378 {
3379 let mut state = spec.state.lock();
3380 state.order = full_order;
3381 state.fills = full_fills;
3382 }
3383
3384 spec.recovery.retry().await;
3385 let reports = spec.reports();
3386 apply_recovery_reports(&node, &reports);
3387 let event_count = kernel
3388 .cache
3389 .borrow()
3390 .order(&ClientOrderId::from("client-1"))
3391 .unwrap()
3392 .event_count();
3393 apply_recovery_reports(&node, &reports);
3394
3395 let cache = kernel.cache.borrow();
3396 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
3397 let positions = cache.positions_refs(None, None, None, None, None);
3398 assert_eq!(order.status(), OrderStatus::Filled);
3399 assert_eq!(order.filled_qty(), Quantity::from("1.00000000"));
3400 assert_eq!(order.event_count(), event_count);
3401 assert_eq!(
3402 order
3403 .trade_ids()
3404 .iter()
3405 .map(|id| id.as_str())
3406 .collect::<Vec<_>>(),
3407 vec!["trade-alpha", "trade-beta"]
3408 );
3409 assert_eq!(positions.len(), 1);
3410 assert_eq!(positions[0].quantity, Quantity::from("1.00000000"));
3411 assert_eq!(positions[0].avg_px_open, rust_decimal_macros::dec!(112));
3412 assert_eq!(positions[0].commissions(), vec![Money::from("0.10 USD")]);
3413 assert_eq!(positions[0].event_count(), 2);
3414 }
3415
3416 #[rstest]
3417 fn test_prepare_user_order_report_retains_trigger_metadata() {
3418 let mut update = make_user_order_update("0", "1", "0", "0", CbStatus::Open);
3419 update.order_type = CbType::StopLimit;
3420
3421 let context = OrderContext {
3422 price: Some(Price::from("100.50")),
3423 trigger_price: Some(Price::from("99.00")),
3424 trigger_type: Some(TriggerType::LastPrice),
3425 post_only: true,
3426 submitted_product_id: Some(Ustr::from("BTC-USDC")),
3427 };
3428
3429 let report = prepare_user_order_report(make_carrier(update), Some(context)).unwrap();
3430 assert_eq!(report.price, Some(Price::from("100.50")));
3431 assert_eq!(report.trigger_price, Some(Price::from("99.00")));
3432 assert_eq!(report.trigger_type, Some(TriggerType::LastPrice));
3433 assert_eq!(
3434 report.instrument_id,
3435 InstrumentId::from("BTC-USDC.COINBASE")
3436 );
3437 assert!(report.post_only);
3438 }
3439
3440 #[rstest]
3441 #[case::partial_cancel("0.4", CbStatus::Cancelled, "CANCELLED")]
3442 #[case::empty_cancel("0", CbStatus::Cancelled, "CANCELLED")]
3443 #[case::empty_expire("0", CbStatus::Expired, "EXPIRED")]
3444 #[tokio::test]
3445 async fn test_terminal_recovery_retains_original_qty(
3446 #[case] cumulative: &str,
3447 #[case] status: CbStatus,
3448 #[case] rest_status: &str,
3449 ) {
3450 let mut spec = RecoverySpec::new().await;
3451 {
3452 let mut state = spec.state.lock();
3453 state.order["order"]["status"] = rest_status.into();
3454 state.order["order"]["filled_size"] = cumulative.into();
3455 let fills = state.fills["fills"].as_array_mut().unwrap();
3456 if cumulative == "0" {
3457 fills.clear();
3458 } else {
3459 fills.remove(0);
3460 }
3461 }
3462
3463 let update = make_user_order_update(cumulative, "0", "100", "0.04", status);
3464 spec.recovery.handle_update(make_carrier(update)).await;
3465 let (report, fills) = spec.bundle();
3466 assert_eq!(report.quantity, Quantity::from("1.00000000"));
3467 assert_eq!(
3468 report.filled_qty.as_decimal(),
3469 cumulative.parse::<Decimal>().unwrap()
3470 );
3471 assert_eq!(fills.len(), usize::from(cumulative != "0"));
3472 }
3473
3474 #[tokio::test]
3475 async fn test_recovery_uses_rest_alias_and_fee_currency_after_context_removal() {
3476 let mut spec = RecoverySpec::new().await;
3477
3478 let InstrumentAny::CurrencyPair(mut pair) = test_instrument() else {
3479 unreachable!()
3480 };
3481
3482 pair.id = InstrumentId::from("BTC-USDC.COINBASE");
3483 pair.raw_symbol = Symbol::from("BTC-USDC");
3484 pair.quote_currency = Currency::USDC();
3485 let instrument = InstrumentAny::CurrencyPair(pair);
3486 spec.recovery.http_client.cache_instrument(&instrument);
3487 spec.recovery.instruments = Arc::new(AHashMap::from([("BTC-USDC".into(), instrument)]));
3488 {
3489 let mut state = spec.state.lock();
3490 state.order["order"]["product_id"] = "BTC-USDC".into();
3491 for fill in state.fills["fills"].as_array_mut().unwrap() {
3492 fill["product_id"] = "BTC-USDC".into();
3493 }
3494 }
3495
3496 let update = make_user_order_update("1", "0", "112", "0.10", CbStatus::Filled);
3497 spec.recovery
3498 .handle_update(make_carrier(update.clone()))
3499 .await;
3500 let (first_order, first_fills) = spec.bundle();
3501 spec.recovery.handle_update(make_carrier(update)).await;
3502 let (replay_order, replay_fills) = spec.bundle();
3503
3504 assert_eq!(
3505 first_order.instrument_id,
3506 InstrumentId::from("BTC-USDC.COINBASE")
3507 );
3508 assert_eq!(replay_order.instrument_id, first_order.instrument_id);
3509 assert_eq!(first_fills.len(), 2);
3510 assert_eq!(replay_fills.len(), 2);
3511
3512 for (first, replay) in first_fills.iter().zip(&replay_fills) {
3513 assert_eq!(first.instrument_id, first_order.instrument_id);
3514 assert_eq!(replay.instrument_id, first.instrument_id);
3515 assert_eq!(first.commission.currency, Currency::USDC());
3516 assert_eq!(replay.commission, first.commission);
3517 assert_eq!(replay.trade_id, first.trade_id);
3518 }
3519 }
3520
3521 #[tokio::test]
3522 async fn test_recovery_deduplicates_rest_rows_before_coverage() {
3523 let mut spec = RecoverySpec::new().await;
3524 {
3525 let mut state = spec.state.lock();
3526 let duplicate = state.fills["fills"][1].clone();
3527 state.fills["fills"].as_array_mut().unwrap().push(duplicate);
3528 }
3529
3530 let update = make_user_order_update("1", "0", "112", "0.10", CbStatus::Filled);
3531 spec.recovery.handle_update(make_carrier(update)).await;
3532 let (_, fills) = spec.bundle();
3533 assert_eq!(fills.len(), 2);
3534 assert_eq!(
3535 fills
3536 .iter()
3537 .map(|fill| fill.last_qty.as_decimal())
3538 .sum::<Decimal>(),
3539 rust_decimal_macros::dec!(1)
3540 );
3541 }
3542
3543 #[tokio::test]
3544 async fn test_discovery_does_not_hide_an_unparsable_fill() {
3545 let mut spec = RecoverySpec::new().await;
3546 spec.state.lock().fills["fills"][0]["size"] = "invalid".into();
3547 spec.recovery.retry().await;
3548 assert_eq!(spec.recovery.pending.lock().len(), 1);
3549 assert!(spec.rx.try_recv().is_err());
3550
3551 spec.state.lock().fills["fills"][0]["size"] = "0.6".into();
3552 spec.recovery.retry().await;
3553 let (_, fills) = spec.bundle();
3554 assert_eq!(fills.len(), 2);
3555 assert!(spec.recovery.pending.lock().is_empty());
3556 }
3557
3558 #[tokio::test]
3559 async fn test_pending_terminal_is_not_replaced_by_stale_update() {
3560 let spec = RecoverySpec::new().await;
3561 let terminal = make_carrier(make_user_order_update(
3562 "1",
3563 "0",
3564 "112",
3565 "0.10",
3566 CbStatus::Filled,
3567 ));
3568 let stale = make_carrier(make_user_order_update(
3569 "0.4",
3570 "0.6",
3571 "100",
3572 "0.04",
3573 CbStatus::Open,
3574 ));
3575 spec.recovery.queue(*terminal.report.clone());
3576 spec.recovery.queue(*stale.report);
3577 let pending = spec.recovery.pending.lock();
3578 assert_eq!(pending[&VenueOrderId::from("venue-1")], *terminal.report);
3579 }
3580
3581 #[tokio::test]
3582 async fn test_recovery_timer_publishes_pending_terminal_without_user_message() {
3583 let mut spec = RecoverySpec::new().await;
3584 let carrier = make_carrier(make_user_order_update(
3585 "1",
3586 "0",
3587 "112",
3588 "0.10",
3589 CbStatus::Filled,
3590 ));
3591 spec.recovery.queue(*carrier.report);
3592 let (_tx, rx) = tokio::sync::mpsc::unbounded_channel();
3593
3594 let reports = {
3595 let run = spec.recovery.run(
3596 rx,
3597 Duration::from_millis(1),
3598 get_atomic_clock_realtime(),
3599 false,
3600 );
3601 tokio::pin!(run);
3602
3603 let collect = async {
3604 let mut reports = Vec::new();
3605
3606 loop {
3607 let ExecutionEvent::Report(report) = spec.rx.recv().await.unwrap() else {
3608 panic!("expected execution report");
3609 };
3610
3611 let complete = matches!(&report, ExecutionReport::Fill(fill) if fill.trade_id.as_str() == "trade-beta");
3612 reports.push(report);
3613 if complete {
3614 return reports;
3615 }
3616 }
3617 };
3618
3619 tokio::select! {
3620 () = &mut run => panic!("recovery loop stopped while the input channel is open"),
3621 result = tokio::time::timeout(Duration::from_secs(2), collect) => result.unwrap(),
3622 }
3623 };
3624
3625 assert_eq!(reports.len(), 3);
3626
3627 let trade_ids: Vec<_> = reports
3628 .iter()
3629 .filter_map(|report| match report {
3630 ExecutionReport::Fill(fill) => Some(fill.trade_id.as_str()),
3631 _ => None,
3632 })
3633 .collect();
3634
3635 assert_eq!(trade_ids, vec!["trade-alpha", "trade-beta"]);
3636 assert!(spec.recovery.pending.lock().is_empty());
3637 }
3638
3639 #[tokio::test]
3640 async fn test_recovery_preserves_new_target_queued_during_request() {
3641 let mut spec = RecoverySpec::new().await;
3642 let full_order = spec.state.lock().order.clone();
3643 let full_fills = spec.state.lock().fills.clone();
3644 let partial = make_carrier(make_user_order_update(
3645 "0.4",
3646 "0.6",
3647 "100",
3648 "0.04",
3649 CbStatus::Open,
3650 ));
3651 let terminal = make_carrier(make_user_order_update(
3652 "1",
3653 "0",
3654 "112",
3655 "0.10",
3656 CbStatus::Filled,
3657 ));
3658 spec.recovery.queue(*partial.report);
3659 let pending = Arc::clone(&spec.recovery.pending);
3660 {
3661 let mut state = spec.state.lock();
3662 state.order["order"]["status"] = "OPEN".into();
3663 state.order["order"]["filled_size"] = "0.4".into();
3664 state.fills["fills"].as_array_mut().unwrap().remove(0);
3665 state.on_order = Some(Box::new(move || {
3666 pending
3667 .lock()
3668 .insert(terminal.report.venue_order_id, *terminal.report);
3669 }));
3670 }
3671
3672 let error = spec.recovery.recover().await.unwrap_err();
3673 assert_eq!(
3674 error.to_string(),
3675 "recovery target changed while reading REST"
3676 );
3677 assert!(spec.rx.try_recv().is_err());
3678 assert_eq!(
3679 spec.recovery.pending.lock()[&VenueOrderId::from("venue-1")].filled_qty,
3680 Quantity::from("1.00000000")
3681 );
3682
3683 {
3684 let mut state = spec.state.lock();
3685 state.order = full_order;
3686 state.fills = full_fills;
3687 }
3688
3689 spec.recovery.retry().await;
3690 let (terminal, fills) = spec.bundle();
3691 assert_eq!(terminal.order_status, OrderStatus::Filled);
3692 assert_eq!(fills.len(), 2);
3693 assert!(spec.recovery.pending.lock().is_empty());
3694 }
3695
3696 #[tokio::test]
3697 async fn test_recovery_retains_pending_on_closed_event_receiver() {
3698 let mut spec = RecoverySpec::new().await;
3699 spec.rx.close();
3700 let carrier = make_carrier(make_user_order_update(
3701 "1",
3702 "0",
3703 "112",
3704 "0.10",
3705 CbStatus::Filled,
3706 ));
3707 spec.recovery.handle_update(carrier).await;
3708 assert_eq!(spec.recovery.pending.lock().len(), 1);
3709
3710 let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
3711 spec.recovery.emitter.set_sender(tx);
3712 spec.rx = rx;
3713 spec.recovery.retry().await;
3714 let (terminal, fills) = spec.bundle();
3715 assert_eq!(terminal.order_status, OrderStatus::Filled);
3716 assert_eq!(fills.len(), 2);
3717 assert!(spec.recovery.pending.lock().is_empty());
3718 }
3719
3720 #[tokio::test]
3721 async fn test_server_time_uses_venue_timestamp_and_rejects_missing_field() {
3722 let spec = RecoverySpec::new().await;
3723 spec.state.lock().order["iso"] = "2024-01-02T03:04:05.123Z".into();
3724 let time = spec
3725 .recovery
3726 .http_client
3727 .request_server_time()
3728 .await
3729 .unwrap();
3730 assert_eq!(
3731 time,
3732 crate::http::parse::parse_rfc3339_timestamp("2024-01-02T03:04:05.123Z").unwrap()
3733 );
3734
3735 spec.state.lock().order["iso"] = serde_json::Value::Null;
3736 let error = spec
3737 .recovery
3738 .http_client
3739 .request_server_time()
3740 .await
3741 .unwrap_err();
3742 assert_eq!(error.to_string(), "server time response is missing iso");
3743 }
3744
3745 #[rstest]
3746 #[case::older_partial("1", CbStatus::Filled)]
3747 #[case::active_before_cancel("0.4", CbStatus::Cancelled)]
3748 #[tokio::test]
3749 async fn test_recovery_waits_for_rest_to_reach_user_state(
3750 #[case] expected_qty: &str,
3751 #[case] status: CbStatus,
3752 ) {
3753 let mut spec = RecoverySpec::new().await;
3754 {
3755 let mut state = spec.state.lock();
3756 state.order["order"]["status"] = "OPEN".into();
3757 state.order["order"]["filled_size"] = "0.4".into();
3758 state.fills["fills"].as_array_mut().unwrap().remove(0);
3759 }
3760
3761 let carrier = make_carrier(make_user_order_update(
3762 expected_qty,
3763 "0",
3764 "112",
3765 "0.10",
3766 status,
3767 ));
3768 spec.recovery.queue(*carrier.report);
3769 let error = spec.recovery.recover().await.unwrap_err();
3770 assert_eq!(
3771 error.to_string(),
3772 "REST order state has not reached the user-channel update"
3773 );
3774 assert_eq!(spec.recovery.pending.lock().len(), 1);
3775 assert!(spec.rx.try_recv().is_err());
3776 }
3777
3778 #[tokio::test]
3779 async fn test_query_order_uses_ordered_rest_recovery() {
3780 let mut spec = RecoverySpec::new().await;
3781 let cache = Rc::new(RefCell::new(Cache::default()));
3782
3783 let core = ExecutionClientCore::new(
3784 TraderId::from("TRADER-001"),
3785 ClientId::from("COINBASE-TEST"),
3786 *COINBASE_VENUE,
3787 OmsType::Netting,
3788 AccountId::from("COINBASE-001"),
3789 AccountType::Cash,
3790 None,
3791 cache,
3792 );
3793
3794 let config = CoinbaseExecutionClientConfig {
3795 api_key: Some("test-key".into()),
3796 api_secret: Some("test-secret".into()),
3797 ..Default::default()
3798 };
3799
3800 let mut client = CoinbaseExecutionClient::new(core, config).unwrap();
3801 client.http_client = spec.recovery.http_client.clone();
3802 client.emitter = spec.recovery.emitter.clone();
3803 client.instruments_cache = Arc::clone(&spec.recovery.instruments);
3804
3805 let command = QueryOrder::new(
3806 TraderId::from("TRADER-001"),
3807 None,
3808 StrategyId::from("S-001"),
3809 InstrumentId::from("BTC-USD.COINBASE"),
3810 ClientOrderId::from("client-1"),
3811 Some(VenueOrderId::from("venue-1")),
3812 UUID4::new(),
3813 UnixNanos::default(),
3814 None,
3815 None,
3816 );
3817 client.query_order(command).unwrap();
3818 tokio::time::timeout(Duration::from_secs(2), client.await_pending_tasks())
3819 .await
3820 .unwrap()
3821 .unwrap();
3822 let (report, fills) = spec.bundle();
3823
3824 assert_eq!(report.order_status, OrderStatus::Filled);
3825 assert_eq!(
3826 fills
3827 .iter()
3828 .map(|fill| fill.trade_id.as_str())
3829 .collect::<Vec<_>>(),
3830 vec!["trade-alpha", "trade-beta"]
3831 );
3832 client.await_pending_tasks().await.unwrap();
3833 }
3834
3835 #[rstest]
3836 #[case::visible(false)]
3837 #[case::delayed_closing_fill(true)]
3838 #[tokio::test]
3839 async fn test_recovery_orders_interleaved_fills_across_orders(#[case] delayed: bool) {
3840 let mut spec = RecoverySpec::new().await;
3841 {
3842 let mut state = spec.state.lock();
3843 let mut closing = state.order.clone();
3844 closing["order"]["order_id"] = "venue-2".into();
3845 closing["order"]["client_order_id"] = "client-2".into();
3846 closing["order"]["side"] = "SELL".into();
3847 closing["order"]["filled_size"] = "0.4".into();
3848 closing["order"]["average_filled_price"] = "110.00".into();
3849 closing["order"]["last_fill_time"] = "2024-01-15T10:01:00Z".into();
3850 closing["order"]["order_configuration"]["limit_limit_gtc"]["base_size"] = "0.4".into();
3851 state.orders.insert("venue-2".into(), closing);
3852 state.order["order"]["last_fill_time"] = "2024-01-15T10:02:00Z".into();
3853 state.fills["fills"][1]["trade_time"] = "2024-01-15T10:00:00Z".into();
3854 state.fills["fills"][0]["trade_time"] = "2024-01-15T10:02:00Z".into();
3855 let mut fill = state.fills["fills"][1].clone();
3856 fill["order_id"] = "venue-2".into();
3857 fill["trade_id"] = "trade-gamma".into();
3858 fill["side"] = "SELL".into();
3859 fill["price"] = "110.00".into();
3860 fill["commission"] = "0.08".into();
3861 fill["trade_time"] = "2024-01-15T10:01:00Z".into();
3862 state.fills["fills"].as_array_mut().unwrap().insert(0, fill);
3863 }
3864
3865 let node = recovery_node();
3866
3867 if delayed {
3868 let closing = spec.state.lock().fills["fills"]
3869 .as_array_mut()
3870 .unwrap()
3871 .remove(0);
3872 spec.recovery.retry().await;
3873 assert!(spec.rx.try_recv().is_err());
3874 assert_eq!(spec.recovery.pending.lock().len(), 2);
3875 spec.state.lock().fills["fills"]
3876 .as_array_mut()
3877 .unwrap()
3878 .push(closing);
3879 }
3880
3881 spec.recovery.retry().await;
3882 let reports = spec.reports();
3883
3884 let trades: Vec<_> = reports
3885 .iter()
3886 .filter_map(|report| match report {
3887 ExecutionReport::Fill(fill) => Some(fill.trade_id.as_str()),
3888 _ => None,
3889 })
3890 .collect();
3891
3892 assert_eq!(trades, vec!["trade-alpha", "trade-gamma", "trade-beta"]);
3893 apply_recovery_reports(&node, &reports);
3894 let events_before: Vec<_> = ["client-1", "client-2"]
3895 .iter()
3896 .map(|id| {
3897 node.kernel()
3898 .cache
3899 .borrow()
3900 .order(&ClientOrderId::from(*id))
3901 .unwrap()
3902 .event_count()
3903 })
3904 .collect();
3905
3906 spec.recovery.retry().await;
3907 apply_recovery_reports(&node, &spec.reports());
3908 let cache = node.kernel().cache.borrow();
3909 let positions = cache.positions_open_refs(None, None, None, None, None);
3910 assert_eq!(positions.len(), 1);
3911 assert_eq!(positions[0].quantity, Quantity::from("0.600000"));
3912 assert_eq!(positions[0].avg_px_open, rust_decimal_macros::dec!(120));
3913
3914 for (id, events) in ["client-1", "client-2"].iter().zip(events_before) {
3915 let order = cache.order(&ClientOrderId::from(*id)).unwrap();
3916 assert_eq!(order.status(), OrderStatus::Filled);
3917 assert_eq!(order.event_count(), events);
3918 assert_eq!(
3919 order
3920 .events()
3921 .iter()
3922 .filter(|event| matches!(event, OrderEventAny::FillVoided(_)))
3923 .count(),
3924 0
3925 );
3926 }
3927 }
3928
3929 #[rstest]
3930 #[case::already_published(true)]
3931 #[case::startup_only(false)]
3932 #[tokio::test]
3933 async fn test_stale_recovery_cannot_void_newer_fills(#[case] published: bool) {
3934 let mut spec = RecoverySpec::new().await;
3935 let node = recovery_node();
3936 spec.recovery.retry().await;
3937 apply_recovery_reports(&node, &spec.reports());
3938 if !published {
3939 spec.recovery.published.lock().await.clear();
3940 }
3941
3942 let before = node
3943 .kernel()
3944 .cache
3945 .borrow()
3946 .order(&ClientOrderId::from("client-1"))
3947 .unwrap()
3948 .event_count();
3949 {
3950 let mut state = spec.state.lock();
3951 state.order["order"]["status"] = "OPEN".into();
3952 state.order["order"]["filled_size"] = "0.4".into();
3953 state.order["order"]["average_filled_price"] = "100.00".into();
3954 state.fills["fills"].as_array_mut().unwrap().remove(0);
3955 }
3956
3957 let partial = make_carrier(make_user_order_update(
3958 "0.4",
3959 "0.6",
3960 "100",
3961 "0.04",
3962 CbStatus::Open,
3963 ));
3964 spec.recovery.queue(*partial.report);
3965 let result = spec.recovery.recover().await;
3966
3967 if published {
3968 assert_eq!(
3969 result.unwrap_err().to_string(),
3970 "REST order state precedes published recovery"
3971 );
3972 assert!(spec.rx.try_recv().is_err());
3973 } else {
3974 result.unwrap();
3975 apply_recovery_reports(&node, &spec.reports());
3976 }
3977
3978 let cache = node.kernel().cache.borrow();
3979 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
3980 let positions = cache.positions_open_refs(None, None, None, None, None);
3981 assert_eq!(order.filled_qty(), Quantity::from("1.000000"));
3982 assert_eq!(order.status(), OrderStatus::Filled);
3983 assert_eq!(order.event_count(), before);
3984 assert_eq!(
3985 order
3986 .events()
3987 .iter()
3988 .filter(|event| matches!(event, OrderEventAny::FillVoided(_)))
3989 .count(),
3990 0
3991 );
3992 assert_eq!(positions.len(), 1);
3993 assert_eq!(positions[0].quantity, Quantity::from("1.000000"));
3994 assert_eq!(positions[0].avg_px_open, rust_decimal_macros::dec!(112));
3995 }
3996
3997 #[rstest]
3998 #[case::closed_order(false)]
3999 #[case::query_pending(true)]
4000 #[tokio::test]
4001 async fn test_idle_recovery_discovers_fills_visible_after_empty_scan(
4002 #[case] query_pending: bool,
4003 ) {
4004 let mut spec = RecoverySpec::new().await;
4005 let complete = spec.state.lock().fills.clone();
4006 let scanned = Arc::new(tokio::sync::Notify::new());
4007 {
4008 let scanned = Arc::clone(&scanned);
4009 let mut state = spec.state.lock();
4010 state.fills["fills"] = serde_json::Value::Array(Vec::new());
4011 state.discover_orders = false;
4012 state.on_discovery = Some(Box::new(move || scanned.notify_one()));
4013 }
4014
4015 let state = Arc::clone(&spec.state);
4016 let pending = Arc::clone(&spec.recovery.pending);
4017 let target = make_carrier(make_user_order_update(
4018 "1",
4019 "0",
4020 "112",
4021 "0.10",
4022 CbStatus::Filled,
4023 ));
4024
4025 let make_visible = async move {
4026 scanned.notified().await;
4027 state.lock().fills = complete;
4028
4029 if query_pending {
4030 pending
4031 .lock()
4032 .insert(target.report.venue_order_id, *target.report);
4033 }
4034 };
4035
4036 let (_tx, rx) = tokio::sync::mpsc::unbounded_channel();
4037 let run = spec.recovery.run(
4038 rx,
4039 Duration::from_millis(1),
4040 get_atomic_clock_realtime(),
4041 false,
4042 );
4043
4044 let collect = async {
4045 make_visible.await;
4046 let mut trades = Vec::new();
4047
4048 loop {
4049 let ExecutionEvent::Report(report) = spec.rx.recv().await.unwrap() else {
4050 panic!("expected execution report");
4051 };
4052
4053 match report {
4054 ExecutionReport::Fill(fill) => {
4055 trades.push(fill.trade_id);
4056 if trades.len() == 2 {
4057 return trades;
4058 }
4059 }
4060 ExecutionReport::Order(_) => {}
4061 other => panic!("unexpected recovery report: {other:?}"),
4062 }
4063 }
4064 };
4065
4066 tokio::pin!(run);
4067 let trades = tokio::select! {
4068 () = &mut run => panic!("recovery loop stopped"),
4069 result = tokio::time::timeout(Duration::from_secs(2), collect) => result.unwrap(),
4070 };
4071 assert_eq!(
4072 trades.iter().map(|id| id.as_str()).collect::<Vec<_>>(),
4073 vec!["trade-alpha", "trade-beta"]
4074 );
4075 }
4076
4077 #[rstest]
4078 #[case::zero_total("0", false)]
4079 #[case::partial_total("0.4", true)]
4080 #[case::empty_targeted_history("0", true)]
4081 #[tokio::test]
4082 async fn test_discovered_trades_survive_stale_totals_and_later_empty_scans(
4083 #[case] filled_qty: &str,
4084 #[case] targeted: bool,
4085 ) {
4086 let mut spec = RecoverySpec::new().await;
4087 let complete = spec.state.lock().fills.clone();
4088 let order = spec.state.lock().order.clone();
4089 {
4090 let mut state = spec.state.lock();
4091 state.order["order"]["status"] = "OPEN".into();
4092 state.order["order"]["filled_size"] = filled_qty.into();
4093
4094 if targeted {
4095 let mut fills = complete.clone();
4096 fills["fills"]
4097 .as_array_mut()
4098 .unwrap()
4099 .retain(|fill| filled_qty == "0.4" && fill["trade_id"] == "trade-alpha");
4100 state.fills_targeted = Some(fills);
4101 }
4102 }
4103
4104 spec.recovery.retry().await;
4105 assert!(spec.rx.try_recv().is_err());
4106 assert_eq!(
4107 spec.recovery.observed.lock()[&VenueOrderId::from("venue-1")].len(),
4108 2
4109 );
4110 spec.state.lock().fills["fills"] = serde_json::Value::Array(Vec::new());
4111 spec.recovery.retry().await;
4112 assert!(spec.rx.try_recv().is_err());
4113 assert_eq!(
4114 spec.recovery.observed.lock()[&VenueOrderId::from("venue-1")].len(),
4115 2
4116 );
4117 {
4118 let mut state = spec.state.lock();
4119 state.order = order;
4120 state.fills = complete;
4121 state.fills_targeted = None;
4122 }
4123
4124 spec.recovery.retry().await;
4125 let (report, fills) = spec.bundle();
4126 assert_eq!(report.filled_qty, Quantity::from("1.00000000"));
4127 assert_eq!(
4128 fills
4129 .iter()
4130 .map(|fill| fill.trade_id.as_str())
4131 .collect::<Vec<_>>(),
4132 vec!["trade-alpha", "trade-beta"]
4133 );
4134 assert!(spec.recovery.observed.lock().is_empty());
4135 assert!(spec.recovery.pending.lock().is_empty());
4136 }
4137
4138 #[tokio::test]
4139 async fn test_recovery_preserves_newer_cached_amendment() {
4140 let mut spec = RecoverySpec::new().await;
4141 {
4142 let mut state = spec.state.lock();
4143 state.order["order"]["status"] = "OPEN".into();
4144 state.order["order"]["filled_size"] = "0.4".into();
4145 state.order["order"]["average_filled_price"] = "100.00".into();
4146 state.fills["fills"].as_array_mut().unwrap().remove(0);
4147 }
4148
4149 let node = recovery_node();
4150 spec.attach(&node);
4151 spec.recovery.retry().await;
4152 apply_recovery_reports(&node, &spec.reports());
4153 let mut amended = spec
4154 .recovery
4155 .http_client
4156 .request_order_status_report(
4157 AccountId::from("COINBASE-001"),
4158 None,
4159 Some(VenueOrderId::from("venue-1")),
4160 )
4161 .await
4162 .unwrap();
4163 amended.quantity = Quantity::from("2.00000000");
4164 amended.price = Some(Price::from("10100.00"));
4165 amended.ts_last = UnixNanos::from(amended.ts_last.as_u64() + 1);
4166 node.kernel()
4167 .exec_engine
4168 .borrow_mut()
4169 .reconcile_order_status_report(&amended);
4170 {
4171 let cache = node.kernel().cache.borrow();
4172 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4173 assert_eq!(order.quantity(), amended.quantity);
4174 assert_eq!(order.price(), amended.price);
4175 }
4176
4177 spec.recovery.retry().await;
4178 apply_recovery_reports(&node, &spec.reports());
4179 let cache = node.kernel().cache.borrow();
4180 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4181 assert_eq!(order.quantity(), amended.quantity);
4182 assert_eq!(order.price(), amended.price);
4183 assert_eq!(order.filled_qty(), Quantity::from("0.40000000"));
4184 }
4185
4186 #[tokio::test]
4187 async fn test_external_recovery_emits_order_status_before_real_fills() {
4188 let mut spec = RecoverySpec::new().await;
4189 spec.recovery.retry().await;
4190 let reports = spec.reports();
4191
4192 let ExecutionReport::Order(accepted) = &reports[0] else {
4193 panic!("order status must precede external fills")
4194 };
4195
4196 let ExecutionReport::Order(terminal) = &reports[3] else {
4197 panic!("final order status must follow external fills")
4198 };
4199
4200 let fills: Vec<_> = reports[1..3]
4201 .iter()
4202 .map(|report| {
4203 let ExecutionReport::Fill(fill) = report else {
4204 panic!("expected canonical fill report")
4205 };
4206
4207 assert_eq!(fill.account_id, accepted.account_id);
4208 assert_eq!(fill.instrument_id, accepted.instrument_id);
4209 assert_eq!(fill.venue_order_id, accepted.venue_order_id);
4210 fill.trade_id
4211 })
4212 .collect();
4213
4214 assert_eq!(reports.len(), 4);
4215 assert_eq!(accepted.order_status, OrderStatus::Accepted);
4216 assert_eq!(accepted.quantity, terminal.quantity);
4217 assert_eq!(accepted.price, terminal.price);
4218 assert_eq!(accepted.filled_qty, Quantity::from("0.00000000"));
4219 assert_eq!(accepted.avg_px, None);
4220 assert_eq!(accepted.ts_last, accepted.ts_accepted);
4221 assert_eq!(terminal.order_status, OrderStatus::Filled);
4222 assert_eq!(terminal.filled_qty, Quantity::from("1.00000000"));
4223 assert_eq!(
4224 fills,
4225 vec![TradeId::from("trade-alpha"), TradeId::from("trade-beta")]
4226 );
4227 let node = recovery_node();
4228 apply_recovery_reports(&node, &reports);
4229 let cache = node.kernel().cache.borrow();
4230 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4231 assert_eq!(order.status(), OrderStatus::Filled);
4232 assert_eq!(order.filled_qty(), terminal.filled_qty);
4233 assert_eq!(
4234 order.trade_ids().into_iter().copied().collect::<Vec<_>>(),
4235 fills
4236 );
4237 }
4238
4239 #[rstest]
4240 #[case::partial_cancel("CANCELLED", OrderStatus::Canceled, "0.4")]
4241 #[case::partial_expire("EXPIRED", OrderStatus::Expired, "0.4")]
4242 #[case::empty_cancel("CANCELLED", OrderStatus::Canceled, "0")]
4243 #[case::empty_expire("EXPIRED", OrderStatus::Expired, "0")]
4244 #[tokio::test]
4245 async fn test_unknown_recovery_applies_native_terminal_after_fills(
4246 #[case] status: &str,
4247 #[case] expected: OrderStatus,
4248 #[case] filled: &str,
4249 #[values(false, true)] known: bool,
4250 #[values(false, true)] missing_client_id: bool,
4251 ) {
4252 let mut spec = RecoverySpec::new().await;
4253
4254 let client_order_id = ClientOrderId::from(if missing_client_id {
4255 "venue-1"
4256 } else {
4257 "client-1"
4258 });
4259
4260 if missing_client_id {
4261 spec.state.lock().order["order"]["client_order_id"] = "".into();
4262 }
4263
4264 {
4265 let mut state = spec.state.lock();
4266 state.order["order"]["status"] = status.into();
4267 state.order["order"]["filled_size"] = filled.into();
4268 state.order["order"]["average_filled_price"] = "100".into();
4269 let fills = state.fills["fills"].as_array_mut().unwrap();
4270 fills.remove(0);
4271 if filled == "0" {
4272 fills.clear();
4273 }
4274 }
4275
4276 let node = recovery_node();
4277
4278 if known {
4279 let mut report = spec
4280 .recovery
4281 .http_client
4282 .request_order_status_report(
4283 AccountId::from("COINBASE-001"),
4284 None,
4285 Some(VenueOrderId::from("venue-1")),
4286 )
4287 .await
4288 .unwrap();
4289 report.quantity = Quantity::from("2.00000000");
4290 report.price = Some(Price::from("10100.00"));
4291 report.filled_qty = Quantity::from("0.00000000");
4292 report.avg_px = None;
4293 report.order_status = OrderStatus::Accepted;
4294 node.kernel()
4295 .exec_engine
4296 .borrow_mut()
4297 .reconcile_order_status_report(&report);
4298 }
4299
4300 spec.attach(&node);
4301
4302 let status = if expected == OrderStatus::Canceled {
4303 CbStatus::Cancelled
4304 } else {
4305 CbStatus::Expired
4306 };
4307
4308 let mut update = make_user_order_update(filled, "0", "100", "0.04", status);
4309 if missing_client_id {
4310 update.client_order_id.clear();
4311 }
4312
4313 let carrier = make_carrier(update);
4314 spec.recovery.handle_update(carrier).await;
4315 spec.apply(&node);
4316 let before = node
4317 .kernel()
4318 .cache
4319 .borrow()
4320 .order(&client_order_id)
4321 .unwrap()
4322 .event_count();
4323 spec.recovery.retry().await;
4324 spec.apply(&node);
4325 let cache = node.kernel().cache.borrow();
4326 let order = cache.order(&client_order_id).unwrap();
4327 assert_eq!(order.status(), expected);
4328 assert_eq!(
4329 order.quantity(),
4330 Quantity::from(if known { "2.00000000" } else { "1.00000000" })
4331 );
4332 assert_eq!(
4333 order.price(),
4334 Some(Price::from(if known { "10100.00" } else { "10000.00" }))
4335 );
4336 assert_eq!(
4337 order.filled_qty().as_decimal(),
4338 filled.parse::<Decimal>().unwrap()
4339 );
4340 assert_eq!(order.event_count(), before);
4341 assert_eq!(order.trade_ids().len(), usize::from(filled != "0"));
4342 assert!(spec.recovery.lifecycle.lock().pending.is_empty());
4343 }
4344
4345 #[rstest]
4346 #[tokio::test]
4347 async fn test_recovery_accepts_cached_submitted_order_without_replacing_terms(
4348 #[values(false, true)] fills_lag: bool,
4349 ) {
4350 let mut spec = RecoverySpec::new().await;
4351 {
4352 let mut state = spec.state.lock();
4353 state.order["order"]["status"] = "OPEN".into();
4354 state.order["order"]["filled_size"] = if fills_lag { "0.4" } else { "0" }.into();
4355 if fills_lag {
4356 state.fills["fills"].as_array_mut().unwrap().remove(0);
4357 state.fail_fills = true;
4358 } else {
4359 state.fills["fills"].as_array_mut().unwrap().clear();
4360 }
4361 }
4362
4363 let mut order = OrderTestBuilder::new(OrderType::Limit)
4364 .trader_id(TraderId::from("TRADER-001"))
4365 .strategy_id(StrategyId::from("STRATEGY-007"))
4366 .instrument_id(InstrumentId::from("BTC-USD.COINBASE"))
4367 .client_order_id(ClientOrderId::from("client-1"))
4368 .side(OrderSide::Buy)
4369 .quantity(Quantity::from("2.00000000"))
4370 .price(Price::from("10100.00"))
4371 .build();
4372
4373 let factory = OrderEventFactory::new(
4374 TraderId::from("TRADER-001"),
4375 AccountId::from("COINBASE-001"),
4376 AccountType::Cash,
4377 None,
4378 );
4379 let submitted = factory.generate_order_submitted(&order, UnixNanos::from(10));
4380 order.apply(submitted).unwrap();
4381 let node = recovery_node();
4382 node.kernel()
4383 .cache
4384 .borrow_mut()
4385 .add_order(order, None, None, false)
4386 .unwrap();
4387 spec.attach(&node);
4388 let carrier = make_carrier(make_user_order_update(
4389 if fills_lag { "0.4" } else { "0" },
4390 if fills_lag { "0.6" } else { "1" },
4391 "100",
4392 "0.04",
4393 CbStatus::Open,
4394 ));
4395 spec.recovery.handle_update(carrier).await;
4396 spec.apply(&node);
4397 {
4398 let cache = node.kernel().cache.borrow();
4399 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4400 assert_eq!(order.status(), OrderStatus::Accepted);
4401 assert_eq!(order.quantity(), Quantity::from("2.00000000"));
4402 assert_eq!(order.price(), Some(Price::from("10100.00")));
4403 assert_eq!(order.filled_qty(), Quantity::from("0.00000000"));
4404 }
4405
4406 if fills_lag {
4407 assert_eq!(spec.recovery.pending.lock().len(), 1);
4408 } else {
4409 assert!(spec.recovery.pending.lock().is_empty());
4410 }
4411 }
4412
4413 #[rstest]
4414 #[case::cancel(OrderStatus::PendingCancel)]
4415 #[case::update(OrderStatus::PendingUpdate)]
4416 #[tokio::test]
4417 async fn test_recovery_preserves_pending_command_on_unchanged_open_report(
4418 #[case] expected_status: OrderStatus,
4419 ) {
4420 let mut spec = RecoverySpec::new().await;
4421 {
4422 let mut state = spec.state.lock();
4423 state.order["order"]["status"] = "OPEN".into();
4424 state.order["order"]["filled_size"] = "0".into();
4425 state.fills["fills"].as_array_mut().unwrap().clear();
4426 }
4427
4428 let report = spec
4429 .recovery
4430 .http_client
4431 .request_order_status_report(
4432 AccountId::from("COINBASE-001"),
4433 None,
4434 Some(VenueOrderId::from("venue-1")),
4435 )
4436 .await
4437 .unwrap();
4438 let node = recovery_node();
4439 node.kernel()
4440 .exec_engine
4441 .borrow_mut()
4442 .reconcile_order_status_report(&report);
4443 let order = node
4444 .kernel()
4445 .cache
4446 .borrow()
4447 .order(&ClientOrderId::from("client-1"))
4448 .unwrap()
4449 .clone();
4450 let ts = UnixNanos::from(report.ts_last.as_u64() + 1);
4451
4452 let pending = match expected_status {
4453 OrderStatus::PendingCancel => OrderEventAny::PendingCancel(OrderPendingCancel::new(
4454 order.trader_id(),
4455 order.strategy_id(),
4456 order.instrument_id(),
4457 order.client_order_id(),
4458 order.account_id(),
4459 UUID4::new(),
4460 ts,
4461 ts,
4462 false,
4463 order.venue_order_id(),
4464 )),
4465 OrderStatus::PendingUpdate => OrderEventAny::PendingUpdate(OrderPendingUpdate::new(
4466 order.trader_id(),
4467 order.strategy_id(),
4468 order.instrument_id(),
4469 order.client_order_id(),
4470 order.account_id(),
4471 UUID4::new(),
4472 ts,
4473 ts,
4474 false,
4475 order.venue_order_id(),
4476 )),
4477 _ => unreachable!(),
4478 };
4479
4480 node.kernel().exec_engine.borrow_mut().process(&pending);
4481 let before = node
4482 .kernel()
4483 .cache
4484 .borrow()
4485 .order(&order.client_order_id())
4486 .unwrap()
4487 .event_count();
4488 node.kernel()
4489 .exec_engine
4490 .borrow_mut()
4491 .reconcile_order_status_report(&report);
4492 {
4493 let cache = node.kernel().cache.borrow();
4494 let order = cache.order(&order.client_order_id()).unwrap();
4495 assert_eq!(order.status(), expected_status);
4496 assert_eq!(order.event_count(), before);
4497 }
4498
4499 spec.attach(&node);
4500 spec.recovery.queue(report);
4501 spec.recovery.retry().await;
4502 spec.apply(&node);
4503 let cache = node.kernel().cache.borrow();
4504 let order = cache.order(&order.client_order_id()).unwrap();
4505 assert_eq!(order.status(), expected_status);
4506 assert_eq!(order.event_count(), before);
4507 assert_eq!(order.quantity(), Quantity::from("1.00000000"));
4508 assert_eq!(order.price(), Some(Price::from("10000.00")));
4509 assert_eq!(order.filled_qty(), Quantity::from("0.00000000"));
4510 assert!(order.trade_ids().is_empty());
4511 assert!(
4512 cache
4513 .positions_refs(None, None, None, None, None)
4514 .is_empty()
4515 );
4516 }
4517
4518 #[tokio::test]
4519 async fn test_partial_user_amendment_survives_snapshot_and_rest_replay() {
4520 let mut spec = RecoverySpec::new().await;
4521 {
4522 let mut state = spec.state.lock();
4523 state.order["order"]["status"] = "OPEN".into();
4524 state.order["order"]["filled_size"] = "0.4".into();
4525 state.order["order"]["average_filled_price"] = "100".into();
4526 state.fills["fills"].as_array_mut().unwrap().remove(0);
4527 }
4528
4529 let node = recovery_node();
4530 spec.attach(&node);
4531 spec.recovery.retry().await;
4532 spec.apply(&node);
4533
4534 let context = OrderContext {
4535 price: Some(Price::from("10100.00")),
4536 ..Default::default()
4537 };
4538
4539 spec.recovery
4540 .order_contexts
4541 .lock()
4542 .insert("client-1".into(), context);
4543 let update = make_user_order_update("0.4", "1.6", "100", "0.04", CbStatus::Open);
4544 spec.recovery.handle_update(make_carrier(update)).await;
4545 spec.apply(&node);
4546 {
4547 let cache = node.kernel().cache.borrow();
4548 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4549 assert_eq!(order.quantity(), Quantity::from("2.00000000"));
4550 assert_eq!(order.price(), Some(Price::from("10100.00")));
4551 }
4552
4553 let stale = make_user_order_update("0.4", "0.6", "100", "0.04", CbStatus::Open);
4554 spec.recovery
4555 .handle_update(make_carrier_with_kind(stale, true))
4556 .await;
4557 spec.apply(&node);
4558 let cache = node.kernel().cache.borrow();
4559 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4560 assert_eq!(order.quantity(), Quantity::from("2.00000000"));
4561 assert_eq!(order.price(), Some(Price::from("10100.00")));
4562 assert_eq!(order.filled_qty(), Quantity::from("0.40000000"));
4563 assert_eq!(
4564 order.trade_ids().into_iter().copied().collect::<Vec<_>>(),
4565 vec![TradeId::from("trade-alpha")]
4566 );
4567 }
4568
4569 #[rstest]
4570 #[case::partial("OPEN", "2", "0.5", OrderStatus::PartiallyFilled)]
4571 #[case::filled_increase("FILLED", "1", "0.5", OrderStatus::Filled)]
4572 #[case::filled_decrease("FILLED", "1", "2", OrderStatus::Filled)]
4573 #[tokio::test]
4574 async fn test_recovery_updates_missed_quantity_amendment_before_fills(
4575 #[case] status: &str,
4576 #[case] quantity: &str,
4577 #[case] cached_quantity: &str,
4578 #[case] expected_status: OrderStatus,
4579 ) {
4580 let mut spec = RecoverySpec::new().await;
4581 {
4582 let mut state = spec.state.lock();
4583 state.order["order"]["status"] = status.into();
4584 state.order["order"]["order_configuration"]["limit_limit_gtc"]["base_size"] =
4585 quantity.into();
4586 }
4587
4588 let mut order = OrderTestBuilder::new(OrderType::Limit)
4589 .trader_id(TraderId::from("TRADER-001"))
4590 .strategy_id(StrategyId::from("STRATEGY-007"))
4591 .instrument_id(InstrumentId::from("BTC-USD.COINBASE"))
4592 .client_order_id(ClientOrderId::from("client-1"))
4593 .side(OrderSide::Buy)
4594 .quantity(cached_quantity.parse().unwrap())
4595 .price(Price::from("10100.00"))
4596 .build();
4597
4598 let factory = OrderEventFactory::new(
4599 TraderId::from("TRADER-001"),
4600 AccountId::from("COINBASE-001"),
4601 AccountType::Cash,
4602 None,
4603 );
4604 let accepted = factory.generate_order_accepted(
4605 &order,
4606 VenueOrderId::from("venue-1"),
4607 UnixNanos::from(1),
4608 UnixNanos::from(1),
4609 );
4610 order.apply(accepted).unwrap();
4611 let before = order.event_count();
4612 let node = recovery_node();
4613 node.kernel()
4614 .cache
4615 .borrow_mut()
4616 .add_order(order, None, None, false)
4617 .unwrap();
4618 spec.attach(&node);
4619 spec.recovery.retry().await;
4620 spec.apply(&node);
4621 let cache = node.kernel().cache.borrow();
4622 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4623 let events = order.events();
4624 let update_index = events
4625 .iter()
4626 .position(|event| matches!(event, OrderEventAny::Updated(_)))
4627 .unwrap();
4628 let fill_index = events
4629 .iter()
4630 .position(|event| matches!(event, OrderEventAny::Filled(_)))
4631 .unwrap();
4632 let positions = cache.positions_open_refs(None, None, None, None, None);
4633 assert!(update_index < fill_index);
4634 assert_eq!(order.status(), expected_status);
4635 assert_eq!(
4636 order.quantity().as_decimal(),
4637 quantity.parse::<Decimal>().unwrap()
4638 );
4639 assert_eq!(order.price(), Some(Price::from("10100.00")));
4640 assert_eq!(order.filled_qty(), Quantity::from("1.00000000"));
4641 assert_eq!(order.event_count(), before + 3);
4642 assert_eq!(
4643 order.trade_ids().into_iter().copied().collect::<Vec<_>>(),
4644 vec![TradeId::from("trade-alpha"), TradeId::from("trade-beta")]
4645 );
4646 assert_eq!(positions.len(), 1);
4647 assert_eq!(positions[0].quantity, Quantity::from("1.00000000"));
4648 assert_eq!(positions[0].avg_px_open, Decimal::from(112));
4649 assert_eq!(positions[0].commissions(), vec![Money::from("0.10 USD")]);
4650 }
4651
4652 #[rstest]
4653 #[tokio::test]
4654 async fn test_quote_market_recovery_normalizes_base_units_after_terminal_totals(
4655 #[values(false, true)] known: bool,
4656 #[values(false, true)] snapshot: bool,
4657 #[values(false, true)] partial: bool,
4658 ) {
4659 let mut spec = RecoverySpec::new().await;
4660 {
4661 let mut state = spec.state.lock();
4662 state.order["order"]["order_type"] = "MARKET".into();
4663 state.order["order"]["time_in_force"] = "IMMEDIATE_OR_CANCEL".into();
4664 state.order["order"]["order_configuration"] =
4665 serde_json::json!({"market_market_ioc": {"quote_size": "112"}});
4666 state.order["order"]["size_in_quote"] = true.into();
4667 state.fills["fills"][0]["size_in_quote"] = true.into();
4668 state.fills["fills"][0]["size"] = "72".into();
4669 state.fills["fills"][1]["size_in_quote"] = true.into();
4670 state.fills["fills"][1]["size"] = "40".into();
4671 }
4672
4673 let node = recovery_node();
4674
4675 if known {
4676 let mut order = OrderTestBuilder::new(OrderType::Market)
4677 .trader_id(TraderId::from("TRADER-001"))
4678 .strategy_id(StrategyId::from("STRATEGY-007"))
4679 .instrument_id(InstrumentId::from("BTC-USD.COINBASE"))
4680 .client_order_id(ClientOrderId::from("client-1"))
4681 .side(OrderSide::Buy)
4682 .quantity(Quantity::from("112.00"))
4683 .quote_quantity(true)
4684 .time_in_force(TimeInForce::Ioc)
4685 .build();
4686
4687 let factory = OrderEventFactory::new(
4688 TraderId::from("TRADER-001"),
4689 AccountId::from("COINBASE-001"),
4690 AccountType::Cash,
4691 None,
4692 );
4693 let accepted = factory.generate_order_accepted(
4694 &order,
4695 VenueOrderId::from("venue-1"),
4696 UnixNanos::from(1),
4697 UnixNanos::from(1),
4698 );
4699 order.apply(accepted).unwrap();
4700 node.kernel()
4701 .cache
4702 .borrow_mut()
4703 .add_order(order, None, None, false)
4704 .unwrap();
4705 spec.recovery
4706 .order_contexts
4707 .lock()
4708 .insert("client-1".into(), OrderContext::default());
4709 }
4710
4711 spec.attach(&node);
4712 let final_order = spec.state.lock().order.clone();
4713 let final_fills = spec.state.lock().fills.clone();
4714
4715 if partial {
4716 {
4717 let mut state = spec.state.lock();
4718 state.order["order"]["status"] = "OPEN".into();
4719 state.order["order"]["filled_size"] = "0.4".into();
4720 state.order["order"]["average_filled_price"] = "100".into();
4721 state.fills["fills"].as_array_mut().unwrap().remove(0);
4722 }
4723
4724 let mut update = make_user_order_update("0.4", "0", "100", "0.04", CbStatus::Open);
4725 update.order_type = CbType::Market;
4726 spec.recovery.handle_update(make_carrier(update)).await;
4727 assert!(spec.rx.try_recv().is_err());
4728 assert_eq!(spec.recovery.pending.lock().len(), 1);
4729 }
4730
4731 {
4732 let mut state = spec.state.lock();
4733 state.order = final_order;
4734 state.fills = final_fills;
4735 }
4736
4737 let mut update = make_user_order_update("1", "0", "112", "0.10", CbStatus::Filled);
4738 update.order_type = CbType::Market;
4739 spec.recovery
4740 .handle_update(make_carrier_with_kind(update, snapshot))
4741 .await;
4742 spec.apply(&node);
4743 let before = node
4744 .kernel()
4745 .cache
4746 .borrow()
4747 .order(&ClientOrderId::from("client-1"))
4748 .unwrap()
4749 .event_count();
4750 spec.recovery.retry().await;
4751 spec.apply(&node);
4752 let cache = node.kernel().cache.borrow();
4753 let order = cache.order(&ClientOrderId::from("client-1")).unwrap();
4754 let positions = cache.positions_refs(None, None, None, None, None);
4755 assert_eq!(order.status(), OrderStatus::Filled);
4756 assert_eq!(order.quantity(), Quantity::from("1.00000000"));
4757 assert_eq!(order.filled_qty(), Quantity::from("1.00000000"));
4758 assert!(!order.is_quote_quantity());
4759 assert_eq!(
4760 order.trade_ids().into_iter().copied().collect::<Vec<_>>(),
4761 vec![TradeId::from("trade-alpha"), TradeId::from("trade-beta")]
4762 );
4763 assert_eq!(order.event_count(), before);
4764 assert_eq!(positions.len(), 1);
4765 assert_eq!(positions[0].quantity, Quantity::from("1.00000000"));
4766 assert_eq!(positions[0].avg_px_open, Decimal::from(112));
4767 assert_eq!(positions[0].commissions(), vec![Money::from("0.10 USD")]);
4768 }
4769
4770 fn apply_recovery_reports(node: &LiveNode, reports: &[ExecutionReport]) {
4771 for report in reports {
4772 node.kernel()
4773 .exec_engine
4774 .borrow_mut()
4775 .reconcile_execution_report(report);
4776 }
4777 }
4778
4779 fn recovery_node() -> LiveNode {
4780 let node = LiveNode::builder(TraderId::from("TRADER-001"), Environment::Live)
4781 .unwrap()
4782 .build()
4783 .unwrap();
4784 let kernel = node.kernel();
4785 kernel
4786 .cache
4787 .borrow_mut()
4788 .add_instrument(test_instrument())
4789 .unwrap();
4790 let account_id = AccountId::from("COINBASE-001");
4791
4792 let state = AccountState::new(
4793 account_id,
4794 AccountType::Cash,
4795 vec![AccountBalance::new(
4796 Money::from("100000 USD"),
4797 Money::from("0 USD"),
4798 Money::from("100000 USD"),
4799 )],
4800 vec![],
4801 true,
4802 UUID4::new(),
4803 UnixNanos::default(),
4804 UnixNanos::default(),
4805 None,
4806 );
4807 kernel
4808 .cache
4809 .borrow_mut()
4810 .add_account(AccountAny::Cash(CashAccount::new(state, true, false)))
4811 .unwrap();
4812 node
4813 }
4814}