1use std::{
29 collections::{BTreeMap, BTreeSet},
30 future::Future,
31 ops::ControlFlow,
32 sync::{
33 Arc,
34 atomic::{AtomicBool, AtomicU64, Ordering},
35 },
36 time::Duration,
37};
38
39use ahash::{AHashMap, AHashSet};
40use anyhow::Context;
41use async_trait::async_trait;
42use futures_util::{StreamExt, stream};
43#[cfg(test)]
44use nautilus_common::live::get_runtime;
45use nautilus_common::{
46 clients::{ExecutionClient, ExecutionReportTask},
47 enums::{LogColor, LogLevel},
48 live::runner::{get_exec_event_sender, try_get_exec_event_sender},
49 log_debug,
50 messages::{
51 ExecutionReport,
52 execution::{
53 BatchCancelOrders, BatchModifyOrders, CancelAllOrders, CancelOrder,
54 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
55 GeneratePositionStatusReports, ModifyOrder, QueryAccount, QueryOrder, SubmitOrder,
56 SubmitOrderList,
57 },
58 },
59 msgbus::{self, TypedHandler},
60};
61use nautilus_core::{
62 DurationNanos, UUID4, UnixNanos,
63 datetime::unix_nanos_to_iso8601,
64 params::Params,
65 string::secret::SecretString,
66 time::{AtomicTime, get_atomic_clock_realtime},
67};
68use nautilus_execution::reconciliation::{
69 inferred_reconciliation_trade_ids, is_inferred_reconciliation_trade_id_format,
70};
71use nautilus_live::{
72 ExecutionClientCore, ExecutionEventEmitter, SocketControlFactory,
73 book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS,
74 execution::failure::CommandFailure,
75 task::{TaskGroup, TaskGroupGuard, TaskJoinOutcome, TaskSlot, TaskSpawner, finish_task},
76};
77use nautilus_model::{
78 accounts::AccountAny,
79 enums::{
80 AccountType, ContingencyType, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
81 TimeInForce,
82 },
83 events::{OrderAccepted, OrderDeniedReason, OrderEventAny, OrderFilled, OrderTriggered},
84 identifiers::{
85 AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, TradeId, TraderId, Venue,
86 VenueOrderId,
87 },
88 instruments::{Instrument, InstrumentAny},
89 orders::{LIMIT_ORDER_TYPES, Order, OrderAny, STOP_ORDER_TYPES, TRIGGERABLE_ORDER_TYPES},
90 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
91 types::{AccountBalance, MarginBalance, Quantity},
92};
93use nautilus_network::error::SendError;
94use parking_lot::Mutex;
95use rust_decimal::Decimal;
96use serde::Deserialize;
97use tokio_util::sync::CancellationToken;
98use zeroize::Zeroizing;
99
100use crate::{
101 common::{
102 consts::{
103 DISCONNECT_TIMEOUT, LIGHTER_ERROR_CODE_INVALID_NONCE, LIGHTER_MAX_BATCH_TX,
104 SCHEDULED_CANCEL_MAX_DELAY_SECS, SCHEDULED_CANCEL_MIN_DELAY_SECS,
105 SCHEDULED_CANCEL_REQUEST_TIMEOUT,
106 },
107 credential::{Credential, scrub_auth},
108 deployment,
109 enums::{
110 LighterAccountTier, LighterCancelAllTimeInForce, LighterGroupingType,
111 LighterOrderStatus, LighterPositionMarginMode, LighterProductType,
112 LighterSelfTradeBehaviorMode, LighterSelfTradeEqualityMode, LighterTxStatus,
113 LighterTxType,
114 },
115 parse::{parse_price_from_ticks, parse_quantity_from_ticks, quantity_from_decimal},
116 rate_limit::{LighterTxRateLimiter, await_tx_quota, build_tx_rate_limiter, resolve_quota},
117 symbol::{MarketRegistry, product_type_from_instrument_id},
118 },
119 config::{
120 LighterExecutionClientConfig, validate_execution_venue, validate_execution_ws_url,
121 validate_market_order_slippage_bps, validate_proxy_url, validate_scheduled_cancel_timeout,
122 },
123 http::{
124 client::{LIGHTER_REST_PAGE_SIZE, LighterHttpClient, LighterRawHttpClient},
125 error::LighterHttpError,
126 models::{
127 LighterAccountDetail, LighterOrder, LighterSendTxRequest, LighterTrade, LighterTx,
128 },
129 query::{
130 LighterAccountActiveOrdersQuery, LighterAccountInactiveOrdersQuery,
131 LighterSortDirection, LighterTradeSortBy, LighterTradesQuery,
132 },
133 },
134 signing::{
135 auth_token::{build_auth_token_for, fresh_k},
136 nonce::NonceError,
137 tx::{
138 ApproveIntegratorTxInfo, CancelAllOrdersTxInfo, CancelOrderTxInfo,
139 CreateGroupedOrdersTxInfo, CreateOrderTxInfo, L2TxAttributes, ModifyOrderTxInfo,
140 OrderInfo, TxContext, TxInfoJson, UpdateLeverageTxInfo, sign_tx,
141 },
142 },
143 websocket::{
144 LighterWsError, USER_STREAMS_ENDPOINT,
145 client::{LighterWebSocketClient, RetainedTaskSlot, TaskRetentionGuard},
146 dispatch::{
147 DeferredOrderEvents, LIGHTER_INSTRUMENT_CACHE, MAX_RECONCILIATION_PAGES, OrderIdentity,
148 OrderReportLookup, OrderShapeSnapshot, PendingOrderAction, PendingSendTx,
149 PendingSendTxKind, PendingTerminal, TradeDedupSource, WsDispatchState,
150 cache_instruments_for_reports, collect_order, collect_order_while_pending,
151 derive_market_order_price_ticks, evict_terminal_mappings, fills_for_order,
152 lookup_create_order_status_report, nautilus_to_lighter_order_type,
153 nautilus_to_lighter_tif, order_expiry_for, parse_http_order_to_report, price_to_ticks,
154 quantity_to_ticks, tx_hash_matches,
155 },
156 messages::{
157 AccountStream, ExecutionReport as LighterExecutionReport, LighterWsChannel,
158 LighterWsTx, NautilusWsMessage, SendTxRejectionSource, TX_BATCH_ID_PREFIX,
159 },
160 parse::{
161 LighterCommissionError, OpenFrameContext, ParsedOrderEvent, is_order_triggered,
162 lighter_order_shape, parse_lighter_order_event, parse_lighter_order_filled,
163 parse_lighter_order_updated, parse_lighter_trade_id, parse_ws_fill_report,
164 parse_ws_order_status_report,
165 },
166 },
167};
168
169const DEFAULT_TX_EXPIRY_MS: i64 = 5 * 60 * 1_000;
172
173const ACKED_ORDER_LOOKUP_DELAY: Duration = Duration::from_secs(2);
175const ACKED_ORDER_PROBE_ATTEMPTS: usize = 3;
176const TERMINAL_FILL_DELAY: Duration = Duration::from_secs(2);
177const TERMINAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(2);
178const TERMINAL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(7);
179const ORDER_EVENTS_PATTERN: &str = "events.order.*";
180const BATCH_ACK_TIMEOUT: Duration = Duration::from_secs(10);
181const ACCOUNT_CANCEL_CONFIRM_TIMEOUT: Duration = Duration::from_secs(10);
182
183const STRATEGY_REASON_MAX_CHARS: usize = 512;
184
185const AUTH_TOKEN_REFRESH_INTERVAL: std::time::Duration =
190 std::time::Duration::from_secs(6 * 60 * 60);
191
192const _: () = assert!(
194 AUTH_TOKEN_REFRESH_INTERVAL.as_secs()
195 < crate::signing::auth_token::DEFAULT_AUTH_TOKEN_TTL_SECS as u64,
196 "AUTH_TOKEN_REFRESH_INTERVAL must stay below DEFAULT_AUTH_TOKEN_TTL_SECS",
197);
198
199const AUTH_TOKEN_REFRESH_RETRY_WINDOW: Duration = Duration::from_secs(60 * 60);
202const AUTH_TOKEN_REFRESH_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(30);
203
204const AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY: Duration = Duration::from_secs(5 * 60);
206const AUTH_TOKEN_REFRESH_BACKOFF: AuthTokenRefreshBackoff = AuthTokenRefreshBackoff {
207 initial_delay: AUTH_TOKEN_REFRESH_RETRY_INITIAL_DELAY,
208 max_delay: AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY,
209 window: AUTH_TOKEN_REFRESH_RETRY_WINDOW,
210};
211const NONCE_REFRESH_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(1);
212const NONCE_REFRESH_RETRY_MAX_DELAY: Duration = Duration::from_secs(30);
213const ACCOUNT_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(10);
216const REFERRAL_ATTRIBUTION_TIMEOUT: Duration = Duration::from_secs(10);
217
218const SENDTX_BARE_ERROR_WINDOW_MS: u64 = 1_000;
223const INTEGRATOR_AUTO_APPROVAL_MAX_TTL_MS: i64 = 5 * 365 * 24 * 60 * 60 * 1_000;
224const INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK: u32 = 0;
225const NONCE_CONNECTION_EPOCH_UNAVAILABLE: u64 = u64::MAX;
226
227#[derive(Debug)]
228pub struct LighterExecutionClient {
229 core: ExecutionClientCore,
230 clock: &'static AtomicTime,
231 config: LighterExecutionClientConfig,
232 emitter: ExecutionEventEmitter,
233 credential: Option<Credential>,
234 http_client: LighterHttpClient,
235 ws_client: LighterWebSocketClient,
236 tx_rate_limiter: Arc<LighterTxRateLimiter>,
237 tx_send_sequencer: TxSendSequencer,
238 nonce_submission_gate: Arc<tokio::sync::RwLock<()>>,
239 nonce_ready_connection_epoch: Arc<AtomicU64>,
240 registry: Arc<MarketRegistry>,
241 socket_factory: SocketControlFactory,
242 pending_tasks: TaskGroup,
243 ws_stream_handle: TaskSlot<()>,
244 terminal_recovery_handle: TaskSlot<()>,
245 terminal_producers_stopped: CancellationToken,
246 fills_applied: Option<AppliedFillSubscription>,
247 ws_disconnect_handle: TaskSlot<Result<(), LighterWsError>>,
248 ws_handler_retained: Arc<RetainedTaskSlot>,
249 auth_refresh_handle: TaskSlot<()>,
250 scheduled_cancel_handle: TaskSlot<()>,
251 scheduled_cancel_notify: Arc<tokio::sync::Notify>,
252 shutdown_errors: Vec<String>,
253 cancellation_token: CancellationToken,
254 dispatch: WsDispatchState,
255 nonce_recovery_inflight: Arc<AtomicBool>,
256 auth_refresh_notify: Arc<tokio::sync::Notify>,
257}
258
259impl LighterExecutionClient {
260 fn log_report_receipt(count: usize, report_type: &str, log_level: LogLevel) {
261 let plural = if count == 1 { "" } else { "s" };
262 let message = format!("Received {count} {report_type}{plural}");
263
264 match log_level {
265 LogLevel::Off => {}
266 LogLevel::Trace => log::trace!("{message}"),
267 LogLevel::Debug => log::debug!("{message}"),
268 LogLevel::Info => log::info!("{message}"),
269 LogLevel::Warning => log::warn!("{message}"),
270 LogLevel::Error => log::error!("{message}"),
271 }
272 }
273
274 pub fn new(
287 core: ExecutionClientCore,
288 config: LighterExecutionClientConfig,
289 ) -> anyhow::Result<Self> {
290 validate_market_order_slippage_bps(config.market_order_slippage_bps)?;
291 validate_scheduled_cancel_timeout(config.scheduled_cancel_timeout_secs)?;
292 validate_execution_venue(config.venue)?;
293 validate_execution_ws_url(config.base_url_ws.as_deref())?;
294 validate_proxy_url(config.proxy_url.as_ref())?;
295 anyhow::ensure!(
296 core.venue == config.resolved_venue(),
297 "Lighter execution core venue {} does not match configured venue {}",
298 core.venue,
299 config.resolved_venue(),
300 );
301
302 anyhow::ensure!(
303 config.account_id.get_issuer() == core.venue,
304 "Lighter account ID issuer {} does not match configured venue {}",
305 config.account_id.get_issuer(),
306 core.venue,
307 );
308
309 let credential = Credential::resolve_for_deployment(
310 config.private_key.clone().map(SecretString::into_inner),
311 config.account_index,
312 config.api_key_index,
313 config.deployment,
314 config.environment,
315 )
316 .context("failed to resolve Lighter credentials")?;
317
318 let registry = Arc::new(MarketRegistry::new_with_venue_and_settlement_currency(
319 core.venue,
320 config.settlement_currency(),
321 ));
322 let socket_factory = SocketControlFactory::new(core.client_id, Some(core.venue));
323
324 let tx_rate_limiter = build_tx_rate_limiter(config.sendtx_quota_per_min);
327
328 let raw_http = LighterRawHttpClient::new_with_quotas(
329 config.environment,
330 Some(config.http_url()),
331 config.http_timeout_secs,
332 config
333 .proxy_url
334 .as_ref()
335 .map(|value| value.expose_secret().to_owned()),
336 resolve_quota(config.rest_quota_per_min),
337 Some(Arc::clone(&tx_rate_limiter)),
338 )
339 .context("failed to construct Lighter raw HTTP client")?;
340
341 let http_client =
342 LighterHttpClient::from_raw_with_registry(raw_http, Arc::clone(®istry));
343
344 let ws_client = Self::create_ws_client(&config, Arc::clone(®istry), &socket_factory);
345
346 let clock = get_atomic_clock_realtime();
347
348 let mut emitter = ExecutionEventEmitter::new(
349 clock,
350 core.trader_id,
351 core.account_id,
352 AccountType::Margin,
353 None,
354 );
355
356 if let Some(sender) = try_get_exec_event_sender() {
357 emitter.set_sender(sender);
358 }
359
360 let pending_tasks = TaskGroup::new();
361
362 Ok(Self {
363 core,
364 clock,
365 config,
366 emitter,
367 credential,
368 http_client,
369 ws_client,
370 tx_rate_limiter,
371 tx_send_sequencer: TxSendSequencer::new(),
372 nonce_submission_gate: Arc::new(tokio::sync::RwLock::new(())),
373 nonce_ready_connection_epoch: Arc::new(AtomicU64::new(0)),
374 registry,
375 socket_factory,
376 cancellation_token: pending_tasks.cancellation_token(),
377 pending_tasks,
378 ws_stream_handle: TaskSlot::new(),
379 terminal_recovery_handle: TaskSlot::new(),
380 terminal_producers_stopped: CancellationToken::new(),
381 fills_applied: None,
382 ws_disconnect_handle: TaskSlot::new(),
383 ws_handler_retained: Arc::new(RetainedTaskSlot::new()),
384 auth_refresh_handle: TaskSlot::new(),
385 scheduled_cancel_handle: TaskSlot::new(),
386 scheduled_cancel_notify: Arc::new(tokio::sync::Notify::new()),
387 shutdown_errors: Vec::new(),
388 dispatch: WsDispatchState::new(),
389 nonce_recovery_inflight: Arc::new(AtomicBool::new(false)),
390 auth_refresh_notify: Arc::new(tokio::sync::Notify::new()),
391 })
392 }
393
394 #[must_use]
396 pub fn config(&self) -> &LighterExecutionClientConfig {
397 &self.config
398 }
399
400 #[must_use]
402 pub fn has_credentials(&self) -> bool {
403 self.credential.is_some()
404 }
405
406 #[must_use]
409 pub fn pending_tasks_all_finished(&self) -> bool {
410 self.pending_tasks.all_finished()
411 }
412
413 pub async fn cancel_all_orders_account(&self) -> anyhow::Result<String> {
430 self.send_cancel_all(LighterCancelAllTimeInForce::Immediate, 0)
431 .await
432 }
433
434 pub async fn schedule_cancel(&self, time: Option<u64>) -> anyhow::Result<String> {
454 let (time_in_force, scheduled_time_ms) = match time {
455 Some(time) => {
456 let now = self.clock.get_time_ms();
457 let delay = time
458 .checked_sub(now)
459 .context("scheduled cancellation is in the past")?;
460 anyhow::ensure!(
461 (SCHEDULED_CANCEL_MIN_DELAY_SECS * 1_000
462 ..=SCHEDULED_CANCEL_MAX_DELAY_SECS * 1_000)
463 .contains(&delay),
464 "scheduled cancellation must be between five minutes and fifteen days ahead",
465 );
466 let time = i64::try_from(time)
467 .context("scheduled cancellation timestamp overflows milliseconds")?;
468 (LighterCancelAllTimeInForce::Scheduled, time)
469 }
470 None => (LighterCancelAllTimeInForce::AbortScheduled, 0),
471 };
472
473 self.send_cancel_all(time_in_force, scheduled_time_ms).await
474 }
475
476 async fn send_cancel_all(
477 &self,
478 time_in_force: LighterCancelAllTimeInForce,
479 scheduled_time_ms: i64,
480 ) -> anyhow::Result<String> {
481 let credential = self
482 .credential
483 .as_ref()
484 .context("Lighter account cancellation requires credentials")?;
485 self.fanout_dispatch_context(credential)
486 .context("failed to spawn Lighter account cancellation")?
487 .send_cancel_all(
488 time_in_force,
489 scheduled_time_ms,
490 false,
491 self.cancellation_token.clone(),
492 )
493 .await
494 }
495
496 async fn start_scheduled_cancel(&mut self) -> anyhow::Result<()> {
497 let Some(timeout_secs) = self.config.scheduled_cancel_timeout_secs else {
498 return Ok(());
499 };
500
501 let credential = self
502 .credential
503 .as_ref()
504 .context("scheduled cancellation requires credentials")?;
505 let context = self.fanout_dispatch_context(credential)?;
506 context
507 .refresh_scheduled_cancel(timeout_secs)
508 .await
509 .context("failed to arm Lighter scheduled cancellation")?;
510 let cancellation = self.cancellation_token.clone();
511 let notify = Arc::clone(&self.scheduled_cancel_notify);
512 let refresh_interval =
513 Duration::from_secs(((timeout_secs - SCHEDULED_CANCEL_MIN_DELAY_SECS) / 3).min(60));
514
515 self.scheduled_cancel_handle.spawn(async move {
516 let mut interval = tokio::time::interval_at(tokio::time::Instant::now() + refresh_interval, refresh_interval);
517 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
518
519 loop {
520 tokio::select! {
521 biased;
522 () = cancellation.cancelled() => break,
523 () = notify.notified() => {},
524 _ = interval.tick() => {},
525 }
526
527 if !context.ws_client.is_active()
528 || context.nonce_ready_connection_epoch.load(Ordering::Acquire) != context.ws_client.connection_epoch() {
529 continue;
530 }
531
532 tokio::select! {
533 biased;
534 () = cancellation.cancelled() => break,
535 result = context.refresh_scheduled_cancel(timeout_secs) => {
536 if let Err(e) = result {
537 log::error!("Scheduled cancellation refresh failed; the last deadline may expire: {e:#}");
538 }
539 }
540 }
541 }
542 })?;
543
544 log::info!("Scheduled cancellation armed: timeout_secs={timeout_secs}");
545 Ok(())
546 }
547
548 fn create_ws_client(
549 config: &LighterExecutionClientConfig,
550 registry: Arc<MarketRegistry>,
551 socket_factory: &SocketControlFactory,
552 ) -> LighterWebSocketClient {
553 let ws_client = LighterWebSocketClient::new(
554 Some(config.ws_url()),
555 config.environment,
556 registry,
557 config.transport_backend,
558 config.ws_timeout_secs,
559 Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
561 config
562 .proxy_url
563 .as_ref()
564 .map(|value| value.expose_secret().to_owned()),
565 );
566
567 ws_client.with_socket_control(socket_factory.control(USER_STREAMS_ENDPOINT))
568 }
569
570 fn take_ws_client(&mut self) -> LighterWebSocketClient {
571 let cached_instruments = self.ws_client.instruments_cache();
572 let ws_cache = cached_instruments
573 .iter()
574 .map(|entry| (*entry.key(), entry.value().clone()))
575 .collect();
576 let replacement = Self::create_ws_client(
577 &self.config,
578 Arc::clone(&self.registry),
579 &self.socket_factory,
580 );
581 replacement.cache_instruments(ws_cache);
582
583 std::mem::replace(&mut self.ws_client, replacement)
584 }
585
586 fn spawn_task<F>(&self, description: &'static str, fut: F)
587 where
588 F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
589 {
590 let future = async move {
591 if let Err(e) = fut.await {
592 log::warn!("{description} failed: {e:?}");
593 }
594 };
595
596 if let Err(e) = self.pending_tasks.spawn(future) {
597 log::warn!("Skipping Lighter {description} after shutdown began: {e}");
598 }
599 }
600
601 fn subscribe_applied_fills(&mut self) {
602 if self.fills_applied.is_some() {
603 return;
604 }
605
606 let core = self.core.clone();
607 let dispatch = self.dispatch.clone();
608 let emitter = self.emitter.clone();
609
610 let handler = TypedHandler::from(move |event: &OrderEventAny| {
611 let OrderEventAny::Filled(fill) = event else {
612 return;
613 };
614
615 if fill.account_id != core.account_id || fill.trader_id != core.trader_id {
616 return;
617 }
618
619 let Some(identity) = dispatch.order_identity(&fill.client_order_id) else {
620 return;
621 };
622
623 if identity.strategy_id != fill.strategy_id
624 || identity.instrument_id != fill.instrument_id
625 {
626 return;
627 }
628
629 let cache = core.cache();
630
631 let Some(order) = cache.order(&fill.client_order_id) else {
632 return;
633 };
634
635 if !order.is_duplicate_fill(fill) {
636 return;
637 }
638
639 let events = dispatch.order_events.lock();
640
641 if is_inferred_reconciliation_trade_id_format(&fill.trade_id) {
642 identity.sync_fills(&order);
643 }
644
645 if identity.has_inferred_fills() {
646 return;
647 }
648
649 identity.publish_fill(fill);
650 identity.publication_pending(Some(&order));
651 dispatch.mark_trade_reconciled(fill.trade_id);
652 release_terminal(fill.client_order_id, false, &dispatch, &emitter, &events);
653 });
654
655 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), Some(10));
656 self.fills_applied = Some(AppliedFillSubscription(handler));
657 }
658
659 fn begin_session_shutdown(&mut self) {
660 self.pending_tasks.begin_shutdown();
661 self.cancellation_token.cancel();
662 self.ws_client.begin_shutdown();
663 self.core.set_disconnected();
664
665 if self.ws_disconnect_handle.is_none()
666 && (self.ws_client.is_active() || self.ws_stream_handle.is_some())
667 {
668 let ws_client = self.take_ws_client();
669 let retained = Arc::clone(&self.ws_handler_retained);
670
671 if let Err(e) = self
672 .ws_disconnect_handle
673 .spawn(ws_client.disconnect_with_task_retention(retained))
674 {
675 log::error!("Failed to start Lighter WebSocket disconnect task: {e}");
676 }
677 }
678 }
679
680 fn session_tasks_finished(&self) -> bool {
681 self.ws_stream_handle.is_none()
682 && self.terminal_recovery_handle.is_none()
683 && self.ws_disconnect_handle.is_none()
684 && self.ws_handler_retained.is_empty()
685 && self.auth_refresh_handle.is_none()
686 && self.scheduled_cancel_handle.is_none()
687 && self.pending_tasks.is_empty()
688 && self.shutdown_errors.is_empty()
689 }
690
691 async fn finish_session_shutdown(&mut self) -> anyhow::Result<()> {
692 self.pending_tasks.begin_shutdown();
693
694 Self::finish_disconnect_task(
695 &mut self.ws_disconnect_handle,
696 "WebSocket disconnect",
697 &mut self.shutdown_errors,
698 )
699 .await;
700
701 if let Err(e) = self.ws_handler_retained.finish().await {
702 self.shutdown_errors.push(e.to_string());
703 }
704 Self::finish_owned_task(
705 &mut self.ws_stream_handle,
706 "execution consumer",
707 &mut self.shutdown_errors,
708 )
709 .await;
710
711 Self::finish_owned_task(
712 &mut self.auth_refresh_handle,
713 "auth-token refresh",
714 &mut self.shutdown_errors,
715 )
716 .await;
717
718 if let Err(e) = self
719 .pending_tasks
720 .finish_shutdown(Duration::from_secs(1), DISCONNECT_TIMEOUT)
721 .await
722 {
723 self.shutdown_errors
724 .push(format!("client-owned tasks failed: {e}"));
725 }
726
727 Self::finish_owned_task(
728 &mut self.scheduled_cancel_handle,
729 "scheduled cancellation refresh",
730 &mut self.shutdown_errors,
731 )
732 .await;
733
734 if self.ws_disconnect_handle.is_none()
735 && self.ws_handler_retained.is_empty()
736 && self.ws_stream_handle.is_none()
737 && self.pending_tasks.is_empty()
738 {
739 self.terminal_producers_stopped.cancel();
740
741 if let Some(outcome) = finish_task(
742 &mut self.terminal_recovery_handle,
743 TERMINAL_SHUTDOWN_TIMEOUT + DISCONNECT_TIMEOUT,
744 DISCONNECT_TIMEOUT,
745 )
746 .await
747 && !matches!(outcome, TaskJoinOutcome::Completed(()))
748 {
749 self.shutdown_errors
750 .push(format!("terminal recovery task failed: {outcome:?}"));
751 }
752 }
753
754 if self.session_tasks_finished() {
755 self.fills_applied = None;
756 }
757
758 let stale = self.dispatch.take_pending_sendtx();
759 warn_pending_sendtx_unknown(&stale, "session shutdown");
760 self.nonce_recovery_inflight.store(false, Ordering::Release);
761
762 if !self.shutdown_errors.is_empty() {
763 let errors = std::mem::take(&mut self.shutdown_errors);
764 anyhow::bail!("Failed to terminate Lighter tasks: {}", errors.join("; "));
765 }
766 Ok(())
767 }
768
769 async fn finish_owned_task(
770 slot: &mut TaskSlot<()>,
771 description: &str,
772 errors: &mut Vec<String>,
773 ) {
774 let Some(outcome) = finish_task(slot, DISCONNECT_TIMEOUT, DISCONNECT_TIMEOUT).await else {
775 return;
776 };
777
778 match outcome {
779 TaskJoinOutcome::Completed(()) => {
780 log::debug!("Lighter {description} task completed");
781 }
782 TaskJoinOutcome::Aborted => {
783 log::debug!("Lighter {description} task cancelled");
784 }
785 TaskJoinOutcome::Failed(e) => {
786 errors.push(format!("{description} task failed: {e}"));
787 }
788 TaskJoinOutcome::Incomplete => {
789 errors.push(format!("{description} task did not stop after abort"));
790 }
791 }
792 }
793
794 async fn finish_disconnect_task(
795 slot: &mut TaskSlot<Result<(), LighterWsError>>,
796 description: &str,
797 errors: &mut Vec<String>,
798 ) {
799 let Some(outcome) = finish_task(slot, DISCONNECT_TIMEOUT, DISCONNECT_TIMEOUT).await else {
800 return;
801 };
802
803 match outcome {
804 TaskJoinOutcome::Completed(Ok(())) | TaskJoinOutcome::Aborted => {}
805 TaskJoinOutcome::Completed(Err(e)) => {
806 errors.push(format!("{description} failed: {e}"));
807 }
808 TaskJoinOutcome::Failed(e) => {
809 errors.push(format!("{description} task failed: {e}"));
810 }
811 TaskJoinOutcome::Incomplete => {
812 errors.push(format!("{description} task did not stop after abort"));
813 }
814 }
815 }
816
817 async fn ensure_instruments_initialized_async(&self) -> anyhow::Result<()> {
818 if self.core.instruments_initialized() {
819 return Ok(());
820 }
821
822 let instruments = self
823 .http_client
824 .request_instruments()
825 .await
826 .context("failed to request Lighter instruments")?;
827
828 let ws_cache: Vec<(i64, InstrumentAny)> = instruments
829 .iter()
830 .filter_map(|instrument| {
831 self.registry
832 .market_index(&instrument.id())
833 .map(|market_index| (market_index, instrument.clone()))
834 })
835 .collect();
836 self.ws_client.cache_instruments(ws_cache);
837 cache_instruments_for_reports(&instruments);
838
839 log::debug!(
840 "Bootstrapped {} Lighter instruments ({} registry entries)",
841 instruments.len(),
842 self.registry.len(),
843 );
844
845 self.core.set_instruments_initialized();
846 Ok(())
847 }
848
849 async fn await_account_streams_ready(&self, timeout_secs: f64) -> anyhow::Result<()> {
850 let timeout = Duration::from_secs_f64(timeout_secs);
851 self.dispatch.account_streams_ready.await_all(timeout).await
852 }
853
854 async fn refresh_nonce(&self) -> anyhow::Result<()> {
855 let Some(credential) = &self.credential else {
856 return Ok(());
857 };
858 self.nonce_ready_connection_epoch
859 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
860 let _refresh_guard = self.nonce_submission_gate.write().await;
861
862 let response = self
863 .http_client
864 .get_next_nonce(credential.account_index(), credential.api_key_index())
865 .await
866 .context("failed to fetch Lighter nextNonce")?;
867
868 self.dispatch.nonce_manager.refresh(
869 credential.account_index(),
870 credential.api_key_index(),
871 response.nonce,
872 );
873
874 self.nonce_recovery_inflight.store(false, Ordering::Release);
876
877 log::debug!(
878 "Refreshed Lighter nonce baseline: account_index={}, api_key_index={}, next_nonce={}",
879 credential.account_index(),
880 credential.api_key_index(),
881 response.nonce,
882 );
883 Ok(())
884 }
885
886 async fn sync_nonce_from_venue(&self) -> anyhow::Result<()> {
887 let Some(credential) = &self.credential else {
888 return Ok(());
889 };
890
891 let _refresh_guard = self.nonce_submission_gate.write().await;
892 let response = self
893 .http_client
894 .get_next_nonce(credential.account_index(), credential.api_key_index())
895 .await
896 .context("failed to fetch Lighter nextNonce")?;
897 self.dispatch.nonce_manager.sync_from_venue(
898 credential.account_index(),
899 credential.api_key_index(),
900 response.nonce,
901 )?;
902 Ok(())
903 }
904
905 async fn fetch_account_detail(&self) -> Option<LighterAccountDetail> {
906 let Some(credential) = &self.credential else {
907 return None;
908 };
909 let account_index = credential.account_index();
910
911 match tokio::time::timeout(
912 ACCOUNT_SNAPSHOT_TIMEOUT,
913 self.http_client.get_account_detail(account_index),
914 )
915 .await
916 {
917 Ok(Ok(detail)) => Some(detail),
918 Ok(Err(e)) => {
919 log::warn!(
920 "Failed to fetch Lighter account detail for account_index={account_index}; \
921 continuing startup without account metadata: {e}"
922 );
923 None
924 }
925 Err(_) => {
926 log::warn!(
927 "Lighter account detail timed out after {}s for account_index={account_index}; \
928 continuing startup without account metadata",
929 ACCOUNT_SNAPSHOT_TIMEOUT.as_secs()
930 );
931 None
932 }
933 }
934 }
935
936 fn log_account_tier(&self, detail: &LighterAccountDetail) {
941 let account_index = detail.account_index;
942 let code = detail.account_type;
943 let tier = LighterAccountTier::from_code(code);
944 let standard_rest = LighterAccountTier::Standard
945 .documented_rest_quota_per_min()
946 .unwrap_or(60);
947 let (active_rest, cross_check) =
948 tier_quota_report(tier, self.config.rest_quota_per_min, standard_rest);
949
950 log_debug!(
951 "Lighter execution account {account_index} reported tier {tier} \
952 (account_type={code}); active REST quota {active_rest} req/min",
953 color = LogColor::Blue
954 );
955
956 match cross_check {
957 Some(TierCrossCheck::AboveTier { documented }) => {
958 log::warn!(
959 "Configured Lighter rest_quota_per_min={active_rest} exceeds the {tier} tier \
960 limit of {documented} req/min; the venue may reject requests unless the \
961 caller IP is registered for the higher limit"
962 );
963 }
964 Some(TierCrossCheck::RaiseHint { documented }) => {
965 log_debug!(
966 "Lighter {tier} tier permits up to {documented} REST req/min; set \
967 rest_quota_per_min (and register the caller IP with Lighter) to use it",
968 color = LogColor::Blue
969 );
970 }
971 None => {}
972 }
973 }
974
975 fn integrator_account_index(&self) -> Option<u64> {
976 deployment::integrator_account_index(self.config.deployment, self.config.environment)
977 }
978
979 async fn apply_referral_attribution(
980 &self,
981 detail: &LighterAccountDetail,
982 ) -> anyhow::Result<()> {
983 let Some(referral_code) =
984 deployment::referral_code(self.config.deployment, self.config.environment)
985 else {
986 return Ok(());
987 };
988
989 let Some(credential) = &self.credential else {
990 return Ok(());
991 };
992
993 anyhow::ensure!(
994 !detail.l1_address.trim().is_empty(),
995 "Lighter account detail returned an empty L1 address"
996 );
997
998 let auth_token = build_auth_token_for(credential)
999 .context("failed to mint Lighter auth token for referral attribution")?;
1000 let referral_code = Zeroizing::new(referral_code.to_string());
1001
1002 self.http_client
1003 .use_referral(&detail.l1_address, referral_code.as_str(), &auth_token)
1004 .await
1005 .context("failed to apply Lighter referral code")?;
1006
1007 Ok(())
1008 }
1009
1010 async fn is_maker_only_api_key(&self, credential: &Credential) -> anyhow::Result<bool> {
1014 let auth_token = build_auth_token_for(credential)
1015 .context("failed to mint Lighter auth token for maker-only check")?;
1016 let response = self
1017 .http_client
1018 .get_maker_only_api_keys(credential.account_index(), auth_token)
1019 .await
1020 .context("failed to query getMakerOnlyApiKeys")?;
1021 let api_key_index = i64::from(credential.api_key_index());
1022 Ok(response.api_key_indexes.contains(&api_key_index))
1023 }
1024
1025 async fn submit_integrator_auto_approval(&self) -> anyhow::Result<()> {
1026 let Some(integrator_account_index) = self.integrator_account_index() else {
1027 return Ok(());
1028 };
1029
1030 let Some(credential) = &self.credential else {
1031 return Ok(());
1032 };
1033
1034 let mut maker_only_check_failed = false;
1035
1036 match self.is_maker_only_api_key(credential).await {
1037 Ok(true) => {
1038 log::warn!(
1039 "Skipping Lighter integrator auto-approval: api_key_index={} is maker-only; \
1040 ensure the account has been approved by a non-maker-only key",
1041 credential.api_key_index(),
1042 );
1043 return Ok(());
1044 }
1045 Ok(false) => {}
1046 Err(e) => {
1047 maker_only_check_failed = true;
1048
1049 log::debug!(
1050 "Unable to determine whether the Lighter API key is maker-only; proceeding \
1051 with integrator auto-approval: {e:?}"
1052 );
1053 }
1054 }
1055
1056 let mut approval =
1057 self.prepare_integrator_auto_approval(credential, integrator_account_index)?;
1058
1059 let request = LighterSendTxRequest::new(
1060 LighterTxType::ApproveIntegrator as u8,
1061 approval.tx_info.clone(),
1062 );
1063
1064 approval.send_reservation.wait_for_turn().await;
1065
1066 let response = match self.http_client.send_tx(&request).await {
1067 Ok(response) => response,
1068 Err(e) => {
1069 if lighter_http_error_is_definite_api_rejection(&e) {
1070 let _ = self.dispatch.nonce_manager.ack_failure_if_latest(
1071 credential.account_index(),
1072 approval.api_key_index,
1073 approval.nonce,
1074 );
1075 }
1076 approval.send_reservation.release();
1077 let hint = if maker_only_check_failed {
1078 " (maker-only pre-flight check failed earlier; venue may reject with 62007 \
1079 if this key is maker-only)"
1080 } else {
1081 ""
1082 };
1083 return Err(anyhow::Error::new(e).context(format!(
1084 "failed to submit Lighter integrator approval nonce={} api_key_index={}{hint}",
1085 approval.nonce, approval.api_key_index,
1086 )));
1087 }
1088 };
1089
1090 let _ = self.dispatch.nonce_manager.ack_success(
1091 credential.account_index(),
1092 approval.api_key_index,
1093 approval.nonce,
1094 );
1095 approval.send_reservation.release();
1096
1097 log::debug!(
1098 "Submitted Lighter integrator approval: integrator={}, nonce={}, \
1099 api_key_index={}, approval_expiry={}, tx_hash={}",
1100 integrator_account_index,
1101 approval.nonce,
1102 approval.api_key_index,
1103 approval.approval_expiry,
1104 response.tx_hash,
1105 );
1106 Ok(())
1107 }
1108
1109 fn prepare_integrator_auto_approval(
1110 &self,
1111 credential: &Credential,
1112 integrator_account_index: u64,
1113 ) -> anyhow::Result<PreparedIntegratorApproval> {
1114 let ReservedTxContext {
1115 context,
1116 send_reservation,
1117 } = self.build_tx_context(credential)?;
1118
1119 let now_ms = (self.clock.get_time_ns().as_u64() as i64) / 1_000_000;
1120 let approval_expiry = now_ms.saturating_add(INTEGRATOR_AUTO_APPROVAL_MAX_TTL_MS);
1121 let nonce = context.nonce;
1122 let api_key_index = context.api_key_index;
1123
1124 let tx = ApproveIntegratorTxInfo {
1125 context,
1126 integrator_account_index: integrator_account_index as i64,
1127 max_perps_taker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1128 max_perps_maker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1129 max_spot_taker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1130 max_spot_maker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1131 approval_expiry,
1132 skip_nonce: 0,
1133 };
1134
1135 let signed = sign_tx(
1136 &tx,
1137 self.config.chain_id(),
1138 &credential.private_key()?,
1139 fresh_k(),
1140 )?;
1141
1142 let tx_info = TxInfoJson::approve_integrator(&tx, &signed, "");
1143
1144 Ok(PreparedIntegratorApproval {
1145 tx_info,
1146 nonce,
1147 api_key_index,
1148 approval_expiry,
1149 send_reservation,
1150 })
1151 }
1152
1153 async fn spawn_ws_consumer(&mut self) -> anyhow::Result<()> {
1154 let mut ws_guard = TaskRetentionGuard::new(
1158 self.ws_client.clone(),
1159 Arc::clone(&self.ws_handler_retained),
1160 );
1161 ws_guard
1162 .client_mut()
1163 .connect_with_cancellation(self.cancellation_token.clone())
1164 .await
1165 .context("failed to connect to Lighter WebSocket")?;
1166
1167 let post_connect = async {
1171 ws_guard
1172 .client_mut()
1173 .wait_until_active()
1174 .await
1175 .context("Lighter WebSocket did not reach active state")?;
1176
1177 if let Some(credential) = &self.credential {
1178 let auth_token = build_auth_token_for(credential)
1179 .context("failed to mint Lighter auth token")?;
1180 let account_index = credential.account_index();
1181
1182 ws_guard
1183 .client_mut()
1184 .set_execution_context(self.core.account_id, account_index)
1185 .await
1186 .map_err(|e| anyhow::anyhow!("failed to set Lighter execution context: {e}"))?;
1187
1188 let channels = auth_token_rotation_channels(account_index);
1189
1190 for channel in channels {
1191 ws_guard
1192 .client_mut()
1193 .subscribe_account(channel.clone(), auth_token.clone())
1194 .await
1195 .map_err(|e| {
1196 anyhow::anyhow!(
1197 "failed to subscribe to Lighter account channel {channel:?}: {e}",
1198 )
1199 })?;
1200 }
1201
1202 log::debug!("Subscribed to Lighter account streams: account_index={account_index}",);
1203 } else {
1204 log::warn!(
1205 "Lighter execution client has no credentials: account streams not subscribed; \
1206 typed execution reports will not flow"
1207 );
1208 }
1209
1210 Ok::<(), anyhow::Error>(())
1211 };
1212
1213 if let Err(e) = post_connect.await {
1214 log::warn!("Lighter post-connect setup failed, tearing down WS: {e}");
1215 let ws_client = ws_guard.disarm();
1216 let mut rollback_errors = Vec::new();
1217
1218 if let Err(e) = ws_client
1219 .disconnect_with_task_retention(Arc::clone(&self.ws_handler_retained))
1220 .await
1221 {
1222 rollback_errors.push(e.to_string());
1223 }
1224
1225 if let Err(e) = self.ws_handler_retained.finish().await {
1226 rollback_errors.push(e.to_string());
1227 }
1228
1229 if rollback_errors.is_empty() {
1230 return Err(e);
1231 }
1232 return Err(e.context(format!(
1233 "Lighter post-connect rollback failed: {}",
1234 rollback_errors.join("; ")
1235 )));
1236 }
1237
1238 let mut ws_client = ws_guard.disarm();
1239 self.ws_client.set_task_slot(ws_client.take_task_slot());
1240
1241 let cancellation_token = self.cancellation_token.clone();
1242 let emitter = self.emitter.clone();
1243 let dispatch = self.dispatch.clone();
1244 let credential_for_loop = self.credential.clone();
1245 let http_client_for_loop = self.http_client.clone();
1246 let registry_for_loop = Arc::clone(&self.registry);
1247 let auth_refresh_notify = Arc::clone(&self.auth_refresh_notify);
1248 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
1249 let nonce_ready_connection_epoch = Arc::clone(&self.nonce_ready_connection_epoch);
1250 let scheduled_cancel_notify = Arc::clone(&self.scheduled_cancel_notify);
1251 let pending_tasks_for_loop = self
1252 .pending_tasks
1253 .spawner()
1254 .context("Lighter task admission is closed")?;
1255 let account_id_for_loop = self.core.account_id;
1256 let clock_for_loop = self.clock;
1257
1258 let orders_cached = {
1259 let cache = self.core.cache();
1260 cache
1261 .orders_open_refs(
1262 Some(&self.core.venue),
1263 None,
1264 None,
1265 Some(&self.core.account_id),
1266 None,
1267 )
1268 .into_iter()
1269 .map(|order| order.clone())
1270 .collect()
1271 };
1272
1273 dispatch.seed_order_restorations(orders_cached);
1274
1275 let nonce_refresh_retry = NonceRefreshRetry {
1276 http_client: self.http_client.clone(),
1277 dispatch: self.dispatch.clone(),
1278 credential: self.credential.clone(),
1279 submission_gate: Arc::clone(&self.nonce_submission_gate),
1280 ready_connection_epoch: Arc::clone(&self.nonce_ready_connection_epoch),
1281 ws_client: ws_client.clone(),
1282 cancellation_token: self.cancellation_token.clone(),
1283 pending_tasks: pending_tasks_for_loop.clone(),
1284 };
1285
1286 let child_context = credential_for_loop
1287 .as_ref()
1288 .map(|credential| AckedOrderProbeContext {
1289 http_client: http_client_for_loop.clone(),
1290 registry: Arc::clone(®istry_for_loop),
1291 credential: credential.clone(),
1292 dispatch: dispatch.clone(),
1293 account_id: account_id_for_loop,
1294 clock: clock_for_loop,
1295 emitter: emitter.clone(),
1296 connection_epoch: ws_client.connection_epoch_atomic(),
1297 cancellation_token: cancellation_token.clone(),
1298 pending_tasks: pending_tasks_for_loop.clone(),
1299 });
1300
1301 if let Some(context) = child_context.clone() {
1302 self.terminal_producers_stopped = CancellationToken::new();
1303
1304 self.terminal_recovery_handle.spawn(run_terminal_recovery(
1305 context,
1306 self.terminal_producers_stopped.clone(),
1307 ))?;
1308 }
1309
1310 self.ws_stream_handle.spawn(async move {
1311 log::debug!("Lighter execution WebSocket consumption loop started");
1312
1313 loop {
1314 tokio::select! {
1315 biased;
1316 () = cancellation_token.cancelled() => {
1317 log::debug!("Lighter execution consumption loop cancelled");
1318 break;
1319 }
1320 msg_opt = ws_client.next_event() => {
1321 match msg_opt {
1322 Some(NautilusWsMessage::ExecutionReports(reports)) => {
1323 let mut order_count = 0_usize;
1324 let mut fill_count = 0_usize;
1325 let trader_id = emitter.trader_id();
1326 let account_index = credential_for_loop
1327 .as_ref()
1328 .map(|c| c.account_index());
1329
1330 for report in reports {
1331 match report {
1332 LighterExecutionReport::Order(order) => {
1333 order_count += 1;
1334 dispatch_lighter_order(
1335 &order,
1336 &dispatch,
1337 &emitter,
1338 ®istry_for_loop,
1339 account_id_for_loop,
1340 trader_id,
1341 clock_for_loop.get_time_ns(),
1342 );
1343
1344 if let Some(context) = &child_context {
1345 confirm_executed_modify(&order, context);
1346 }
1347 }
1348 LighterExecutionReport::Fill(trade) => {
1349 fill_count += 1;
1350 dispatch_lighter_trade(
1351 &trade,
1352 &dispatch,
1353 &emitter,
1354 ®istry_for_loop,
1355 account_id_for_loop,
1356 trader_id,
1357 account_index,
1358 clock_for_loop.get_time_ns(),
1359 );
1360 }
1361 }
1362 }
1363
1364 if let Some(context) = &child_context {
1365 spawn_child_recoveries(context);
1366 }
1367 log::debug!(
1368 "Lighter execution batch: orders={order_count} fills={fill_count}",
1369 );
1370 }
1371 Some(NautilusWsMessage::AccountTx { connection_epoch, txs }) => {
1372 if let Some(context) = &child_context {
1373 handle_account_tx(connection_epoch, &txs, context);
1374 }
1375 }
1376 Some(NautilusWsMessage::PositionSnapshot {
1377 reports,
1378 skipped_market_ids,
1379 }) => {
1380 for r in &reports {
1383 if let Some(idx) =
1384 registry_for_loop.market_index(&r.instrument_id)
1385 {
1386 dispatch.note_active_market(idx);
1387 }
1388 }
1389 let position_count = reports.len();
1390 let retained_positions: Vec<InstrumentId> = skipped_market_ids
1391 .iter()
1392 .filter_map(|market_id| {
1393 registry_for_loop.instrument_id(*market_id)
1394 })
1395 .collect();
1396 let removed = dispatch.replace_position_snapshot(
1397 &reports,
1398 &retained_positions,
1399 &skipped_market_ids,
1400 );
1401 log::debug!(
1402 "Lighter position snapshot: positions={position_count}, skipped_markets={}, removed={}",
1403 skipped_market_ids.len(),
1404 removed.len(),
1405 );
1406 emit_lighter_position_reports(
1407 reports,
1408 removed,
1409 &emitter,
1410 account_id_for_loop,
1411 clock_for_loop.get_time_ns(),
1412 );
1413 }
1414 Some(NautilusWsMessage::PositionUpdate {
1415 reports,
1416 closed_market_ids,
1417 skipped_market_ids,
1418 }) => {
1419 let mut covered_market_ids = closed_market_ids.clone();
1420
1421 for report in &reports {
1422 if let Some(market_id) =
1423 registry_for_loop.market_index(&report.instrument_id)
1424 {
1425 dispatch.note_active_market(market_id);
1426 covered_market_ids.push(market_id);
1427 }
1428 }
1429 let position_count = reports.len();
1430 let closed_positions: Vec<InstrumentId> = closed_market_ids
1431 .iter()
1432 .filter_map(|market_id| {
1433 registry_for_loop.instrument_id(*market_id)
1434 })
1435 .collect();
1436 let removed = dispatch.apply_position_update(
1437 &reports,
1438 &closed_positions,
1439 &covered_market_ids,
1440 &skipped_market_ids,
1441 );
1442 log::debug!(
1443 "Lighter position update: positions={position_count}, closed_markets={}, skipped_markets={}, removed={}",
1444 closed_market_ids.len(),
1445 skipped_market_ids.len(),
1446 removed.len(),
1447 );
1448 emit_lighter_position_reports(
1449 reports,
1450 removed,
1451 &emitter,
1452 account_id_for_loop,
1453 clock_for_loop.get_time_ns(),
1454 );
1455 }
1456 Some(NautilusWsMessage::AccountState(state)) => {
1457 log::debug!(
1458 "Lighter AccountState: balances={} margins={}",
1459 state.balances.len(),
1460 state.margins.len(),
1461 );
1462 dispatch.cache_account_state((*state).clone());
1468 emitter.send_account_state(*state);
1469 }
1470 Some(NautilusWsMessage::Reconnected { connection_epoch }) => {
1471 dispatch.terminals_recover.notify_one();
1472 dispatch.invalidate_position_snapshot();
1473 dispatch.stop_cancel_probes_before(connection_epoch);
1474 nonce_ready_connection_epoch.store(
1475 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
1476 Ordering::Release,
1477 );
1478 log::debug!("Lighter WebSocket reconnected (execution stream)");
1486 auth_refresh_notify.notify_one();
1487
1488 let refresh_guard = nonce_submission_gate.write().await;
1502 let disconnected_epoch = connection_epoch.saturating_sub(1);
1503 let stale = dispatch.drain_pending_sendtx(disconnected_epoch);
1504 warn_pending_sendtx_unknown(&stale, "reconnect");
1505
1506 if let Some(credential) = &credential_for_loop {
1507 match http_client_for_loop
1508 .get_next_nonce(
1509 credential.account_index(),
1510 credential.api_key_index(),
1511 )
1512 .await
1513 {
1514 Ok(response) => {
1515 dispatch.nonce_manager.refresh(
1516 credential.account_index(),
1517 credential.api_key_index(),
1518 response.nonce,
1519 );
1520 nonce_ready_connection_epoch
1521 .store(connection_epoch, Ordering::Release);
1522 drop(refresh_guard);
1523 scheduled_cancel_notify.notify_one();
1524 log::debug!(
1525 "Refreshed Lighter nonce after reconnect: \
1526 account_index={}, next_nonce={}",
1527 credential.account_index(),
1528 response.nonce,
1529 );
1530 }
1531 Err(e) => {
1532 log::warn!(
1533 "Failed to refresh Lighter nonce after reconnect: {e}",
1534 );
1535 nonce_refresh_retry
1536 .clone()
1537 .spawn(connection_epoch);
1538 }
1539 }
1540 }
1541 }
1542 Some(message @ (NautilusWsMessage::SendTxAck { .. } | NautilusWsMessage::SendTxBatchResult { .. })) => {
1543 let account_index = credential_for_loop.as_ref().map(|c| c.account_index());
1544 let acked = match message {
1545 NautilusWsMessage::SendTxAck { connection_epoch, tx_hash, code } => {
1546 handle_send_tx_ack_for_connection(&dispatch, account_index, connection_epoch, code, tx_hash.as_deref()).into_iter().collect()
1547 }
1548 NautilusWsMessage::SendTxBatchResult { connection_epoch, id, code, message, tx_hashes } => {
1549 let (acked, resync) = handle_send_tx_batch_result(
1550 &dispatch, &emitter, account_index, connection_epoch,
1551 clock_for_loop.get_time_ns(), &id, code, &message, &tx_hashes,
1552 );
1553
1554 if resync && ws_client.connection_epoch() == connection_epoch {
1555 nonce_ready_connection_epoch.store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
1556
1557 nonce_refresh_retry.clone().spawn(connection_epoch);
1558 }
1559 acked
1560 }
1561 _ => unreachable!("matched transaction response"),
1562 };
1563
1564 for pending in acked {
1565 let Some(credential) = credential_for_loop.clone() else { continue; };
1566
1567 if matches!(pending.kind, PendingSendTxKind::Modify { .. }) { continue; }
1568 spawn_acked_order_probe(
1569 &pending,
1570 AckedOrderProbeContext {
1571 http_client: http_client_for_loop.clone(),
1572 registry: Arc::clone(®istry_for_loop),
1573 credential,
1574 dispatch: dispatch.clone(),
1575 account_id: account_id_for_loop,
1576 clock: clock_for_loop,
1577 emitter: emitter.clone(),
1578 connection_epoch: ws_client.connection_epoch_atomic(),
1579 cancellation_token: cancellation_token.clone(),
1580 pending_tasks: pending_tasks_for_loop.clone(),
1581 },
1582 );
1583 }
1584 }
1585 Some(NautilusWsMessage::SendTxRejected {
1586 connection_epoch,
1587 source,
1588 code,
1589 message,
1590 tx_hash,
1591 }) => {
1592 let account_index = credential_for_loop
1593 .as_ref()
1594 .map(|c| c.account_index());
1595 let needs_nonce_resync = handle_send_tx_rejection_for_connection(
1596 &dispatch,
1597 &emitter,
1598 account_index,
1599 connection_epoch,
1600 clock_for_loop.get_time_ns(),
1601 source,
1602 code,
1603 &message,
1604 tx_hash.as_deref(),
1605 );
1606
1607 if needs_nonce_resync
1611 && let Some(credential) = &credential_for_loop
1612 {
1613 if ws_client.connection_epoch() != connection_epoch {
1614 log::warn!(
1615 "Skipping stale Lighter nonce rejection from connection \
1616 epoch {connection_epoch}",
1617 );
1618 continue;
1619 }
1620 nonce_ready_connection_epoch.store(
1621 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
1622 Ordering::Release,
1623 );
1624 let _refresh_guard = nonce_submission_gate.write().await;
1625
1626 match http_client_for_loop
1627 .get_next_nonce(
1628 credential.account_index(),
1629 credential.api_key_index(),
1630 )
1631 .await
1632 {
1633 Ok(response) => {
1634 dispatch.nonce_manager.refresh(
1635 credential.account_index(),
1636 credential.api_key_index(),
1637 response.nonce,
1638 );
1639 nonce_ready_connection_epoch
1640 .store(connection_epoch, Ordering::Release);
1641 log::debug!(
1642 "Hard-refreshed Lighter nonce after invalid-nonce \
1643 rejection: account_index={}, next_nonce={}",
1644 credential.account_index(),
1645 response.nonce,
1646 );
1647 }
1648 Err(e) => {
1649 log::warn!(
1650 "Failed to refresh Lighter nonce after \
1651 invalid-nonce rejection: {e}",
1652 );
1653 nonce_refresh_retry
1654 .clone()
1655 .spawn(connection_epoch);
1656 }
1657 }
1658 }
1659 }
1660 Some(NautilusWsMessage::Raw(value)) => {
1661 log::debug!("Unhandled Lighter raw frame on execution stream: {value}");
1662 }
1663 Some(NautilusWsMessage::AccountStreamFirstFrame(stream)) => {
1664 match stream {
1670 AccountStream::Orders => {
1671 dispatch.account_streams_ready.mark_orders();
1672 }
1673 AccountStream::Trades => {
1674 dispatch.account_streams_ready.mark_trades();
1675 }
1676 AccountStream::Positions => {
1677 dispatch.account_streams_ready.mark_positions();
1678 }
1679 AccountStream::Assets => {
1680 dispatch.account_streams_ready.mark_assets();
1681 }
1682 AccountStream::UserStats => {
1683 dispatch.account_streams_ready.mark_user_stats();
1684 }
1685 }
1686 }
1687 Some(
1691 NautilusWsMessage::Trades(_)
1692 | NautilusWsMessage::Quote(_)
1693 | NautilusWsMessage::Deltas(_)
1694 | NautilusWsMessage::Depth(_)
1695 | NautilusWsMessage::Bar(_)
1696 | NautilusWsMessage::MarkPrice(_)
1697 | NautilusWsMessage::IndexPrice(_)
1698 | NautilusWsMessage::FundingRate(_),
1699 ) => {}
1700 None => {
1701 log::debug!("Lighter execution next_event returned None");
1702 tokio::select! {
1703 () = cancellation_token.cancelled() => {
1704 log::debug!(
1705 "Lighter execution consumption loop cancelled"
1706 );
1707 break;
1708 }
1709 () = tokio::time::sleep(Duration::from_secs(1)) => {}
1710 }
1711 }
1712 }
1713 }
1714 }
1715 }
1716
1717 log::debug!("Lighter execution WebSocket consumption loop finished");
1718 })?;
1719
1720 if let Some(credential) = &self.credential {
1721 self.spawn_auth_token_refresh(credential.clone())?;
1722 }
1723
1724 Ok(())
1725 }
1726
1727 fn spawn_auth_token_refresh(&mut self, credential: Credential) -> anyhow::Result<()> {
1728 let ws_client = self.ws_client.clone();
1729 let cancellation_token = self.cancellation_token.clone();
1730 let account_index = credential.account_index();
1731 let channels = auth_token_rotation_channels(account_index);
1732 let refresh_notify = Arc::clone(&self.auth_refresh_notify);
1733
1734 self.auth_refresh_handle.spawn(async move {
1735 log::debug!(
1736 "Lighter auth-token refresh task started: interval={}s, account_index={account_index}",
1737 AUTH_TOKEN_REFRESH_INTERVAL.as_secs(),
1738 );
1739
1740 let mut next_refresh_delay = AUTH_TOKEN_REFRESH_INTERVAL;
1741
1742 loop {
1743 tokio::select! {
1744 biased;
1745 () = cancellation_token.cancelled() => {
1746 log::debug!("Lighter auth-token refresh task cancelled");
1747 break;
1748 }
1749 () = refresh_notify.notified() => {
1750 log::debug!(
1751 "Refreshing Lighter auth token immediately after reconnect: account_index={account_index}",
1752 );
1753 }
1754 () = tokio::time::sleep(next_refresh_delay) => {}
1755 }
1756
1757 let outcome = refresh_auth_token_until_rotated(
1758 &credential,
1759 &channels,
1760 &cancellation_token,
1761 AUTH_TOKEN_REFRESH_BACKOFF,
1762 build_auth_token_for,
1763 |channel, token| {
1764 let ws_client = ws_client.clone();
1765 async move { ws_client.subscribe_account(channel, token).await }
1766 },
1767 )
1768 .await;
1769
1770 if let Some(delay) = auth_token_refresh_next_delay(outcome) {
1771 if outcome == AuthTokenRefreshOutcome::Exhausted {
1772 log::error!(
1773 "Lighter auth-token rotation exhausted retry window; retrying again in {}s: account_index={account_index}",
1774 delay.as_secs(),
1775 );
1776 }
1777 next_refresh_delay = delay;
1778 } else {
1779 log::debug!("Lighter auth-token refresh task cancelled");
1780 break;
1781 }
1782 }
1783 })?;
1784 Ok(())
1785 }
1786
1787 fn resolve_slippage_bps(&self, params: Option<&Params>) -> anyhow::Result<u32> {
1789 let Some(value) = params.and_then(|p| p.get("market_order_slippage_bps")) else {
1790 return Ok(self.config.market_order_slippage_bps);
1791 };
1792
1793 let value = value
1794 .as_u64()
1795 .context("Lighter market_order_slippage_bps must be an unsigned integer")?;
1796 let value =
1797 u32::try_from(value).context("Lighter market_order_slippage_bps exceeds u32 range")?;
1798 validate_market_order_slippage_bps(value)?;
1799 Ok(value)
1800 }
1801
1802 fn build_tx_context(&self, credential: &Credential) -> anyhow::Result<ReservedTxContext> {
1803 let connection_epoch = self.ws_client.connection_epoch();
1804 anyhow::ensure!(
1805 self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
1806 TxPreparationError::NonceRefreshPending(connection_epoch),
1807 );
1808 let nonce_guard = Arc::clone(&self.nonce_submission_gate)
1809 .try_read_owned()
1810 .context("Lighter nonce refresh is in progress")?;
1811 anyhow::ensure!(
1812 self.ws_client.connection_epoch() == connection_epoch
1813 && self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
1814 TxPreparationError::StateChanged,
1815 );
1816
1817 let send_reservation = match self.tx_send_sequencer.reserve(
1818 credential.account_index(),
1819 credential.api_key_index(),
1820 || {
1821 self.dispatch
1822 .nonce_manager
1823 .next_nonce(credential.account_index(), credential.api_key_index())
1824 },
1825 ) {
1826 Ok(reservation) => reservation,
1827 Err(e @ NonceError::SkipWindowExhausted { .. }) => {
1828 self.spawn_nonce_window_recovery(credential);
1829 anyhow::bail!("failed to allocate Lighter nonce: {e}");
1830 }
1831 Err(e) => anyhow::bail!("failed to allocate Lighter nonce: {e}"),
1832 };
1833
1834 let nonce = send_reservation.nonce;
1835
1836 let now_ns = self.clock.get_time_ns().as_u64() as i64;
1837 let expired_at = (now_ns / 1_000_000).saturating_add(DEFAULT_TX_EXPIRY_MS);
1838 let send_reservation = send_reservation.with_nonce_ownership(nonce_guard, connection_epoch);
1839
1840 let context = TxContext {
1841 account_index: credential.account_index(),
1842 api_key_index: credential.api_key_index(),
1843 nonce,
1844 expired_at,
1845 };
1846
1847 Ok(ReservedTxContext {
1848 context,
1849 send_reservation,
1850 })
1851 }
1852
1853 fn spawn_nonce_window_recovery(&self, credential: &Credential) {
1854 if self.nonce_recovery_inflight.swap(true, Ordering::AcqRel) {
1855 return;
1856 }
1857
1858 let inflight = Arc::clone(&self.nonce_recovery_inflight);
1859 let http_client = self.http_client.clone();
1860 let dispatch = self.dispatch.clone();
1861 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
1862 let account_index = credential.account_index();
1863 let api_key_index = credential.api_key_index();
1864
1865 self.spawn_task("nonce_window_recovery", async move {
1866 let _refresh_guard = nonce_submission_gate.write().await;
1867 let result = http_client
1868 .get_next_nonce(account_index, api_key_index)
1869 .await;
1870 inflight.store(false, Ordering::Release);
1871
1872 match result {
1873 Ok(response) => {
1874 let _ = dispatch.nonce_manager.sync_from_venue(
1877 account_index,
1878 api_key_index,
1879 response.nonce,
1880 );
1881 log::debug!(
1882 "Resynced Lighter nonce baseline after skip-window exhaustion: \
1883 account_index={account_index}, api_key_index={api_key_index}, \
1884 next_nonce={}",
1885 response.nonce,
1886 );
1887 }
1888 Err(e) => {
1889 log::error!(
1890 "Failed to resync Lighter nonce after skip-window exhaustion: {e}",
1891 );
1892 }
1893 }
1894 Ok(())
1895 });
1896 }
1897
1898 fn dispatch_create_order_plan(
1899 &self,
1900 plan: &CreateOrderPlan,
1901 credential: &Credential,
1902 ) -> anyhow::Result<()> {
1903 let context = self.fanout_dispatch_context(credential)?;
1904 let prepared = context.sign_create_order(plan)?;
1905 self.spawn_task("submit_order", async move {
1906 context.send_create_order(prepared).await;
1907 Ok(())
1908 });
1909
1910 Ok(())
1911 }
1912
1913 fn prepare_create_order_plan(
1914 &self,
1915 order: &OrderAny,
1916 params: Option<&Params>,
1917 ) -> anyhow::Result<CreateOrderPlan> {
1918 let slippage_bps = self.resolve_slippage_bps(params)?;
1919 let attributes = self.resolve_create_attributes(params)?;
1920 let instrument_id = order.instrument_id();
1921 let market_index = self.registry.market_index(&instrument_id).ok_or_else(|| {
1922 anyhow::anyhow!("no Lighter market_index registered for instrument {instrument_id}")
1923 })?;
1924
1925 let instrument = self.core.cache().try_instrument(&instrument_id)?.clone();
1926
1927 let order_kind = nautilus_to_lighter_order_type(order.order_type())?;
1928
1929 let tif = nautilus_to_lighter_tif(
1930 order.order_type(),
1931 order.time_in_force(),
1932 order.is_post_only(),
1933 )?;
1934 let now_ms = (self.clock.get_time_ns().as_u64() / 1_000_000) as i64;
1935 let order_expiry = order_expiry_for(
1936 order.order_type(),
1937 &order.time_in_force(),
1938 order.expire_time(),
1939 now_ms,
1940 self.config.use_gtd,
1941 )?;
1942
1943 let base_amount = quantity_to_ticks(&order.quantity(), instrument.size_precision())?;
1944
1945 anyhow::ensure!(
1947 base_amount > 0,
1948 "quantity `{}` rounds to 0 ticks at size_precision {}",
1949 order.quantity(),
1950 instrument.size_precision(),
1951 );
1952 let price_precision = instrument.price_precision();
1953 let is_buy = matches!(order.order_side(), OrderSide::Buy);
1954
1955 let price_ticks = match order.order_type() {
1958 OrderType::Market => {
1959 let quote = self
1960 .core
1961 .cache()
1962 .quote(&instrument_id)
1963 .copied()
1964 .ok_or_else(|| {
1965 anyhow::anyhow!(
1966 "no cached quote for {instrument_id}: subscribe to quotes before submitting MARKET orders",
1967 )
1968 })?;
1969 let base = if is_buy {
1970 quote.ask_price.as_decimal()
1971 } else {
1972 quote.bid_price.as_decimal()
1973 };
1974 derive_market_order_price_ticks(base, is_buy, price_precision, slippage_bps)?
1975 }
1976 OrderType::StopMarket | OrderType::MarketIfTouched => {
1977 let trigger = order.trigger_price().ok_or_else(|| {
1978 anyhow::anyhow!("{:?} orders require a trigger_price", order.order_type(),)
1979 })?;
1980 derive_market_order_price_ticks(
1981 trigger.as_decimal(),
1982 is_buy,
1983 price_precision,
1984 slippage_bps,
1985 )?
1986 }
1987 _ => order
1988 .price()
1989 .map(|p| price_to_ticks(&p, price_precision))
1990 .transpose()?
1991 .unwrap_or(0),
1992 };
1993
1994 let trigger_price_ticks = order
1995 .trigger_price()
1996 .map(|p| price_to_ticks(&p, price_precision))
1997 .transpose()?
1998 .unwrap_or(0);
1999
2000 if matches!(
2003 order.order_type(),
2004 OrderType::StopMarket
2005 | OrderType::StopLimit
2006 | OrderType::MarketIfTouched
2007 | OrderType::LimitIfTouched
2008 ) {
2009 anyhow::ensure!(
2010 trigger_price_ticks > 0,
2011 "trigger_price `{:?}` rounds to 0 ticks at precision {price_precision}",
2012 order.trigger_price(),
2013 );
2014 }
2015 validate_order_amount(&instrument, order.quantity(), price_ticks, price_precision)?;
2016
2017 Ok(CreateOrderPlan {
2018 order: order.clone(),
2019 market_index,
2020 base_amount,
2021 price: price_ticks,
2022 order_type: order_kind as u8,
2023 time_in_force: tif as u8,
2024 trigger_price: trigger_price_ticks,
2025 order_expiry,
2026 attributes,
2027 })
2028 }
2029
2030 fn resolve_create_attributes(&self, params: Option<&Params>) -> anyhow::Result<L2TxAttributes> {
2031 let mut attributes = integrator_attributes(self.integrator_account_index());
2032
2033 let behavior = match params.and_then(|p| p.get("self_trade_behavior_mode")) {
2034 Some(value) => value
2035 .as_str()
2036 .context("Lighter self_trade_behavior_mode must be a string")?
2037 .parse::<LighterSelfTradeBehaviorMode>()
2038 .context("invalid Lighter self_trade_behavior_mode")?,
2039 None => LighterSelfTradeBehaviorMode::default(),
2040 };
2041
2042 let equality = match params.and_then(|p| p.get("self_trade_equality_mode")) {
2043 Some(value) => value
2044 .as_str()
2045 .context("Lighter self_trade_equality_mode must be a string")?
2046 .parse::<LighterSelfTradeEqualityMode>()
2047 .context("invalid Lighter self_trade_equality_mode")?,
2048 None => LighterSelfTradeEqualityMode::default(),
2049 };
2050
2051 attributes.self_trade_behavior_mode = Some(behavior);
2052 attributes.self_trade_equality_mode = Some(equality);
2053 attributes.validate()?;
2054 Ok(attributes)
2055 }
2056
2057 fn prepare_create_group_plan(
2058 &self,
2059 orders: &[OrderAny],
2060 params: Option<&Params>,
2061 ) -> anyhow::Result<CreateGroupPlan> {
2062 let (grouping_type, indices) = grouped_order_indices(orders)?;
2063 let mut plans = Vec::with_capacity(indices.len());
2064 for index in indices {
2065 let order = &orders[index];
2066 let instrument = self
2067 .core
2068 .cache()
2069 .instrument(&order.instrument_id())
2070 .cloned();
2071
2072 if let Some(reason) = local_submit_denial_reason(order, instrument.as_ref()) {
2073 return Err(reason);
2074 }
2075
2076 anyhow::ensure!(
2077 !is_lighter_spot_order(order, instrument.as_ref()),
2078 "Lighter grouped orders require perpetual markets"
2079 );
2080 plans.push(self.prepare_create_order_plan(order, params)?);
2081 }
2082
2083 prepare_group_bodies(grouping_type, &mut plans, self.config.use_gtd)?;
2084 Ok(CreateGroupPlan {
2085 grouping_type,
2086 orders: plans,
2087 })
2088 }
2089
2090 fn fanout_dispatch_context(
2091 &self,
2092 credential: &Credential,
2093 ) -> anyhow::Result<FanoutDispatchContext> {
2094 let pending_tasks = self
2095 .pending_tasks
2096 .spawner()
2097 .context("Lighter task admission is closed")?;
2098 Ok(FanoutDispatchContext {
2099 clock: self.clock,
2100 chain_id: self.config.chain_id(),
2101 integrator_account_index: self.integrator_account_index(),
2102 emitter: self.emitter.clone(),
2103 credential: credential.clone(),
2104 http_client: self.http_client.clone(),
2105 registry: Arc::clone(&self.registry),
2106 ws_client: self.ws_client.clone(),
2107 tx_rate_limiter: Arc::clone(&self.tx_rate_limiter),
2108 tx_send_sequencer: self.tx_send_sequencer.clone(),
2109 nonce_submission_gate: Arc::clone(&self.nonce_submission_gate),
2110 nonce_ready_connection_epoch: Arc::clone(&self.nonce_ready_connection_epoch),
2111 dispatch: self.dispatch.clone(),
2112 nonce_recovery_inflight: Arc::clone(&self.nonce_recovery_inflight),
2113 pending_tasks,
2114 })
2115 }
2116
2117 fn dispatch_signed_cancel_order(&self, cmd: &CancelOrder, credential: &Credential) {
2118 let context = match self.fanout_dispatch_context(credential) {
2119 Ok(context) => context,
2120 Err(e) => {
2121 log::warn!("Skipping Lighter cancel_order after shutdown began: {e}");
2122 self.deny_cancel_commands(std::slice::from_ref(cmd), &e);
2123 return;
2124 }
2125 };
2126
2127 let emit_cancel_rejected = self.can_emit_order_cancel_rejected(&cmd.client_order_id);
2128
2129 let prepared = match self
2130 .prepare_cancel_order_plan(cmd)
2131 .and_then(|plan| context.sign_cancel_order(&plan))
2132 {
2133 Ok(prepared) => prepared,
2134 Err(e) => {
2135 let reason = format!("Lighter cancel_order failed: {e}");
2136
2137 log::warn!("{reason} for {}", cmd.client_order_id);
2138 self.emitter.emit_order_cancel_denied_event(
2139 cmd.strategy_id,
2140 cmd.instrument_id,
2141 cmd.client_order_id,
2142 cmd.venue_order_id,
2143 &reason,
2144 self.clock.get_time_ns(),
2145 Some(cmd.command_id),
2146 );
2147
2148 return;
2149 }
2150 };
2151
2152 self.dispatch.register_cancel_command(
2153 cmd.client_order_id,
2154 cmd.command_id,
2155 emit_cancel_rejected,
2156 );
2157
2158 self.spawn_task("cancel_order", async move {
2159 context
2160 .send_cancel_order(prepared, emit_cancel_rejected)
2161 .await;
2162 Ok(())
2163 });
2164 }
2165
2166 fn dispatch_cancel_orders(&self, cancels: &[CancelOrder]) -> anyhow::Result<()> {
2167 let credential = self
2168 .credential
2169 .as_ref()
2170 .ok_or_else(|| {
2171 anyhow::anyhow!("Lighter execution client cannot cancel without credentials")
2172 })
2173 .inspect_err(|e| self.deny_cancel_commands(cancels, e))?;
2174
2175 if cancels.is_empty() {
2176 log::debug!("batch_cancel_orders called with empty cancel list");
2177 return Ok(());
2178 }
2179
2180 let context = self
2181 .fanout_dispatch_context(credential)
2182 .inspect_err(|e| self.deny_cancel_commands(cancels, e))?;
2183
2184 let mut plans = Vec::with_capacity(cancels.len());
2185 let mut failures = Vec::new();
2186
2187 let command_ids: AHashMap<_, _> = cancels
2188 .iter()
2189 .map(|cancel| (cancel.client_order_id, cancel.command_id))
2190 .collect();
2191
2192 for cancel in cancels {
2193 if command_ids
2194 .get(&cancel.client_order_id)
2195 .is_some_and(|command_id| *command_id != cancel.command_id)
2196 {
2197 let (strategy_id, instrument_id, venue_order_id) =
2198 self.core.get_order(&cancel.client_order_id).map_or(
2199 (
2200 cancel.strategy_id,
2201 cancel.instrument_id,
2202 cancel.venue_order_id,
2203 ),
2204 |order| {
2205 (
2206 order.strategy_id(),
2207 order.instrument_id(),
2208 order.venue_order_id(),
2209 )
2210 },
2211 );
2212
2213 self.emitter.emit_order_cancel_denied_event(
2214 strategy_id,
2215 instrument_id,
2216 cancel.client_order_id,
2217 venue_order_id,
2218 "Duplicate cancel target in batch",
2219 self.clock.get_time_ns(),
2220 Some(cancel.command_id),
2221 );
2222 continue;
2223 }
2224
2225 match self.prepare_cancel_order_plan(cancel) {
2226 Ok(plan) => {
2227 let group = self
2228 .prepare_oco_cancel_plans(cancel)
2229 .unwrap_or_else(|| vec![plan]);
2230 plans.extend(group.into_iter().map(|plan| {
2231 let emit = self.can_emit_order_cancel_rejected(&plan.client_order_id);
2232 (plan, emit)
2233 }));
2234 }
2235 Err(e) => failures.push((cancel, format!("Lighter cancel_order failed: {e}"))),
2236 }
2237 }
2238
2239 for (plan, _) in &mut plans {
2240 plan.command_id = command_ids.get(&plan.client_order_id).copied();
2241 }
2242
2243 let mut seen = AHashSet::new();
2244 plans.retain(|(plan, _)| seen.insert(plan.client_order_id));
2245 for (cancel, reason) in failures {
2246 if !seen.contains(&cancel.client_order_id) {
2247 self.emitter.emit_order_cancel_denied_event(
2248 cancel.strategy_id,
2249 cancel.instrument_id,
2250 cancel.client_order_id,
2251 cancel.venue_order_id,
2252 &reason,
2253 self.clock.get_time_ns(),
2254 Some(cancel.command_id),
2255 );
2256 }
2257 }
2258
2259 for (plan, emit) in &plans {
2260 if let Some(command_id) = plan.command_id {
2261 self.dispatch
2262 .register_cancel_command(plan.client_order_id, command_id, *emit);
2263 }
2264 }
2265
2266 if plans.is_empty() {
2267 return Ok(());
2268 }
2269
2270 self.spawn_task("batch_cancel_orders", async move {
2271 context
2272 .dispatch_batch_plans(
2273 plans
2274 .into_iter()
2275 .map(|(plan, emit)| BatchOrderPlan::Cancel(plan, emit))
2276 .collect(),
2277 )
2278 .await;
2279 Ok(())
2280 });
2281
2282 Ok(())
2283 }
2284
2285 fn can_emit_order_cancel_rejected(&self, client_order_id: &ClientOrderId) -> bool {
2286 self.core
2287 .cache()
2288 .order(client_order_id)
2289 .is_none_or(|order| order.is_pending_cancel())
2290 }
2291
2292 fn prepare_cancel_order_plan(&self, cmd: &CancelOrder) -> anyhow::Result<CancelOrderPlan> {
2293 let market_index = self
2294 .registry
2295 .market_index(&cmd.instrument_id)
2296 .ok_or_else(|| {
2297 anyhow::anyhow!(
2298 "no Lighter market_index registered for instrument {}",
2299 cmd.instrument_id,
2300 )
2301 })?;
2302
2303 self.core.cache().try_order(&cmd.client_order_id)?;
2304
2305 let voi = cmd
2308 .venue_order_id
2309 .or_else(|| self.dispatch.lookup_venue_order_id(&cmd.client_order_id))
2310 .ok_or_else(|| {
2311 anyhow::anyhow!(
2312 "cannot cancel Lighter order {}: venue order_id not yet known \
2313 (await OrderAccepted before issuing cancel)",
2314 cmd.client_order_id,
2315 )
2316 })?;
2317
2318 let venue_index: i64 = voi
2319 .as_str()
2320 .parse()
2321 .with_context(|| format!("Lighter venue_order_id `{voi}` is not an integer index"))?;
2322
2323 Ok(CancelOrderPlan {
2324 command_id: Some(cmd.command_id),
2325 client_order_id: cmd.client_order_id,
2326 strategy_id: cmd.strategy_id,
2327 instrument_id: cmd.instrument_id,
2328 venue_order_id: Some(voi),
2329 market_index,
2330 venue_index,
2331 })
2332 }
2333
2334 fn deny_modify_commands(&self, commands: &[ModifyOrder], error: &anyhow::Error) {
2335 let reason = error.to_string();
2336
2337 for command in commands {
2338 self.emitter.emit_order_modify_denied_event(
2339 command.strategy_id,
2340 command.instrument_id,
2341 command.client_order_id,
2342 command.venue_order_id,
2343 &reason,
2344 self.clock.get_time_ns(),
2345 Some(command.command_id),
2346 );
2347 }
2348 }
2349
2350 fn deny_cancel_commands(&self, commands: &[CancelOrder], error: &anyhow::Error) {
2351 let reason = error.to_string();
2352
2353 for command in commands {
2354 self.emitter.emit_order_cancel_denied_event(
2355 command.strategy_id,
2356 command.instrument_id,
2357 command.client_order_id,
2358 command.venue_order_id,
2359 &reason,
2360 self.clock.get_time_ns(),
2361 Some(command.command_id),
2362 );
2363 }
2364 }
2365
2366 fn dispatch_signed_modify_order(&self, cmd: &ModifyOrder, credential: &Credential) {
2367 let probe_context = match self.fanout_dispatch_context(credential) {
2368 Ok(context) => context.probe_context(),
2369 Err(e) => {
2370 log::warn!("Lighter modify probe unavailable: {e}");
2371 self.deny_modify_commands(std::slice::from_ref(cmd), &e);
2372 return;
2373 }
2374 };
2375
2376 if !self.admit_modify_order(cmd) {
2377 return;
2378 }
2379
2380 let prepared = match self.prepare_signed_modify_order(cmd, credential) {
2381 Ok(prepared) => prepared,
2382 Err(e) => {
2383 let mut events = self.dispatch.order_events.lock();
2384 self.dispatch
2385 .clear_pending_modify_if_command(&cmd.client_order_id, cmd.command_id);
2386 let reason = format!("Lighter modify_order failed: {e}");
2387 log::warn!("{reason} for {}", cmd.client_order_id);
2388 self.emitter.emit_order_modify_denied_event(
2389 cmd.strategy_id,
2390 cmd.instrument_id,
2391 cmd.client_order_id,
2392 cmd.venue_order_id,
2393 &reason,
2394 self.clock.get_time_ns(),
2395 Some(cmd.command_id),
2396 );
2397
2398 if let Some(deferred) = events.remove(&cmd.client_order_id) {
2399 flush_modify_events(
2400 cmd.client_order_id,
2401 deferred,
2402 &self.dispatch,
2403 &self.emitter,
2404 &mut events,
2405 );
2406 }
2407
2408 return;
2409 }
2410 };
2411 let PreparedModifyOrder {
2412 command_id,
2413 client_order_id,
2414 strategy_id,
2415 instrument_id,
2416 venue_order_id,
2417 tx_info,
2418 nonce,
2419 api_key_index,
2420 tx_hash,
2421 mut send_reservation,
2422 } = prepared;
2423 let connection_epoch = send_reservation.connection_epoch;
2424
2425 let ws_client = self.ws_client.clone();
2426 let dispatch = self.dispatch.clone();
2427 let credential = credential.clone();
2428 let emitter = self.emitter.clone();
2429 let clock = self.clock;
2430
2431 let tx_rate_limiter = self.tx_rate_limiter.clone();
2432
2433 self.spawn_task("modify_order", async move {
2434 send_reservation.wait_for_turn().await;
2435 await_tx_quota(&tx_rate_limiter).await;
2436
2437 let pending = PendingSendTx {
2438 batch_id: None,
2439 connection_epoch,
2440 kind: PendingSendTxKind::Modify {
2441 command_id,
2442 strategy_id,
2443 instrument_id,
2444 client_order_id,
2445 venue_order_id,
2446 },
2447 submitted_at: clock.get_time_ns(),
2448 nonce,
2449 api_key_index,
2450 tx_hash,
2451 };
2452
2453 dispatch.enqueue_pending_sendtx(pending.clone());
2454 spawn_acked_order_probe(&pending, probe_context);
2455
2456 if let Err(e) = ws_client
2457 .send_tx_on_connection(LighterTxType::ModifyOrder as u8, tx_info, connection_epoch)
2458 .await
2459 {
2460 let failure = classify_lighter_ws_command_failure("modify_order", &e);
2461 let reason = command_failure_reason(&failure);
2462 if matches!(&failure, CommandFailure::Ambiguous(_)) {
2463 log::warn!(
2464 "Lighter modify_order dispatch outcome unknown for {client_order_id}: {reason}; \
2465 retaining pending state for venue reconciliation; diagnostic={e:?}",
2466 );
2467 } else {
2468 log::warn!("{reason} for {client_order_id}; diagnostic={e:?}");
2469 dispatch.remove_pending_sendtx_by_nonce(connection_epoch, nonce);
2470 rollback_tx_dispatch(&dispatch, &credential, None, nonce);
2471 reject_modify_order(
2472 &dispatch,
2473 &emitter,
2474 &pending,
2475 &failure,
2476 clock.get_time_ns(),
2477 false,
2478 );
2479 }
2480 }
2481 send_reservation.release();
2482 Ok(())
2483 });
2484 }
2485
2486 fn admit_modify_order(&self, cmd: &ModifyOrder) -> bool {
2487 let _events = self.dispatch.order_events.lock();
2488
2489 let publication_pending = self
2490 .dispatch
2491 .order_identity(&cmd.client_order_id)
2492 .is_some_and(|identity| {
2493 identity
2494 .publication_pending(self.core.cache().order(&cmd.client_order_id).as_deref())
2495 });
2496
2497 if !publication_pending && self.dispatch.set_pending_modify_action(cmd) {
2498 return true;
2499 }
2500
2501 self.emitter.emit_order_modify_denied_event(
2502 cmd.strategy_id,
2503 cmd.instrument_id,
2504 cmd.client_order_id,
2505 cmd.venue_order_id,
2506 "Lighter permits one unresolved modification per order; wait for its update or rejection",
2507 self.clock.get_time_ns(),
2508 Some(cmd.command_id),
2509 );
2510 false
2511 }
2512
2513 fn prepare_signed_modify_order(
2514 &self,
2515 cmd: &ModifyOrder,
2516 credential: &Credential,
2517 ) -> anyhow::Result<PreparedModifyOrder> {
2518 let plan = self.prepare_modify_order_plan(cmd)?;
2519 let context = self.fanout_dispatch_context(credential)?;
2520 let reserved = self.build_tx_context(credential)?;
2521 context.sign_modify_order(&plan, reserved)
2522 }
2523
2524 fn prepare_modify_order_plan(&self, cmd: &ModifyOrder) -> anyhow::Result<ModifyOrderPlan> {
2525 let market_index = self
2526 .registry
2527 .market_index(&cmd.instrument_id)
2528 .ok_or_else(|| {
2529 anyhow::anyhow!(
2530 "no Lighter market_index registered for instrument {}",
2531 cmd.instrument_id,
2532 )
2533 })?;
2534
2535 let voi = cmd
2536 .venue_order_id
2537 .or_else(|| self.dispatch.lookup_venue_order_id(&cmd.client_order_id))
2538 .ok_or_else(|| {
2539 anyhow::anyhow!(
2540 "cannot modify Lighter order {}: venue order_id not yet known \
2541 (await OrderAccepted before issuing modify)",
2542 cmd.client_order_id,
2543 )
2544 })?;
2545
2546 let venue_index: i64 = voi
2547 .as_str()
2548 .parse()
2549 .with_context(|| format!("Lighter venue_order_id `{voi}` is not an integer index"))?;
2550
2551 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
2552 let instrument = self
2553 .core
2554 .cache()
2555 .try_instrument(&cmd.instrument_id)?
2556 .clone();
2557
2558 let new_qty = cmd.quantity.unwrap_or(order.quantity());
2559 anyhow::ensure!(
2560 !(order.parent_order_id().is_some()
2561 && self
2562 .dispatch
2563 .snapshot_for(&cmd.client_order_id)
2564 .is_some_and(|shape| shape.quantity.is_zero())),
2565 "Lighter cannot modify an OTO child while it waits for parent execution"
2566 );
2567 let price_precision = instrument.price_precision();
2568 let new_trigger = cmd.trigger_price.or(order.trigger_price());
2569 let triggered = order.is_triggered() == Some(true)
2570 || self.dispatch.triggered_was_emitted(&cmd.client_order_id);
2571 anyhow::ensure!(
2572 !triggered || cmd.trigger_price.is_none(),
2573 "Lighter cannot modify a trigger price after the order has triggered"
2574 );
2575
2576 let price_ticks = match order.order_type() {
2579 OrderType::StopMarket | OrderType::MarketIfTouched => {
2580 let trigger = new_trigger.ok_or_else(|| {
2581 anyhow::anyhow!("{:?} orders require a trigger_price", order.order_type())
2582 })?;
2583 let is_buy = matches!(order.order_side(), OrderSide::Buy);
2584 let slippage_bps = self.resolve_slippage_bps(cmd.params.as_ref())?;
2585
2586 derive_market_order_price_ticks(
2587 trigger.as_decimal(),
2588 is_buy,
2589 price_precision,
2590 slippage_bps,
2591 )?
2592 }
2593 _ => {
2594 let new_price = cmd.price.or(order.price()).ok_or_else(|| {
2595 anyhow::anyhow!("modify_order requires a price (none on order or command)")
2596 })?;
2597
2598 price_to_ticks(&new_price, price_precision)?
2599 }
2600 };
2601
2602 let base_amount = quantity_to_ticks(&new_qty, instrument.size_precision())?;
2603 anyhow::ensure!(
2604 base_amount > 0,
2605 "quantity `{new_qty}` rounds to 0 ticks at size_precision {}",
2606 instrument.size_precision(),
2607 );
2608
2609 let reduced_qty = self
2611 .dispatch
2612 .order_identity(&cmd.client_order_id)
2613 .map_or(Decimal::ZERO, |identity| identity.reduced_qty());
2614 let reduced_qty = quantity_from_decimal(reduced_qty, instrument.size_precision())?;
2615 let reduced_amount = quantity_to_ticks(&reduced_qty, instrument.size_precision())?;
2616 let base_amount = base_amount
2617 .checked_add(reduced_amount)
2618 .context("Lighter modified quantity overflow")?;
2619
2620 let trigger_price_ticks = match new_trigger.filter(|_| !triggered) {
2621 Some(trigger) if !trigger.is_zero() => price_to_ticks(&trigger, price_precision)?,
2622 _ => 0,
2623 };
2624
2625 if !triggered
2626 && matches!(
2627 order.order_type(),
2628 OrderType::StopMarket
2629 | OrderType::StopLimit
2630 | OrderType::MarketIfTouched
2631 | OrderType::LimitIfTouched
2632 )
2633 {
2634 anyhow::ensure!(
2635 trigger_price_ticks > 0,
2636 "trigger_price `{new_trigger:?}` rounds to 0 ticks at precision {price_precision}",
2637 );
2638 }
2639 validate_order_amount(&instrument, new_qty, price_ticks, price_precision)?;
2640 self.dispatch.set_pending_modify_shape(
2641 cmd,
2642 OrderShapeSnapshot {
2643 quantity: parse_quantity_from_ticks(base_amount, instrument.size_precision())?,
2644 price: LIMIT_ORDER_TYPES
2645 .contains(&order.order_type())
2646 .then(|| parse_price_from_ticks(price_ticks, price_precision))
2647 .transpose()?,
2648 trigger_price: STOP_ORDER_TYPES
2649 .contains(&order.order_type())
2650 .then(|| {
2651 if triggered {
2652 Ok(None)
2653 } else {
2654 parse_price_from_ticks(trigger_price_ticks, price_precision).map(Some)
2655 }
2656 })
2657 .transpose()?
2658 .flatten(),
2659 },
2660 );
2661
2662 Ok(ModifyOrderPlan {
2663 command_id: cmd.command_id,
2664 client_order_id: cmd.client_order_id,
2665 strategy_id: cmd.strategy_id,
2666 instrument_id: cmd.instrument_id,
2667 venue_order_id: Some(voi),
2668 market_index,
2669 venue_index,
2670 base_amount,
2671 price: price_ticks,
2672 trigger_price: trigger_price_ticks,
2673 })
2674 }
2675
2676 pub fn update_leverage(
2695 &self,
2696 instrument_id: InstrumentId,
2697 initial_margin_fraction: u16,
2698 margin_mode: LighterPositionMarginMode,
2699 ) -> anyhow::Result<()> {
2700 let credential = self.credential.as_ref().ok_or_else(|| {
2701 anyhow::anyhow!("Lighter execution client cannot update leverage without credentials")
2702 })?;
2703
2704 let market_index = self.registry.market_index(&instrument_id).ok_or_else(|| {
2705 anyhow::anyhow!("no Lighter market_index registered for instrument {instrument_id}")
2706 })?;
2707
2708 anyhow::ensure!(
2709 (1..=10_000).contains(&initial_margin_fraction),
2710 "initial_margin_fraction must be in 1..=10_000, was {initial_margin_fraction}",
2711 );
2712
2713 let ReservedTxContext {
2714 context,
2715 mut send_reservation,
2716 } = self.build_tx_context(credential)?;
2717
2718 let connection_epoch = send_reservation.connection_epoch;
2719
2720 let captured_nonce = context.nonce;
2721 let captured_api_key_index = context.api_key_index;
2722 let mut rollback_guard =
2723 TxDispatchGuard::new(self.dispatch.clone(), credential, None, captured_nonce);
2724
2725 let tx = UpdateLeverageTxInfo {
2726 context,
2727 market_index,
2728 initial_margin_fraction,
2729 margin_mode: margin_mode as u8,
2730 skip_nonce: 0,
2731 };
2732
2733 let signed = sign_tx(
2734 &tx,
2735 self.config.chain_id(),
2736 &credential.private_key()?,
2737 fresh_k(),
2738 )?;
2739
2740 let tx_info_str = TxInfoJson::update_leverage(&tx, &signed);
2741 let tx_info = serde_json::value::RawValue::from_string(tx_info_str)
2742 .context("failed to wrap signed Lighter update_leverage tx_info JSON")?;
2743
2744 rollback_guard.disarm();
2745 let captured_tx_hash = signed.tx_hash_hex();
2746
2747 let ws_client = self.ws_client.clone();
2748 let dispatch = self.dispatch.clone();
2749 let credential = credential.clone();
2750 let clock = self.clock;
2751
2752 let tx_rate_limiter = self.tx_rate_limiter.clone();
2753
2754 self.spawn_task("update_leverage", async move {
2755 send_reservation.wait_for_turn().await;
2756 await_tx_quota(&tx_rate_limiter).await;
2757 dispatch.enqueue_pending_sendtx(PendingSendTx {
2758 batch_id: None,
2759 connection_epoch,
2760 kind: PendingSendTxKind::Other,
2761 submitted_at: clock.get_time_ns(),
2762 nonce: captured_nonce,
2763 api_key_index: captured_api_key_index,
2764 tx_hash: captured_tx_hash,
2765 });
2766
2767 if let Err(e) = ws_client
2768 .send_tx_on_connection(
2769 LighterTxType::UpdateLeverage as u8,
2770 tx_info,
2771 connection_epoch,
2772 )
2773 .await
2774 {
2775 let failure = classify_lighter_ws_command_failure("update_leverage", &e);
2776 let reason = command_failure_reason(&failure);
2777 if matches!(&failure, CommandFailure::Ambiguous(_)) {
2778 log::warn!(
2779 "Lighter update_leverage dispatch outcome unknown for {instrument_id}: \
2780 {reason}; retaining pending nonce for venue reconciliation; diagnostic={e:?}",
2781 );
2782 } else {
2783 log::error!("{reason} for {instrument_id}; diagnostic={e:?}");
2784 dispatch.remove_pending_sendtx_by_nonce(connection_epoch, captured_nonce);
2785 rollback_tx_dispatch(&dispatch, &credential, None, captured_nonce);
2786 }
2787 }
2788 send_reservation.release();
2789 Ok(())
2790 });
2791
2792 Ok(())
2793 }
2794}
2795
2796fn emit_lighter_position_reports(
2797 reports: Vec<PositionStatusReport>,
2798 removed: Vec<InstrumentId>,
2799 emitter: &ExecutionEventEmitter,
2800 account_id: AccountId,
2801 now: UnixNanos,
2802) {
2803 for report in reports {
2804 log::debug!(
2805 "Lighter PositionStatusReport: instrument={} side={:?} qty={}",
2806 report.instrument_id,
2807 report.position_side,
2808 report.quantity,
2809 );
2810 emitter.send_position_report(report);
2811 }
2812
2813 for instrument_id in removed {
2815 let flat = PositionStatusReport::new(
2816 account_id,
2817 instrument_id,
2818 PositionSide::Flat,
2819 Quantity::zero(0),
2820 now,
2821 now,
2822 Some(UUID4::new()),
2823 None,
2824 None,
2825 );
2826 emitter.send_position_report(flat);
2827 }
2828}
2829
2830#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2831enum AuthTokenRefreshOutcome {
2832 Rotated,
2833 Cancelled,
2834 Exhausted,
2835}
2836
2837#[derive(Debug, Clone, Copy)]
2838struct AuthTokenRefreshBackoff {
2839 initial_delay: Duration,
2840 max_delay: Duration,
2841 window: Duration,
2842}
2843
2844#[derive(Clone)]
2845struct NonceRefreshRetry {
2846 http_client: LighterHttpClient,
2847 dispatch: WsDispatchState,
2848 credential: Option<Credential>,
2849 submission_gate: Arc<tokio::sync::RwLock<()>>,
2850 ready_connection_epoch: Arc<AtomicU64>,
2851 ws_client: LighterWebSocketClient,
2852 cancellation_token: CancellationToken,
2853 pending_tasks: TaskSpawner,
2854}
2855
2856impl NonceRefreshRetry {
2857 fn spawn(self, connection_epoch: u64) {
2858 let pending_tasks = self.pending_tasks.clone();
2859
2860 let future = async move {
2861 let Some(credential) = self.credential else {
2862 return;
2863 };
2864 let mut retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
2865
2866 loop {
2867 tokio::select! {
2868 () = self.cancellation_token.cancelled() => return,
2869 () = tokio::time::sleep(retry_delay) => {}
2870 }
2871
2872 if self.ws_client.connection_epoch() != connection_epoch
2873 || self.ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
2874 {
2875 return;
2876 }
2877
2878 let _refresh_guard = tokio::select! {
2879 () = self.cancellation_token.cancelled() => return,
2880 guard = self.submission_gate.write() => guard,
2881 };
2882
2883 if self.ws_client.connection_epoch() != connection_epoch
2884 || self.ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
2885 {
2886 return;
2887 }
2888
2889 let result = tokio::select! {
2890 () = self.cancellation_token.cancelled() => return,
2891 result = self.http_client.get_next_nonce(
2892 credential.account_index(),
2893 credential.api_key_index(),
2894 ) => result,
2895 };
2896
2897 match result {
2898 Ok(response) => {
2899 if self.ws_client.connection_epoch() != connection_epoch {
2900 return;
2901 }
2902 self.dispatch.nonce_manager.refresh(
2903 credential.account_index(),
2904 credential.api_key_index(),
2905 response.nonce,
2906 );
2907 self.ready_connection_epoch
2908 .store(connection_epoch, Ordering::Release);
2909 log::info!(
2910 "Recovered Lighter nonce refresh: account_index={}, \
2911 connection_epoch={connection_epoch}, next_nonce={}",
2912 credential.account_index(),
2913 response.nonce,
2914 );
2915 return;
2916 }
2917 Err(e) => {
2918 log::warn!(
2919 "Failed to retry Lighter nonce refresh for connection epoch \
2920 {connection_epoch}: {e}",
2921 );
2922 }
2923 }
2924
2925 retry_delay = retry_delay
2926 .saturating_mul(2)
2927 .min(NONCE_REFRESH_RETRY_MAX_DELAY);
2928 }
2929 };
2930
2931 if let Err(e) = pending_tasks.spawn(future) {
2932 log::debug!("Skipping Lighter nonce refresh retry during shutdown: {e}");
2933 }
2934 }
2935}
2936
2937fn auth_token_rotation_channels(account_index: i64) -> [LighterWsChannel; 6] {
2938 [
2939 LighterWsChannel::AccountAllOrders(account_index),
2940 LighterWsChannel::AccountAllTrades(account_index),
2941 LighterWsChannel::AccountAllPositions(account_index),
2942 LighterWsChannel::AccountAllAssets(account_index),
2943 LighterWsChannel::AccountTx(account_index),
2944 LighterWsChannel::UserStats(account_index),
2945 ]
2946}
2947
2948async fn refresh_auth_token_until_rotated<MintToken, Subscribe, SubscribeFuture>(
2949 credential: &Credential,
2950 channels: &[LighterWsChannel],
2951 cancellation_token: &CancellationToken,
2952 backoff: AuthTokenRefreshBackoff,
2953 mut mint_token: MintToken,
2954 mut subscribe: Subscribe,
2955) -> AuthTokenRefreshOutcome
2956where
2957 MintToken: FnMut(&Credential) -> anyhow::Result<SecretString>,
2958 Subscribe: FnMut(LighterWsChannel, SecretString) -> SubscribeFuture,
2959 SubscribeFuture: Future<Output = Result<(), crate::websocket::error::LighterWsError>>,
2960{
2961 let retry_started = tokio::time::Instant::now();
2962 let mut retry_delay = backoff.initial_delay.min(backoff.max_delay);
2963 let mut attempt = 1_u32;
2964
2965 loop {
2966 match rotate_auth_token_once(credential, channels, &mut mint_token, &mut subscribe).await {
2967 Ok(()) => {
2968 log::debug!(
2969 "Lighter auth-token rotated for account_index={}, attempts={attempt}",
2970 credential.account_index(),
2971 );
2972 return AuthTokenRefreshOutcome::Rotated;
2973 }
2974 Err(e) => {
2975 log::warn!("Lighter auth-token rotation attempt {attempt} failed: {e:#}");
2976 }
2977 }
2978
2979 let Some(remaining) = backoff.window.checked_sub(retry_started.elapsed()) else {
2980 log::error!(
2981 "Lighter auth-token rotation retry window exhausted: account_index={}, attempts={attempt}",
2982 credential.account_index(),
2983 );
2984 return AuthTokenRefreshOutcome::Exhausted;
2985 };
2986
2987 if remaining.as_nanos() == 0 {
2988 log::error!(
2989 "Lighter auth-token rotation retry window exhausted: account_index={}, attempts={attempt}",
2990 credential.account_index(),
2991 );
2992 return AuthTokenRefreshOutcome::Exhausted;
2993 }
2994
2995 let delay = retry_delay.min(remaining);
2996 log::warn!(
2997 "Retrying Lighter auth-token rotation in {:.3}s: account_index={}, attempts={attempt}",
2998 delay.as_secs_f64(),
2999 credential.account_index(),
3000 );
3001
3002 if !sleep_or_auth_token_refresh_cancelled(delay, cancellation_token).await {
3003 return AuthTokenRefreshOutcome::Cancelled;
3004 }
3005
3006 retry_delay = next_auth_token_refresh_retry_delay(retry_delay, backoff.max_delay);
3007 attempt = attempt.saturating_add(1);
3008 }
3009}
3010
3011async fn rotate_auth_token_once<MintToken, Subscribe, SubscribeFuture>(
3012 credential: &Credential,
3013 channels: &[LighterWsChannel],
3014 mint_token: &mut MintToken,
3015 subscribe: &mut Subscribe,
3016) -> anyhow::Result<()>
3017where
3018 MintToken: FnMut(&Credential) -> anyhow::Result<SecretString>,
3019 Subscribe: FnMut(LighterWsChannel, SecretString) -> SubscribeFuture,
3020 SubscribeFuture: Future<Output = Result<(), crate::websocket::error::LighterWsError>>,
3021{
3022 let token =
3023 mint_token(credential).context("failed to mint Lighter auth token during rotation")?;
3024 let mut first_error = None;
3025
3026 for channel in channels {
3027 if let Err(e) = subscribe(channel.clone(), token.clone()).await {
3028 log::debug!("Lighter auth-token rotation: re-subscribe failed for {channel:?}: {e}",);
3029 first_error.get_or_insert_with(|| format!("{channel:?}: {e}"));
3030 }
3031 }
3032
3033 if let Some(error) = first_error {
3034 anyhow::bail!("failed to re-subscribe Lighter account channels: {error}");
3035 }
3036
3037 Ok(())
3038}
3039
3040async fn sleep_or_auth_token_refresh_cancelled(
3041 duration: Duration,
3042 cancellation_token: &CancellationToken,
3043) -> bool {
3044 tokio::select! {
3045 () = cancellation_token.cancelled() => false,
3046 () = tokio::time::sleep(duration) => true,
3047 }
3048}
3049
3050fn auth_token_refresh_next_delay(outcome: AuthTokenRefreshOutcome) -> Option<Duration> {
3051 match outcome {
3052 AuthTokenRefreshOutcome::Rotated => Some(AUTH_TOKEN_REFRESH_INTERVAL),
3053 AuthTokenRefreshOutcome::Cancelled => None,
3054 AuthTokenRefreshOutcome::Exhausted => Some(AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY),
3055 }
3056}
3057
3058fn next_auth_token_refresh_retry_delay(current: Duration, max: Duration) -> Duration {
3059 current.checked_mul(2).unwrap_or(max).min(max)
3060}
3061
3062#[derive(Debug)]
3063struct ReservedTxContext {
3064 context: TxContext,
3065 send_reservation: TxSendReservation,
3066}
3067
3068#[derive(Debug, Clone)]
3069struct TxSendSequencer {
3070 state: Arc<Mutex<TxSendSequencerState>>,
3071 version: Arc<AtomicU64>,
3072 changed: tokio::sync::watch::Sender<u64>,
3073}
3074
3075impl TxSendSequencer {
3076 fn new() -> Self {
3077 let (changed, _) = tokio::sync::watch::channel(0);
3078 Self {
3079 state: Arc::new(Mutex::new(TxSendSequencerState::default())),
3080 version: Arc::new(AtomicU64::new(0)),
3081 changed,
3082 }
3083 }
3084
3085 fn reserve(
3086 &self,
3087 account_index: i64,
3088 api_key_index: u8,
3089 allocate_nonce: impl FnOnce() -> Result<i64, NonceError>,
3090 ) -> Result<TxSendReservation, NonceError> {
3091 let key = TxSendKey {
3092 account_index,
3093 api_key_index,
3094 };
3095
3096 let mut state = self.state.lock();
3097 let nonce = allocate_nonce()?;
3098 *state
3099 .pending
3100 .entry(key)
3101 .or_default()
3102 .entry(nonce)
3103 .or_default() += 1;
3104 drop(state);
3105 self.notify_waiters();
3106 Ok(TxSendReservation {
3107 sequencer: self.clone(),
3108 key,
3109 nonce,
3110 released: false,
3111 nonce_guard: None,
3112 connection_epoch: 0,
3113 })
3114 }
3115
3116 async fn wait_for_turn(&self, key: TxSendKey, nonce: i64) {
3117 let mut changed = self.changed.subscribe();
3118
3119 loop {
3120 if self.ready_to_send(key, nonce) {
3121 return;
3122 }
3123
3124 if changed.changed().await.is_err() {
3125 tokio::task::yield_now().await;
3126 }
3127 }
3128 }
3129
3130 fn release(&self, key: TxSendKey, nonce: i64) {
3131 let mut state = self.state.lock();
3132
3133 let should_notify = if let Some(pending) = state.pending.get_mut(&key) {
3134 let removed = if let Some(count) = pending.get_mut(&nonce) {
3135 *count -= 1;
3136 if *count == 0 {
3137 pending.remove(&nonce);
3138 }
3139
3140 true
3141 } else {
3142 false
3143 };
3144
3145 if pending.is_empty() {
3146 state.pending.remove(&key);
3147 }
3148 removed
3149 } else {
3150 false
3151 };
3152 drop(state);
3153
3154 if should_notify {
3155 self.notify_waiters();
3156 }
3157 }
3158
3159 fn ready_to_send(&self, key: TxSendKey, nonce: i64) -> bool {
3160 let state = self.state.lock();
3161 state
3162 .pending
3163 .get(&key)
3164 .and_then(|pending| pending.first_key_value())
3165 .is_none_or(|(first, _)| *first >= nonce)
3166 }
3167
3168 fn notify_waiters(&self) {
3169 let version = self.version.fetch_add(1, Ordering::AcqRel) + 1;
3170 let _ = self.changed.send(version);
3171 }
3172}
3173
3174#[derive(Debug, Default)]
3175struct TxSendSequencerState {
3176 pending: BTreeMap<TxSendKey, BTreeMap<i64, usize>>,
3177}
3178
3179#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
3180struct TxSendKey {
3181 account_index: i64,
3182 api_key_index: u8,
3183}
3184
3185#[derive(Debug)]
3186struct TxSendReservation {
3187 sequencer: TxSendSequencer,
3188 key: TxSendKey,
3189 nonce: i64,
3190 released: bool,
3191 nonce_guard: Option<tokio::sync::OwnedRwLockReadGuard<()>>,
3192 connection_epoch: u64,
3193}
3194
3195impl TxSendReservation {
3196 fn with_nonce_ownership(
3197 mut self,
3198 nonce_guard: tokio::sync::OwnedRwLockReadGuard<()>,
3199 connection_epoch: u64,
3200 ) -> Self {
3201 self.nonce_guard = Some(nonce_guard);
3202 self.connection_epoch = connection_epoch;
3203 self
3204 }
3205
3206 async fn wait_for_turn(&self) {
3207 self.sequencer.wait_for_turn(self.key, self.nonce).await;
3208 }
3209
3210 fn release(&mut self) {
3211 if self.released {
3212 return;
3213 }
3214
3215 self.sequencer.release(self.key, self.nonce);
3216 self.nonce_guard.take();
3217 self.released = true;
3218 }
3219}
3220
3221impl Drop for TxSendReservation {
3222 fn drop(&mut self) {
3223 self.release();
3224 }
3225}
3226
3227#[cfg(test)]
3228async fn wait_for_tx_send_reservations(reservations: &[&TxSendReservation]) {
3229 let Some(first) = reservations.first() else {
3230 return;
3231 };
3232
3233 debug_assert!(
3234 reservations
3235 .iter()
3236 .all(|reservation| reservation.key == first.key),
3237 "batch send reservations must share one nonce stream",
3238 );
3239
3240 let nonce = reservations
3241 .iter()
3242 .map(|reservation| reservation.nonce)
3243 .min()
3244 .expect("reservations is non-empty");
3245 first.sequencer.wait_for_turn(first.key, nonce).await;
3246}
3247
3248struct PreparedCreateOrder {
3249 order: OrderAny,
3250 client_order_index: i64,
3251 tx_info: Box<serde_json::value::RawValue>,
3252 nonce: i64,
3253 api_key_index: u8,
3254 tx_hash: String,
3255 send_reservation: TxSendReservation,
3256}
3257
3258struct CreateOrderPlan {
3259 order: OrderAny,
3260 market_index: i64,
3261 base_amount: i64,
3262 price: u32,
3263 order_type: u8,
3264 time_in_force: u8,
3265 trigger_price: u32,
3266 order_expiry: i64,
3267 attributes: L2TxAttributes,
3268}
3269
3270impl CreateOrderPlan {
3271 fn order_info(&self, client_order_index: i64) -> OrderInfo {
3272 OrderInfo {
3273 market_index: self.market_index,
3274 client_order_index,
3275 base_amount: self.base_amount,
3276 price: self.price,
3277 is_ask: self.order.order_side() == OrderSide::Sell,
3278 order_type: self.order_type,
3279 time_in_force: self.time_in_force,
3280 reduce_only: self.order.is_reduce_only(),
3281 trigger_price: self.trigger_price,
3282 order_expiry: self.order_expiry,
3283 }
3284 }
3285}
3286
3287struct CreateGroupPlan {
3288 grouping_type: LighterGroupingType,
3289 orders: Vec<CreateOrderPlan>,
3290}
3291
3292struct PreparedCancelOrder {
3293 command_id: Option<UUID4>,
3294 client_order_id: ClientOrderId,
3295 strategy_id: StrategyId,
3296 instrument_id: InstrumentId,
3297 venue_order_id: Option<VenueOrderId>,
3298 tx_info: Box<serde_json::value::RawValue>,
3299 nonce: i64,
3300 api_key_index: u8,
3301 tx_hash: String,
3302 send_reservation: TxSendReservation,
3303}
3304
3305struct CancelOrderPlan {
3306 command_id: Option<UUID4>,
3307 client_order_id: ClientOrderId,
3308 strategy_id: StrategyId,
3309 instrument_id: InstrumentId,
3310 venue_order_id: Option<VenueOrderId>,
3311 market_index: i64,
3312 venue_index: i64,
3313}
3314
3315#[derive(Clone)]
3316struct FanoutDispatchContext {
3317 clock: &'static AtomicTime,
3318 chain_id: u32,
3319 integrator_account_index: Option<u64>,
3320 emitter: ExecutionEventEmitter,
3321 credential: Credential,
3322 http_client: LighterHttpClient,
3323 registry: Arc<MarketRegistry>,
3324 ws_client: LighterWebSocketClient,
3325 tx_rate_limiter: Arc<LighterTxRateLimiter>,
3326 tx_send_sequencer: TxSendSequencer,
3327 nonce_submission_gate: Arc<tokio::sync::RwLock<()>>,
3328 nonce_ready_connection_epoch: Arc<AtomicU64>,
3329 dispatch: WsDispatchState,
3330 nonce_recovery_inflight: Arc<AtomicBool>,
3331 pending_tasks: TaskSpawner,
3332}
3333
3334impl FanoutDispatchContext {
3335 async fn refresh_scheduled_cancel(&self, timeout_secs: u64) -> anyhow::Result<()> {
3336 let deadline = self
3337 .clock
3338 .get_time_ms()
3339 .checked_add(timeout_secs * 1_000)
3340 .and_then(|deadline| i64::try_from(deadline).ok())
3341 .context("scheduled cancellation deadline overflows milliseconds")?;
3342 let cancellation = self.pending_tasks.cancellation_token().child_token();
3343 let _guard = cancellation.clone().drop_guard();
3344 tokio::time::timeout(
3345 SCHEDULED_CANCEL_REQUEST_TIMEOUT,
3346 self.send_cancel_all(
3347 LighterCancelAllTimeInForce::Scheduled,
3348 deadline,
3349 true,
3350 cancellation,
3351 ),
3352 )
3353 .await
3354 .context("scheduled cancellation request timed out")??;
3355 Ok(())
3356 }
3357
3358 async fn send_cancel_all(
3359 &self,
3360 time_in_force: LighterCancelAllTimeInForce,
3361 scheduled_time_ms: i64,
3362 require_execution: bool,
3363 cancellation: CancellationToken,
3364 ) -> anyhow::Result<String> {
3365 let credential = &self.credential;
3366 let ReservedTxContext {
3367 context,
3368 mut send_reservation,
3369 } = self.build_tx_context()?;
3370 let mut rollback_guard =
3371 TxDispatchGuard::new(self.dispatch.clone(), credential, None, context.nonce);
3372
3373 let tx = CancelAllOrdersTxInfo {
3374 context,
3375 time_in_force: time_in_force as u8,
3376 scheduled_time_ms,
3377 skip_nonce: 0,
3378 };
3379
3380 let signed = sign_tx(&tx, self.chain_id, &credential.private_key()?, fresh_k())?;
3381 let tx_info = TxInfoJson::cancel_all_orders(&tx, &signed);
3382 let tx_info = serde_json::value::RawValue::from_string(tx_info)
3383 .context("failed to wrap signed Lighter account cancellation JSON")?;
3384 let tx_hash = signed.tx_hash_hex();
3385 let connection_epoch = send_reservation.connection_epoch;
3386 let ws_client = self.ws_client.clone();
3387 let dispatch = self.dispatch.clone();
3388 let credential = credential.clone();
3389 let clock = self.clock;
3390 let tx_rate_limiter = self.tx_rate_limiter.clone();
3391 let dispatched_hash = tx_hash.clone();
3392 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3393
3394 self.pending_tasks
3395 .spawn(async move {
3396 {
3397 let ready = async {
3398 send_reservation.wait_for_turn().await;
3399 await_tx_quota(&tx_rate_limiter).await;
3400 };
3401
3402 tokio::pin!(ready);
3403 tokio::select! {
3404 biased;
3405 () = cancellation.cancelled() => {
3406 if matches!(dispatch.nonce_manager.ack_failure_if_latest(
3408 context.account_index, context.api_key_index, context.nonce,
3409 ), Ok(true)) {
3410 rollback_guard.disarm();
3411 return;
3412 }
3413 ready.await;
3414 }
3415 () = &mut ready => {},
3416 }
3417 }
3418
3419 dispatch.enqueue_pending_sendtx(PendingSendTx {
3420 batch_id: None,
3421 connection_epoch,
3422 kind: PendingSendTxKind::Other,
3423 submitted_at: clock.get_time_ns(),
3424 nonce: context.nonce,
3425 api_key_index: context.api_key_index,
3426 tx_hash: dispatched_hash.clone(),
3427 });
3428
3429 rollback_guard.disarm();
3430 let response = ws_client.send_tx_on_connection(
3431 LighterTxType::CancelAllOrders as u8, tx_info, connection_epoch,
3432 ).await;
3433
3434 if let Err(e) = &response {
3435 let failure =
3436 classify_lighter_ws_command_failure("cancel_all_orders_account", e);
3437 log::warn!(
3438 "Lighter account cancellation transaction {dispatched_hash} dispatch failed, nonce={}: {failure:?}; diagnostic={e}",
3439 context.nonce,
3440 );
3441
3442 if !matches!(failure, CommandFailure::Ambiguous(_)) {
3443 dispatch.remove_pending_sendtx_by_hash(connection_epoch, &dispatched_hash);
3444 rollback_tx_dispatch(&dispatch, &credential, None, context.nonce);
3445 }
3446 }
3447
3448 send_reservation.release();
3449 let _ = response_tx.send(response);
3450 })
3451 .context("failed to spawn Lighter account cancellation")?;
3452
3453 let response = response_rx.await.with_context(|| {
3454 format!("Lighter account cancellation transaction {tx_hash} dispatch outcome unknown after session shutdown")
3455 })?;
3456
3457 response.with_context(|| {
3458 format!("failed to submit Lighter account cancellation transaction {tx_hash}")
3459 })?;
3460
3461 let confirmation = async {
3462 loop {
3463 match self.http_client.get_tx(tx_hash.clone()).await {
3464 Ok(tx) => {
3465 anyhow::ensure!(
3466 tx.status != LighterTxStatus::Failed,
3467 "Lighter account cancellation transaction {tx_hash} failed: {}",
3468 tx.event_info,
3469 );
3470
3471 if require_execution && tx.status != LighterTxStatus::Executed {
3472 tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY).await;
3473 continue;
3474 }
3475
3476 let event = parse_acked_tx_event(&tx.event_info).with_context(|| {
3477 format!("failed to parse Lighter account cancellation transaction {tx_hash} event")
3478 })?;
3479
3480 anyhow::ensure!(
3481 event.app_error.is_empty(),
3482 "Lighter account cancellation transaction {tx_hash} failed: {}",
3483 event.app_error,
3484 );
3485
3486 anyhow::ensure!(
3487 !require_execution
3488 || (tx_hash_matches(&tx.hash, &tx_hash)
3489 && tx.tx_type == LighterTxType::CancelAllOrders as u8
3490 && tx.account_index == self.credential.account_index()
3491 && tx.api_key_index == self.credential.api_key_index()
3492 && tx.nonce == context.nonce),
3493 "Lighter scheduled cancellation transaction identity mismatch",
3494 );
3495 return Ok(tx_hash.clone());
3496 }
3497 Err(LighterHttpError::Venue { code: 21500, .. }) => {}
3498 Err(e) => {
3499 return Err(e).with_context(|| {
3500 format!("failed to look up Lighter account cancellation transaction {tx_hash}")
3501 });
3502 }
3503 }
3504
3505 tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY).await;
3506 }
3507 };
3508
3509 tokio::time::timeout(
3510 if require_execution {
3511 SCHEDULED_CANCEL_REQUEST_TIMEOUT
3512 } else {
3513 ACCOUNT_CANCEL_CONFIRM_TIMEOUT
3514 },
3515 confirmation,
3516 )
3517 .await
3518 .with_context(|| {
3519 format!("Lighter account cancellation transaction {tx_hash} confirmation timed out")
3520 })?
3521 }
3522
3523 async fn dispatch_batch_plans(&self, plans: Vec<BatchOrderPlan>) {
3524 for chunk in plans.chunks(LIGHTER_MAX_BATCH_TX) {
3525 let mut batch = Vec::with_capacity(chunk.len());
3526 let mut remaining = chunk.iter().peekable();
3527 let mut retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
3528
3529 while let Some(plan) = remaining.peek() {
3530 match self.sign_batch_plan(plan) {
3531 Ok(prepared) => {
3532 batch.push(prepared);
3533 retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
3534 }
3535 Err(e) if should_retry_batch_signing(&e) => {
3536 self.send_batch(std::mem::take(&mut batch)).await;
3537 tokio::time::sleep(retry_delay).await;
3538 retry_delay = (retry_delay * 2).min(NONCE_REFRESH_RETRY_MAX_DELAY);
3539 continue;
3540 }
3541 Err(e) => self.reject_batch_plan(plan, &e.to_string()),
3542 }
3543
3544 remaining.next();
3545 }
3546
3547 self.send_batch(batch).await;
3548 }
3549 }
3550
3551 fn probe_context(&self) -> AckedOrderProbeContext {
3552 AckedOrderProbeContext {
3553 http_client: self.http_client.clone(),
3554 registry: Arc::clone(&self.registry),
3555 credential: self.credential.clone(),
3556 dispatch: self.dispatch.clone(),
3557 account_id: self.emitter.account_id(),
3558 clock: self.clock,
3559 emitter: self.emitter.clone(),
3560 connection_epoch: self.ws_client.connection_epoch_atomic(),
3561 cancellation_token: self.pending_tasks.cancellation_token(),
3562 pending_tasks: self.pending_tasks.clone(),
3563 }
3564 }
3565
3566 fn sign_batch_plan(&self, plan: &BatchOrderPlan) -> anyhow::Result<PreparedBatchTx> {
3567 match plan {
3568 BatchOrderPlan::Create(plan) => self.sign_create_order(plan).map(PreparedBatchTx::from),
3569 BatchOrderPlan::CreateGroup(plan) => self.sign_create_group(plan),
3570 BatchOrderPlan::Modify(plan) => {
3571 let reserved = self.build_tx_context()?;
3572 self.sign_modify_order(plan, reserved)
3573 .map(PreparedBatchTx::from)
3574 }
3575 BatchOrderPlan::Cancel(plan, emit) => self
3576 .sign_cancel_order(plan)
3577 .map(|prepared| PreparedBatchTx::from((prepared, *emit))),
3578 }
3579 }
3580
3581 fn reject_batch_plan(&self, plan: &BatchOrderPlan, error: &str) {
3582 match plan {
3583 BatchOrderPlan::CreateGroup(plan) => {
3584 for leg in &plan.orders {
3585 self.emitter.emit_order_denied(
3586 &leg.order,
3587 &format!("Lighter grouped submit failed: {error}"),
3588 );
3589 }
3590 }
3591 BatchOrderPlan::Create(plan) => {
3592 let reason = OrderDeniedReason::SubmitFailed {
3593 detail: format!("Lighter submit_order_list failed: {error}"),
3594 };
3595
3596 self.emitter
3597 .emit_order_denied(&plan.order, &reason.to_string());
3598 }
3599 BatchOrderPlan::Modify(plan) => {
3600 let mut events = self.dispatch.order_events.lock();
3601
3602 if self
3603 .dispatch
3604 .clear_pending_modify_if_command(&plan.client_order_id, plan.command_id)
3605 {
3606 self.emitter.emit_order_modify_denied_event(
3607 plan.strategy_id,
3608 plan.instrument_id,
3609 plan.client_order_id,
3610 plan.venue_order_id,
3611 &format!("Lighter modify_order failed: {error}"),
3612 self.clock.get_time_ns(),
3613 Some(plan.command_id),
3614 );
3615
3616 if let Some(deferred) = events.remove(&plan.client_order_id) {
3617 flush_modify_events(
3618 plan.client_order_id,
3619 deferred,
3620 &self.dispatch,
3621 &self.emitter,
3622 &mut events,
3623 );
3624 }
3625 }
3626 }
3627 BatchOrderPlan::Cancel(plan, _) => {
3628 let Some(command_id) = plan.command_id else {
3629 return;
3630 };
3631
3632 self.dispatch
3633 .clear_pending_cancel_if_command(&plan.client_order_id, command_id);
3634 let reason = format!("Lighter cancel_order failed: {error}");
3635
3636 self.emitter.emit_order_cancel_denied_event(
3637 plan.strategy_id,
3638 plan.instrument_id,
3639 plan.client_order_id,
3640 plan.venue_order_id,
3641 &reason,
3642 self.clock.get_time_ns(),
3643 Some(command_id),
3644 );
3645 }
3646 }
3647 }
3648
3649 fn build_tx_context(&self) -> anyhow::Result<ReservedTxContext> {
3650 let connection_epoch = self.ws_client.connection_epoch();
3651 anyhow::ensure!(
3652 self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
3653 TxPreparationError::NonceRefreshPending(connection_epoch),
3654 );
3655 let nonce_guard = Arc::clone(&self.nonce_submission_gate)
3656 .try_read_owned()
3657 .context("Lighter nonce refresh is in progress")?;
3658 anyhow::ensure!(
3659 self.ws_client.connection_epoch() == connection_epoch
3660 && self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
3661 TxPreparationError::StateChanged,
3662 );
3663
3664 let send_reservation = match self.tx_send_sequencer.reserve(
3665 self.credential.account_index(),
3666 self.credential.api_key_index(),
3667 || {
3668 self.dispatch.nonce_manager.next_nonce(
3669 self.credential.account_index(),
3670 self.credential.api_key_index(),
3671 )
3672 },
3673 ) {
3674 Ok(reservation) => reservation,
3675 Err(e @ NonceError::SkipWindowExhausted { .. }) => {
3676 self.spawn_nonce_window_recovery();
3677 let reason = format!("failed to allocate Lighter nonce: {e}");
3678 return Err(anyhow::Error::new(e).context(reason));
3679 }
3680 Err(e) => anyhow::bail!("failed to allocate Lighter nonce: {e}"),
3681 };
3682
3683 let nonce = send_reservation.nonce;
3684
3685 let now_ns = self.clock.get_time_ns().as_u64() as i64;
3686 let expired_at = (now_ns / 1_000_000).saturating_add(DEFAULT_TX_EXPIRY_MS);
3687 let send_reservation = send_reservation.with_nonce_ownership(nonce_guard, connection_epoch);
3688
3689 Ok(ReservedTxContext {
3690 context: TxContext {
3691 account_index: self.credential.account_index(),
3692 api_key_index: self.credential.api_key_index(),
3693 nonce,
3694 expired_at,
3695 },
3696 send_reservation,
3697 })
3698 }
3699
3700 fn spawn_nonce_window_recovery(&self) {
3701 if self.nonce_recovery_inflight.swap(true, Ordering::AcqRel) {
3702 return;
3703 }
3704
3705 let inflight = Arc::clone(&self.nonce_recovery_inflight);
3706 let http_client = self.http_client.clone();
3707 let dispatch = self.dispatch.clone();
3708 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
3709 let account_index = self.credential.account_index();
3710 let api_key_index = self.credential.api_key_index();
3711
3712 let future = async move {
3713 let _refresh_guard = nonce_submission_gate.write().await;
3714 let result = http_client
3715 .get_next_nonce(account_index, api_key_index)
3716 .await;
3717 inflight.store(false, Ordering::Release);
3718
3719 match result {
3720 Ok(response) => {
3721 let _ = dispatch.nonce_manager.sync_from_venue(
3722 account_index,
3723 api_key_index,
3724 response.nonce,
3725 );
3726 log::debug!(
3727 "Resynced Lighter nonce baseline after skip-window exhaustion: \
3728 account_index={account_index}, api_key_index={api_key_index}, \
3729 next_nonce={}",
3730 response.nonce,
3731 );
3732 }
3733 Err(e) => {
3734 log::error!("Failed to resync Lighter nonce after skip-window exhaustion: {e}");
3735 }
3736 }
3737 };
3738
3739 if let Err(e) = self.pending_tasks.spawn(future) {
3740 self.nonce_recovery_inflight.store(false, Ordering::Release);
3741 log::warn!("Skipping Lighter nonce recovery after shutdown began: {e}");
3742 }
3743 }
3744
3745 fn sign_create_order(&self, plan: &CreateOrderPlan) -> anyhow::Result<PreparedCreateOrder> {
3746 let cloid = plan.order.client_order_id();
3747 let client_order_index = self.dispatch.register_create_identity(&plan.order)?;
3748
3749 let ReservedTxContext {
3750 context,
3751 send_reservation,
3752 } = match self.build_tx_context() {
3753 Ok(context) => context,
3754 Err(e) => {
3755 self.dispatch.forget_cloid(client_order_index);
3756 self.dispatch.forget_order_identity(&cloid);
3757 return Err(e);
3758 }
3759 };
3760
3761 let nonce = context.nonce;
3762 let api_key_index = context.api_key_index;
3763 let mut rollback_guard = TxDispatchGuard::new(
3764 self.dispatch.clone(),
3765 &self.credential,
3766 Some(client_order_index),
3767 nonce,
3768 )
3769 .with_order_identity(cloid);
3770
3771 let tx = CreateOrderTxInfo {
3772 context,
3773 order: plan.order_info(client_order_index),
3774 attributes: plan.attributes,
3775 };
3776
3777 let signed = sign_tx(
3778 &tx,
3779 self.chain_id,
3780 &self.credential.private_key()?,
3781 fresh_k(),
3782 )?;
3783
3784 let tx_info =
3785 serde_json::value::RawValue::from_string(TxInfoJson::create_order(&tx, &signed))
3786 .context("failed to wrap signed Lighter tx_info JSON")?;
3787
3788 rollback_guard.disarm();
3789
3790 Ok(PreparedCreateOrder {
3791 order: plan.order.clone(),
3792 client_order_index,
3793 tx_info,
3794 nonce,
3795 api_key_index,
3796 tx_hash: signed.tx_hash_hex(),
3797 send_reservation,
3798 })
3799 }
3800
3801 fn sign_create_group(&self, plan: &CreateGroupPlan) -> anyhow::Result<PreparedBatchTx> {
3802 let reserved = self.build_tx_context()?;
3803 let mut guards = Vec::with_capacity(plan.orders.len());
3804 let mut orders = Vec::with_capacity(plan.orders.len());
3805 let mut bodies = Vec::with_capacity(plan.orders.len());
3806
3807 let mut nonce_guard = TxDispatchGuard::new(
3808 self.dispatch.clone(),
3809 &self.credential,
3810 None,
3811 reserved.context.nonce,
3812 );
3813
3814 for leg in &plan.orders {
3815 let index = self.dispatch.register_create_identity(&leg.order)?;
3816 guards.push(
3817 TxDispatchGuard::new(
3818 self.dispatch.clone(),
3819 &self.credential,
3820 Some(index),
3821 reserved.context.nonce,
3822 )
3823 .with_order_identity(leg.order.client_order_id()),
3824 );
3825 bodies.push(leg.order_info(index));
3826 orders.push((leg.order.clone(), index));
3827 }
3828
3829 let tx = CreateGroupedOrdersTxInfo {
3830 context: reserved.context,
3831 grouping_type: plan.grouping_type,
3832 orders: bodies,
3833 attributes: plan.orders[0].attributes,
3834 };
3835
3836 let signed = sign_tx(
3837 &tx,
3838 self.chain_id,
3839 &self.credential.private_key()?,
3840 fresh_k(),
3841 )?;
3842 let tx_info = serde_json::value::RawValue::from_string(TxInfoJson::create_grouped_orders(
3843 &tx, &signed,
3844 ))
3845 .context("failed to wrap signed Lighter grouped tx_info JSON")?;
3846
3847 for guard in &mut guards {
3848 guard.disarm();
3849 }
3850
3851 nonce_guard.disarm();
3852 Ok(PreparedBatchTx {
3853 tx_type: LighterTxType::CreateGroupedOrders as u8,
3854 tx_info,
3855 pending: PendingSendTx {
3856 batch_id: None,
3857 connection_epoch: reserved.send_reservation.connection_epoch,
3858 kind: PendingSendTxKind::CreateGroup { orders },
3859 submitted_at: UnixNanos::default(),
3860 nonce: reserved.context.nonce,
3861 api_key_index: reserved.context.api_key_index,
3862 tx_hash: signed.tx_hash_hex(),
3863 },
3864 send_reservation: reserved.send_reservation,
3865 })
3866 }
3867
3868 async fn send_create_order(&self, prepared: PreparedCreateOrder) {
3869 let PreparedCreateOrder {
3870 order,
3871 client_order_index,
3872 tx_info,
3873 nonce,
3874 api_key_index,
3875 tx_hash,
3876 mut send_reservation,
3877 } = prepared;
3878 let connection_epoch = send_reservation.connection_epoch;
3879 let client_order_id = order.client_order_id();
3880
3881 self.emitter.emit_order_submitted(&order);
3882 log::debug!("Lighter submit_order: queueing CreateOrder tx for {client_order_id}");
3883 send_reservation.wait_for_turn().await;
3884 await_tx_quota(&self.tx_rate_limiter).await;
3885 self.dispatch.enqueue_pending_sendtx(PendingSendTx {
3886 batch_id: None,
3887 connection_epoch,
3888 kind: PendingSendTxKind::Create {
3889 order: Box::new(order.clone()),
3890 client_order_index,
3891 },
3892 submitted_at: self.clock.get_time_ns(),
3893 nonce,
3894 api_key_index,
3895 tx_hash,
3896 });
3897
3898 if let Err(e) = self
3899 .ws_client
3900 .send_tx_on_connection(LighterTxType::CreateOrder as u8, tx_info, connection_epoch)
3901 .await
3902 {
3903 let failure = classify_lighter_ws_command_failure("submit_order", &e);
3904 let reason = command_failure_reason(&failure);
3905 if matches!(&failure, CommandFailure::Ambiguous(_)) {
3906 log::warn!(
3907 "Lighter submit_order dispatch outcome unknown for {client_order_id}: {reason}; \
3908 retaining pending state for venue reconciliation; diagnostic={e:?}",
3909 );
3910 } else {
3911 log::warn!("{reason} for {client_order_id}; diagnostic={e:?}");
3912 self.dispatch
3913 .remove_pending_sendtx_by_nonce(connection_epoch, nonce);
3914 rollback_tx_dispatch_create(
3915 &self.dispatch,
3916 &self.credential,
3917 Some(client_order_index),
3918 &client_order_id,
3919 nonce,
3920 );
3921 self.emitter
3922 .emit_order_rejected(&order, reason, self.clock.get_time_ns(), false);
3923 }
3924 }
3925 send_reservation.release();
3926 }
3927
3928 fn sign_modify_order(
3929 &self,
3930 plan: &ModifyOrderPlan,
3931 reserved: ReservedTxContext,
3932 ) -> anyhow::Result<PreparedModifyOrder> {
3933 let ReservedTxContext {
3934 context,
3935 send_reservation,
3936 } = reserved;
3937
3938 let captured_nonce = context.nonce;
3939 let captured_api_key_index = context.api_key_index;
3940
3941 let mut rollback_guard = TxDispatchGuard::new(
3942 self.dispatch.clone(),
3943 &self.credential,
3944 None,
3945 captured_nonce,
3946 );
3947
3948 let tx = ModifyOrderTxInfo {
3949 context,
3950 market_index: plan.market_index,
3951 index: plan.venue_index,
3952 base_amount: plan.base_amount,
3953 price: plan.price,
3954 trigger_price: plan.trigger_price,
3955 attributes: integrator_attributes(self.integrator_account_index),
3956 };
3957
3958 let signed = sign_tx(
3959 &tx,
3960 self.chain_id,
3961 &self.credential.private_key()?,
3962 fresh_k(),
3963 )?;
3964
3965 let tx_info_str = TxInfoJson::modify_order(&tx, &signed);
3966 let tx_info = serde_json::value::RawValue::from_string(tx_info_str)
3967 .context("failed to wrap signed Lighter modify tx_info JSON")?;
3968
3969 self.dispatch.set_pending_modify_nonce(
3970 &plan.client_order_id,
3971 plan.command_id,
3972 send_reservation.connection_epoch,
3973 captured_nonce,
3974 );
3975 rollback_guard.disarm();
3976
3977 Ok(PreparedModifyOrder {
3978 command_id: plan.command_id,
3979 client_order_id: plan.client_order_id,
3980 strategy_id: plan.strategy_id,
3981 instrument_id: plan.instrument_id,
3982 venue_order_id: plan.venue_order_id,
3983 tx_info,
3984 nonce: captured_nonce,
3985 api_key_index: captured_api_key_index,
3986 tx_hash: signed.tx_hash_hex(),
3987 send_reservation,
3988 })
3989 }
3990
3991 fn sign_cancel_order(&self, plan: &CancelOrderPlan) -> anyhow::Result<PreparedCancelOrder> {
3992 let ReservedTxContext {
3993 context,
3994 send_reservation,
3995 } = self.build_tx_context()?;
3996
3997 let nonce = context.nonce;
3998 let api_key_index = context.api_key_index;
3999 let mut rollback_guard =
4000 TxDispatchGuard::new(self.dispatch.clone(), &self.credential, None, nonce);
4001
4002 let tx = CancelOrderTxInfo {
4003 context,
4004 market_index: plan.market_index,
4005 index: plan.venue_index,
4006 skip_nonce: 0,
4007 };
4008
4009 let signed = sign_tx(
4010 &tx,
4011 self.chain_id,
4012 &self.credential.private_key()?,
4013 fresh_k(),
4014 )?;
4015
4016 let tx_info =
4017 serde_json::value::RawValue::from_string(TxInfoJson::cancel_order(&tx, &signed))
4018 .context("failed to wrap signed Lighter cancel tx_info JSON")?;
4019
4020 rollback_guard.disarm();
4021
4022 Ok(PreparedCancelOrder {
4023 command_id: plan.command_id,
4024 client_order_id: plan.client_order_id,
4025 strategy_id: plan.strategy_id,
4026 instrument_id: plan.instrument_id,
4027 venue_order_id: plan.venue_order_id,
4028 tx_info,
4029 nonce,
4030 api_key_index,
4031 tx_hash: signed.tx_hash_hex(),
4032 send_reservation,
4033 })
4034 }
4035
4036 async fn send_cancel_order(
4037 &self,
4038 mut prepared: PreparedCancelOrder,
4039 emit_cancel_rejected: bool,
4040 ) {
4041 let connection_epoch = prepared.send_reservation.connection_epoch;
4042 prepared.send_reservation.wait_for_turn().await;
4043 await_tx_quota(&self.tx_rate_limiter).await;
4044 self.enqueue_cancel_order(&prepared, emit_cancel_rejected);
4045 let result = self
4046 .ws_client
4047 .send_tx_on_connection(
4048 LighterTxType::CancelOrder as u8,
4049 prepared.tx_info.clone(),
4050 connection_epoch,
4051 )
4052 .await;
4053
4054 match result {
4055 Ok(()) => {
4056 if let Some(command_id) = prepared.command_id {
4057 self.dispatch.promote_pending_cancel(
4058 &prepared.client_order_id,
4059 command_id,
4060 connection_epoch,
4061 prepared.nonce,
4062 emit_cancel_rejected,
4063 );
4064 }
4065 }
4066 Err(e) => self.handle_cancel_send_failure(&prepared, &e, emit_cancel_rejected),
4067 }
4068
4069 prepared.send_reservation.release();
4070 }
4071
4072 async fn send_batch(&self, mut batch: Vec<PreparedBatchTx>) {
4073 let mut batch_ids = Vec::new();
4074
4075 while !batch.is_empty() {
4076 let count = batch
4079 .windows(2)
4080 .position(|pair| {
4081 pair[1].pending.nonce != pair[0].pending.nonce + 1
4082 || pair[1].send_reservation.connection_epoch
4083 != pair[0].send_reservation.connection_epoch
4084 })
4085 .map_or(batch.len(), |index| index + 1);
4086
4087 let group: Vec<_> = batch.drain(..count).collect();
4088 let id = self.send_batch_group(group).await;
4089 batch_ids.push(id);
4090 }
4091
4092 let wait = async {
4094 while self.dispatch.pending_sendtx.lock().iter().any(|pending| {
4095 pending
4096 .batch_id
4097 .as_ref()
4098 .is_some_and(|id| batch_ids.contains(id))
4099 }) {
4100 tokio::time::sleep(Duration::from_millis(10)).await;
4101 }
4102 };
4103
4104 if tokio::time::timeout(BATCH_ACK_TIMEOUT, wait).await.is_err() {
4105 log::warn!("Lighter batch acknowledgement timed out: ids={batch_ids:?}");
4106 }
4107 }
4108
4109 async fn send_batch_group(&self, mut group: Vec<PreparedBatchTx>) -> String {
4110 let connection_epoch = group[0].send_reservation.connection_epoch;
4111 let id = format!("{TX_BATCH_ID_PREFIX}{}", group[0].pending.tx_hash);
4112 group[0].send_reservation.wait_for_turn().await;
4113 await_tx_quota(&self.tx_rate_limiter).await;
4114
4115 let data = crate::websocket::messages::LighterWsSendTxBatch {
4116 id: id.clone(),
4117 tx_types: serde_json::to_string(
4118 &group
4119 .iter()
4120 .map(|prepared| prepared.tx_type)
4121 .collect::<Vec<_>>(),
4122 )
4123 .expect("transaction types serialize"),
4124 tx_infos: serde_json::to_string(
4125 &group
4126 .iter()
4127 .map(|prepared| prepared.tx_info.get())
4128 .collect::<Vec<_>>(),
4129 )
4130 .expect("transaction strings serialize"),
4131 };
4132
4133 for prepared in &mut group {
4134 prepared.pending.batch_id = Some(id.clone());
4135 prepared.pending.submitted_at = self.clock.get_time_ns();
4136 self.enqueue_batch_tx(prepared);
4137 }
4138
4139 let result = self
4140 .ws_client
4141 .send_tx_batch_on_connection(data, connection_epoch)
4142 .await;
4143
4144 match result {
4145 Ok(()) => {
4146 for prepared in &group {
4147 promote_cancel_submission(&self.dispatch, &prepared.pending);
4148 }
4149 }
4150 Err(e) => {
4151 for prepared in group.iter().rev() {
4152 self.handle_batch_send_failure(prepared, &e);
4153 }
4154 }
4155 }
4156
4157 for prepared in &mut group {
4158 prepared.send_reservation.release();
4159 }
4160
4161 id
4162 }
4163
4164 fn enqueue_batch_tx(&self, prepared: &PreparedBatchTx) {
4165 for (order, _) in prepared.pending.kind.creates() {
4166 self.emitter.emit_order_submitted(order);
4167 }
4168
4169 self.dispatch
4170 .enqueue_pending_sendtx(prepared.pending.clone());
4171 if matches!(prepared.pending.kind, PendingSendTxKind::Modify { .. }) {
4172 spawn_acked_order_probe(&prepared.pending, self.probe_context());
4173 }
4174 }
4175
4176 fn handle_batch_send_failure(&self, prepared: &PreparedBatchTx, error: &LighterWsError) {
4177 let action = match prepared.pending.kind {
4178 PendingSendTxKind::Create { .. } | PendingSendTxKind::CreateGroup { .. } => {
4179 "submit_order"
4180 }
4181 PendingSendTxKind::Modify { .. } => "modify_order",
4182 PendingSendTxKind::Cancel { .. } => "cancel_order",
4183 PendingSendTxKind::Other => {
4184 unreachable!("order batches contain only order transactions")
4185 }
4186 };
4187
4188 let failure = classify_lighter_ws_command_failure(action, error);
4189 let reason = command_failure_reason(&failure);
4190
4191 if !matches!(failure, CommandFailure::NotSent(_)) {
4192 promote_cancel_submission(&self.dispatch, &prepared.pending);
4193 }
4194
4195 if matches!(failure, CommandFailure::Ambiguous(_)) {
4196 log::warn!("{reason}; retaining batch transaction for reconciliation");
4197 return;
4198 }
4199
4200 if let Some(pending) = self.dispatch.remove_pending_sendtx_by_hash(
4201 prepared.pending.connection_epoch,
4202 &prepared.pending.tx_hash,
4203 ) {
4204 reject_pending_sendtx(
4205 &self.dispatch,
4206 &self.emitter,
4207 Some(self.credential.account_index()),
4208 &pending,
4209 &failure,
4210 self.clock.get_time_ns(),
4211 false,
4212 );
4213 }
4214 }
4215
4216 fn enqueue_cancel_order(&self, prepared: &PreparedCancelOrder, emit_cancel_rejected: bool) {
4217 self.dispatch.enqueue_pending_sendtx(PendingSendTx {
4218 batch_id: None,
4219 connection_epoch: prepared.send_reservation.connection_epoch,
4220 kind: PendingSendTxKind::Cancel {
4221 command_id: prepared.command_id,
4222 emit_cancel_rejected,
4223 strategy_id: prepared.strategy_id,
4224 instrument_id: prepared.instrument_id,
4225 client_order_id: prepared.client_order_id,
4226 venue_order_id: prepared.venue_order_id,
4227 },
4228 submitted_at: self.clock.get_time_ns(),
4229 nonce: prepared.nonce,
4230 api_key_index: prepared.api_key_index,
4231 tx_hash: prepared.tx_hash.clone(),
4232 });
4233 }
4234
4235 fn handle_cancel_send_failure(
4236 &self,
4237 prepared: &PreparedCancelOrder,
4238 error: &LighterWsError,
4239 emit_cancel_rejected: bool,
4240 ) {
4241 let failure = classify_lighter_ws_command_failure("cancel_order", error);
4242 let reason = command_failure_reason(&failure);
4243
4244 if !matches!(failure, CommandFailure::NotSent(_))
4245 && let Some(command_id) = prepared.command_id
4246 {
4247 self.dispatch.promote_pending_cancel(
4248 &prepared.client_order_id,
4249 command_id,
4250 prepared.send_reservation.connection_epoch,
4251 prepared.nonce,
4252 emit_cancel_rejected,
4253 );
4254 }
4255
4256 if matches!(failure, CommandFailure::Ambiguous(_)) {
4257 log::warn!(
4258 "Lighter cancel_order outcome unknown for {}: {reason}; retaining pending state for venue reconciliation",
4259 prepared.client_order_id
4260 );
4261 return;
4262 }
4263
4264 log::warn!(
4265 "{reason} for {}; diagnostic={error:?}",
4266 prepared.client_order_id
4267 );
4268
4269 self.dispatch.remove_pending_sendtx_by_nonce(
4270 prepared.send_reservation.connection_epoch,
4271 prepared.nonce,
4272 );
4273 rollback_tx_dispatch(&self.dispatch, &self.credential, None, prepared.nonce);
4274
4275 let Some(command_id) = prepared.command_id else {
4276 return;
4277 };
4278
4279 self.dispatch
4280 .clear_pending_cancel_if_command(&prepared.client_order_id, command_id);
4281
4282 match &failure {
4283 CommandFailure::NotSent(reason) => self.emitter.emit_order_cancel_denied_event(
4284 prepared.strategy_id,
4285 prepared.instrument_id,
4286 prepared.client_order_id,
4287 prepared.venue_order_id,
4288 reason,
4289 self.clock.get_time_ns(),
4290 Some(command_id),
4291 ),
4292 CommandFailure::VenueRejected(reason) => self.emitter.emit_order_cancel_rejected_event(
4293 prepared.strategy_id,
4294 prepared.instrument_id,
4295 prepared.client_order_id,
4296 prepared.venue_order_id,
4297 reason,
4298 self.clock.get_time_ns(),
4299 Some(command_id),
4300 ),
4301 _ => {}
4302 }
4303 }
4304}
4305
4306fn should_retry_batch_signing(error: &anyhow::Error) -> bool {
4307 error.is::<TxPreparationError>()
4308 || error.is::<tokio::sync::TryLockError>()
4309 || matches!(
4310 error.downcast_ref::<NonceError>(),
4311 Some(NonceError::SkipWindowExhausted { .. })
4312 )
4313}
4314
4315#[derive(Debug, thiserror::Error)]
4316enum TxPreparationError {
4317 #[error("Lighter nonce refresh is pending for connection epoch {0}")]
4318 NonceRefreshPending(u64),
4319 #[error("Lighter connection or nonce state changed during transaction preparation")]
4320 StateChanged,
4321}
4322
4323struct PreparedModifyOrder {
4324 command_id: UUID4,
4325 client_order_id: ClientOrderId,
4326 strategy_id: StrategyId,
4327 instrument_id: InstrumentId,
4328 venue_order_id: Option<VenueOrderId>,
4329 tx_info: Box<serde_json::value::RawValue>,
4330 nonce: i64,
4331 api_key_index: u8,
4332 tx_hash: String,
4333 send_reservation: TxSendReservation,
4334}
4335
4336struct ModifyOrderPlan {
4337 command_id: UUID4,
4338 client_order_id: ClientOrderId,
4339 strategy_id: StrategyId,
4340 instrument_id: InstrumentId,
4341 venue_order_id: Option<VenueOrderId>,
4342 market_index: i64,
4343 venue_index: i64,
4344 base_amount: i64,
4345 price: u32,
4346 trigger_price: u32,
4347}
4348
4349enum BatchOrderPlan {
4350 Create(Box<CreateOrderPlan>),
4351 CreateGroup(CreateGroupPlan),
4352 Modify(ModifyOrderPlan),
4353 Cancel(CancelOrderPlan, bool),
4354}
4355
4356struct PreparedBatchTx {
4357 tx_type: u8,
4358 tx_info: Box<serde_json::value::RawValue>,
4359 pending: PendingSendTx,
4360 send_reservation: TxSendReservation,
4361}
4362
4363impl From<PreparedCreateOrder> for PreparedBatchTx {
4364 fn from(prepared: PreparedCreateOrder) -> Self {
4365 Self {
4366 tx_type: LighterTxType::CreateOrder as u8,
4367 tx_info: prepared.tx_info,
4368 pending: PendingSendTx {
4369 batch_id: None,
4370 connection_epoch: prepared.send_reservation.connection_epoch,
4371 kind: PendingSendTxKind::Create {
4372 order: Box::new(prepared.order),
4373 client_order_index: prepared.client_order_index,
4374 },
4375 submitted_at: UnixNanos::default(),
4376 nonce: prepared.nonce,
4377 api_key_index: prepared.api_key_index,
4378 tx_hash: prepared.tx_hash,
4379 },
4380 send_reservation: prepared.send_reservation,
4381 }
4382 }
4383}
4384
4385impl From<PreparedModifyOrder> for PreparedBatchTx {
4386 fn from(prepared: PreparedModifyOrder) -> Self {
4387 Self {
4388 tx_type: LighterTxType::ModifyOrder as u8,
4389 tx_info: prepared.tx_info,
4390 pending: PendingSendTx {
4391 batch_id: None,
4392 connection_epoch: prepared.send_reservation.connection_epoch,
4393 kind: PendingSendTxKind::Modify {
4394 command_id: prepared.command_id,
4395 client_order_id: prepared.client_order_id,
4396 strategy_id: prepared.strategy_id,
4397 instrument_id: prepared.instrument_id,
4398 venue_order_id: prepared.venue_order_id,
4399 },
4400 submitted_at: UnixNanos::default(),
4401 nonce: prepared.nonce,
4402 api_key_index: prepared.api_key_index,
4403 tx_hash: prepared.tx_hash,
4404 },
4405 send_reservation: prepared.send_reservation,
4406 }
4407 }
4408}
4409
4410impl From<(PreparedCancelOrder, bool)> for PreparedBatchTx {
4411 fn from((prepared, emit_cancel_rejected): (PreparedCancelOrder, bool)) -> Self {
4412 Self {
4413 tx_type: LighterTxType::CancelOrder as u8,
4414 tx_info: prepared.tx_info,
4415 pending: PendingSendTx {
4416 batch_id: None,
4417 connection_epoch: prepared.send_reservation.connection_epoch,
4418 kind: PendingSendTxKind::Cancel {
4419 command_id: prepared.command_id,
4420 emit_cancel_rejected,
4421 client_order_id: prepared.client_order_id,
4422 strategy_id: prepared.strategy_id,
4423 instrument_id: prepared.instrument_id,
4424 venue_order_id: prepared.venue_order_id,
4425 },
4426 submitted_at: UnixNanos::default(),
4427 nonce: prepared.nonce,
4428 api_key_index: prepared.api_key_index,
4429 tx_hash: prepared.tx_hash,
4430 },
4431 send_reservation: prepared.send_reservation,
4432 }
4433 }
4434}
4435
4436struct PreparedIntegratorApproval {
4437 tx_info: String,
4438 nonce: i64,
4439 api_key_index: u8,
4440 approval_expiry: i64,
4441 send_reservation: TxSendReservation,
4442}
4443
4444#[derive(Debug, PartialEq, Eq)]
4448enum TierCrossCheck {
4449 AboveTier { documented: u32 },
4450 RaiseHint { documented: u32 },
4451}
4452
4453fn tier_quota_report(
4458 tier: LighterAccountTier,
4459 rest_quota_per_min: Option<u32>,
4460 standard_rest: u32,
4461) -> (u32, Option<TierCrossCheck>) {
4462 let configured = rest_quota_per_min.filter(|&n| n > 0);
4463 let active_rest = configured.unwrap_or(standard_rest);
4464 let cross_check = match (tier.documented_rest_quota_per_min(), configured) {
4465 (Some(documented), Some(configured)) if configured > documented => {
4466 Some(TierCrossCheck::AboveTier { documented })
4467 }
4468 (Some(documented), None) if documented > standard_rest => {
4469 Some(TierCrossCheck::RaiseHint { documented })
4470 }
4471 _ => None,
4472 };
4473 (active_rest, cross_check)
4474}
4475
4476struct TxDispatchGuard {
4477 dispatch: WsDispatchState,
4478 account_index: i64,
4479 api_key_index: u8,
4480 client_order_index: Option<i64>,
4481 client_order_id: Option<ClientOrderId>,
4482 nonce: i64,
4483 armed: bool,
4484}
4485
4486impl TxDispatchGuard {
4487 fn new(
4488 dispatch: WsDispatchState,
4489 credential: &Credential,
4490 client_order_index: Option<i64>,
4491 nonce: i64,
4492 ) -> Self {
4493 Self {
4494 dispatch,
4495 account_index: credential.account_index(),
4496 api_key_index: credential.api_key_index(),
4497 client_order_index,
4498 client_order_id: None,
4499 nonce,
4500 armed: true,
4501 }
4502 }
4503
4504 fn with_order_identity(mut self, client_order_id: ClientOrderId) -> Self {
4505 self.client_order_id = Some(client_order_id);
4506 self
4507 }
4508
4509 fn disarm(&mut self) {
4510 self.armed = false;
4511 }
4512}
4513
4514impl Drop for TxDispatchGuard {
4515 fn drop(&mut self) {
4516 if self.armed {
4517 rollback_tx_dispatch_indices(
4518 &self.dispatch,
4519 self.account_index,
4520 self.api_key_index,
4521 self.client_order_index,
4522 self.client_order_id.as_ref(),
4523 self.nonce,
4524 );
4525 }
4526 }
4527}
4528
4529fn handle_send_tx_ack_for_connection(
4534 dispatch: &WsDispatchState,
4535 account_index: Option<i64>,
4536 connection_epoch: u64,
4537 code: i64,
4538 tx_hash: Option<&str>,
4539) -> Option<PendingSendTx> {
4540 let popped = match tx_hash {
4541 Some(hash) => {
4542 let matched = dispatch.remove_pending_sendtx_by_hash(connection_epoch, hash);
4543 if matched.is_none() {
4544 log::warn!("Lighter sendTx ack unmatched: tx_hash={hash} code={code}");
4545 }
4546 matched
4547 }
4548 None => dispatch.pop_pending_sendtx_if_only(connection_epoch),
4549 };
4550
4551 if let (Some(pending), Some(account_index)) = (&popped, account_index) {
4552 let _ =
4553 dispatch
4554 .nonce_manager
4555 .ack_success(account_index, pending.api_key_index, pending.nonce);
4556 }
4557
4558 if let Some(pending) = &popped {
4559 promote_cancel_submission(dispatch, pending);
4560 }
4561
4562 log::debug!(
4563 "Lighter sendTx ack: code={code} tx_hash={tx_hash:?} popped_nonce={:?}",
4564 popped.as_ref().map(|p| p.nonce),
4565 );
4566
4567 popped
4568}
4569
4570fn promote_cancel_submission(dispatch: &WsDispatchState, pending: &PendingSendTx) {
4571 if let PendingSendTxKind::Cancel {
4572 command_id: Some(command_id),
4573 client_order_id,
4574 emit_cancel_rejected,
4575 ..
4576 } = pending.kind
4577 {
4578 dispatch.promote_pending_cancel(
4579 &client_order_id,
4580 command_id,
4581 pending.connection_epoch,
4582 pending.nonce,
4583 emit_cancel_rejected,
4584 );
4585 }
4586}
4587
4588#[expect(
4589 clippy::too_many_arguments,
4590 reason = "batch response shares the single-transaction rejection sink"
4591)]
4592fn handle_send_tx_batch_result(
4593 dispatch: &WsDispatchState,
4594 emitter: &ExecutionEventEmitter,
4595 account_index: Option<i64>,
4596 connection_epoch: u64,
4597 now: UnixNanos,
4598 id: &str,
4599 code: i64,
4600 message: &str,
4601 tx_hashes: &[String],
4602) -> (Vec<PendingSendTx>, bool) {
4603 let mut pending: Vec<_> = dispatch
4604 .pending_sendtx
4605 .lock()
4606 .iter()
4607 .filter(|pending| {
4608 pending.connection_epoch == connection_epoch && pending.batch_id.as_deref() == Some(id)
4609 })
4610 .cloned()
4611 .collect();
4612
4613 if pending.is_empty() {
4614 log::warn!("Ignoring unmatched Lighter batch response: id={id}");
4615 return (Vec::new(), false);
4616 }
4617
4618 if code == 200 {
4619 let acked = pending
4620 .iter()
4621 .filter(|pending| tx_hashes.contains(&pending.tx_hash))
4622 .filter_map(|pending| {
4623 handle_send_tx_ack_for_connection(
4624 dispatch,
4625 account_index,
4626 connection_epoch,
4627 code,
4628 Some(&pending.tx_hash),
4629 )
4630 })
4631 .collect();
4632
4633 if pending
4634 .iter()
4635 .any(|pending| !tx_hashes.contains(&pending.tx_hash))
4636 {
4637 log::warn!(
4638 "Incomplete Lighter batch acknowledgement: id={id}; retaining unmatched transactions for reconciliation"
4639 );
4640 }
4641
4642 return (acked, false);
4643 }
4644
4645 pending.sort_unstable_by_key(|pending| std::cmp::Reverse(pending.nonce));
4647 let mut resync = false;
4648 for pending in pending {
4649 resync |= handle_send_tx_rejection_for_connection(
4650 dispatch,
4651 emitter,
4652 account_index,
4653 connection_epoch,
4654 now,
4655 SendTxRejectionSource::Ack,
4656 Some(code),
4657 message,
4658 Some(&pending.tx_hash),
4659 );
4660 }
4661
4662 (Vec::new(), resync)
4663}
4664
4665fn spawn_acked_order_probe(pending: &PendingSendTx, context: AckedOrderProbeContext) {
4666 if let PendingSendTxKind::CreateGroup { orders } = &pending.kind {
4667 for (order, index) in orders {
4668 let mut leg = pending.clone();
4669
4670 leg.kind = PendingSendTxKind::Create {
4671 order: Box::new(order.clone()),
4672 client_order_index: *index,
4673 };
4674
4675 spawn_acked_order_probe(&leg, context.clone());
4676 }
4677
4678 return;
4679 }
4680
4681 let action = matches!(
4682 pending.kind,
4683 PendingSendTxKind::Modify { .. } | PendingSendTxKind::Cancel { .. }
4684 )
4685 .then(|| pending.clone());
4686 let probe = AckedOrderProbe::from_pending(pending);
4687 if action.is_none() && probe.is_none() {
4688 return;
4689 }
4690
4691 let pending_tasks = context.pending_tasks.clone();
4692
4693 let future = async move {
4694 tokio::select! {
4695 () = context.cancellation_token.cancelled() => return,
4696 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
4697 }
4698
4699 if let Some(pending) = action {
4700 let result = if let PendingSendTxKind::Cancel {
4701 client_order_id, ..
4702 } = pending.kind
4703 {
4704 let Some(token) = context.dispatch.pending_cancel_token(
4705 &client_order_id,
4706 pending.connection_epoch,
4707 pending.nonce,
4708 ) else {
4709 return;
4710 };
4711
4712 tokio::select! {
4713 biased;
4714 () = context.cancellation_token.cancelled() => return,
4715 () = token.cancelled() => return,
4716 result = probe_acked_action(&pending,&context) => result,
4717 }
4718 } else {
4719 probe_acked_action(&pending, &context).await
4720 };
4721
4722 if let Err(e) = result {
4723 log::warn!("Lighter acknowledged action probe failed: {e}");
4724 }
4725
4726 return;
4727 }
4728
4729 if let Some(probe) = probe
4730 && let Err(e) = probe_acked_create(&probe, &context).await
4731 {
4732 log::warn!("Lighter acknowledged order probe failed: {e:?}");
4733 }
4734 };
4735
4736 if let Err(e) = pending_tasks.spawn(future) {
4737 log::debug!("Skipping Lighter acknowledged-order probe during shutdown: {e}");
4738 }
4739}
4740
4741#[derive(Clone)]
4742struct AckedOrderProbeContext {
4743 http_client: LighterHttpClient,
4744 registry: Arc<MarketRegistry>,
4745 credential: Credential,
4746 dispatch: WsDispatchState,
4747 account_id: AccountId,
4748 clock: &'static AtomicTime,
4749 emitter: ExecutionEventEmitter,
4750 connection_epoch: Arc<AtomicU64>,
4751 cancellation_token: CancellationToken,
4752 pending_tasks: TaskSpawner,
4753}
4754
4755fn handle_account_tx(connection_epoch: u64, txs: &[LighterWsTx], context: &AckedOrderProbeContext) {
4756 if connection_epoch != context.connection_epoch.load(Ordering::Acquire) {
4757 return;
4758 }
4759
4760 for tx in txs {
4761 let Some(pending) = context.dispatch.pending_modify_by_hash(&tx.hash) else {
4762 continue;
4763 };
4764
4765 let PendingSendTxKind::Modify {
4766 client_order_id, ..
4767 } = pending.kind
4768 else {
4769 unreachable!("pending modification owns a modify transaction")
4770 };
4771
4772 let failure = match tx_application_error(tx.status, &tx.event_info) {
4773 Ok(failure) => failure,
4774 Err(e) => {
4775 log::warn!("Lighter transaction update unavailable: {e}");
4776 continue;
4777 }
4778 };
4779
4780 if failure.is_none() && tx.status != LighterTxStatus::Executed {
4781 continue;
4782 }
4783
4784 let _ = context.dispatch.nonce_manager.ack_success(
4785 context.credential.account_index(),
4786 pending.api_key_index,
4787 pending.nonce,
4788 );
4789
4790 if let Some(detail) = failure {
4791 let reason = format!("Lighter modify transaction failed: {detail}");
4792 reject_modify_order(
4793 &context.dispatch,
4794 &context.emitter,
4795 &pending,
4796 &CommandFailure::venue_rejected(&reason),
4797 context.clock.get_time_ns(),
4798 true,
4799 );
4800 continue;
4801 }
4802
4803 context
4804 .dispatch
4805 .set_pending_modify_executed(&client_order_id, &pending);
4806 let order = context
4807 .dispatch
4808 .order_events
4809 .lock()
4810 .get(&client_order_id)
4811 .and_then(|events| events.order.clone());
4812
4813 if let Some(order) = order {
4814 confirm_executed_modify(&order, context);
4815 }
4816 }
4817}
4818
4819fn confirm_executed_modify(order: &LighterOrder, context: &AckedOrderProbeContext) {
4820 let venue_order_id = VenueOrderId::new(&order.order_id);
4821
4822 let Some(cloid) = context
4823 .dispatch
4824 .resolve_client_order_index_for_venue(order.client_order_index, venue_order_id)
4825 else {
4826 return;
4827 };
4828
4829 let Some(pending) = context.dispatch.pending_modify_executed(&cloid) else {
4830 return;
4831 };
4832
4833 if let Err(e) = confirm_modify_order(&pending, context, order) {
4834 log::debug!("Lighter executed modification awaits matching order state: {e}");
4835 }
4836}
4837
4838async fn probe_acked_action(
4839 pending: &PendingSendTx,
4840 context: &AckedOrderProbeContext,
4841) -> anyhow::Result<()> {
4842 let (client_order_id, _) = acked_action_identity(pending)?;
4843 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
4844 if !acked_action_is_pending(pending, context) {
4845 return Ok(());
4846 }
4847
4848 match context.http_client.get_tx(pending.tx_hash.clone()).await {
4849 Ok(tx) => match resolve_acked_action(pending, context, tx).await {
4850 Ok(true) => return Ok(()),
4851 Ok(false) => {}
4852 Err(e) => log::warn!("Lighter action confirmation unavailable: {e}"),
4853 },
4854 Err(LighterHttpError::Venue { code: 21500, .. }) => {}
4855 Err(e) => log::warn!("Lighter acknowledged action transaction lookup failed: {e}"),
4856 }
4857
4858 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
4859 tokio::select! {
4860 () = context.cancellation_token.cancelled() => return Ok(()),
4861 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
4862 }
4863 }
4864 }
4865
4866 log::warn!(
4867 "Lighter acknowledged action outcome unresolved for {client_order_id}; retaining pending action for reconciliation"
4868 );
4869 Ok(())
4870}
4871
4872fn acked_action_identity(
4873 pending: &PendingSendTx,
4874) -> anyhow::Result<(ClientOrderId, LighterTxType)> {
4875 match pending.kind {
4876 PendingSendTxKind::Cancel {
4877 client_order_id, ..
4878 } => Ok((client_order_id, LighterTxType::CancelOrder)),
4879 PendingSendTxKind::Modify {
4880 client_order_id, ..
4881 } => Ok((client_order_id, LighterTxType::ModifyOrder)),
4882 _ => anyhow::bail!("acknowledged action requires a cancel or modify transaction"),
4883 }
4884}
4885
4886fn acked_action_is_pending(pending: &PendingSendTx, context: &AckedOrderProbeContext) -> bool {
4887 match pending.kind {
4888 PendingSendTxKind::Cancel {
4889 command_id: Some(command_id),
4890 client_order_id,
4891 ..
4892 } => {
4893 context.connection_epoch.load(Ordering::Acquire) == pending.connection_epoch
4894 && context.dispatch.pending_cancel_identity(&client_order_id)
4895 == Some((command_id, (pending.connection_epoch, pending.nonce)))
4896 }
4897 PendingSendTxKind::Modify {
4898 client_order_id, ..
4899 } => {
4900 context.dispatch.pending_modify_nonce(&client_order_id)
4901 == Some((pending.connection_epoch, pending.nonce))
4902 }
4903 _ => false,
4904 }
4905}
4906
4907async fn resolve_acked_action(
4908 pending: &PendingSendTx,
4909 context: &AckedOrderProbeContext,
4910 tx: LighterTx,
4911) -> anyhow::Result<bool> {
4912 let (_, tx_type) = acked_action_identity(pending)?;
4913 anyhow::ensure!(
4914 tx_hash_matches(&tx.hash, &pending.tx_hash)
4915 && tx.tx_type == tx_type as u8
4916 && tx.account_index == context.credential.account_index()
4917 && tx.api_key_index == pending.api_key_index
4918 && tx.nonce == pending.nonce,
4919 "Lighter transaction lookup did not match acknowledged action identity"
4920 );
4921
4922 let is_modify = tx_type == LighterTxType::ModifyOrder;
4923
4924 if !acked_action_is_pending(pending, context) {
4925 return Ok(true);
4926 }
4927
4928 if let Some(detail) = tx_application_error(tx.status, &tx.event_info)? {
4929 let action = if is_modify { "modify" } else { "cancel" };
4930 let reason = format!("Lighter {action} transaction failed: {detail}");
4931 log::warn!("{reason} for nonce={}", pending.nonce);
4932
4933 match pending.kind {
4934 PendingSendTxKind::Modify { .. } => reject_modify_order(
4935 &context.dispatch,
4936 &context.emitter,
4937 pending,
4938 &CommandFailure::venue_rejected(&reason),
4939 context.clock.get_time_ns(),
4940 true,
4941 ),
4942 PendingSendTxKind::Cancel { .. } => reject_cancel_order(
4943 &context.dispatch,
4944 &context.emitter,
4945 pending,
4946 &CommandFailure::venue_rejected(&reason),
4947 context.clock.get_time_ns(),
4948 ),
4949 _ => unreachable!("validated acknowledged action"),
4950 }
4951
4952 return Ok(true);
4954 }
4955
4956 if tx.status == LighterTxStatus::Executed {
4957 return recover_acked_action_report(pending, context).await;
4958 }
4959
4960 Ok(false)
4961}
4962
4963async fn recover_acked_action_report(
4964 pending: &PendingSendTx,
4965 context: &AckedOrderProbeContext,
4966) -> anyhow::Result<bool> {
4967 let (instrument_id, client_order_id, venue_order_id) = match pending.kind {
4968 PendingSendTxKind::Cancel {
4969 instrument_id,
4970 client_order_id,
4971 venue_order_id,
4972 ..
4973 }
4974 | PendingSendTxKind::Modify {
4975 instrument_id,
4976 client_order_id,
4977 venue_order_id,
4978 ..
4979 } => (instrument_id, client_order_id, venue_order_id),
4980 _ => return Ok(true),
4981 };
4982
4983 let lookup = OrderReportLookup::new(
4984 &context.registry,
4985 &context.dispatch,
4986 Some(instrument_id),
4987 Some(&client_order_id),
4988 venue_order_id.as_ref(),
4989 )?;
4990
4991 anyhow::ensure!(
4992 matches!(pending.kind, PendingSendTxKind::Modify { .. })
4993 || !context.dispatch.has_pending_modify(&client_order_id),
4994 "Lighter cancellation waits for modification confirmation"
4995 );
4996 let order = collect_order_while_pending(
4997 &context.http_client,
4998 &context.registry,
4999 &context.credential,
5000 &lookup,
5001 || acked_action_is_pending(pending, context),
5002 )
5003 .await?;
5004
5005 if !acked_action_is_pending(pending, context) {
5006 return Ok(true);
5007 }
5008
5009 let Some(order) = order else {
5010 return Ok(false);
5011 };
5012
5013 if matches!(pending.kind, PendingSendTxKind::Modify { .. }) {
5014 confirm_modify_order(pending, context, &order)?;
5015 return Ok(true);
5016 }
5017
5018 recover_report_fills(&order, context).await?;
5019
5020 let events = context.dispatch.order_events.lock();
5021 if !acked_action_is_pending(pending, context) {
5022 return Ok(true);
5023 }
5024
5025 if context
5026 .dispatch
5027 .terminals_pending
5028 .lock()
5029 .contains_key(&client_order_id)
5030 || events.get(&client_order_id).is_some_and(|deferred| {
5031 deferred
5032 .order
5033 .as_ref()
5034 .is_some_and(|order| order.status.is_terminal())
5035 })
5036 || context
5037 .dispatch
5038 .order_identity(&client_order_id)
5039 .is_some_and(|identity| position_fills_pending(&identity, &events, &context.dispatch))
5040 {
5041 return Ok(false);
5042 }
5043
5044 let report = lookup.parse(
5045 &order,
5046 &context.registry,
5047 context.account_id,
5048 context.clock.get_time_ns(),
5049 &context.dispatch,
5050 )?;
5051 let report = lookup.translate(&context.dispatch, report);
5052 if context
5053 .dispatch
5054 .order_identity(&client_order_id)
5055 .is_some_and(|identity| {
5056 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
5057 })
5058 {
5059 return Ok(false);
5060 }
5061
5062 let resolved = report.order_status.is_closed();
5063
5064 let current = if resolved {
5065 context.dispatch.clear_pending_cancel_if_nonce(
5066 &client_order_id,
5067 pending.connection_epoch,
5068 pending.nonce,
5069 )
5070 } else {
5071 context.dispatch.pending_cancel_nonce(&client_order_id)
5072 == Some((pending.connection_epoch, pending.nonce))
5073 };
5074
5075 if !current {
5076 return Ok(true);
5077 }
5078
5079 if resolved {
5080 context
5081 .dispatch
5082 .clear_pending_order_action(&client_order_id);
5083 }
5084
5085 let report = if report.order_status.is_closed() {
5086 report
5087 } else {
5088 context.dispatch.preserve_pending_order_status(report)
5089 };
5090
5091 context.emitter.send_order_status_report(report);
5092 Ok(resolved)
5093}
5094
5095fn confirm_modify_order(
5096 pending: &PendingSendTx,
5097 context: &AckedOrderProbeContext,
5098 order: &crate::http::models::LighterOrder,
5099) -> anyhow::Result<()> {
5100 let (cloid, _) = acked_action_identity(pending)?;
5101 let mut events = context.dispatch.order_events.lock();
5102 if !acked_action_is_pending(pending, context) {
5103 return Ok(());
5104 }
5105
5106 let identity = context
5107 .dispatch
5108 .order_identity(&cloid)
5109 .context("missing modified order identity")?;
5110 let instrument = LIGHTER_INSTRUMENT_CACHE
5111 .get(&identity.instrument_id)
5112 .context("missing modified order instrument")?;
5113 let shape = lighter_order_shape(order, &instrument, identity.order_type)?;
5114 let mut requested_shape = shape.clone();
5115 requested_shape.quantity =
5116 quantity_from_decimal(order.initial_base_amount, instrument.size_precision())?;
5117 anyhow::ensure!(
5118 context.dispatch.pending_modify_shape(&cloid).as_ref() == Some(&requested_shape),
5119 "Lighter executed modification shape is not available yet"
5120 );
5121 identity.record_reduction(order.initial_base_amount - shape.quantity.as_decimal());
5122 let updated = parse_lighter_order_updated(
5123 order,
5124 &instrument,
5125 &identity,
5126 cloid,
5127 context.account_id,
5128 context.emitter.trader_id(),
5129 context.clock.get_time_ns(),
5130 )?;
5131 context.dispatch.store_snapshot(cloid, shape);
5132 identity.publish_update(updated.event_id);
5133 context
5134 .emitter
5135 .send_order_event(OrderEventAny::Updated(updated));
5136 context
5137 .dispatch
5138 .clear_pending_modify_if_nonce(&cloid, pending.connection_epoch, pending.nonce);
5139 let mut deferred = events.remove(&cloid).unwrap_or_default();
5140 if deferred.order.is_none() {
5141 deferred.store_order(order);
5142 }
5143
5144 flush_modify_events(
5145 cloid,
5146 deferred,
5147 &context.dispatch,
5148 &context.emitter,
5149 &mut events,
5150 );
5151 drop(events);
5152 spawn_child_recoveries(context);
5153 Ok(())
5154}
5155
5156fn flush_modify_events(
5157 cloid: ClientOrderId,
5158 deferred: DeferredOrderEvents,
5159 dispatch: &WsDispatchState,
5160 emitter: &ExecutionEventEmitter,
5161 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
5162) {
5163 if dispatch
5164 .order_identity(&cloid)
5165 .is_some_and(|identity| identity.child_fill_baseline().is_some())
5166 && let Some(order) = deferred
5167 .order
5168 .as_ref()
5169 .filter(|order| is_order_triggered(order))
5170 {
5171 dispatch_deferred_order(cloid, order, dispatch, emitter);
5172 }
5173
5174 events.insert(cloid, deferred);
5175 publish_child_recovery(cloid, events, dispatch, emitter);
5176}
5177
5178fn flush_order_terminal(
5179 cloid: ClientOrderId,
5180 deferred: DeferredOrderEvents,
5181 dispatch: &WsDispatchState,
5182 emitter: &ExecutionEventEmitter,
5183 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
5184) {
5185 for fill in deferred.fills {
5186 if dispatch.mark_trade_seen(fill.trade_id) {
5187 dispatch_lighter_fill(fill, emitter, dispatch, events);
5188 }
5189 }
5190
5191 let Some(order) = deferred.order else {
5192 return;
5193 };
5194
5195 dispatch_deferred_order(cloid, &order, dispatch, emitter);
5196}
5197
5198fn dispatch_deferred_order(
5199 cloid: ClientOrderId,
5200 order: &crate::http::models::LighterOrder,
5201 dispatch: &WsDispatchState,
5202 emitter: &ExecutionEventEmitter,
5203) {
5204 let Some(identity) = dispatch
5205 .order_identities
5206 .get(&cloid)
5207 .map(|entry| entry.value().clone())
5208 else {
5209 return;
5210 };
5211
5212 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id) else {
5213 return;
5214 };
5215
5216 let ts_init = get_atomic_clock_realtime().get_time_ns();
5217
5218 if order.status == crate::common::enums::LighterOrderStatus::Filled {
5219 dispatch_owned_order(
5220 order,
5221 &instrument,
5222 &identity,
5223 cloid,
5224 VenueOrderId::new(&order.order_id),
5225 dispatch,
5226 emitter,
5227 emitter.account_id(),
5228 emitter.trader_id(),
5229 ts_init,
5230 );
5231 return;
5232 }
5233
5234 let context = OpenFrameContext {
5235 accepted_already_emitted: dispatch.accepted_was_emitted(&cloid),
5236 triggered_already_emitted: dispatch.triggered_was_emitted(&cloid),
5237 shape_changed: false,
5238 };
5239
5240 match parse_lighter_order_event(
5241 order,
5242 &instrument,
5243 &identity,
5244 cloid,
5245 emitter.account_id(),
5246 emitter.trader_id(),
5247 context,
5248 ts_init,
5249 ) {
5250 Ok(Some(event)) => {
5251 let Some(event) = defer_terminal(order, event, &identity, cloid, dispatch) else {
5252 return;
5253 };
5254
5255 dispatch_tracked_order_event(
5256 event,
5257 cloid,
5258 VenueOrderId::new(&order.order_id),
5259 &identity,
5260 emitter.account_id(),
5261 emitter.trader_id(),
5262 emitter,
5263 dispatch,
5264 ts_init,
5265 );
5266 }
5267 Ok(None) => {}
5268 Err(e) => log::error!("Failed to parse deferred Lighter order for {cloid}: {e}"),
5269 }
5270}
5271
5272async fn probe_acked_create(
5273 probe: &AckedOrderProbe,
5274 context: &AckedOrderProbeContext,
5275) -> anyhow::Result<()> {
5276 let AckedOrderProbe {
5277 order,
5278 client_order_index,
5279 connection_epoch,
5280 nonce,
5281 ..
5282 } = probe;
5283
5284 let client_order_id = order.client_order_id();
5285 let mut transaction_executed = false;
5286
5287 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
5288 if context.connection_epoch.load(Ordering::Acquire) != *connection_epoch {
5289 log::warn!(
5290 "Lighter acknowledged create outcome unresolved after reconnect for {client_order_id}; retaining identity for reconciliation",
5291 );
5292 return Ok(());
5293 }
5294
5295 if !context.dispatch.create_submission_is_pending(
5296 &client_order_id,
5297 *client_order_index,
5298 *nonce,
5299 ) {
5300 return Ok(());
5301 }
5302
5303 if probe_acked_create_report(probe, context, attempt).await {
5304 return Ok(());
5305 }
5306
5307 match probe_acked_create_tx(probe, context, attempt).await? {
5308 ControlFlow::Break(()) => return Ok(()),
5309 ControlFlow::Continue(executed) => transaction_executed |= executed,
5310 }
5311
5312 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
5313 tokio::select! {
5314 () = context.cancellation_token.cancelled() => return Ok(()),
5315 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
5316 }
5317 }
5318 }
5319
5320 if transaction_executed {
5321 let _ = context.dispatch.confirm_create_submission(
5322 &client_order_id,
5323 *client_order_index,
5324 *nonce,
5325 );
5326 log::warn!(
5327 "Lighter acknowledged create transaction executed without a queryable order for {client_order_id}; retaining identity for reconciliation",
5328 );
5329 } else {
5330 log::warn!(
5331 "Lighter acknowledged create outcome unresolved after {ACKED_ORDER_PROBE_ATTEMPTS} transaction lookups for {client_order_id}; retaining identity for reconciliation",
5332 );
5333 }
5334 Ok(())
5335}
5336
5337async fn probe_acked_create_report(
5338 probe: &AckedOrderProbe,
5339 context: &AckedOrderProbeContext,
5340 attempt: usize,
5341) -> bool {
5342 let AckedOrderProbe {
5343 order,
5344 client_order_index,
5345 nonce,
5346 ..
5347 } = probe;
5348
5349 let client_order_id = order.client_order_id();
5350
5351 let report = match lookup_create_order_status_report(
5352 &context.http_client,
5353 &context.registry,
5354 &context.credential,
5355 context.account_id,
5356 order.instrument_id(),
5357 client_order_id,
5358 *client_order_index,
5359 *nonce,
5360 &context.dispatch,
5361 context.clock,
5362 )
5363 .await
5364 {
5365 Ok(report) => report,
5366 Err(e) => {
5367 log::warn!(
5368 "Lighter acknowledged create order lookup failed for {client_order_id} on attempt {attempt}: {e:?}",
5369 );
5370 None
5371 }
5372 };
5373
5374 if let Some(report) = report {
5375 let _events = context.dispatch.order_events.lock();
5376 if context
5377 .dispatch
5378 .order_identity(&client_order_id)
5379 .is_some_and(|identity| {
5380 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
5381 })
5382 {
5383 return false;
5384 }
5385
5386 if context.dispatch.observe_create_submission(
5387 &client_order_id,
5388 *client_order_index,
5389 *nonce,
5390 report.venue_order_id,
5391 ) {
5392 context.dispatch.seed_accepted_from_report(&report);
5393 context.emitter.send_order_status_report(report);
5394 }
5395
5396 return true;
5397 }
5398
5399 false
5400}
5401
5402async fn probe_acked_create_tx(
5403 probe: &AckedOrderProbe,
5404 context: &AckedOrderProbeContext,
5405 attempt: usize,
5406) -> anyhow::Result<ControlFlow<(), bool>> {
5407 let AckedOrderProbe {
5408 order,
5409 client_order_index,
5410 connection_epoch,
5411 nonce,
5412 api_key_index,
5413 tx_hash,
5414 } = probe;
5415
5416 let client_order_id = order.client_order_id();
5417
5418 let tx = match context.http_client.get_tx(tx_hash.clone()).await {
5419 Ok(tx) => tx,
5420 Err(LighterHttpError::Venue { code: 21500, .. }) => {
5421 return Ok(ControlFlow::Continue(false));
5422 }
5423 Err(e) => {
5424 log::warn!(
5425 "Lighter acknowledged create transaction lookup failed for {client_order_id} on attempt {attempt}: {e}",
5426 );
5427 return Ok(ControlFlow::Continue(false));
5428 }
5429 };
5430
5431 validate_acked_create_tx(
5432 &tx,
5433 context.credential.account_index(),
5434 *api_key_index,
5435 *client_order_index,
5436 *nonce,
5437 tx_hash,
5438 )?;
5439
5440 let event = parse_acked_tx_event(&tx.event_info).unwrap_or_else(|e| {
5441 log::warn!(
5442 "Lighter create transaction carried invalid event_info for {client_order_id}: {e}",
5443 );
5444 AckedTxEvent::default()
5445 });
5446
5447 if tx.status == LighterTxStatus::Failed || !event.app_error.is_empty() {
5448 let detail = acked_tx_error_detail(&event.app_error);
5449 let reason = format!(
5450 "Lighter sequencer rejected acknowledged create transaction {tx_hash}: {detail}",
5451 );
5452 reject_create_order(
5453 &context.dispatch,
5454 &context.emitter,
5455 order,
5456 *client_order_index,
5457 *nonce,
5458 &reason,
5459 context.clock.get_time_ns(),
5460 lighter_reason_indicates_post_only_rejection(detail),
5461 Some((&context.connection_epoch, *connection_epoch)),
5462 );
5463 return Ok(ControlFlow::Break(()));
5464 }
5465
5466 Ok(ControlFlow::Continue(
5467 tx.status == LighterTxStatus::Executed,
5468 ))
5469}
5470
5471#[derive(Deserialize)]
5472struct AckedCreateTxInfo {
5473 #[serde(rename = "ClientOrderIndex")]
5474 client_order_index: i64,
5475}
5476
5477#[derive(Deserialize)]
5478struct AckedCreateGroupTxInfo {
5479 #[serde(rename = "Orders")]
5480 orders: Vec<AckedCreateTxInfo>,
5481}
5482
5483#[derive(Default, Deserialize)]
5484struct AckedTxEvent {
5485 #[serde(default, rename = "ae")]
5486 app_error: String,
5487}
5488
5489fn validate_acked_create_tx(
5490 tx: &crate::http::models::LighterTx,
5491 account_index: i64,
5492 api_key_index: u8,
5493 client_order_index: i64,
5494 nonce: i64,
5495 tx_hash: &str,
5496) -> anyhow::Result<()> {
5497 anyhow::ensure!(
5498 tx_hash_matches(&tx.hash, tx_hash)
5499 && matches!(tx.tx_type, value if value == LighterTxType::CreateOrder as u8 || value == LighterTxType::CreateGroupedOrders as u8)
5500 && tx.account_index == account_index
5501 && tx.api_key_index == api_key_index
5502 && tx.nonce == nonce,
5503 "Lighter transaction lookup did not match acknowledged create identity",
5504 );
5505
5506 if tx.tx_type == LighterTxType::CreateGroupedOrders as u8 {
5507 let info: AckedCreateGroupTxInfo = serde_json::from_str(&tx.info)
5508 .context("failed to parse Lighter grouped transaction info")?;
5509 anyhow::ensure!(
5510 info.orders
5511 .iter()
5512 .any(|order| order.client_order_index == client_order_index),
5513 "Lighter transaction lookup did not include acknowledged client_order_index {client_order_index}"
5514 );
5515 return Ok(());
5516 }
5517
5518 let info: AckedCreateTxInfo = serde_json::from_str(&tx.info)
5519 .context("failed to parse Lighter create transaction info")?;
5520 anyhow::ensure!(
5521 info.client_order_index == client_order_index,
5522 "Lighter transaction lookup returned client_order_index {} for acknowledged create {client_order_index}",
5523 info.client_order_index,
5524 );
5525 Ok(())
5526}
5527
5528fn parse_acked_tx_event(event_info: &str) -> anyhow::Result<AckedTxEvent> {
5529 if event_info.trim().is_empty() {
5530 return Ok(AckedTxEvent::default());
5531 }
5532
5533 serde_json::from_str(event_info).context("failed to parse Lighter transaction event")
5534}
5535
5536fn tx_application_error(
5537 status: LighterTxStatus,
5538 event_info: &str,
5539) -> anyhow::Result<Option<String>> {
5540 let event = match parse_acked_tx_event(event_info) {
5541 Ok(event) => event,
5542 Err(e) if status == LighterTxStatus::Failed => {
5543 log::warn!("Lighter failed transaction carried invalid event_info: {e}");
5544 AckedTxEvent::default()
5545 }
5546 Err(e) => return Err(e),
5547 };
5548
5549 Ok(
5550 (status == LighterTxStatus::Failed || !event.app_error.is_empty())
5551 .then(|| acked_tx_error_detail(&event.app_error).to_owned()),
5552 )
5553}
5554
5555fn acked_tx_error_detail(app_error: &str) -> &str {
5556 if app_error.is_empty() {
5557 "transaction failed without an application error"
5558 } else {
5559 app_error
5560 }
5561}
5562
5563#[derive(Debug, Clone)]
5564struct AckedOrderProbe {
5565 order: Box<OrderAny>,
5566 client_order_index: i64,
5567 connection_epoch: u64,
5568 nonce: i64,
5569 api_key_index: u8,
5570 tx_hash: String,
5571}
5572
5573impl AckedOrderProbe {
5574 fn from_pending(pending: &PendingSendTx) -> Option<Self> {
5575 let PendingSendTxKind::Create {
5576 order,
5577 client_order_index,
5578 } = &pending.kind
5579 else {
5580 return None;
5581 };
5582
5583 Some(Self {
5584 order: order.clone(),
5585 client_order_index: *client_order_index,
5586 connection_epoch: pending.connection_epoch,
5587 nonce: pending.nonce,
5588 api_key_index: pending.api_key_index,
5589 tx_hash: pending.tx_hash.clone(),
5590 })
5591 }
5592}
5593
5594#[expect(
5603 clippy::too_many_arguments,
5604 reason = "shared terminal create transition keeps cleanup and event attribution together"
5605)]
5606fn reject_create_order(
5607 dispatch: &WsDispatchState,
5608 emitter: &ExecutionEventEmitter,
5609 order: &OrderAny,
5610 client_order_index: i64,
5611 nonce: i64,
5612 reason: &str,
5613 now: UnixNanos,
5614 due_post_only: bool,
5615 connection_epoch: Option<(&AtomicU64, u64)>,
5616) -> bool {
5617 let cloid = order.client_order_id();
5618 if !dispatch.reject_create_submission(&cloid, client_order_index, nonce, connection_epoch) {
5619 log::warn!(
5620 "Ignored stale Lighter create rejection for cloid={cloid} client_order_index={client_order_index} nonce={nonce}",
5621 );
5622 return false;
5623 }
5624 emitter.emit_order_rejected(order, reason, now, due_post_only);
5625 true
5626}
5627
5628#[expect(
5629 clippy::too_many_arguments,
5630 reason = "consumer-loop sink that flattens one SendTxRejected message without a wrapper struct"
5631)]
5632fn handle_send_tx_rejection_for_connection(
5633 dispatch: &WsDispatchState,
5634 emitter: &ExecutionEventEmitter,
5635 account_index: Option<i64>,
5636 connection_epoch: u64,
5637 now: UnixNanos,
5638 source: SendTxRejectionSource,
5639 code: Option<i64>,
5640 message: &str,
5641 tx_hash: Option<&str>,
5642) -> bool {
5643 let needs_nonce_resync = code == Some(LIGHTER_ERROR_CODE_INVALID_NONCE);
5644 let failure = venue_rejection_failure(code, message);
5645
5646 let pending = match tx_hash {
5647 Some(hash) => dispatch.remove_pending_sendtx_by_hash(connection_epoch, hash),
5648 None => match source {
5649 SendTxRejectionSource::Ack => dispatch.pop_pending_sendtx_if_only(connection_epoch),
5650 SendTxRejectionSource::BareError => dispatch.pop_pending_sendtx_if_only_within(
5651 connection_epoch,
5652 now,
5653 SENDTX_BARE_ERROR_WINDOW_MS,
5654 ),
5655 },
5656 };
5657 let Some(pending) = pending else {
5658 log::warn!(
5659 "Lighter sendTx rejection unattributed (source={source:?} code={code:?}): {message:?}",
5660 );
5661 return needs_nonce_resync;
5662 };
5663
5664 reject_pending_sendtx(
5665 dispatch,
5666 emitter,
5667 account_index,
5668 &pending,
5669 &failure,
5670 now,
5671 lighter_reason_indicates_post_only_rejection(message),
5672 );
5673 needs_nonce_resync
5674}
5675
5676fn reject_pending_sendtx(
5677 dispatch: &WsDispatchState,
5678 emitter: &ExecutionEventEmitter,
5679 account_index: Option<i64>,
5680 pending: &PendingSendTx,
5681 failure: &CommandFailure,
5682 now: UnixNanos,
5683 due_post_only: bool,
5684) {
5685 let reason = command_failure_reason(failure);
5686 log::warn!(
5687 "{reason}, nonce={} api_key_index={}",
5688 pending.nonce,
5689 pending.api_key_index
5690 );
5691
5692 if let Some(account_index) = account_index {
5693 let _ = dispatch.nonce_manager.ack_failure_if_latest(
5694 account_index,
5695 pending.api_key_index,
5696 pending.nonce,
5697 );
5698 }
5699
5700 for (order, client_order_index) in pending.kind.creates() {
5701 reject_create_order(
5702 dispatch,
5703 emitter,
5704 order,
5705 client_order_index,
5706 pending.nonce,
5707 reason,
5708 now,
5709 due_post_only,
5710 None,
5711 );
5712 }
5713
5714 match &pending.kind {
5715 PendingSendTxKind::Create { .. } | PendingSendTxKind::CreateGroup { .. } => {}
5716 PendingSendTxKind::Cancel { .. } => {
5717 reject_cancel_order(dispatch, emitter, pending, failure, now);
5718 }
5719 PendingSendTxKind::Modify { .. } => {
5720 reject_modify_order(dispatch, emitter, pending, failure, now, false);
5721 }
5722 PendingSendTxKind::Other => {}
5723 }
5724}
5725
5726fn reject_cancel_order(
5727 dispatch: &WsDispatchState,
5728 emitter: &ExecutionEventEmitter,
5729 pending: &PendingSendTx,
5730 failure: &CommandFailure,
5731 now: UnixNanos,
5732) {
5733 let PendingSendTxKind::Cancel {
5734 command_id: Some(command_id),
5735 strategy_id,
5736 instrument_id,
5737 client_order_id,
5738 venue_order_id,
5739 ..
5740 } = pending.kind
5741 else {
5742 return;
5743 };
5744
5745 dispatch.clear_pending_cancel_if_command(&client_order_id, command_id);
5746
5747 match failure {
5748 CommandFailure::NotSent(reason) => emitter.emit_order_cancel_denied_event(
5749 strategy_id,
5750 instrument_id,
5751 client_order_id,
5752 venue_order_id,
5753 reason,
5754 now,
5755 Some(command_id),
5756 ),
5757 CommandFailure::VenueRejected(reason) => emitter.emit_order_cancel_rejected_event(
5758 strategy_id,
5759 instrument_id,
5760 client_order_id,
5761 venue_order_id,
5762 reason,
5763 now,
5764 Some(command_id),
5765 ),
5766 _ => {}
5767 }
5768}
5769
5770fn reject_modify_order(
5771 dispatch: &WsDispatchState,
5772 emitter: &ExecutionEventEmitter,
5773 pending: &PendingSendTx,
5774 failure: &CommandFailure,
5775 now: UnixNanos,
5776 require_pending: bool,
5777) {
5778 let PendingSendTxKind::Modify {
5779 command_id,
5780 strategy_id,
5781 instrument_id,
5782 client_order_id,
5783 venue_order_id,
5784 } = pending.kind
5785 else {
5786 return;
5787 };
5788
5789 let mut events = dispatch.order_events.lock();
5790 let current = dispatch.clear_pending_modify_if_nonce(
5791 &client_order_id,
5792 pending.connection_epoch,
5793 pending.nonce,
5794 );
5795
5796 if require_pending && !current {
5797 return;
5798 }
5799
5800 match failure {
5801 CommandFailure::NotSent(reason) => emitter.emit_order_modify_denied_event(
5802 strategy_id,
5803 instrument_id,
5804 client_order_id,
5805 venue_order_id,
5806 reason,
5807 now,
5808 Some(command_id),
5809 ),
5810 CommandFailure::VenueRejected(reason) => emitter.emit_order_modify_rejected_event(
5811 strategy_id,
5812 instrument_id,
5813 client_order_id,
5814 venue_order_id,
5815 reason,
5816 now,
5817 Some(command_id),
5818 ),
5819 _ => {}
5820 }
5821
5822 if current && let Some(deferred) = events.remove(&client_order_id) {
5823 flush_modify_events(client_order_id, deferred, dispatch, emitter, &mut events);
5824 }
5825}
5826
5827fn lighter_reason_indicates_post_only_rejection(reason: &str) -> bool {
5828 let normalized: String = reason
5829 .chars()
5830 .filter_map(|ch| {
5831 if ch == '-' || ch == '_' || ch.is_whitespace() {
5832 None
5833 } else {
5834 Some(ch.to_ascii_lowercase())
5835 }
5836 })
5837 .collect();
5838
5839 normalized.contains("postonly") || normalized.contains("postwouldexecute")
5840}
5841
5842fn lighter_http_error_is_definite_api_rejection(error: &LighterHttpError) -> bool {
5843 match error {
5844 LighterHttpError::RateLimit { .. } | LighterHttpError::Venue { .. } => true,
5845 LighterHttpError::Http { status, .. } => *status < 500,
5846 LighterHttpError::Network(_)
5847 | LighterHttpError::HistoryIncomplete { .. }
5848 | LighterHttpError::Parse(_) => false,
5849 }
5850}
5851
5852fn rollback_tx_dispatch(
5853 dispatch: &WsDispatchState,
5854 credential: &Credential,
5855 client_order_index: Option<i64>,
5856 nonce: i64,
5857) {
5858 rollback_tx_dispatch_indices(
5859 dispatch,
5860 credential.account_index(),
5861 credential.api_key_index(),
5862 client_order_index,
5863 None,
5864 nonce,
5865 );
5866}
5867
5868fn rollback_tx_dispatch_create(
5869 dispatch: &WsDispatchState,
5870 credential: &Credential,
5871 client_order_index: Option<i64>,
5872 client_order_id: &ClientOrderId,
5873 nonce: i64,
5874) {
5875 rollback_tx_dispatch_indices(
5876 dispatch,
5877 credential.account_index(),
5878 credential.api_key_index(),
5879 client_order_index,
5880 Some(client_order_id),
5881 nonce,
5882 );
5883}
5884
5885fn rollback_tx_dispatch_indices(
5889 dispatch: &WsDispatchState,
5890 account_index: i64,
5891 api_key_index: u8,
5892 client_order_index: Option<i64>,
5893 client_order_id: Option<&ClientOrderId>,
5894 nonce: i64,
5895) {
5896 let _ = dispatch
5897 .nonce_manager
5898 .ack_failure_if_latest(account_index, api_key_index, nonce);
5899
5900 if let Some(client_order_index) = client_order_index {
5901 dispatch.forget_cloid(client_order_index);
5902 }
5903
5904 if let Some(cloid) = client_order_id {
5905 dispatch.forget_order_identity(cloid);
5906 }
5907}
5908
5909fn integrator_attributes(integrator_account_index: Option<u64>) -> L2TxAttributes {
5910 integrator_account_index.map_or_else(L2TxAttributes::default, |integrator_account_index| {
5911 L2TxAttributes {
5912 integrator_account_index,
5913 ..Default::default()
5914 }
5915 })
5916}
5917
5918fn command_failure_reason(failure: &CommandFailure) -> &str {
5919 match failure {
5920 CommandFailure::NotSent(reason)
5921 | CommandFailure::Ambiguous(reason)
5922 | CommandFailure::VenueRejected(reason) => reason,
5923 }
5924}
5925
5926fn warn_pending_sendtx_unknown(pending_sendtx: &[PendingSendTx], context: &str) {
5927 if pending_sendtx.is_empty() {
5928 return;
5929 }
5930
5931 log::warn!(
5932 "Discarded {} pending sendTx entries during {context}; order state recovers via \
5933 reconciliation",
5934 pending_sendtx.len(),
5935 );
5936
5937 for pending in pending_sendtx {
5938 for (order, _) in pending.kind.creates() {
5939 log::warn!(
5940 "Lighter sendTx outcome unknown during {context} for {}",
5941 order.client_order_id(),
5942 );
5943 }
5944 }
5945}
5946
5947fn classify_lighter_ws_command_failure(action: &str, error: &LighterWsError) -> CommandFailure {
5948 let fallback = format!("Lighter {action} dispatch failed");
5949 let reason = sanitize_strategy_reason(&format!("{fallback}: {error}"))
5950 .unwrap_or_else(|| fallback.clone());
5951
5952 match error {
5953 LighterWsError::SendTxOutcomeUnknown(_)
5954 | LighterWsError::Network(_)
5955 | LighterWsError::Parse(_)
5956 | LighterWsError::Transport(SendError::WriteTimeout | SendError::BrokenPipe(_)) => {
5957 CommandFailure::ambiguous(reason)
5958 }
5959 LighterWsError::Transport(
5960 SendError::InvalidInput(_)
5961 | SendError::BufferFull
5962 | SendError::Closed
5963 | SendError::Timeout
5964 | SendError::ConnectionChanged,
5965 )
5966 | LighterWsError::Authentication(_)
5967 | LighterWsError::Client(_) => CommandFailure::not_sent(reason),
5968 }
5969}
5970
5971fn venue_rejection_failure(code: Option<i64>, message: &str) -> CommandFailure {
5972 let clean_message = sanitize_strategy_reason(message);
5973 let reason = match (code, clean_message) {
5974 (Some(code), Some(message)) => format!("LIGHTER_{code}: {message}"),
5975 (Some(code), None) => format!("LIGHTER_{code}"),
5976 (None, Some(message)) => message,
5977 (None, None) => "Lighter venue rejected sendTx".to_string(),
5978 };
5979 let reason = sanitize_strategy_reason(&reason)
5980 .unwrap_or_else(|| "Lighter venue rejected sendTx".to_string());
5981 CommandFailure::venue_rejected(reason)
5982}
5983
5984fn sanitize_strategy_reason(input: &str) -> Option<String> {
5985 let mut output = String::new();
5986 let mut inside_markup = false;
5987 let mut pending_space = false;
5988 let mut char_count = 0;
5989 let mut chars = input.chars().peekable();
5990
5991 while let Some(ch) = chars.next() {
5992 if inside_markup {
5993 if ch == '>' {
5994 inside_markup = false;
5995 }
5996 continue;
5997 }
5998
5999 match ch {
6000 '<' if chars.peek().is_some_and(|next| {
6001 next.is_ascii_alphabetic() || matches!(next, '/' | '!' | '?')
6002 }) =>
6003 {
6004 inside_markup = true;
6005 pending_space = !output.is_empty();
6006 }
6007 _ if ch.is_control() || is_unicode_format_control(ch) || ch.is_whitespace() => {
6008 pending_space = !output.is_empty();
6009 }
6010 _ => {
6011 if pending_space && char_count < STRATEGY_REASON_MAX_CHARS {
6012 output.push(' ');
6013 char_count += 1;
6014 }
6015 pending_space = false;
6016
6017 if char_count == STRATEGY_REASON_MAX_CHARS {
6018 break;
6019 }
6020 output.push(ch);
6021 char_count += 1;
6022 }
6023 }
6024 }
6025
6026 let output = output.trim().to_string();
6027 (!output.is_empty()).then_some(output)
6028}
6029
6030fn is_unicode_format_control(ch: char) -> bool {
6031 matches!(
6032 ch,
6033 '\u{00ad}'
6034 | '\u{061c}'
6035 | '\u{06dd}'
6036 | '\u{070f}'
6037 | '\u{08e2}'
6038 | '\u{180e}'
6039 | '\u{feff}'
6040 | '\u{fff9}'..='\u{fffb}'
6041 | '\u{0600}'..='\u{0605}'
6042 | '\u{0890}'..='\u{0891}'
6043 | '\u{200b}'..='\u{200f}'
6044 | '\u{202a}'..='\u{202e}'
6045 | '\u{2060}'..='\u{206f}'
6046 | '\u{110bd}'
6047 | '\u{110cd}'
6048 | '\u{13430}'..='\u{1343f}'
6049 | '\u{1bca0}'..='\u{1bca3}'
6050 | '\u{1d173}'..='\u{1d17a}'
6051 | '\u{e0001}'
6052 | '\u{e0020}'..='\u{e007f}'
6053 )
6054}
6055
6056fn format_between_timestamps(
6061 start: Option<UnixNanos>,
6062 end: Option<UnixNanos>,
6063 ts_now: UnixNanos,
6064) -> Option<String> {
6065 let (start, end) = match (start, end) {
6066 (None, None) => return None,
6067 (Some(s), Some(e)) => (s, e),
6068 (Some(s), None) => (s, ts_now),
6069 (None, Some(e)) => (UnixNanos::from(0), e),
6070 };
6071 let start_secs = start.as_u64() / 1_000_000_000;
6072 let end_secs = end.as_u64() / 1_000_000_000;
6073 Some(format!("{start_secs}-{end_secs}"))
6074}
6075
6076#[async_trait(?Send)]
6077impl ExecutionClient for LighterExecutionClient {
6078 fn is_connected(&self) -> bool {
6079 self.core.is_connected()
6080 }
6081
6082 fn client_id(&self) -> ClientId {
6083 self.core.client_id
6084 }
6085
6086 fn account_id(&self) -> AccountId {
6087 self.core.account_id
6088 }
6089
6090 fn venue(&self) -> Venue {
6091 self.core.venue
6092 }
6093
6094 fn oms_type(&self) -> OmsType {
6095 self.core.oms_type
6096 }
6097
6098 fn get_account(&self) -> Option<AccountAny> {
6099 self.core.cache().account_owned(&self.core.account_id)
6100 }
6101
6102 fn generate_account_state(
6103 &self,
6104 balances: Vec<AccountBalance>,
6105 margins: Vec<MarginBalance>,
6106 reported: bool,
6107 ts_event: UnixNanos,
6108 info: Option<Params>,
6109 ) -> anyhow::Result<()> {
6110 self.emitter
6111 .emit_account_state(balances, margins, reported, ts_event, info);
6112 Ok(())
6113 }
6114
6115 fn start(&mut self) -> anyhow::Result<()> {
6116 let sender = get_exec_event_sender();
6117 self.emitter.set_sender(sender);
6118 self.subscribe_applied_fills();
6119
6120 if self.core.is_started() {
6121 return Ok(());
6122 }
6123
6124 self.core.set_started();
6125
6126 log::info!(
6127 "Started Lighter execution client: client_id={}, account_id={}, environment={:?}, has_credentials={}",
6128 self.core.client_id,
6129 self.core.account_id,
6130 self.config.environment,
6131 self.has_credentials(),
6132 );
6133
6134 Ok(())
6135 }
6136
6137 fn stop(&mut self) -> anyhow::Result<()> {
6138 if self.core.is_stopped() && self.core.is_disconnected() && self.session_tasks_finished() {
6139 return Ok(());
6140 }
6141
6142 log::info!("Stopping Lighter execution client {}", self.core.client_id);
6143
6144 self.begin_session_shutdown();
6145
6146 self.core.set_stopped();
6147
6148 log::info!("Lighter execution client stopped");
6149 Ok(())
6150 }
6151
6152 fn reset(&mut self) -> anyhow::Result<()> {
6153 log::debug!("Resetting Lighter execution client {}", self.core.client_id);
6154 self.begin_session_shutdown();
6155 Ok(())
6156 }
6157
6158 fn dispose(&mut self) -> anyhow::Result<()> {
6159 log::debug!("Disposing Lighter execution client {}", self.core.client_id);
6160 self.stop()
6161 }
6162
6163 async fn connect(&mut self) -> anyhow::Result<()> {
6164 if self.core.is_connected() && self.pending_tasks.is_open() {
6165 return Ok(());
6166 }
6167
6168 if !self.has_credentials() {
6172 anyhow::bail!(
6173 "Lighter execution client requires credentials; \
6174 set private_key, account_index, and api_key_index in the config \
6175 or use the deployment-specific credential environment variables"
6176 );
6177 }
6178
6179 log::info!(
6180 "Connecting Lighter execution client {}",
6181 self.core.client_id
6182 );
6183
6184 if !self.session_tasks_finished() || !self.pending_tasks.is_open() {
6187 self.begin_session_shutdown();
6188 self.finish_session_shutdown().await?;
6189 }
6190
6191 if !self.pending_tasks.is_open() {
6192 self.pending_tasks
6193 .start_generation()
6194 .map_err(|e| anyhow::anyhow!("Failed to start Lighter task generation: {e}"))?;
6195 self.cancellation_token = self.pending_tasks.cancellation_token();
6196 }
6197
6198 let ws_client = self.ws_client.clone();
6199 let setup_guard = TaskGroupGuard::new(&[&self.pending_tasks], move || {
6200 ws_client.begin_shutdown();
6201 });
6202 self.auth_refresh_notify = Arc::new(tokio::sync::Notify::new());
6203 self.scheduled_cancel_notify = Arc::new(tokio::sync::Notify::new());
6204
6205 self.dispatch.account_streams_ready.reset();
6211 self.dispatch.clear_position_cache();
6212 self.dispatch.clear_account_state_cache();
6213
6214 self.ensure_instruments_initialized_async().await?;
6215 self.refresh_nonce().await?;
6216
6217 self.nonce_ready_connection_epoch
6219 .store(self.ws_client.connection_epoch(), Ordering::Release);
6220 let account_detail = self.fetch_account_detail().await;
6221
6222 if let Some(detail) = &account_detail {
6223 self.log_account_tier(detail);
6224
6225 let _ = tokio::time::timeout(
6226 REFERRAL_ATTRIBUTION_TIMEOUT,
6227 self.apply_referral_attribution(detail),
6228 )
6229 .await;
6230 }
6231
6232 if let Err(e) = self.submit_integrator_auto_approval().await {
6233 let is_unapproved = e.chain().any(|cause| {
6238 matches!(
6239 cause.downcast_ref::<LighterHttpError>(),
6240 Some(LighterHttpError::Venue { code: 21149, .. }),
6241 )
6242 });
6243
6244 if is_unapproved {
6245 self.nonce_ready_connection_epoch
6246 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
6247 return Err(e.context(
6248 "Lighter account is not integrator-approved (venue 21149); \
6249 orders cannot be placed",
6250 ));
6251 }
6252
6253 log::warn!("Lighter integrator approval failed; continuing startup: {e:?}");
6254 }
6255
6256 self.nonce_ready_connection_epoch
6257 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
6258
6259 if let Err(e) = self.sync_nonce_from_venue().await {
6260 log::debug!(
6261 "Failed to sync Lighter nonce after integrator approval; continuing startup: {e:?}"
6262 );
6263 }
6264
6265 self.subscribe_applied_fills();
6266
6267 if let Err(e) = self.spawn_ws_consumer().await {
6268 self.begin_session_shutdown();
6269 return match self.finish_session_shutdown().await {
6270 Ok(()) => Err(e),
6271 Err(shutdown_error) => Err(anyhow::anyhow!(
6272 "{e}; failed to roll back partial Lighter connection: {shutdown_error}"
6273 )),
6274 };
6275 }
6276 self.nonce_ready_connection_epoch
6277 .store(self.ws_client.connection_epoch(), Ordering::Release);
6278
6279 let ready = async {
6280 self.await_account_streams_ready(30.0).await?;
6281 self.start_scheduled_cancel().await
6282 }
6283 .await;
6284
6285 if let Err(e) = ready {
6286 log::warn!("Connect failed after WS started, tearing down: {e}");
6287 self.begin_session_shutdown();
6288
6289 if let Err(shutdown_error) = self.finish_session_shutdown().await {
6290 log::warn!("Failed to finish partial Lighter connection: {shutdown_error}");
6291 }
6292
6293 return Err(e);
6294 }
6295
6296 setup_guard.disarm();
6297 self.core.set_connected();
6298
6299 log::info!("Connected: client_id={}", self.core.client_id);
6300 Ok(())
6301 }
6302
6303 async fn disconnect(&mut self) -> anyhow::Result<()> {
6304 if self.core.is_disconnected() && self.session_tasks_finished() {
6305 return Ok(());
6306 }
6307
6308 log::info!(
6309 "Disconnecting Lighter execution client {}",
6310 self.core.client_id
6311 );
6312
6313 self.begin_session_shutdown();
6314 let tasks_result = self.finish_session_shutdown().await;
6315
6316 self.core.set_disconnected();
6317
6318 log::info!("Disconnected: client_id={}", self.core.client_id);
6319 tasks_result
6320 }
6321
6322 fn submit_order(&self, cmd: SubmitOrder) -> anyhow::Result<()> {
6323 let credential = self.credential.as_ref().ok_or_else(|| {
6324 anyhow::anyhow!("Lighter execution client cannot submit without credentials")
6325 })?;
6326
6327 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
6328
6329 if order.is_closed() {
6330 log::warn!("Cannot submit closed order {}", order.client_order_id());
6331 return Ok(());
6332 }
6333
6334 let cached_instrument = self
6335 .core
6336 .cache()
6337 .instrument(&order.instrument_id())
6338 .cloned();
6339
6340 if let Some(reason) = local_submit_denial_reason(&order, cached_instrument.as_ref()) {
6341 self.emitter.emit_order_denied(&order, &reason.to_string());
6342 return Ok(());
6343 }
6344
6345 let plan = match self.prepare_create_order_plan(&order, cmd.params.as_ref()) {
6346 Ok(plan) => plan,
6347 Err(e) => {
6348 let reason = OrderDeniedReason::ValidationFailed {
6349 detail: format!("Lighter submit_order failed: {e}"),
6350 };
6351 self.emitter.emit_order_denied(&order, &reason.to_string());
6352 return Ok(());
6353 }
6354 };
6355
6356 if let Err(e) = self.dispatch_create_order_plan(&plan, credential) {
6357 let reason = OrderDeniedReason::SubmitFailed {
6358 detail: format!("Lighter submit_order failed: {e}"),
6359 };
6360 self.emitter.emit_order_denied(&order, &reason.to_string());
6361 }
6362
6363 Ok(())
6364 }
6365
6366 fn submit_order_list(&self, cmd: SubmitOrderList) -> anyhow::Result<()> {
6367 let credential = self.credential.as_ref().ok_or_else(|| {
6368 anyhow::anyhow!("Lighter execution client cannot submit without credentials")
6369 })?;
6370
6371 if cmd.order_list.client_order_ids.is_empty() {
6372 log::debug!("submit_order_list called with empty order list");
6373 return Ok(());
6374 }
6375
6376 let orders = self.core.get_orders_for_list(&cmd.order_list)?;
6377
6378 if orders.iter().any(is_grouped_order) {
6379 let result = self
6380 .prepare_create_group_plan(&orders, cmd.params.as_ref())
6381 .and_then(|plan| {
6382 let context = self.fanout_dispatch_context(credential)?;
6383 self.spawn_task("submit_grouped_orders", async move {
6384 context
6385 .dispatch_batch_plans(vec![BatchOrderPlan::CreateGroup(plan)])
6386 .await;
6387 Ok(())
6388 });
6389
6390 Ok(())
6391 });
6392
6393 if let Err(e) = result {
6394 let reason = grouped_submit_denial_reason(&e);
6395 for order in &orders {
6396 self.emitter.emit_order_denied(order, &reason);
6397 }
6398 }
6399 return Ok(());
6400 }
6401
6402 let mut plans = Vec::with_capacity(orders.len());
6403
6404 for order in orders {
6405 if order.is_closed() {
6406 log::warn!("Cannot submit closed order {}", order.client_order_id());
6407 continue;
6408 }
6409
6410 let cached_instrument = self
6411 .core
6412 .cache()
6413 .instrument(&order.instrument_id())
6414 .cloned();
6415
6416 if let Some(reason) = local_submit_denial_reason(&order, cached_instrument.as_ref()) {
6417 self.emitter.emit_order_denied(&order, &reason.to_string());
6418 continue;
6419 }
6420
6421 match self.prepare_create_order_plan(&order, cmd.params.as_ref()) {
6422 Ok(plan) => plans.push(plan),
6423 Err(e) => {
6424 let reason = OrderDeniedReason::ValidationFailed {
6425 detail: format!("Lighter submit_order_list failed: {e}"),
6426 }
6427 .to_string();
6428
6429 self.emitter.emit_order_denied(&order, &reason);
6430 }
6431 }
6432 }
6433
6434 if plans.is_empty() {
6435 log::warn!(
6436 "Lighter submit_order_list: no supported orders to dispatch for {}",
6437 cmd.order_list.id,
6438 );
6439 return Ok(());
6440 }
6441
6442 let context = self.fanout_dispatch_context(credential)?;
6443 self.spawn_task("submit_order_list", async move {
6444 context
6445 .dispatch_batch_plans(
6446 plans
6447 .into_iter()
6448 .map(|plan| BatchOrderPlan::Create(Box::new(plan)))
6449 .collect(),
6450 )
6451 .await;
6452 Ok(())
6453 });
6454
6455 Ok(())
6456 }
6457
6458 fn modify_order(&self, cmd: ModifyOrder) -> anyhow::Result<()> {
6459 let credential = self
6460 .credential
6461 .as_ref()
6462 .ok_or_else(|| {
6463 anyhow::anyhow!("Lighter execution client cannot modify without credentials")
6464 })
6465 .inspect_err(|e| self.deny_modify_commands(std::slice::from_ref(&cmd), e))?;
6466
6467 self.dispatch_signed_modify_order(&cmd, credential);
6468 Ok(())
6469 }
6470
6471 fn batch_modify_orders(&self, cmd: BatchModifyOrders) -> anyhow::Result<()> {
6472 let credential = self
6473 .credential
6474 .as_ref()
6475 .ok_or_else(|| {
6476 anyhow::anyhow!("Lighter execution client cannot modify without credentials")
6477 })
6478 .inspect_err(|e| self.deny_modify_commands(&cmd.modifies, e))?;
6479 let context = self
6480 .fanout_dispatch_context(credential)
6481 .inspect_err(|e| self.deny_modify_commands(&cmd.modifies, e))?;
6482 let mut plans = Vec::with_capacity(cmd.modifies.len());
6483 for modify in &cmd.modifies {
6484 if !self.admit_modify_order(modify) {
6485 continue;
6486 }
6487
6488 match self.prepare_modify_order_plan(modify) {
6489 Ok(plan) => plans.push(BatchOrderPlan::Modify(plan)),
6490 Err(e) => {
6491 let mut events = self.dispatch.order_events.lock();
6492 self.dispatch.clear_pending_modify_if_command(
6493 &modify.client_order_id,
6494 modify.command_id,
6495 );
6496 self.emitter.emit_order_modify_denied_event(
6497 modify.strategy_id,
6498 modify.instrument_id,
6499 modify.client_order_id,
6500 modify.venue_order_id,
6501 &format!("Lighter modify_order failed: {e}"),
6502 self.clock.get_time_ns(),
6503 Some(modify.command_id),
6504 );
6505
6506 if let Some(deferred) = events.remove(&modify.client_order_id) {
6507 flush_modify_events(
6508 modify.client_order_id,
6509 deferred,
6510 &self.dispatch,
6511 &self.emitter,
6512 &mut events,
6513 );
6514 }
6515 }
6516 }
6517 }
6518
6519 if !plans.is_empty() {
6520 self.spawn_task("batch_modify_orders", async move {
6521 context.dispatch_batch_plans(plans).await;
6522 Ok(())
6523 });
6524 }
6525
6526 Ok(())
6527 }
6528
6529 fn cancel_order(&self, cmd: CancelOrder) -> anyhow::Result<()> {
6530 let credential = self
6531 .credential
6532 .as_ref()
6533 .ok_or_else(|| {
6534 anyhow::anyhow!("Lighter execution client cannot cancel without credentials")
6535 })
6536 .inspect_err(|e| self.deny_cancel_commands(std::slice::from_ref(&cmd), e))?;
6537
6538 if let Some(plans) = self.prepare_oco_cancel_plans(&cmd) {
6539 let context = self.fanout_dispatch_context(credential).inspect_err(|e| {
6540 self.deny_cancel_commands(std::slice::from_ref(&cmd), e);
6541 })?;
6542
6543 let plans = plans
6544 .into_iter()
6545 .map(|plan| {
6546 let emit = self.can_emit_order_cancel_rejected(&plan.client_order_id);
6547 if let Some(command_id) = plan.command_id {
6548 self.dispatch.register_cancel_command(
6549 plan.client_order_id,
6550 command_id,
6551 emit,
6552 );
6553 }
6554
6555 BatchOrderPlan::Cancel(plan, emit)
6556 })
6557 .collect();
6558
6559 self.spawn_task("cancel_oco_orders", async move {
6560 context.dispatch_batch_plans(plans).await;
6561 Ok(())
6562 });
6563
6564 return Ok(());
6565 }
6566
6567 self.dispatch_signed_cancel_order(&cmd, credential);
6568 Ok(())
6569 }
6570
6571 fn cancel_all_orders(&self, cmd: CancelAllOrders) -> anyhow::Result<()> {
6572 let cancels: Vec<_> = self
6575 .core
6576 .cache()
6577 .orders_open(
6578 None,
6579 Some(&cmd.instrument_id),
6580 None,
6581 Some(&self.core.account_id),
6582 cmd.order_side,
6583 )
6584 .into_iter()
6585 .map(|order| {
6586 cancel_order_from_cancel_all(&cmd, order.client_order_id(), order.strategy_id())
6587 })
6588 .collect();
6589
6590 if cancels.is_empty() {
6591 return Ok(());
6592 }
6593
6594 self.dispatch_cancel_orders(&cancels)
6595 }
6596
6597 fn batch_cancel_orders(&self, cmd: BatchCancelOrders) -> anyhow::Result<()> {
6598 self.dispatch_cancel_orders(&cmd.cancels)
6599 }
6600
6601 fn query_account(&self, _cmd: QueryAccount) -> anyhow::Result<()> {
6602 let cached = self.dispatch.snapshot_account_state();
6608 match cached {
6609 Some(state) => {
6610 log::debug!("Lighter query_account replaying cached AccountState");
6611 self.emitter.send_account_state(state);
6612 }
6613 None => {
6614 log::warn!(
6615 "Lighter query_account: no AccountState cached yet \
6616 (account_all_assets stream has not pushed since connect)",
6617 );
6618 }
6619 }
6620 Ok(())
6621 }
6622
6623 fn query_order(&self, cmd: QueryOrder) -> anyhow::Result<()> {
6624 let credential = self
6625 .credential
6626 .as_ref()
6627 .ok_or_else(|| anyhow::anyhow!("Lighter query_order requires credentials"))?
6628 .clone();
6629 let registry = Arc::clone(&self.registry);
6630 let http_client = self.http_client.clone();
6631 let emitter = self.emitter.clone();
6632 let core_account_id = self.core.account_id;
6633 let dispatch = self.dispatch.clone();
6634 let clock = self.clock;
6635
6636 if let Some(order) = self.core.cache().order_owned(&cmd.client_order_id) {
6637 self.dispatch.seed_order_restoration(order);
6638 }
6639
6640 let _ = order_shape_pending(&self.core, &self.dispatch, &cmd.client_order_id);
6641 let child_context = self.child_probe_context();
6642 self.spawn_task("query_order", async move {
6643 let lookup = OrderReportLookup::new(
6644 ®istry,
6645 &dispatch,
6646 Some(cmd.instrument_id),
6647 Some(&cmd.client_order_id),
6648 cmd.venue_order_id.as_ref(),
6649 )?;
6650
6651 let report = if let Some(order) =
6652 collect_order(&http_client, ®istry, &credential, &lookup).await?
6653 {
6654 if !dispatch.restore_cached_order(
6655 VenueOrderId::new(&order.order_id),
6656 order.client_order_index,
6657 cmd.instrument_id,
6658 ) {
6659 return Ok(());
6660 }
6661
6662 if let Some(context) = &child_context {
6663 recover_report_fills(&order, context).await?;
6664 }
6665
6666 let parsed = lookup.parse(
6667 &order,
6668 ®istry,
6669 core_account_id,
6670 clock.get_time_ns(),
6671 &dispatch,
6672 )?;
6673 let report = lookup.finish(&dispatch, parsed);
6674 if dispatch
6675 .order_identity(&cmd.client_order_id)
6676 .is_some_and(|identity| identity.child_fill_baseline().is_some())
6677 {
6678 if let Some(context) = &child_context {
6679 spawn_child_recovery(cmd.client_order_id, context);
6680 }
6681
6682 return Ok(());
6683 }
6684
6685 Some((order.client_order_index, report))
6686 } else {
6687 None
6688 };
6689
6690 dispatch_query_order_report(report, &cmd, &dispatch, &emitter);
6691 Ok(())
6692 });
6693 Ok(())
6694 }
6695
6696 fn generate_order_status_report_task(
6697 &self,
6698 cmd: &GenerateOrderStatusReport,
6699 ) -> Option<ExecutionReportTask<Option<OrderStatusReport>>> {
6700 let credential = self.credential.clone();
6701 let http = self.http_client.clone();
6702 let registry = Arc::clone(&self.registry);
6703 let account_id = self.core.account_id;
6704 let clock = self.clock;
6705 let core = self.core.clone();
6706 let dispatch = self.dispatch.clone();
6707 let cancellation_token = self.cancellation_token.clone();
6708 let command = cmd.clone();
6709 let child_context = self.child_probe_context();
6710 let collection_context = child_context.clone();
6711 let lookup_registry = Arc::clone(®istry);
6712
6713 let lookup = OrderReportLookup::new(
6714 ®istry,
6715 &dispatch,
6716 cmd.instrument_id,
6717 cmd.client_order_id.as_ref(),
6718 cmd.venue_order_id.as_ref(),
6719 );
6720 Some(ExecutionReportTask::new(
6721 async move {
6722 let Some(credential) = credential else {
6723 log::warn!("Lighter generate_order_status_report: no credentials");
6724 return Ok(None);
6725 };
6726
6727 if command.client_order_id.is_none() && command.venue_order_id.is_none() {
6728 log::warn!(
6729 "Lighter generate_order_status_report: must supply client_order_id or venue_order_id"
6730 );
6731 return Ok(None);
6732 }
6733
6734 let lookup = lookup?;
6735 let order = collect_order(&http, &lookup_registry, &credential, &lookup).await?;
6736 if let Some(order) = &order
6737 && order.status == LighterOrderStatus::Filled
6738 && let Some(context) = &collection_context
6739 {
6740 recover_report_fills(order, context).await?;
6741 }
6742
6743 Ok(order.map(|order| (lookup, order)))
6744 },
6745 move |collected| {
6746 anyhow::ensure!(
6747 !cancellation_token.is_cancelled(),
6748 "Lighter report session stopped"
6749 );
6750
6751 let report = collected
6752 .map(
6753 |(lookup, order)| -> anyhow::Result<Option<OrderStatusReport>> {
6754 let parsed = lookup.parse(
6755 &order,
6756 ®istry,
6757 account_id,
6758 clock.get_time_ns(),
6759 &dispatch,
6760 )?;
6761 let report = &parsed.0;
6762 let pending = core
6763 .cache()
6764 .client_order_id(&report.venue_order_id)
6765 .copied()
6766 .is_some_and(|cloid| dispatch.has_pending_modify(&cloid));
6767
6768 if !pending {
6769 restore_reconciled_order(
6770 &core,
6771 &dispatch,
6772 report.venue_order_id,
6773 order.client_order_index,
6774 report.order_status.is_closed(),
6775 );
6776 dispatch.record_order_reduction(
6777 order.client_order_index,
6778 report.venue_order_id,
6779 order.initial_base_amount - report.quantity.as_decimal(),
6780 );
6781 }
6782
6783 let report = lookup.finish(&dispatch, parsed);
6784 if is_unowned_child_cancel(&dispatch, order.client_order_index, &report)
6785 {
6786 return Ok(None);
6787 }
6788
6789 if let Some(context) = &child_context {
6790 request_child_recovery(&core, context, &report);
6791 }
6792
6793 ensure_order_report_ready(&core, &dispatch, &report)?;
6794 Ok(Some(report))
6795 },
6796 )
6797 .transpose()?
6798 .flatten();
6799
6800 if let Some(report) = &report {
6801 dispatch.seed_accepted_from_report(report);
6802 }
6803
6804 Ok(report)
6805 },
6806 ))
6807 }
6808
6809 fn generate_order_status_reports_task(
6810 &self,
6811 cmd: &GenerateOrderStatusReports,
6812 ) -> Option<ExecutionReportTask<Vec<OrderStatusReport>>> {
6813 let client = self.report_client();
6814 let core = self.core.clone();
6815 let dispatch = self.dispatch.clone();
6816 let cancellation_token = self.cancellation_token.clone();
6817 let child_context = self.child_probe_context();
6818 let command = cmd.clone();
6819 let collection_command = command.clone();
6820 let collection_context = child_context.clone();
6821 Some(ExecutionReportTask::new(
6822 async move {
6823 let sweep = client
6824 .collect_order_status_reports(&collection_command)
6825 .await;
6826
6827 if let Some(context) = &collection_context {
6828 recover_filled_reports(&sweep, context).await;
6829 }
6830
6831 Ok(sweep)
6832 },
6833 move |sweep| {
6834 anyhow::ensure!(
6835 !cancellation_token.is_cancelled(),
6836 "Lighter report session stopped"
6837 );
6838
6839 if let Some(context) = &child_context {
6840 request_child_report_recoveries(&core, context, &sweep);
6841 }
6842
6843 finish_order_reports(&core, &dispatch, sweep, &command, None)
6844 },
6845 ))
6846 }
6847
6848 fn generate_fill_reports_task(
6849 &self,
6850 cmd: &GenerateFillReports,
6851 ) -> Option<ExecutionReportTask<Vec<FillReport>>> {
6852 let client = self.report_client();
6853 let instruments = self.report_instruments(cmd.instrument_id);
6854 let core = self.core.clone();
6855 let dispatch = self.dispatch.clone();
6856 let cancellation_token = self.cancellation_token.clone();
6857 let child_context = self.child_probe_context();
6858 let command = cmd.clone();
6859 let cmd_log_level = cmd.log_receipt_level;
6860 Some(ExecutionReportTask::new(
6861 async move {
6862 let mut markets = BTreeSet::new();
6863 let result = client
6864 .collect_fill_reports(&command, &instruments, &mut markets)
6865 .await;
6866 Ok((markets, result))
6867 },
6868 move |(markets, result)| {
6869 anyhow::ensure!(
6870 !cancellation_token.is_cancelled(),
6871 "Lighter report session stopped"
6872 );
6873 let reports = finish_fill_reports(&core, &dispatch, markets, result)?.reports;
6874
6875 if let Some(context) = &child_context {
6876 spawn_child_recoveries(context);
6877 }
6878
6879 Self::log_report_receipt(reports.len(), "FillReport", cmd_log_level);
6880 Ok(reports)
6881 },
6882 ))
6883 }
6884
6885 fn generate_position_status_reports_task(
6886 &self,
6887 cmd: &GeneratePositionStatusReports,
6888 ) -> Option<ExecutionReportTask<Vec<PositionStatusReport>>> {
6889 let core = self.core.clone();
6890 let dispatch = self.dispatch.clone();
6891 let cancellation_token = self.cancellation_token.clone();
6892 let registry = Arc::clone(&self.registry);
6893 let command = cmd.clone();
6894 Some(ExecutionReportTask::new(async { Ok(()) }, move |()| {
6895 anyhow::ensure!(
6896 !cancellation_token.is_cancelled(),
6897 "Lighter report session stopped"
6898 );
6899 let (reports, complete, _) =
6900 cached_position_reports(&core, &dispatch, ®istry, &command)?;
6901 anyhow::ensure!(
6902 complete,
6903 "Lighter position snapshot does not cover the requested instrument scope"
6904 );
6905 Self::log_report_receipt(
6906 reports.len(),
6907 "PositionStatusReport",
6908 command.log_receipt_level,
6909 );
6910 Ok(reports)
6911 }))
6912 }
6913
6914 async fn generate_order_status_report(
6915 &self,
6916 cmd: &GenerateOrderStatusReport,
6917 ) -> anyhow::Result<Option<OrderStatusReport>> {
6918 let Some(credential) = &self.credential else {
6919 log::warn!("Lighter generate_order_status_report: no credentials");
6920 return Ok(None);
6921 };
6922
6923 if cmd.client_order_id.is_none() && cmd.venue_order_id.is_none() {
6924 log::warn!(
6925 "Lighter generate_order_status_report: must supply client_order_id or venue_order_id",
6926 );
6927 return Ok(None);
6928 }
6929
6930 let lookup = OrderReportLookup::new(
6931 &self.registry,
6932 &self.dispatch,
6933 cmd.instrument_id,
6934 cmd.client_order_id.as_ref(),
6935 cmd.venue_order_id.as_ref(),
6936 )?;
6937
6938 let report = if let Some(order) =
6939 collect_order(&self.http_client, &self.registry, credential, &lookup).await?
6940 {
6941 if order.status == LighterOrderStatus::Filled
6942 && let Some(context) = self.child_probe_context()
6943 {
6944 recover_report_fills(&order, &context).await?;
6945 }
6946
6947 let parsed = lookup.parse(
6948 &order,
6949 &self.registry,
6950 self.core.account_id,
6951 self.clock.get_time_ns(),
6952 &self.dispatch,
6953 )?;
6954
6955 if !self
6956 .core
6957 .cache()
6958 .client_order_id(&parsed.0.venue_order_id)
6959 .is_some_and(|cloid| self.dispatch.has_pending_modify(cloid))
6960 {
6961 restore_reconciled_order(
6962 &self.core,
6963 &self.dispatch,
6964 parsed.0.venue_order_id,
6965 order.client_order_index,
6966 parsed.0.order_status.is_closed(),
6967 );
6968 self.dispatch.record_order_reduction(
6969 order.client_order_index,
6970 parsed.0.venue_order_id,
6971 order.initial_base_amount - parsed.0.quantity.as_decimal(),
6972 );
6973 }
6974
6975 let report = lookup.finish(&self.dispatch, parsed);
6976 if is_unowned_child_cancel(&self.dispatch, order.client_order_index, &report) {
6977 return Ok(None);
6978 }
6979
6980 if let Some(context) = self.child_probe_context() {
6981 request_child_recovery(&self.core, &context, &report);
6982 }
6983
6984 ensure_order_report_ready(&self.core, &self.dispatch, &report)?;
6985 Some(report)
6986 } else {
6987 None
6988 };
6989
6990 if let Some(report) = &report {
6991 self.dispatch.seed_accepted_from_report(report);
6992 }
6993
6994 Ok(report)
6995 }
6996
6997 async fn generate_order_status_reports(
6998 &self,
6999 cmd: &GenerateOrderStatusReports,
7000 ) -> anyhow::Result<Vec<OrderStatusReport>> {
7001 self.collect_order_reports(cmd, None).await
7002 }
7003
7004 async fn generate_fill_reports(
7005 &self,
7006 cmd: GenerateFillReports,
7007 ) -> anyhow::Result<Vec<FillReport>> {
7008 let reports = self.paginate_fill_reports(&cmd).await?.reports;
7009 Self::log_report_receipt(reports.len(), "FillReport", cmd.log_receipt_level);
7010 Ok(reports)
7011 }
7012
7013 async fn generate_position_status_reports(
7014 &self,
7015 cmd: &GeneratePositionStatusReports,
7016 ) -> anyhow::Result<Vec<PositionStatusReport>> {
7017 let (reports, complete, _) = self.cached_position_reports(cmd)?;
7018 anyhow::ensure!(
7019 complete,
7020 "Lighter position snapshot does not cover the requested instrument scope",
7021 );
7022 Self::log_report_receipt(reports.len(), "PositionStatusReport", cmd.log_receipt_level);
7023 Ok(reports)
7024 }
7025
7026 async fn generate_mass_status(
7027 &self,
7028 lookback_mins: Option<u64>,
7029 ) -> anyhow::Result<Option<ExecutionMassStatus>> {
7030 let ts_init = self.clock.get_time_ns();
7031
7032 let lookback_start = lookback_mins
7035 .map(DurationNanos::try_from_mins)
7036 .transpose()?
7037 .map(|lookback| ts_init.saturating_sub(lookback));
7038
7039 let mut order_cmd = GenerateOrderStatusReports::new(
7044 UUID4::new(),
7045 ts_init,
7046 false,
7047 None,
7048 lookback_start,
7049 None,
7050 None,
7051 None,
7052 );
7053 order_cmd.log_receipt_level = LogLevel::Debug;
7054 let fill_cmd = GenerateFillReports::new(
7055 UUID4::new(),
7056 ts_init,
7057 None,
7058 None,
7059 lookback_start,
7060 None,
7061 None,
7062 None,
7063 );
7064 let position_cmd =
7065 GeneratePositionStatusReports::new(UUID4::new(), ts_init, None, None, None, None, None);
7066
7067 let (mut fill_reports, fill_reports_complete, withheld_fills) =
7068 fill_reports_for_mass_status(self.paginate_fill_reports(&fill_cmd).await)
7069 .context("Lighter fill reconciliation failed")?;
7070 Self::log_report_receipt(fill_reports.len(), "FillReport", fill_cmd.log_receipt_level);
7071
7072 let mut withheld_instruments = withheld_fills;
7075 let order_result = self
7076 .collect_order_reports(&order_cmd, Some(&fill_reports))
7077 .await;
7078
7079 let (mut order_reports, mut reports_complete) = match order_result {
7080 Ok(reports) => (reports, true),
7081 Err(e) => {
7082 withheld_instruments.extend(withheld_order_instruments(&e));
7083 log::warn!(
7084 "Lighter order report generation failed: {}",
7085 scrub_auth(&format!("{e:#}")),
7086 );
7087 let partial = partial_order_reports(&e);
7088 let reports = if partial.is_empty() {
7089 let mut active_cmd = GenerateOrderStatusReports::new(
7090 UUID4::new(),
7091 ts_init,
7092 true,
7093 None,
7094 None,
7095 None,
7096 None,
7097 None,
7098 );
7099 active_cmd.log_receipt_level = LogLevel::Debug;
7100 let active_result = self
7101 .collect_order_reports(&active_cmd, Some(&fill_reports))
7102 .await;
7103
7104 match active_result {
7105 Ok(reports) => reports,
7106 Err(e) => {
7107 withheld_instruments.extend(withheld_order_instruments(&e));
7108 log::warn!(
7109 "Lighter active order report generation incomplete: {}",
7110 scrub_auth(&format!("{e:#}")),
7111 );
7112 partial_order_reports(&e)
7113 }
7114 }
7115 } else {
7116 partial
7117 };
7118 (reports, false)
7119 }
7120 };
7121
7122 reports_complete &= fill_reports_complete;
7123 let mut reported_orders: AHashSet<VenueOrderId> = order_reports
7124 .iter()
7125 .map(|report| report.venue_order_id)
7126 .collect();
7127 let mut fill_markets: Vec<i64> = fill_reports
7128 .iter()
7129 .filter(|report| !reported_orders.contains(&report.venue_order_id))
7130 .filter_map(|report| self.registry.market_index(&report.instrument_id))
7131 .collect::<AHashSet<_>>()
7132 .into_iter()
7133 .collect();
7134 fill_markets.sort_unstable();
7135
7136 for market_index in fill_markets {
7137 let Some(instrument_id) = self.registry.instrument_id(market_index) else {
7138 reports_complete = false;
7139 continue;
7140 };
7141 let mut order_cmd = GenerateOrderStatusReports::new(
7142 UUID4::new(),
7143 ts_init,
7144 false,
7145 Some(instrument_id),
7146 lookback_start,
7147 None,
7148 None,
7149 None,
7150 );
7151 order_cmd.log_receipt_level = LogLevel::Debug;
7152 let order_result = self
7153 .collect_order_reports(&order_cmd, Some(&fill_reports))
7154 .await;
7155 reports_complete &= order_result.is_ok();
7156 let reports = match order_result {
7157 Ok(reports) => reports,
7158 Err(e) => {
7159 withheld_instruments.extend(withheld_order_instruments(&e));
7160 log::warn!(
7161 "Lighter order report generation failed: {}",
7162 scrub_auth(&format!("{e:#}")),
7163 );
7164 partial_order_reports(&e)
7165 }
7166 };
7167 reported_orders.extend(reports.iter().map(|report| report.venue_order_id));
7168 order_reports.extend(reports);
7169 }
7170
7171 if fill_reports
7172 .iter()
7173 .any(|report| !reported_orders.contains(&report.venue_order_id))
7174 {
7175 reports_complete = false;
7176 }
7177
7178 for fill_report in &mut fill_reports {
7179 if let Some(client_order_id) = order_reports
7180 .iter()
7181 .find(|order_report| order_report.venue_order_id == fill_report.venue_order_id)
7182 .and_then(|order_report| order_report.client_order_id)
7183 {
7184 fill_report.client_order_id = Some(client_order_id);
7185 }
7186 }
7187
7188 self.publish_terminal_fill_bundles(&mut order_reports, &fill_reports)?;
7189
7190 let position_result = self.cached_position_reports(&position_cmd);
7191
7192 let (mut position_reports, mut position_coverage) = match position_result {
7193 Ok((reports, complete, coverage)) => {
7194 reports_complete &= complete;
7195 (reports, coverage)
7196 }
7197 Err(e) => {
7198 reports_complete = false;
7199 log::warn!("Lighter position report generation failed: {e:#}");
7200 (Vec::new(), None)
7201 }
7202 };
7203
7204 position_reports.retain(|report| !withheld_instruments.contains(&report.instrument_id));
7205
7206 if let Some(skipped) = &mut position_coverage {
7207 skipped.extend(
7208 withheld_instruments
7209 .iter()
7210 .filter_map(|id| self.registry.market_index(id)),
7211 );
7212 }
7213
7214 Self::log_report_receipt(
7215 position_reports.len(),
7216 "PositionStatusReport",
7217 position_cmd.log_receipt_level,
7218 );
7219
7220 if lookback_start.is_some() {
7221 let touched_instruments: AHashSet<InstrumentId> = order_reports
7222 .iter()
7223 .map(|report| report.instrument_id)
7224 .chain(fill_reports.iter().map(|report| report.instrument_id))
7225 .collect();
7226 let position_instruments: AHashSet<InstrumentId> = position_reports
7227 .iter()
7228 .map(|report| report.instrument_id)
7229 .collect();
7230 let cache = self.core.cache();
7231 let mut touched_instruments: Vec<InstrumentId> =
7232 touched_instruments.into_iter().collect();
7233 touched_instruments.sort_unstable();
7234
7235 for instrument_id in touched_instruments {
7236 let Some(instrument) = cache.instrument(&instrument_id) else {
7237 reports_complete = false;
7238 continue;
7239 };
7240
7241 if !matches!(instrument, InstrumentAny::CryptoPerpetual(_)) {
7242 continue;
7243 }
7244 let Some(market_id) = self.registry.market_index(&instrument_id) else {
7245 reports_complete = false;
7246 continue;
7247 };
7248
7249 if !position_coverage
7250 .as_ref()
7251 .is_some_and(|skipped| !skipped.contains(&market_id))
7252 {
7253 reports_complete = false;
7254 continue;
7255 }
7256
7257 if position_instruments.contains(&instrument_id) {
7258 continue;
7259 }
7260
7261 position_reports.push(PositionStatusReport::new(
7262 self.core.account_id,
7263 instrument_id,
7264 PositionSide::Flat,
7265 Quantity::zero(instrument.size_precision()),
7266 ts_init,
7267 ts_init,
7268 Some(UUID4::new()),
7269 None,
7270 None,
7271 ));
7272 }
7273 }
7274
7275 let mut mass_status = ExecutionMassStatus::new(
7276 self.core.client_id,
7277 self.core.account_id,
7278 self.core.venue,
7279 ts_init,
7280 None,
7281 );
7282 mass_status.set_report_window(lookback_start, reports_complete);
7283 mass_status.add_order_reports(order_reports);
7284 mass_status.add_fill_reports(fill_reports);
7285 mass_status.add_position_reports(position_reports);
7286
7287 log::debug!(
7288 "Generated Lighter mass status: {} orders, {} fills, {} positions",
7289 mass_status.order_reports().len(),
7290 mass_status.fill_reports().len(),
7291 mass_status.position_reports().len(),
7292 );
7293
7294 Ok(Some(mass_status))
7295 }
7296}
7297
7298#[derive(Debug)]
7299struct AppliedFillSubscription(TypedHandler<OrderEventAny>);
7300
7301impl Drop for AppliedFillSubscription {
7302 fn drop(&mut self) {
7303 msgbus::unsubscribe_order_events(ORDER_EVENTS_PATTERN.into(), &self.0);
7304 }
7305}
7306
7307struct FillSweep {
7311 reports: Vec<FillReport>,
7312 covers_window: bool,
7313 withheld_instruments: AHashSet<InstrumentId>,
7314}
7315
7316#[derive(Debug, thiserror::Error)]
7317#[error("incomplete Lighter order reports: {detail}")]
7318struct IncompleteOrderReports {
7319 reports: Vec<OrderStatusReport>,
7320 withheld_instruments: AHashSet<InstrumentId>,
7321 detail: String,
7322}
7323
7324fn incomplete_order_reports(
7325 reports: Vec<OrderStatusReport>,
7326 detail: impl Into<String>,
7327 withheld_instruments: AHashSet<InstrumentId>,
7328) -> anyhow::Error {
7329 anyhow::Error::new(IncompleteOrderReports {
7330 reports,
7331 withheld_instruments,
7332 detail: detail.into(),
7333 })
7334}
7335
7336fn partial_order_reports(error: &anyhow::Error) -> Vec<OrderStatusReport> {
7337 error
7338 .downcast_ref::<IncompleteOrderReports>()
7339 .map(|incomplete| incomplete.reports.clone())
7340 .unwrap_or_default()
7341}
7342
7343fn withheld_order_instruments(error: &anyhow::Error) -> AHashSet<InstrumentId> {
7344 error
7345 .downcast_ref::<IncompleteOrderReports>()
7346 .map(|incomplete| incomplete.withheld_instruments.clone())
7347 .unwrap_or_default()
7348}
7349
7350fn is_commission_error(error: &anyhow::Error) -> bool {
7351 error
7352 .chain()
7353 .any(|cause| cause.downcast_ref::<LighterCommissionError>().is_some())
7354}
7355
7356fn fill_reports_for_mass_status(
7357 result: anyhow::Result<FillSweep>,
7358) -> anyhow::Result<(Vec<FillReport>, bool, AHashSet<InstrumentId>)> {
7359 match result {
7360 Ok(sweep) => Ok((
7361 sweep.reports,
7362 sweep.covers_window,
7363 sweep.withheld_instruments,
7364 )),
7365 Err(e) if is_commission_error(&e) => Err(e),
7366 Err(e) => {
7367 log::warn!(
7368 "Lighter fill report generation failed: {}",
7369 scrub_auth(&format!("{e:#}")),
7370 );
7371 Ok((Vec::new(), false, AHashSet::new()))
7372 }
7373 }
7374}
7375
7376impl LighterExecutionClient {
7377 async fn collect_order_reports(
7378 &self,
7379 cmd: &GenerateOrderStatusReports,
7380 fills: Option<&[FillReport]>,
7381 ) -> anyhow::Result<Vec<OrderStatusReport>> {
7382 let sweep = self.report_client().collect_order_status_reports(cmd).await;
7383 if let Some(context) = self.child_probe_context() {
7384 request_child_report_recoveries(&self.core, &context, &sweep);
7385 if fills.is_none() {
7386 recover_filled_reports(&sweep, &context).await;
7387 }
7388 }
7389
7390 finish_order_reports(&self.core, &self.dispatch, sweep, cmd, fills)
7391 }
7392
7393 fn publish_terminal_fill_bundles(
7394 &self,
7395 orders: &mut Vec<OrderStatusReport>,
7396 fills: &[FillReport],
7397 ) -> anyhow::Result<()> {
7398 let mut bundled = AHashSet::new();
7399
7400 for report in orders.iter() {
7401 if !matches!(
7402 report.order_status,
7403 OrderStatus::Canceled | OrderStatus::Expired
7404 ) || report
7405 .client_order_id
7406 .and_then(|id| self.dispatch.order_identity(&id))
7407 .is_none()
7408 || !order_report_ready(&self.core, &self.dispatch, report, fills)
7409 || !fills_for_order(report, fills).any(|fill| fill.ts_event > report.ts_last)
7410 {
7411 continue;
7412 }
7413
7414 let paired = fills_for_order(report, fills).cloned().collect();
7416 let bundle = ExecutionReport::OrderWithFills(Box::new(report.clone()), paired);
7417 self.emitter.try_send_execution_report(bundle)?;
7418 bundled.insert(report.venue_order_id);
7419 }
7420
7421 orders.retain(|report| !bundled.contains(&report.venue_order_id));
7422 Ok(())
7423 }
7424
7425 fn prepare_oco_cancel_plans(&self, cmd: &CancelOrder) -> Option<Vec<CancelOrderPlan>> {
7426 let cache = self.core.cache();
7427 let order = cache.order(&cmd.client_order_id)?;
7428 if order.contingency_type() != Some(ContingencyType::Oco) || order.is_closed() {
7429 return None;
7430 }
7431
7432 let [peer_id] = order.linked_order_ids()? else {
7433 return None;
7434 };
7435
7436 if *peer_id == order.client_order_id() {
7437 return None;
7438 }
7439
7440 let peer = cache.order(peer_id)?;
7441 if peer.is_closed()
7442 || peer.instrument_id() != order.instrument_id()
7443 || peer.strategy_id() != order.strategy_id()
7444 || peer.account_id() != order.account_id()
7445 || validate_oco_links(&order, &peer, order.parent_order_id()).is_err()
7446 {
7447 return None;
7448 }
7449
7450 let original = self.prepare_cancel_order_plan(cmd).ok()?;
7451
7452 let expected_venue = order.venue_order_id().or_else(|| {
7453 self.dispatch
7454 .lookup_venue_order_id(&order.client_order_id())
7455 });
7456
7457 if original.venue_order_id != expected_venue {
7458 return None;
7459 }
7460
7461 let venue_order_id = peer.venue_order_id().or_else(|| {
7462 self.dispatch
7463 .order_identity(peer_id)
7464 .and_then(|identity| identity.venue_order_id())
7465 });
7466
7467 let venue_index = match venue_order_id {
7469 Some(venue) => venue.as_str().parse().ok()?,
7470 None => {
7471 let identity = self.dispatch.order_identity(peer_id)?;
7472 let original = self.dispatch.order_identity(&order.client_order_id())?;
7473 if !identity.shares_submission(&original) {
7474 return None;
7475 }
7476
7477 identity.client_order_index
7478 }
7479 };
7480
7481 Some(vec![
7482 original,
7483 CancelOrderPlan {
7484 command_id: None,
7485 client_order_id: *peer_id,
7486 strategy_id: peer.strategy_id(),
7487 instrument_id: peer.instrument_id(),
7488 venue_order_id,
7489 market_index: self.registry.market_index(&peer.instrument_id())?,
7490 venue_index,
7491 },
7492 ])
7493 }
7494
7495 fn child_probe_context(&self) -> Option<AckedOrderProbeContext> {
7496 let credential = self.credential.as_ref()?;
7497 self.fanout_dispatch_context(credential)
7498 .ok()
7499 .map(|context| context.probe_context())
7500 }
7501
7502 fn report_client(&self) -> LighterReportClient {
7503 LighterReportClient {
7504 http_client: self.http_client.clone(),
7505 credential: self.credential.clone(),
7506 account_id: self.core.account_id,
7507 registry: Arc::clone(&self.registry),
7508 clock: self.clock,
7509 markets: self.dispatch.active_markets.clone(),
7510 dispatch: self.dispatch.clone(),
7511 }
7512 }
7513
7514 fn report_instruments(
7515 &self,
7516 instrument_id: Option<InstrumentId>,
7517 ) -> AHashMap<InstrumentId, InstrumentAny> {
7518 let cache = self.core.cache();
7519 match instrument_id {
7520 Some(instrument_id) => cache
7521 .instrument(&instrument_id)
7522 .map(|instrument| (instrument_id, instrument.clone()))
7523 .into_iter()
7524 .collect(),
7525 None => cache
7526 .instruments(&self.core.venue, None)
7527 .into_iter()
7528 .map(|instrument| (instrument.id(), instrument.clone()))
7529 .collect(),
7530 }
7531 }
7532
7533 fn cached_position_reports(
7534 &self,
7535 cmd: &GeneratePositionStatusReports,
7536 ) -> anyhow::Result<(Vec<PositionStatusReport>, bool, Option<AHashSet<i64>>)> {
7537 cached_position_reports(&self.core, &self.dispatch, &self.registry, cmd)
7538 }
7539
7540 async fn paginate_fill_reports(&self, cmd: &GenerateFillReports) -> anyhow::Result<FillSweep> {
7541 let client = self.report_client();
7542 let instruments = self.report_instruments(cmd.instrument_id);
7543 let mut markets = BTreeSet::new();
7544 let result = client
7545 .collect_fill_reports(cmd, &instruments, &mut markets)
7546 .await;
7547 let result = finish_fill_reports(&self.core, &self.dispatch, markets, result);
7548
7549 if let Some(context) = self.child_probe_context() {
7550 spawn_child_recoveries(&context);
7551 }
7552
7553 result
7554 }
7555}
7556
7557struct LighterReportClient {
7558 http_client: LighterHttpClient,
7559 credential: Option<Credential>,
7560 account_id: AccountId,
7561 registry: Arc<MarketRegistry>,
7562 clock: &'static AtomicTime,
7563 markets: Arc<dashmap::DashSet<i64>>,
7564 dispatch: WsDispatchState,
7565}
7566
7567#[derive(Default)]
7568struct OrderReportSweep {
7569 orders: Vec<(i64, OrderStatusReport)>,
7570 reductions: AHashMap<VenueOrderId, Decimal>,
7571 markets: BTreeSet<i64>,
7572 error: Option<anyhow::Error>,
7573 partial: bool,
7574}
7575
7576impl LighterReportClient {
7577 async fn collect_order_status_reports(
7578 &self,
7579 cmd: &GenerateOrderStatusReports,
7580 ) -> OrderReportSweep {
7581 let mut sweep = OrderReportSweep {
7582 markets: self.markets.iter().map(|market| *market).collect(),
7583 ..Default::default()
7584 };
7585
7586 sweep.error = self.collect_orders(cmd, &mut sweep).await.err();
7587 sweep
7588 }
7589
7590 async fn collect_orders(
7591 &self,
7592 cmd: &GenerateOrderStatusReports,
7593 sweep: &mut OrderReportSweep,
7594 ) -> anyhow::Result<()> {
7595 let Some(credential) = &self.credential else {
7596 log::warn!("Lighter generate_order_status_reports: no credentials");
7597
7598 return Ok(());
7599 };
7600
7601 let auth = build_auth_token_for(credential)
7602 .context("failed to mint Lighter auth token for report fetch")?;
7603 let ts_init = self.clock.get_time_ns();
7604
7605 if cmd.instrument_id.is_none() && sweep.markets.is_empty() {
7614 collect_active_markets_from_inactive_orders(
7615 &self.http_client,
7616 &mut sweep.markets,
7617 credential,
7618 &auth,
7619 format_between_timestamps(cmd.start, cmd.end, ts_init),
7620 )
7621 .await?;
7622 }
7623
7624 sweep
7625 .markets
7626 .extend(self.markets.iter().map(|market| *market));
7627
7628 let market_indices = match cmd.instrument_id {
7629 Some(id) => match self.registry.market_index(&id) {
7630 Some(idx) => vec![idx],
7631 None => {
7632 anyhow::bail!("no Lighter market_index for order report instrument {id}",);
7633 }
7634 },
7635 None => sweep.markets.iter().copied().collect::<Vec<_>>(),
7636 };
7637
7638 if market_indices.is_empty() {
7639 log::debug!(
7640 "Lighter generate_order_status_reports: no active markets yet; returning empty",
7641 );
7642
7643 return Ok(());
7644 }
7645
7646 sweep.partial = true;
7647 self.collect_orders_active(market_indices, credential, &auth, ts_init, sweep)
7648 .await?;
7649
7650 if !cmd.open_only {
7658 self.collect_orders_inactive(cmd, credential, &auth, ts_init, sweep)
7659 .await?;
7660 }
7661
7662 Ok(())
7663 }
7664
7665 async fn collect_orders_active(
7666 &self,
7667 market_indices: Vec<i64>,
7668 credential: &Credential,
7669 auth: &SecretString,
7670 ts_init: UnixNanos,
7671 sweep: &mut OrderReportSweep,
7672 ) -> anyhow::Result<()> {
7673 let mut active_errors = Vec::new();
7674
7675 for market_index in market_indices {
7681 let query = Zeroizing::new(LighterAccountActiveOrdersQuery {
7682 authorization: None,
7683 auth: Some(auth.clone()),
7684 account_index: credential.account_index(),
7685 market_id: market_index,
7686 });
7687
7688 let active = match self.http_client.get_account_active_orders(&query).await {
7689 Ok(response) => response,
7690 Err(e) => {
7691 let detail = format!(
7692 "failed to fetch Lighter active orders for market_index={market_index}: {}",
7693 scrub_auth(&format!("{e:#}")),
7694 );
7695 log::warn!("{detail}",);
7696 active_errors.push(detail);
7697 continue;
7698 }
7699 };
7700
7701 for order in &active.orders {
7702 sweep.markets.insert(order.market_index);
7703
7704 let Some(report) = parse_http_order_to_report(
7705 order,
7706 &self.registry,
7707 self.account_id,
7708 ts_init,
7709 &self.dispatch,
7710 ) else {
7711 let detail = format!(
7712 "failed to parse Lighter active order {} for market_index={market_index}",
7713 order.order_id,
7714 );
7715 log::warn!("{detail}");
7716 active_errors.push(detail);
7717 continue;
7718 };
7719
7720 let reduced_qty = order.initial_base_amount - report.quantity.as_decimal();
7721 if reduced_qty > Decimal::ZERO {
7722 sweep.reductions.insert(report.venue_order_id, reduced_qty);
7723 }
7724
7725 sweep.orders.push((order.client_order_index, report));
7726 }
7727 }
7728
7729 if !active_errors.is_empty() {
7730 anyhow::bail!(active_errors.join("; "));
7731 }
7732
7733 Ok(())
7734 }
7735
7736 async fn collect_orders_inactive(
7737 &self,
7738 cmd: &GenerateOrderStatusReports,
7739 credential: &Credential,
7740 auth: &SecretString,
7741 ts_init: UnixNanos,
7742 sweep: &mut OrderReportSweep,
7743 ) -> anyhow::Result<()> {
7744 sweep
7745 .markets
7746 .extend(self.markets.iter().map(|market| *market));
7747
7748 let inactive_markets: Vec<i64> = match cmd.instrument_id {
7749 Some(id) => self
7750 .registry
7751 .market_index(&id)
7752 .map(|m| vec![m])
7753 .unwrap_or_default(),
7754 None => sweep.markets.iter().copied().collect::<Vec<_>>(),
7755 };
7756
7757 let between_timestamps = format_between_timestamps(cmd.start, cmd.end, ts_init);
7758
7759 for market_id in inactive_markets {
7760 let mut cursor: Option<String> = None;
7761 let mut seen_cursors = AHashSet::new();
7762 let mut pages = 0_usize;
7763 loop {
7764 pages += 1;
7765 if pages > MAX_RECONCILIATION_PAGES {
7766 anyhow::bail!(
7767 "Lighter inactive-order reconciliation exceeded {MAX_RECONCILIATION_PAGES} pages for market_index={market_id}",
7768 );
7769 }
7770
7771 let query = Zeroizing::new(LighterAccountInactiveOrdersQuery {
7772 authorization: None,
7773 auth: Some(auth.clone()),
7774 account_index: credential.account_index(),
7775 market_id: Some(market_id),
7776 ask_filter: None,
7777 between_timestamps: between_timestamps.clone(),
7778 cursor: cursor.clone(),
7779 limit: LIGHTER_REST_PAGE_SIZE,
7780 });
7781
7782 let inactive = self.http_client.get_account_inactive_orders(&query).await.map_err(|e| {
7783 anyhow::anyhow!(
7784 "failed to fetch Lighter inactive orders for market_index={market_id}: {}",
7785 scrub_auth(&format!("{e:#}")),
7786 )
7787 })?;
7788
7789 for order in &inactive.orders {
7790 let Some(report) = parse_http_order_to_report(
7791 order,
7792 &self.registry,
7793 self.account_id,
7794 ts_init,
7795 &self.dispatch,
7796 ) else {
7797 anyhow::bail!(
7798 "failed to parse Lighter inactive order {} for market_index={market_id}",
7799 order.order_id,
7800 );
7801 };
7802
7803 if cmd.start.is_some_and(|start| report.ts_last < start)
7804 || cmd.end.is_some_and(|end| report.ts_last > end)
7805 {
7806 continue;
7807 }
7808
7809 sweep.markets.insert(order.market_index);
7810 sweep.orders.push((order.client_order_index, report));
7811 }
7812
7813 let Some(next) = inactive.next_cursor else {
7814 break;
7815 };
7816
7817 if next.is_empty() {
7818 break;
7819 }
7820
7821 anyhow::ensure!(
7822 seen_cursors.insert(next.clone()),
7823 "Lighter inactive-order reconciliation repeated cursor `{next}` for market_index={market_id}",
7824 );
7825 cursor = Some(next);
7826 }
7827 }
7828
7829 Ok(())
7830 }
7831
7832 async fn collect_fill_reports(
7833 &self,
7834 cmd: &GenerateFillReports,
7835 instruments: &AHashMap<InstrumentId, InstrumentAny>,
7836 markets: &mut BTreeSet<i64>,
7837 ) -> anyhow::Result<FillSweep> {
7838 let Some(credential) = &self.credential else {
7839 log::warn!("Lighter generate_fill_reports: no credentials");
7840 return Ok(FillSweep {
7841 reports: Vec::new(),
7842 covers_window: true,
7843 withheld_instruments: AHashSet::new(),
7844 });
7845 };
7846
7847 collect_fill_reports(
7848 &self.http_client,
7849 &self.registry,
7850 credential,
7851 self.account_id,
7852 self.clock,
7853 cmd,
7854 instruments,
7855 markets,
7856 None,
7857 )
7858 .await
7859 }
7860}
7861
7862#[expect(
7863 clippy::too_many_arguments,
7864 reason = "ordinary and child-scoped recovery share authenticated fill pagination"
7865)]
7866async fn collect_fill_reports(
7867 http_client: &LighterHttpClient,
7868 registry: &Arc<MarketRegistry>,
7869 credential: &Credential,
7870 account_id: AccountId,
7871 clock: &AtomicTime,
7872 cmd: &GenerateFillReports,
7873 instruments: &AHashMap<InstrumentId, InstrumentAny>,
7874 markets: &mut BTreeSet<i64>,
7875 order_index: Option<i64>,
7876) -> anyhow::Result<FillSweep> {
7877 let market_id = match cmd.instrument_id {
7878 Some(instrument_id) => Some(registry.market_index(&instrument_id).ok_or_else(|| {
7879 anyhow::anyhow!("no Lighter market_index for fill instrument {instrument_id}",)
7880 })?),
7881 None => None,
7882 };
7883
7884 let auth = build_auth_token_for(credential)
7885 .context("failed to mint Lighter auth token for fill fetch")?;
7886
7887 let ts_init = clock.get_time_ns();
7888 let mut reports = Vec::new();
7889 let mut cursor: Option<String> = None;
7890 let mut seen_cursors = AHashSet::new();
7891 let mut seen_in_call = AHashSet::new();
7892 let mut pages = 0_usize;
7893 let mut oldest_served: Option<UnixNanos> = None;
7894 let mut covers_window = true;
7895
7896 loop {
7897 pages += 1;
7898 anyhow::ensure!(
7899 pages <= MAX_RECONCILIATION_PAGES,
7900 "Lighter fill reconciliation exceeded {MAX_RECONCILIATION_PAGES} pages",
7901 );
7902
7903 let query = Zeroizing::new(LighterTradesQuery {
7904 authorization: None,
7905 auth: Some(auth.clone()),
7906 market_id,
7907 account_index: Some(credential.account_index()),
7908 order_index,
7909 order_index_str: None,
7910 integrator_account_index: None,
7911 sort_by: LighterTradeSortBy::Timestamp,
7912 sort_dir: Some(LighterSortDirection::Desc),
7913 cursor: cursor.clone(),
7914 from_timestamp: None,
7917 ask_filter: None,
7918 role: None,
7919 trade_type: None,
7920 limit: LIGHTER_REST_PAGE_SIZE,
7921 aggregate: None,
7922 });
7923
7924 let response = match http_client.get_trades(&query).await {
7925 Ok(response) => response,
7926 Err(e) => {
7927 log::warn!(
7931 "Lighter get_trades failed (market_id={:?}, account_index={}, cursor={:?}): {}",
7932 query.market_id,
7933 credential.account_index(),
7934 cursor,
7935 scrub_auth(&format!("{e:#}")),
7936 );
7937 return Err(anyhow::Error::new(e).context("failed to fetch Lighter fills"));
7938 }
7939 };
7940
7941 for trade in &response.trades {
7942 let Some(report) = parse_scoped_fill_report(
7943 trade,
7944 registry,
7945 instruments,
7946 credential.account_index(),
7947 account_id,
7948 cmd,
7949 ts_init,
7950 )?
7951 else {
7952 continue;
7953 };
7954
7955 if !seen_in_call.insert(report.trade_id) {
7956 log::debug!(
7957 "Lighter duplicate trade {} ignored within HTTP fill pagination",
7958 report.trade_id,
7959 );
7960 continue;
7961 }
7962
7963 markets.insert(trade.market_id);
7964 reports.push(report);
7965 }
7966
7967 let page_oldest = response
7968 .trades
7969 .iter()
7970 .filter_map(|trade| u64::try_from(trade.timestamp).ok())
7971 .map(|timestamp_ms| UnixNanos::from(timestamp_ms.saturating_mul(1_000_000)))
7972 .min();
7973
7974 if let Some(page_oldest) = page_oldest {
7975 oldest_served = Some(oldest_served.map_or(page_oldest, |ts| ts.min(page_oldest)));
7976 }
7977
7978 let reached_start_boundary = cmd
7979 .start
7980 .is_some_and(|start| page_oldest.is_some_and(|oldest| oldest < start));
7981
7982 if reached_start_boundary {
7983 break;
7984 }
7985
7986 match response.next_cursor {
7987 Some(next) if !next.is_empty() => {
7988 anyhow::ensure!(
7989 seen_cursors.insert(next.clone()),
7990 "Lighter fill reconciliation repeated cursor `{next}`",
7991 );
7992 cursor = Some(next);
7993 }
7994 _ => {
7995 if let (Some(start), Some(oldest)) = (cmd.start, oldest_served) {
8001 covers_window = false;
8002
8003 log::warn!(
8004 "Lighter fill reports do not cover {} to {}: trade pagination exhausted before the requested start; the venue `export` endpoint serves full history",
8005 unix_nanos_to_iso8601(start),
8006 unix_nanos_to_iso8601(oldest),
8007 );
8008 }
8009
8010 break;
8011 }
8012 }
8013 }
8014
8015 Ok(FillSweep {
8016 reports,
8017 covers_window,
8018 withheld_instruments: AHashSet::new(),
8019 })
8020}
8021
8022fn parse_scoped_fill_report(
8023 trade: &LighterTrade,
8024 registry: &MarketRegistry,
8025 instruments: &AHashMap<InstrumentId, InstrumentAny>,
8026 account_index: i64,
8027 account_id: AccountId,
8028 cmd: &GenerateFillReports,
8029 ts_init: UnixNanos,
8030) -> anyhow::Result<Option<FillReport>> {
8031 let instrument_id = registry.instrument_id(trade.market_id).with_context(|| {
8032 format!(
8033 "no Lighter instrument registered for fill market_index={}",
8034 trade.market_id
8035 )
8036 })?;
8037
8038 let instrument = instruments
8039 .get(&instrument_id)
8040 .with_context(|| format!("Lighter fill instrument {instrument_id} missing from cache"))?;
8041
8042 let Some(report) = parse_ws_fill_report(trade, account_index, instrument, account_id, ts_init)
8043 .context("failed to parse Lighter fill report")?
8044 else {
8045 return Ok(None);
8046 };
8047
8048 if cmd.start.is_some_and(|start| report.ts_event < start)
8049 || cmd.end.is_some_and(|end| report.ts_event > end)
8050 {
8051 return Ok(None);
8052 }
8053
8054 Ok(Some(report))
8055}
8056
8057fn cached_position_reports(
8058 core: &ExecutionClientCore,
8059 dispatch: &WsDispatchState,
8060 registry: &Arc<MarketRegistry>,
8061 cmd: &GeneratePositionStatusReports,
8062) -> anyhow::Result<(Vec<PositionStatusReport>, bool, Option<AHashSet<i64>>)> {
8063 let (mut reports, mut coverage) = dispatch.snapshot_positions_with_coverage();
8067 let pending = execution_instruments_pending(core, dispatch);
8068 reports.retain(|report| !pending.contains(&report.instrument_id));
8069 if let Some(skipped) = &mut coverage {
8070 skipped.extend(pending.iter().filter_map(|id| registry.market_index(id)));
8071 }
8072
8073 let complete = match cmd.instrument_id {
8074 Some(instrument_id) => {
8075 let market_id = registry.market_index(&instrument_id).ok_or_else(|| {
8076 anyhow::anyhow!(
8077 "no Lighter market_index for position report instrument {instrument_id}",
8078 )
8079 })?;
8080
8081 reports.retain(|report| report.instrument_id == instrument_id);
8082 !pending.contains(&instrument_id)
8083 && coverage
8084 .as_ref()
8085 .is_some_and(|skipped| !skipped.contains(&market_id))
8086 }
8087 None => pending.is_empty() && coverage.as_ref().is_some_and(|skipped| skipped.is_empty()),
8088 };
8089
8090 Ok((reports, complete, coverage))
8091}
8092
8093fn execution_instruments_pending(
8094 core: &ExecutionClientCore,
8095 dispatch: &WsDispatchState,
8096) -> AHashSet<InstrumentId> {
8097 let events = dispatch.order_events.lock();
8098 let cache = core.cache();
8099 let mut pending = AHashSet::new();
8100 dispatch.any_order_identity(|cloid, identity, retired| {
8101 if (identity.has_inferred_fills()
8102 && cache.order(cloid).map_or(!retired, |order| {
8103 !order.is_closed() || inferred_reconciliation_trade_ids(&order).is_err()
8104 }))
8105 || dispatch.has_pending_modify(cloid)
8106 || dispatch.terminals_pending.lock().contains_key(cloid)
8107 || events.get(cloid).is_some_and(|deferred| {
8108 !deferred.fills.is_empty()
8109 || deferred
8110 .order
8111 .as_ref()
8112 .is_some_and(|order| order.status.is_terminal())
8113 || deferred.recovery_epoch.is_some()
8114 })
8115 || identity.publication_pending(cache.order(cloid).as_deref())
8116 {
8117 pending.insert(identity.instrument_id);
8118 }
8119
8120 false
8121 });
8122
8123 pending
8124}
8125
8126fn dispatch_query_order_report(
8127 report: Option<(i64, OrderStatusReport)>,
8128 cmd: &QueryOrder,
8129 dispatch: &WsDispatchState,
8130 emitter: &ExecutionEventEmitter,
8131) {
8132 let events = dispatch.order_events.lock();
8133 if dispatch
8134 .terminals_pending
8135 .lock()
8136 .contains_key(&cmd.client_order_id)
8137 || events.get(&cmd.client_order_id).is_some_and(|deferred| {
8138 deferred
8139 .order
8140 .as_ref()
8141 .is_some_and(|order| order.status.is_terminal())
8142 })
8143 || dispatch
8144 .order_identity(&cmd.client_order_id)
8145 .is_some_and(|identity| {
8146 position_fills_pending(&identity, &events, dispatch)
8147 || dispatch.any_order_identity(|_, other, _| {
8148 other.strategy_id == identity.strategy_id
8149 && other.instrument_id == identity.instrument_id
8150 && other.has_publication()
8151 })
8152 })
8153 || dispatch.has_pending_modify(&cmd.client_order_id)
8154 || dispatch
8155 .order_identity(&cmd.client_order_id)
8156 .is_some_and(|identity| identity.has_publication())
8157 {
8158 return;
8159 }
8160
8161 match report {
8162 Some((index, report)) => {
8163 if is_unowned_child_cancel(dispatch, index, &report) {
8164 return;
8165 }
8166
8167 if dispatch
8168 .order_identity(&cmd.client_order_id)
8169 .is_some_and(|identity| !identity.report_fills_covered(&report, &[]))
8170 {
8171 return;
8172 }
8173
8174 log::debug!(
8175 "Lighter query_order returning report for {}",
8176 cmd.client_order_id
8177 );
8178 dispatch.seed_accepted_from_report(&report);
8179 emitter.send_order_status_report(report);
8180 }
8181 None => {
8182 log::warn!(
8183 "Lighter query_order: no order found for {}",
8184 cmd.client_order_id,
8185 );
8186 }
8187 }
8188}
8189
8190fn finish_order_reports(
8191 core: &ExecutionClientCore,
8192 dispatch: &WsDispatchState,
8193 sweep: OrderReportSweep,
8194 cmd: &GenerateOrderStatusReports,
8195 fills: Option<&[FillReport]>,
8196) -> anyhow::Result<Vec<OrderStatusReport>> {
8197 for market in sweep.markets {
8198 dispatch.note_active_market(market);
8199 }
8200
8201 let mut withheld = AHashSet::new();
8202 let mut reports = Vec::with_capacity(sweep.orders.len());
8203 for (client_order_index, report) in sweep.orders {
8204 let pending_cloid = core
8205 .cache()
8206 .client_order_id(&report.venue_order_id)
8207 .copied()
8208 .or_else(|| {
8209 dispatch
8210 .resolve_client_order_index_for_venue(client_order_index, report.venue_order_id)
8211 });
8212
8213 if pending_cloid
8214 .as_ref()
8215 .is_some_and(|cloid| order_shape_pending(core, dispatch, cloid))
8216 {
8217 withheld.insert(report.instrument_id);
8218 continue;
8219 }
8220
8221 restore_reconciled_order(
8222 core,
8223 dispatch,
8224 report.venue_order_id,
8225 client_order_index,
8226 report.order_status.is_closed(),
8227 );
8228
8229 if let Some(reduced_qty) = sweep.reductions.get(&report.venue_order_id) {
8230 dispatch.record_order_reduction(
8231 client_order_index,
8232 report.venue_order_id,
8233 *reduced_qty,
8234 );
8235 }
8236
8237 let report = dispatch.translate_order_cloid(report);
8238 let report = dispatch.preserve_pending_order_status(report);
8239 if is_unowned_child_cancel(dispatch, client_order_index, &report) {
8240 continue;
8241 }
8242
8243 if report
8244 .client_order_id
8245 .as_ref()
8246 .is_some_and(|cloid| order_shape_pending(core, dispatch, cloid))
8247 {
8248 withheld.insert(report.instrument_id);
8249 continue;
8250 }
8251
8252 if !order_report_collectible(core, dispatch, &report, fills) {
8253 withheld.insert(report.instrument_id);
8254 continue;
8255 }
8256
8257 dispatch.seed_accepted_from_report(&report);
8258 reports.push(report);
8259 }
8260
8261 match sweep.error {
8262 Some(e) if sweep.partial => {
8263 return Err(incomplete_order_reports(reports, e.to_string(), withheld));
8264 }
8265 Some(e) => return Err(e),
8266 None if !withheld.is_empty() => {
8267 return Err(incomplete_order_reports(
8268 reports,
8269 "Lighter execution confirmation is pending".to_string(),
8270 withheld,
8271 ));
8272 }
8273 None => {}
8274 }
8275
8276 LighterExecutionClient::log_report_receipt(
8277 reports.len(),
8278 "OrderStatusReport",
8279 cmd.log_receipt_level,
8280 );
8281 Ok(reports)
8282}
8283
8284fn is_unowned_child_cancel(
8285 dispatch: &WsDispatchState,
8286 client_order_index: i64,
8287 report: &OrderStatusReport,
8288) -> bool {
8289 report.order_status == OrderStatus::Canceled
8291 && report.cancel_reason.as_deref() == LighterOrderStatus::CanceledChild.as_cancel_reason()
8292 && report.parent_order_id.is_some()
8293 && report.quantity.is_zero()
8294 && report.filled_qty.is_zero()
8295 && dispatch
8296 .resolve_client_order_index_for_venue(client_order_index, report.venue_order_id)
8297 .is_none()
8298}
8299
8300fn finish_fill_reports(
8301 core: &ExecutionClientCore,
8302 dispatch: &WsDispatchState,
8303 markets: BTreeSet<i64>,
8304 result: anyhow::Result<FillSweep>,
8305) -> anyhow::Result<FillSweep> {
8306 for market in markets {
8307 dispatch.note_active_market(market);
8308 }
8309
8310 let mut sweep = result?;
8311 let mut reports = Vec::with_capacity(sweep.reports.len());
8312
8313 for mut report in sweep.reports {
8314 defer_reconciled_fill(core, dispatch, &mut report);
8315
8316 let report = dispatch.translate_fill_cloid(report);
8319 if report.client_order_id.as_ref().is_some_and(|cloid| {
8320 order_shape_pending(core, dispatch, cloid)
8321 || (dispatch
8322 .order_identity(cloid)
8323 .is_some_and(|identity| identity.child_fill_baseline().is_some())
8324 && core
8325 .cache()
8326 .order(cloid)
8327 .is_none_or(|order| !order.is_closed()))
8328 }) {
8329 sweep.covers_window = false;
8330 sweep.withheld_instruments.insert(report.instrument_id);
8331 continue;
8332 }
8333
8334 if matches!(
8335 dispatch.trade_source(report.trade_id),
8336 Some(TradeDedupSource::Live)
8337 ) {
8338 log::debug!(
8339 "Lighter trade {} ignored in HTTP fill reports after live delivery",
8340 report.trade_id
8341 );
8342 continue;
8343 }
8344
8345 reports.push(report);
8346 }
8347
8348 sweep.reports = reports;
8349 Ok(sweep)
8350}
8351
8352fn defer_reconciled_fill(
8353 core: &ExecutionClientCore,
8354 dispatch: &WsDispatchState,
8355 report: &mut FillReport,
8356) {
8357 let Some((cloid, identity)) = restore_fill_identity(core, dispatch, report) else {
8358 return;
8359 };
8360
8361 report.client_order_id = Some(cloid);
8362
8363 if identity.has_inferred_fills() {
8364 return;
8365 }
8366
8367 let Some(fill) = child_fill_from_report(report, &identity, core.trader_id) else {
8368 return;
8369 };
8370
8371 if core
8372 .cache()
8373 .order(&cloid)
8374 .is_some_and(|order| order.is_duplicate_fill(&fill))
8375 {
8376 return;
8377 }
8378
8379 let mut events = dispatch.order_events.lock();
8380 let baseline = identity.child_fill_baseline();
8381 let new_fill = baseline
8382 .as_ref()
8383 .is_none_or(|baseline| !baseline.trade_ids.contains(&fill.trade_id));
8384 if new_fill
8385 && (baseline.is_some()
8386 || dispatch.has_pending_modify(&cloid)
8387 || position_fills_pending(&identity, &events, dispatch))
8388 {
8389 dispatch.defer_fill(events.entry(cloid).or_default(), fill);
8390 }
8391}
8392
8393fn restore_fill_identity(
8394 core: &ExecutionClientCore,
8395 dispatch: &WsDispatchState,
8396 report: &FillReport,
8397) -> Option<(ClientOrderId, OrderIdentity)> {
8398 let cached_order = {
8399 let cache = core.cache();
8400 cache
8401 .client_order_id(&report.venue_order_id)
8402 .and_then(|cloid| cache.order_owned(cloid))
8403 .filter(|order| {
8404 !order.strategy_id().is_external()
8405 && order.instrument_id() == report.instrument_id
8406 && order.account_id() == Some(report.account_id)
8407 })
8408 }?;
8409
8410 let cloid = cached_order.client_order_id();
8411 if dispatch.order_identity(&cloid).is_none() {
8412 let index = report
8413 .client_order_id
8414 .and_then(|id| id.as_str().parse::<i64>().ok())
8415 .unwrap_or(0);
8416 dispatch.seed_order_restoration(cached_order);
8417 dispatch.restore_cached_order(report.venue_order_id, index, report.instrument_id);
8418 }
8419
8420 dispatch
8421 .order_identity(&cloid)
8422 .map(|identity| (cloid, identity))
8423}
8424
8425fn ensure_order_report_ready(
8426 core: &ExecutionClientCore,
8427 dispatch: &WsDispatchState,
8428 report: &OrderStatusReport,
8429) -> anyhow::Result<()> {
8430 anyhow::ensure!(
8431 order_report_collectible(core, dispatch, report, None),
8432 "Lighter order report deferred until execution state is applied",
8433 );
8434 Ok(())
8435}
8436
8437fn order_report_collectible(
8438 core: &ExecutionClientCore,
8439 dispatch: &WsDispatchState,
8440 report: &OrderStatusReport,
8441 fills: Option<&[FillReport]>,
8442) -> bool {
8443 if fills.is_none()
8445 && matches!(
8446 report.order_status,
8447 OrderStatus::Canceled | OrderStatus::Expired
8448 )
8449 {
8450 order_report_shape_ready(core, dispatch, report)
8451 } else {
8452 order_report_ready(core, dispatch, report, fills.unwrap_or_default())
8453 }
8454}
8455
8456fn order_report_ready(
8457 core: &ExecutionClientCore,
8458 dispatch: &WsDispatchState,
8459 report: &OrderStatusReport,
8460 fills: &[FillReport],
8461) -> bool {
8462 order_report_shape_ready(core, dispatch, report)
8463 && report
8464 .client_order_id
8465 .and_then(|cloid| dispatch.order_identity(&cloid))
8466 .is_none_or(|identity| identity.report_fills_covered(report, fills))
8467}
8468
8469fn order_report_shape_ready(
8470 core: &ExecutionClientCore,
8471 dispatch: &WsDispatchState,
8472 report: &OrderStatusReport,
8473) -> bool {
8474 let Some(cloid) = report.client_order_id else {
8475 return true;
8476 };
8477
8478 let pending = order_shape_pending(core, dispatch, &cloid);
8479
8480 if let Some(identity) = dispatch.order_identity(&cloid)
8481 && identity.child_fill_baseline().is_some()
8482 && core
8483 .cache()
8484 .order(&cloid)
8485 .is_none_or(|order| report.filled_qty != order.filled_qty())
8486 {
8487 return false;
8488 }
8489
8490 if !pending {
8491 return true;
8492 }
8493
8494 report.order_status == OrderStatus::PendingUpdate
8495 && core.cache().order(&cloid).is_some_and(|order| {
8496 order.quantity() == report.quantity
8497 && order.price() == report.price
8498 && order.trigger_price() == report.trigger_price
8499 && order.filled_qty() == report.filled_qty
8500 })
8501}
8502
8503fn order_shape_pending(
8504 core: &ExecutionClientCore,
8505 dispatch: &WsDispatchState,
8506 cloid: &ClientOrderId,
8507) -> bool {
8508 let events = dispatch.order_events.lock();
8509 if dispatch.has_pending_modify(cloid) {
8510 return true;
8511 }
8512
8513 let Some(identity) = dispatch.order_identity(cloid) else {
8514 return false;
8515 };
8516
8517 if let Some(order) = core.cache().order(cloid) {
8518 identity.sync_fills(&order);
8519 }
8520
8521 if identity.has_inferred_fills() {
8522 return true;
8523 }
8524
8525 if dispatch.terminals_pending.lock().contains_key(cloid) {
8526 return true;
8527 }
8528
8529 if position_fills_pending(&identity, &events, dispatch) {
8530 return true;
8531 }
8532
8533 let cache = core.cache();
8534
8535 if dispatch.any_order_identity(|id, other, _| {
8536 other.strategy_id == identity.strategy_id
8537 && other.instrument_id == identity.instrument_id
8538 && other.publication_pending(cache.order(id).as_deref())
8539 }) {
8540 return true;
8541 }
8542
8543 let order = cache.order(cloid);
8544 let pending = identity.publication_pending(order.as_deref());
8545 if !pending && order.as_ref().is_some_and(|order| order.is_closed()) {
8546 dispatch.retire_order_identity(cloid);
8547 }
8548
8549 pending
8550}
8551
8552fn restore_reconciled_order(
8553 core: &ExecutionClientCore,
8554 dispatch: &WsDispatchState,
8555 venue_order_id: VenueOrderId,
8556 client_order_index: i64,
8557 terminal: bool,
8558) {
8559 let cached_order = {
8560 let cache = core.cache();
8561
8562 let Some(cloid) = cache.client_order_id(&venue_order_id).copied() else {
8563 return;
8564 };
8565
8566 let Some(order) = cache.order_owned(&cloid) else {
8567 log::warn!(
8568 "Ignoring reconciled Lighter order missing from cache: cloid={cloid}, venue_order_id={venue_order_id}",
8569 );
8570 return;
8571 };
8572
8573 order
8574 };
8575
8576 if cached_order.instrument_id().venue != core.venue
8577 || cached_order.account_id() != Some(core.account_id)
8578 {
8579 log::warn!(
8580 "Ignoring reconciled Lighter order outside this client: cloid={}, venue_order_id={venue_order_id}",
8581 cached_order.client_order_id(),
8582 );
8583 return;
8584 }
8585
8586 if order_shape_pending(core, dispatch, &cached_order.client_order_id()) {
8587 return;
8588 }
8589
8590 let terminal =
8591 terminal && (cached_order.parent_order_id().is_none() || cached_order.is_closed());
8592
8593 if let Err(e) = dispatch.restore_reconciled_order(
8594 &cached_order,
8595 client_order_index,
8596 venue_order_id,
8597 terminal,
8598 ) {
8599 log::warn!(
8600 "Ignoring conflicting Lighter reconciliation identity: cloid={}, venue_order_id={venue_order_id}, client_order_index={}, error={e}",
8601 cached_order.client_order_id(),
8602 client_order_index,
8603 );
8604 }
8605}
8606
8607fn grouped_submit_denial_reason(error: &anyhow::Error) -> String {
8608 match error.downcast_ref::<OrderDeniedReason>() {
8609 Some(reason) => reason.to_string(),
8610 None => OrderDeniedReason::UnsupportedOrderList {
8611 detail: format!("Lighter grouped submit failed: {error}"),
8612 }
8613 .to_string(),
8614 }
8615}
8616
8617fn local_submit_denial_reason(
8618 order: &OrderAny,
8619 instrument: Option<&InstrumentAny>,
8620) -> Option<anyhow::Error> {
8621 if instrument.is_none() {
8622 return Some(anyhow::anyhow!(
8623 OrderDeniedReason::InstrumentNotFound {
8624 instrument_id: order.instrument_id(),
8625 }
8626 .to_string(),
8627 ));
8628 }
8629
8630 if !is_lighter_supported_order_type(order.order_type()) {
8631 return Some(anyhow::anyhow!(unsupported_lighter_order_type_reason(
8632 order.order_type()
8633 )));
8634 }
8635
8636 if is_lighter_limit_style_order(order.order_type()) && order.price().is_none() {
8637 return Some(anyhow::anyhow!(
8638 OrderDeniedReason::ValidationFailed {
8639 detail: "Lighter limit-style orders require a limit price".to_string(),
8640 }
8641 .to_string(),
8642 ));
8643 }
8644
8645 if order.is_quote_quantity() {
8646 return Some(anyhow::anyhow!(
8647 OrderDeniedReason::unsupported_quote_quantity()
8648 ));
8649 }
8650
8651 if order.display_qty().is_some() {
8652 return Some(anyhow::anyhow!(
8653 OrderDeniedReason::ValidationFailed {
8654 detail: "Lighter orders do not support display_qty iceberg instructions"
8655 .to_string(),
8656 }
8657 .to_string(),
8658 ));
8659 }
8660
8661 if is_lighter_spot_order(order, instrument) && is_lighter_conditional_order(order.order_type())
8662 {
8663 let denied = OrderDeniedReason::UnsupportedOrderType {
8664 order_type: order.order_type(),
8665 };
8666
8667 return Some(anyhow::anyhow!(
8668 "{denied}; Lighter spot markets do not support conditional orders",
8669 ));
8670 }
8671
8672 nautilus_to_lighter_tif(
8673 order.order_type(),
8674 order.time_in_force(),
8675 order.is_post_only(),
8676 )
8677 .err()
8678 .map(|e| {
8679 let denied = OrderDeniedReason::UnsupportedTimeInForce(order.time_in_force());
8680 anyhow::anyhow!("{denied}; {e}")
8681 })
8682}
8683
8684fn unsupported_lighter_order_type_reason(order_type: OrderType) -> String {
8685 let denied = OrderDeniedReason::UnsupportedOrderType { order_type };
8686 format!(
8687 "{denied}; Lighter supports MARKET, LIMIT, STOP_MARKET, STOP_LIMIT, MARKET_IF_TOUCHED, and LIMIT_IF_TOUCHED",
8688 )
8689}
8690
8691fn is_grouped_order(order: &OrderAny) -> bool {
8692 order.contingency_type().is_some() || order.parent_order_id().is_some()
8693}
8694
8695fn grouped_order_indices(orders: &[OrderAny]) -> anyhow::Result<(LighterGroupingType, Vec<usize>)> {
8696 anyhow::ensure!(
8697 (2..=3).contains(&orders.len()),
8698 "Lighter groups require two or three orders"
8699 );
8700 let first = &orders[0];
8701 anyhow::ensure!(
8702 orders
8703 .iter()
8704 .enumerate()
8705 .all(|(index, order)| orders[..index]
8706 .iter()
8707 .all(|other| other.client_order_id() != order.client_order_id())),
8708 "Lighter groups require unique client order IDs"
8709 );
8710 anyhow::ensure!(
8711 orders.iter().all(|order| !order.is_closed()
8712 && order.instrument_id() == first.instrument_id()
8713 && order.strategy_id() == first.strategy_id()
8714 && order.order_list_id() == first.order_list_id()),
8715 "Lighter groups require open orders from one instrument, strategy, and order list"
8716 );
8717 anyhow::ensure!(
8718 !orders
8719 .iter()
8720 .any(|order| order.contingency_type() == Some(ContingencyType::Ouo)),
8721 "Lighter native groups do not support Nautilus OUO quantity updates"
8722 );
8723
8724 let parents: Vec<_> = orders
8725 .iter()
8726 .enumerate()
8727 .filter(|(_, order)| {
8728 order.contingency_type() == Some(ContingencyType::Oto)
8729 && order.parent_order_id().is_none()
8730 })
8731 .map(|(index, _)| index)
8732 .collect();
8733
8734 if parents.is_empty() {
8735 anyhow::ensure!(
8736 orders.len() == 2,
8737 "Lighter OCO requires exactly two siblings"
8738 );
8739 validate_oco_links(&orders[0], &orders[1], None)?;
8740 return Ok((LighterGroupingType::OneCancelsTheOther, vec![0, 1]));
8741 }
8742
8743 anyhow::ensure!(
8744 parents.len() == 1,
8745 "Lighter OTO groups require exactly one parent"
8746 );
8747 let parent_index = parents[0];
8748 let parent = &orders[parent_index];
8749 anyhow::ensure!(
8750 matches!(parent.order_type(), OrderType::Market | OrderType::Limit),
8751 "Lighter OTO parents must be MARKET or LIMIT orders"
8752 );
8753 let children: Vec<_> = orders
8754 .iter()
8755 .enumerate()
8756 .filter(|(index, _)| *index != parent_index)
8757 .collect();
8758 let child_ids: Vec<_> = children
8759 .iter()
8760 .map(|(_, order)| order.client_order_id())
8761 .collect();
8762 let links = parent
8763 .linked_order_ids()
8764 .context("Lighter OTO parent requires linked children")?;
8765 anyhow::ensure!(
8766 links.len() == child_ids.len() && child_ids.iter().all(|id| links.contains(id)),
8767 "Lighter OTO parent links must identify every child exactly once"
8768 );
8769 anyhow::ensure!(
8770 children
8771 .iter()
8772 .all(|(_, order)| order.parent_order_id() == Some(parent.client_order_id())),
8773 "Lighter OTO children must reference their parent"
8774 );
8775
8776 let grouping_type = if children.len() == 1 {
8777 anyhow::ensure!(
8778 children[0].1.contingency_type().is_none()
8779 && children[0].1.linked_order_ids().is_none(),
8780 "Lighter OTO child must not have a sibling contingency"
8781 );
8782 LighterGroupingType::OneTriggersTheOther
8783 } else {
8784 validate_oco_links(children[0].1, children[1].1, Some(parent.client_order_id()))?;
8785 LighterGroupingType::OneTriggersOneCancelsTheOther
8786 };
8787
8788 Ok((
8789 grouping_type,
8790 std::iter::once(parent_index)
8791 .chain(children.iter().map(|(index, _)| *index))
8792 .collect(),
8793 ))
8794}
8795
8796fn validate_oco_links(
8797 first: &OrderAny,
8798 second: &OrderAny,
8799 parent: Option<ClientOrderId>,
8800) -> anyhow::Result<()> {
8801 let orders = [first, second];
8802 for (index, order) in orders.iter().enumerate() {
8803 anyhow::ensure!(
8804 order.contingency_type() == Some(ContingencyType::Oco)
8805 && order.parent_order_id() == parent,
8806 "Lighter OCO siblings must have matching OCO relationships"
8807 );
8808 anyhow::ensure!(
8809 order
8810 .linked_order_ids()
8811 .is_some_and(|links| links == [orders[1 - index].client_order_id()]),
8812 "Lighter OCO siblings must link only to each other"
8813 );
8814 }
8815
8816 Ok(())
8817}
8818
8819fn prepare_group_bodies(
8820 grouping_type: LighterGroupingType,
8821 plans: &mut [CreateOrderPlan],
8822 use_gtd: bool,
8823) -> anyhow::Result<()> {
8824 let has_parent = grouping_type != LighterGroupingType::OneCancelsTheOther;
8825 let parent_quantity = plans[0].order.quantity();
8826 let child_start = usize::from(has_parent);
8827
8828 let default_expiry = plans
8829 .iter()
8830 .find(|plan| {
8831 plan.order_expiry != 0 && !(plan.order.time_in_force() == TimeInForce::Gtd && use_gtd)
8832 })
8833 .map_or(plans[child_start].order_expiry, |plan| plan.order_expiry);
8834
8835 for plan in plans.iter_mut() {
8836 if plan.order_expiry != 0 && !(plan.order.time_in_force() == TimeInForce::Gtd && use_gtd) {
8837 plan.order_expiry = default_expiry;
8838 }
8839 }
8840
8841 let expiry = plans[child_start].order_expiry;
8842 let parent_side = plans[0].order.order_side();
8843 for child in &mut plans[child_start..] {
8844 anyhow::ensure!(
8845 is_lighter_conditional_order(child.order.order_type()) && child.order.is_reduce_only(),
8846 "Lighter group children require reduce-only SL/TP orders"
8847 );
8848 anyhow::ensure!(
8849 child.order.quantity() == parent_quantity,
8850 "Lighter group quantities must match before parent-fill sizing"
8851 );
8852 anyhow::ensure!(
8853 (child.order.order_side() != parent_side) == has_parent,
8854 "Lighter OTO children must oppose their parent; OCO siblings must have the same side"
8855 );
8856 anyhow::ensure!(
8857 child.order_expiry == expiry && expiry != 0,
8858 "Lighter group children require matching nonzero expiries"
8859 );
8860
8861 if has_parent {
8862 child.base_amount = 0;
8863 }
8864 }
8865
8866 anyhow::ensure!(
8867 plans[0].order_expiry == 0 || plans[0].order_expiry == expiry,
8868 "Lighter parent and child expiries must match"
8869 );
8870
8871 if plans.len() - child_start == 2 {
8872 let stop_count = plans[child_start..]
8873 .iter()
8874 .filter(|plan| {
8875 matches!(
8876 plan.order.order_type(),
8877 OrderType::StopMarket | OrderType::StopLimit
8878 )
8879 })
8880 .count();
8881
8882 anyhow::ensure!(
8883 stop_count == 1,
8884 "Lighter OCO requires one stop-loss and one take-profit"
8885 );
8886 }
8887
8888 Ok(())
8889}
8890
8891fn is_lighter_spot_order(order: &OrderAny, instrument: Option<&InstrumentAny>) -> bool {
8892 instrument.is_some_and(|instrument| matches!(instrument, InstrumentAny::CurrencyPair(_)))
8893 || product_type_from_instrument_id(&order.instrument_id()) == Some(LighterProductType::Spot)
8894}
8895
8896fn is_lighter_supported_order_type(order_type: OrderType) -> bool {
8897 matches!(
8898 order_type,
8899 OrderType::Market
8900 | OrderType::Limit
8901 | OrderType::StopMarket
8902 | OrderType::StopLimit
8903 | OrderType::MarketIfTouched
8904 | OrderType::LimitIfTouched
8905 )
8906}
8907
8908fn is_lighter_limit_style_order(order_type: OrderType) -> bool {
8909 matches!(
8910 order_type,
8911 OrderType::Limit | OrderType::StopLimit | OrderType::LimitIfTouched
8912 )
8913}
8914
8915fn is_lighter_conditional_order(order_type: OrderType) -> bool {
8916 matches!(
8917 order_type,
8918 OrderType::StopMarket
8919 | OrderType::StopLimit
8920 | OrderType::MarketIfTouched
8921 | OrderType::LimitIfTouched
8922 )
8923}
8924
8925async fn collect_active_markets_from_inactive_orders(
8926 http_client: &LighterHttpClient,
8927 markets: &mut BTreeSet<i64>,
8928 credential: &Credential,
8929 auth: &SecretString,
8930 between_timestamps: Option<String>,
8931) -> anyhow::Result<()> {
8932 let mut cursor: Option<String> = None;
8933 let mut seen_cursors = AHashSet::new();
8934 let mut orders_seen = 0_usize;
8935 let mut pages = 0_usize;
8936 loop {
8937 pages += 1;
8938 anyhow::ensure!(
8939 pages <= MAX_RECONCILIATION_PAGES,
8940 "Lighter active-market seed exceeded {MAX_RECONCILIATION_PAGES} pages",
8941 );
8942
8943 let query = Zeroizing::new(LighterAccountInactiveOrdersQuery {
8944 authorization: None,
8945 auth: Some(auth.clone()),
8946 account_index: credential.account_index(),
8947 market_id: None,
8948 ask_filter: None,
8949 between_timestamps: between_timestamps.clone(),
8950 cursor: cursor.clone(),
8951 limit: LIGHTER_REST_PAGE_SIZE,
8952 });
8953 let response = http_client
8954 .get_account_inactive_orders(&query)
8955 .await
8956 .context("failed to seed Lighter active markets from inactive orders")?;
8957
8958 for order in &response.orders {
8959 markets.insert(order.market_index);
8960 orders_seen += 1;
8961 }
8962
8963 match response.next_cursor {
8964 Some(next) if !next.is_empty() => {
8965 anyhow::ensure!(
8966 seen_cursors.insert(next.clone()),
8967 "Lighter active-market seed repeated cursor `{next}`",
8968 );
8969 cursor = Some(next);
8970 }
8971 _ => break,
8972 }
8973 }
8974
8975 if orders_seen > 0 {
8976 log::debug!("Seeded Lighter active markets from {orders_seen} inactive order report(s)");
8977 }
8978
8979 Ok(())
8980}
8981
8982fn cancel_order_from_cancel_all(
8983 cmd: &CancelAllOrders,
8984 client_order_id: ClientOrderId,
8985 strategy_id: StrategyId,
8986) -> CancelOrder {
8987 CancelOrder {
8988 trader_id: cmd.trader_id,
8989 client_id: cmd.client_id,
8990 strategy_id,
8991 instrument_id: cmd.instrument_id,
8992 client_order_id,
8993 venue_order_id: None,
8994 command_id: cmd.command_id,
8995 ts_init: cmd.ts_init,
8996 params: cmd.params.clone(),
8997 correlation_id: cmd.correlation_id,
8998 causation_id: cmd.causation_id,
8999 }
9000}
9001
9002fn validate_order_amount(
9003 instrument: &InstrumentAny,
9004 quantity: Quantity,
9005 price_ticks: u32,
9006 price_precision: u8,
9007) -> anyhow::Result<()> {
9008 if let Some(min_quantity) = instrument.min_quantity() {
9009 anyhow::ensure!(
9010 quantity >= min_quantity,
9011 "quantity `{quantity}` below Lighter min_base_amount `{min_quantity}` for {}",
9012 instrument.id(),
9013 );
9014 }
9015
9016 if let Some(min_notional) = instrument.min_notional() {
9017 let price = decimal_from_ticks(price_ticks, price_precision);
9018 let notional = quantity.as_decimal() * price;
9019 anyhow::ensure!(
9020 notional >= min_notional.as_decimal(),
9021 "order notional `{notional}` below Lighter min_quote_amount `{}` for {}",
9022 min_notional.as_decimal(),
9023 instrument.id(),
9024 );
9025 }
9026
9027 Ok(())
9028}
9029
9030fn decimal_from_ticks(ticks: u32, decimals: u8) -> Decimal {
9031 Decimal::from(ticks) / Decimal::from(10_i64.pow(u32::from(decimals)))
9032}
9033
9034fn dispatch_lighter_order(
9038 order: &crate::http::models::LighterOrder,
9039 dispatch: &WsDispatchState,
9040 emitter: &ExecutionEventEmitter,
9041 registry: &Arc<MarketRegistry>,
9042 account_id: AccountId,
9043 trader_id: TraderId,
9044 ts_init: UnixNanos,
9045) {
9046 let mut events = dispatch.order_events.lock();
9047
9048 let instrument_id = match registry.instrument_id(order.market_index) {
9049 Some(id) => id,
9050 None => {
9051 log::debug!(
9052 "Lighter order frame dropped: no instrument for market_index={}",
9053 order.market_index,
9054 );
9055 return;
9056 }
9057 };
9058
9059 dispatch.note_active_market(order.market_index);
9060
9061 let instrument = match LIGHTER_INSTRUMENT_CACHE.get(&instrument_id) {
9062 Some(inst) => inst.value().clone(),
9063 None => {
9064 log::debug!("Lighter order frame dropped: instrument {instrument_id} not in cache",);
9065 return;
9066 }
9067 };
9068
9069 let venue_order_id = VenueOrderId::new(order.order_id.as_str());
9070 if !dispatch.restore_cached_order(venue_order_id, order.client_order_index, instrument_id) {
9071 return;
9072 }
9073
9074 let resolved_cloid =
9075 dispatch.resolve_live_order_cloid(order.client_order_id.as_str(), venue_order_id);
9076
9077 let identity = resolved_cloid.and_then(|cid| {
9078 dispatch
9079 .order_identities
9080 .get(&cid)
9081 .map(|entry| (cid, entry.value().clone()))
9082 });
9083
9084 let Some((cloid, identity)) = identity else {
9085 dispatch_unowned_order(order, &instrument, dispatch, emitter, account_id, ts_init);
9086 return;
9087 };
9088
9089 if identity.instrument_id != instrument_id {
9090 return;
9091 }
9092
9093 if defer_owned_order(order, &identity, cloid, &mut events, dispatch, emitter) {
9094 return;
9095 }
9096
9097 dispatch_owned_order(
9098 order,
9099 &instrument,
9100 &identity,
9101 cloid,
9102 venue_order_id,
9103 dispatch,
9104 emitter,
9105 account_id,
9106 trader_id,
9107 ts_init,
9108 );
9109}
9110
9111fn defer_owned_order(
9112 order: &LighterOrder,
9113 identity: &OrderIdentity,
9114 cloid: ClientOrderId,
9115 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
9116 dispatch: &WsDispatchState,
9117 emitter: &ExecutionEventEmitter,
9118) -> bool {
9119 if dispatch.has_pending_modify(&cloid) {
9120 events.entry(cloid).or_default().store_order(order);
9121 return true;
9122 }
9123
9124 if identity.child_fill_baseline().is_some() && !identity.has_inferred_fills() {
9125 if identity.has_child_publication() {
9126 return true;
9127 }
9128
9129 let deferred = events.entry(cloid).or_default();
9130
9131 let had_terminal = deferred
9132 .order
9133 .as_ref()
9134 .is_some_and(|prior| prior.status.is_terminal());
9135
9136 deferred.store_order(order);
9137 let terminal = order.status.is_terminal();
9138
9139 if terminal || had_terminal {
9140 publish_child_recovery(cloid, events, dispatch, emitter);
9141 return true;
9142 }
9143 }
9144
9145 if position_fills_pending(identity, events, dispatch) && order.status.is_terminal() {
9146 events.entry(cloid).or_default().store_order(order);
9147 return true;
9148 }
9149
9150 false
9151}
9152
9153fn dispatch_unowned_order(
9154 order: &LighterOrder,
9155 instrument: &InstrumentAny,
9156 dispatch: &WsDispatchState,
9157 emitter: &ExecutionEventEmitter,
9158 account_id: AccountId,
9159 ts_init: UnixNanos,
9160) {
9161 match parse_ws_order_status_report(order, instrument, account_id, ts_init) {
9162 Ok(mut report) => {
9163 report = dispatch.apply_order_contingencies(order, report);
9164 report = dispatch.translate_order_cloid(report);
9165 report = dispatch.preserve_pending_order_status(report);
9166 if is_unowned_child_cancel(dispatch, order.client_order_index, &report) {
9167 return;
9168 }
9169
9170 if report
9171 .client_order_id
9172 .and_then(|id| dispatch.order_identity(&id))
9173 .is_some_and(|identity| {
9174 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
9175 })
9176 {
9177 return;
9178 }
9179
9180 if let Some(cloid) = &report.client_order_id {
9181 dispatch.venue_id_map.insert(*cloid, report.venue_order_id);
9182 }
9183
9184 if report.order_status.is_closed() {
9185 evict_terminal_mappings(&report, &dispatch.venue_id_map);
9186 }
9187
9188 log::debug!(
9189 "Lighter OrderStatusReport: voi={} status={:?} cloid={:?}",
9190 report.venue_order_id,
9191 report.order_status,
9192 report.client_order_id,
9193 );
9194 emitter.send_order_status_report(report);
9195 }
9196 Err(e) => {
9197 log::error!(
9198 "Failed to parse Lighter order status report: error={e}, order_id={}",
9199 order.order_id,
9200 );
9201 }
9202 }
9203}
9204
9205#[expect(
9206 clippy::too_many_arguments,
9207 reason = "tracked order publication uses the same identity and emitter context as stream dispatch"
9208)]
9209fn dispatch_owned_order(
9210 order: &crate::http::models::LighterOrder,
9211 instrument: &InstrumentAny,
9212 identity: &OrderIdentity,
9213 cloid: ClientOrderId,
9214 venue_order_id: VenueOrderId,
9215 dispatch: &WsDispatchState,
9216 emitter: &ExecutionEventEmitter,
9217 account_id: AccountId,
9218 trader_id: TraderId,
9219 ts_init: UnixNanos,
9220) {
9221 dispatch.venue_id_map.insert(cloid, venue_order_id);
9222
9223 let is_live_status = matches!(
9228 order.status,
9229 crate::common::enums::LighterOrderStatus::Pending
9230 | crate::common::enums::LighterOrderStatus::Open,
9231 );
9232 let pending_action = dispatch.pending_order_action(&cloid);
9233 if is_live_status && pending_action == Some(PendingOrderAction::Cancel) {
9234 log::debug!(
9235 "Deferring Lighter {:?} frame while cancel is pending for {cloid}",
9236 order.status,
9237 );
9238 return;
9239 }
9240
9241 let current_shape = match lighter_order_shape(order, instrument, identity.order_type) {
9242 Ok(shape) => shape,
9243 Err(e) => {
9244 log::error!(
9245 "Failed to compute Lighter order shape: error={e}, voi={venue_order_id}, cloid={cloid}",
9246 );
9247 return;
9248 }
9249 };
9250
9251 let prior_shape = dispatch.snapshot_for(&cloid);
9252 identity.record_reduction(order.initial_base_amount - current_shape.quantity.as_decimal());
9253 let shape_changed = prior_shape.map_or_else(
9254 || current_shape.quantity.as_decimal() < order.initial_base_amount,
9255 |previous| {
9256 identity.normalize_shape(previous) != identity.normalize_shape(current_shape.clone())
9257 },
9258 );
9259
9260 if shape_changed
9261 && !dispatch.accepted_was_emitted(&cloid)
9262 && (is_live_status || order.status == crate::common::enums::LighterOrderStatus::Filled)
9263 {
9264 ensure_accepted_emitted(
9265 cloid,
9266 venue_order_id,
9267 identity,
9268 account_id,
9269 trader_id,
9270 emitter,
9271 dispatch,
9272 ts_init,
9273 );
9274 }
9275
9276 let fresh_trigger = is_order_triggered(order)
9277 && TRIGGERABLE_ORDER_TYPES.contains(&identity.order_type)
9278 && !dispatch.triggered_was_emitted(&cloid);
9279
9280 if pending_action == Some(PendingOrderAction::Modify) && fresh_trigger && !shape_changed {
9281 log::debug!("Deferring Lighter trigger while modify confirmation is pending for {cloid}",);
9282 return;
9283 }
9284
9285 let open_ctx = OpenFrameContext {
9286 accepted_already_emitted: dispatch.accepted_was_emitted(&cloid),
9287 triggered_already_emitted: dispatch.triggered_was_emitted(&cloid),
9288 shape_changed,
9289 };
9290
9291 let event_opt = match parse_lighter_order_event(
9292 order, instrument, identity, cloid, account_id, trader_id, open_ctx, ts_init,
9293 ) {
9294 Ok(event) => event,
9295 Err(e) => {
9296 log::error!(
9297 "Failed to parse Lighter order event: error={e}, voi={venue_order_id}, cloid={cloid}"
9298 );
9299 return;
9300 }
9301 };
9302
9303 dispatch.store_snapshot(cloid, current_shape);
9304 if let Some(event) = event_opt {
9305 let Some(event) = defer_terminal(order, event, identity, cloid, dispatch) else {
9306 return;
9307 };
9308
9309 dispatch_tracked_order_event(
9310 event,
9311 cloid,
9312 venue_order_id,
9313 identity,
9314 account_id,
9315 trader_id,
9316 emitter,
9317 dispatch,
9318 ts_init,
9319 );
9320 }
9321
9322 if order.status == crate::common::enums::LighterOrderStatus::Filled {
9323 dispatch.clear_pending_order_action(&cloid);
9324 dispatch.retire_order_identity(&cloid);
9325 }
9326}
9327
9328#[expect(
9332 clippy::too_many_arguments,
9333 reason = "consumption-loop dispatch threads identity and emitter context"
9334)]
9335fn dispatch_lighter_trade(
9336 trade: &crate::http::models::LighterTrade,
9337 dispatch: &WsDispatchState,
9338 emitter: &ExecutionEventEmitter,
9339 registry: &Arc<MarketRegistry>,
9340 account_id: AccountId,
9341 trader_id: TraderId,
9342 account_index: Option<i64>,
9343 ts_init: UnixNanos,
9344) {
9345 let mut events = dispatch.order_events.lock();
9346
9347 let Some(account_index) = account_index else {
9348 log::debug!("Lighter trade frame dropped: no credential / account_index available",);
9349 return;
9350 };
9351
9352 let Some(instrument_id) = registry.instrument_id(trade.market_id) else {
9353 log::debug!(
9354 "Lighter trade frame dropped: no instrument for market_id={}",
9355 trade.market_id,
9356 );
9357 return;
9358 };
9359
9360 dispatch.note_active_market(trade.market_id);
9361
9362 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE
9363 .get(&instrument_id)
9364 .map(|instrument| instrument.value().clone())
9365 else {
9366 log::debug!("Lighter trade frame dropped: instrument {instrument_id} not in cache",);
9367 return;
9368 };
9369
9370 let user_is_bidder = trade.bid_account_id == account_index;
9371 let user_is_asker = trade.ask_account_id == account_index;
9372 if !user_is_bidder && !user_is_asker {
9373 return;
9376 }
9377
9378 let trade_id = match parse_lighter_trade_id(trade) {
9381 Ok(id) => id,
9382 Err(e) => {
9383 log::error!("Lighter trade has invalid trade_id: {e}");
9384 return;
9385 }
9386 };
9387
9388 let (raw_client_id, client_order_index, venue_order_id) = if user_is_bidder {
9389 (
9390 trade
9391 .bid_client_id_str
9392 .as_deref()
9393 .map_or_else(|| trade.bid_client_id.to_string(), str::to_string),
9394 trade.bid_client_id,
9395 trade.bid_id_str.as_deref().map_or_else(
9396 || VenueOrderId::new(trade.bid_id.to_string()),
9397 VenueOrderId::new,
9398 ),
9399 )
9400 } else {
9401 (
9402 trade
9403 .ask_client_id_str
9404 .as_deref()
9405 .map_or_else(|| trade.ask_client_id.to_string(), str::to_string),
9406 trade.ask_client_id,
9407 trade.ask_id_str.as_deref().map_or_else(
9408 || VenueOrderId::new(trade.ask_id.to_string()),
9409 VenueOrderId::new,
9410 ),
9411 )
9412 };
9413
9414 if !dispatch.restore_cached_order(venue_order_id, client_order_index, instrument_id) {
9415 return;
9416 }
9417
9418 let resolved_cloid = dispatch.resolve_live_trade_cloid(raw_client_id.as_str(), venue_order_id);
9419 let identity = resolved_cloid.and_then(|cid| {
9420 dispatch
9421 .order_identity(&cid)
9422 .map(|identity| (cid, identity))
9423 });
9424
9425 let Some((cloid, identity)) = identity else {
9426 match parse_ws_fill_report(trade, account_index, &instrument, account_id, ts_init) {
9427 Ok(Some(mut report)) => {
9428 report = dispatch.translate_fill_cloid(report);
9429 log::debug!(
9430 "Lighter FillReport: voi={} qty={} px={} liq={:?} cloid={:?}",
9431 report.venue_order_id,
9432 report.last_qty,
9433 report.last_px,
9434 report.liquidity_side,
9435 report.client_order_id,
9436 );
9437
9438 if dispatch.mark_trade_seen(trade_id) {
9439 emitter.send_fill_report(report);
9440 }
9441 }
9442 Ok(None) => {}
9443 Err(e) => {
9444 log::error!("Failed to parse Lighter fill report: error={e}, trade_id={trade_id}",);
9445 }
9446 }
9447
9448 return;
9449 };
9450
9451 if identity.instrument_id != instrument_id {
9452 return;
9453 }
9454
9455 dispatch_owned_trade(
9456 trade,
9457 &instrument,
9458 &identity,
9459 cloid,
9460 venue_order_id,
9461 &mut events,
9462 dispatch,
9463 emitter,
9464 account_id,
9465 trader_id,
9466 account_index,
9467 trade_id,
9468 ts_init,
9469 );
9470}
9471
9472#[expect(
9473 clippy::too_many_arguments,
9474 reason = "tracked trade publication uses the stream identity and emitter context"
9475)]
9476fn dispatch_owned_trade(
9477 trade: &crate::http::models::LighterTrade,
9478 instrument: &InstrumentAny,
9479 identity: &OrderIdentity,
9480 cloid: ClientOrderId,
9481 venue_order_id: VenueOrderId,
9482 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
9483 dispatch: &WsDispatchState,
9484 emitter: &ExecutionEventEmitter,
9485 account_id: AccountId,
9486 trader_id: TraderId,
9487 account_index: i64,
9488 trade_id: TradeId,
9489 ts_init: UnixNanos,
9490) {
9491 if identity.has_inferred_fills() {
9492 log::warn!(
9493 "Withholding Lighter trade {trade_id} for {cloid}: inferred fills require reconciliation"
9494 );
9495 return;
9496 }
9497
9498 match parse_lighter_order_filled(
9499 trade,
9500 instrument,
9501 identity,
9502 cloid,
9503 account_id,
9504 trader_id,
9505 account_index,
9506 ts_init,
9507 ) {
9508 Ok(Some(filled)) => {
9509 ensure_accepted_emitted(
9510 cloid,
9511 venue_order_id,
9512 identity,
9513 account_id,
9514 trader_id,
9515 emitter,
9516 dispatch,
9517 ts_init,
9518 );
9519
9520 log::debug!(
9521 "Lighter OrderFilled: voi={} qty={} px={} liq={:?} cloid={cloid}",
9522 filled.venue_order_id,
9523 filled.last_qty,
9524 filled.last_px,
9525 filled.liquidity_side,
9526 );
9527
9528 if let Some(baseline) = identity.child_fill_baseline() {
9529 if baseline.trade_ids.contains(&trade_id) {
9530 return;
9531 }
9532
9533 dispatch.defer_fill(events.entry(cloid).or_default(), filled);
9534 publish_child_recovery(cloid, events, dispatch, emitter);
9535 } else if dispatch.has_pending_modify(&cloid)
9536 || position_fills_pending(identity, events, dispatch)
9537 {
9538 dispatch.defer_fill(events.entry(cloid).or_default(), filled);
9539 } else if dispatch.mark_trade_seen(trade_id) {
9540 dispatch_lighter_fill(filled, emitter, dispatch, events);
9541 }
9542 }
9543 Ok(None) => {}
9544 Err(e) => {
9545 log::error!("Failed to parse Lighter typed fill: error={e}, trade_id={trade_id}",);
9546 }
9547 }
9548}
9549
9550fn spawn_child_recoveries(context: &AckedOrderProbeContext) {
9551 let candidates = context
9552 .dispatch
9553 .order_events
9554 .lock()
9555 .iter()
9556 .filter_map(|(cloid, deferred)| (!deferred.fills.is_empty()).then_some(*cloid))
9557 .collect::<Vec<_>>();
9558
9559 for cloid in candidates {
9560 spawn_child_recovery(cloid, context);
9561 }
9562}
9563
9564fn request_child_report_recoveries(
9565 core: &ExecutionClientCore,
9566 context: &AckedOrderProbeContext,
9567 sweep: &OrderReportSweep,
9568) {
9569 for (index, report) in &sweep.orders {
9570 if !core
9571 .cache()
9572 .client_order_id(&report.venue_order_id)
9573 .is_some_and(|cloid| context.dispatch.has_pending_modify(cloid))
9574 {
9575 restore_reconciled_order(
9576 core,
9577 &context.dispatch,
9578 report.venue_order_id,
9579 *index,
9580 report.order_status.is_closed(),
9581 );
9582 }
9583
9584 request_child_recovery(core, context, report);
9585 }
9586}
9587
9588fn request_child_recovery(
9589 core: &ExecutionClientCore,
9590 context: &AckedOrderProbeContext,
9591 report: &OrderStatusReport,
9592) {
9593 let cache = core.cache();
9594
9595 let Some(cloid) = cache.client_order_id(&report.venue_order_id).copied() else {
9596 return;
9597 };
9598
9599 let Some(order) = cache.order(&cloid) else {
9600 return;
9601 };
9602
9603 if order.parent_order_id().is_some()
9604 && !order.is_closed()
9605 && (report.order_status.is_closed() || report.filled_qty != order.filled_qty())
9606 {
9607 spawn_child_recovery(cloid, context);
9608 }
9609}
9610
9611fn spawn_child_recovery(cloid: ClientOrderId, context: &AckedOrderProbeContext) {
9612 let epoch = context.connection_epoch.load(Ordering::Acquire);
9613 {
9614 let mut events = context.dispatch.order_events.lock();
9615
9616 let Some(identity) = context.dispatch.order_identity(&cloid) else {
9617 return;
9618 };
9619
9620 if identity.child_fill_baseline().is_none()
9621 || identity.has_inferred_fills()
9622 || identity.has_child_publication()
9623 || context.dispatch.has_pending_modify(&cloid)
9624 {
9625 return;
9626 }
9627
9628 let deferred = events.entry(cloid).or_default();
9629 if deferred.recovery_epoch == Some(epoch) {
9630 return;
9631 }
9632
9633 deferred.recovery_epoch = Some(epoch);
9634 }
9635
9636 let context = context.clone();
9637 let tasks = context.pending_tasks.clone();
9638 let dispatch = context.dispatch.clone();
9639
9640 let future = async move {
9641 for attempt in 0..ACKED_ORDER_PROBE_ATTEMPTS {
9642 if context.connection_epoch.load(Ordering::Acquire) != epoch {
9643 break;
9644 }
9645
9646 let result = tokio::select! {
9647 () = context.cancellation_token.cancelled() => break,
9648 result = recover_child_order(cloid, epoch, &context) => result,
9649 };
9650
9651 match result {
9652 Ok(true) => break,
9653 Ok(false) => {}
9654 Err(e) => log::warn!(
9655 "Lighter child recovery unavailable for {cloid}: {}",
9656 scrub_auth(&format!("{e:#}"))
9657 ),
9658 }
9659
9660 if attempt + 1 < ACKED_ORDER_PROBE_ATTEMPTS {
9661 tokio::select! {
9662 () = context.cancellation_token.cancelled() => break,
9663 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {},
9664 }
9665 }
9666 }
9667
9668 let mut events = context.dispatch.order_events.lock();
9669 if let Some(deferred) = events.get_mut(&cloid)
9670 && deferred.recovery_epoch == Some(epoch)
9671 {
9672 deferred.recovery_epoch = None;
9673 }
9674 };
9675
9676 if let Err(e) = tasks.spawn(future) {
9677 if let Some(deferred) = dispatch.order_events.lock().get_mut(&cloid) {
9678 deferred.recovery_epoch = None;
9679 }
9680
9681 log::debug!("Skipping Lighter child recovery during shutdown: {e}");
9682 }
9683}
9684
9685async fn recover_child_order(
9686 cloid: ClientOrderId,
9687 epoch: u64,
9688 context: &AckedOrderProbeContext,
9689) -> anyhow::Result<bool> {
9690 let Some(identity) = context.dispatch.order_identity(&cloid) else {
9691 return Ok(true);
9692 };
9693
9694 if context.dispatch.has_pending_modify(&cloid)
9695 || identity.has_inferred_fills()
9696 || identity.has_child_publication()
9697 {
9698 return Ok(true);
9699 }
9700
9701 let Some(venue_order_id) = identity.venue_order_id() else {
9702 return Ok(false);
9703 };
9704
9705 let lookup = OrderReportLookup::new(
9706 &context.registry,
9707 &context.dispatch,
9708 Some(identity.instrument_id),
9709 Some(&cloid),
9710 Some(&venue_order_id),
9711 )?;
9712
9713 let Some(order) = collect_order(
9714 &context.http_client,
9715 &context.registry,
9716 &context.credential,
9717 &lookup,
9718 )
9719 .await?
9720 else {
9721 return Ok(false);
9722 };
9723
9724 anyhow::ensure!(
9725 order.owner_account_index == context.credential.account_index()
9726 && context.registry.instrument_id(order.market_index) == Some(identity.instrument_id)
9727 && identity.matches_venue_order_id(VenueOrderId::new(&order.order_id)),
9728 "Lighter child recovery ownership mismatch"
9729 );
9730 let instrument = LIGHTER_INSTRUMENT_CACHE
9731 .get(&identity.instrument_id)
9732 .context("missing child recovery instrument")?
9733 .value()
9734 .clone();
9735 let report = parse_ws_order_status_report(
9736 &order,
9737 &instrument,
9738 context.account_id,
9739 context.clock.get_time_ns(),
9740 )?;
9741
9742 if !report.order_status.is_closed() {
9743 return Ok(false);
9744 }
9745
9746 let mut instruments = AHashMap::new();
9747 instruments.insert(identity.instrument_id, instrument.clone());
9748
9749 let command = GenerateFillReports::new(
9750 UUID4::new(),
9751 context.clock.get_time_ns(),
9752 Some(identity.instrument_id),
9753 Some(venue_order_id),
9754 None,
9755 None,
9756 None,
9757 None,
9758 );
9759 let sweep = collect_fill_reports(
9760 &context.http_client,
9761 &context.registry,
9762 &context.credential,
9763 context.account_id,
9764 context.clock,
9765 &command,
9766 &instruments,
9767 &mut BTreeSet::new(),
9768 Some(order.order_index),
9769 )
9770 .await?;
9771 let mut events = context.dispatch.order_events.lock();
9772 if context.connection_epoch.load(Ordering::Acquire) != epoch
9773 || context.dispatch.has_pending_modify(&cloid)
9774 || identity.has_inferred_fills()
9775 || identity.has_child_publication()
9776 {
9777 return Ok(true);
9778 }
9779
9780 let deferred = events.entry(cloid).or_default();
9781 deferred.store_order(&order);
9782
9783 for mut fill in sweep.reports {
9784 fill.client_order_id = Some(cloid);
9785 if fill.venue_order_id != venue_order_id {
9786 continue;
9787 }
9788
9789 if let Some(fill) = child_fill_from_report(&fill, &identity, context.emitter.trader_id()) {
9790 context.dispatch.defer_fill(deferred, fill);
9791 }
9792 }
9793
9794 publish_child_recovery(cloid, &mut events, &context.dispatch, &context.emitter);
9795 Ok(identity.has_child_publication())
9796}
9797
9798fn child_fill_from_report(
9799 report: &FillReport,
9800 identity: &OrderIdentity,
9801 trader_id: TraderId,
9802) -> Option<OrderFilled> {
9803 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id)?;
9804 let cloid = report.client_order_id.filter(|_| {
9805 identity.matches_venue_order_id(report.venue_order_id)
9806 && identity.instrument_id == report.instrument_id
9807 })?;
9808
9809 Some(OrderFilled::new(
9810 trader_id,
9811 identity.strategy_id,
9812 identity.instrument_id,
9813 cloid,
9814 report.venue_order_id,
9815 report.account_id,
9816 report.trade_id,
9817 report.order_side,
9818 identity.order_type,
9819 report.last_qty,
9820 report.last_px,
9821 instrument.quote_currency(),
9822 report.liquidity_side,
9823 UUID4::new(),
9824 report.ts_event,
9825 report.ts_init,
9826 false,
9827 None,
9828 Some(report.commission),
9829 None,
9830 ))
9831}
9832
9833fn publish_child_recovery(
9834 cloid: ClientOrderId,
9835 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
9836 dispatch: &WsDispatchState,
9837 emitter: &ExecutionEventEmitter,
9838) {
9839 let Some(identity) = dispatch.order_identity(&cloid) else {
9840 return;
9841 };
9842
9843 if position_modify_pending(&identity, dispatch) {
9844 return;
9845 }
9846
9847 let Some((ids, children)) = collect_child_recoveries(&identity, events, dispatch, emitter)
9848 else {
9849 return;
9850 };
9851
9852 let mut fills = Vec::new();
9853 let mut terminals = Vec::new();
9854
9855 for id in ids {
9856 let Some(mut deferred) = events.remove(&id) else {
9857 continue;
9858 };
9859
9860 fills.extend(deferred.fills.drain(..).map(|fill| {
9861 let sequence = deferred.fill_sequence.remove(&fill.trade_id).unwrap_or(0);
9862 (fill, sequence)
9863 }));
9864
9865 terminals.push((id, deferred));
9866 }
9867
9868 publish_child_updates(&children, &terminals, emitter);
9869
9870 fills.sort_by_key(|(fill, sequence)| (fill.ts_event, *sequence));
9871 for (fill, _) in fills {
9872 if dispatch.mark_trade_seen(fill.trade_id) {
9873 dispatch_lighter_fill(fill, emitter, dispatch, events);
9874 }
9875 }
9876
9877 for child in children {
9878 let id = child.cloid;
9879 let bundle = ExecutionReport::OrderWithFills(Box::new(child.report.clone()), Vec::new());
9880
9881 if let Err(e) = emitter.try_send_execution_report(bundle) {
9882 log::warn!("Lighter child report publication failed for {id}: {e}");
9883
9884 let Some(index) = terminals.iter().position(|(cid, _)| *cid == id) else {
9885 continue;
9886 };
9887
9888 let (_, deferred) = terminals.swap_remove(index);
9889 events.insert(id, deferred);
9890
9891 continue;
9892 }
9893
9894 child.identity.publish_child(&child.report, child.trade_ids);
9895 dispatch.retire_order_identity(&id);
9896 }
9897
9898 for (id, deferred) in terminals {
9899 if dispatch
9900 .order_identity(&id)
9901 .is_some_and(|identity| identity.child_fill_baseline().is_none())
9902 {
9903 flush_order_terminal(id, deferred, dispatch, emitter, events);
9904 }
9905 }
9906}
9907
9908struct ChildRecovery {
9909 cloid: ClientOrderId,
9910 identity: OrderIdentity,
9911 report: OrderStatusReport,
9912 trade_ids: Vec<TradeId>,
9913}
9914
9915fn collect_child_recoveries(
9916 identity: &OrderIdentity,
9917 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
9918 dispatch: &WsDispatchState,
9919 emitter: &ExecutionEventEmitter,
9920) -> Option<(Vec<ClientOrderId>, Vec<ChildRecovery>)> {
9921 let mut children = Vec::new();
9922 let mut ids = Vec::new();
9923
9924 for (id, deferred) in events {
9925 let Some(other) = dispatch.order_identity(id) else {
9926 continue;
9927 };
9928
9929 if other.strategy_id != identity.strategy_id
9930 || other.instrument_id != identity.instrument_id
9931 {
9932 continue;
9933 }
9934
9935 if dispatch.has_pending_modify(id) {
9936 return None;
9937 }
9938
9939 if other.child_fill_baseline().is_none() {
9940 ids.push(*id);
9941 continue;
9942 }
9943
9944 if deferred.fills.is_empty()
9945 && deferred
9946 .order
9947 .as_ref()
9948 .is_none_or(|order| !order.status.is_terminal())
9949 {
9950 continue;
9951 }
9952
9953 let report = child_recovery_report(*id, events, dispatch, emitter)?;
9954 let trade_ids = deferred.fills.iter().map(|fill| fill.trade_id).collect();
9955 children.push(ChildRecovery {
9956 cloid: *id,
9957 identity: other,
9958 report,
9959 trade_ids,
9960 });
9961
9962 ids.push(*id);
9963 }
9964
9965 Some((ids, children))
9966}
9967
9968fn publish_child_updates(
9969 children: &[ChildRecovery],
9970 terminals: &[(ClientOrderId, DeferredOrderEvents)],
9971 emitter: &ExecutionEventEmitter,
9972) {
9973 for child in children {
9974 if child.trade_ids.is_empty() {
9975 continue;
9976 }
9977
9978 let Some(order) = terminals
9979 .iter()
9980 .find(|(cid, _)| *cid == child.cloid)
9981 .and_then(|(_, deferred)| deferred.order.as_ref())
9982 else {
9983 continue;
9984 };
9985
9986 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE.get(&child.identity.instrument_id) else {
9987 continue;
9988 };
9989
9990 if let Ok(updated) = parse_lighter_order_updated(
9991 order,
9992 &instrument,
9993 &child.identity,
9994 child.cloid,
9995 emitter.account_id(),
9996 emitter.trader_id(),
9997 child.report.ts_init,
9998 ) {
9999 emitter.send_order_event(OrderEventAny::Updated(updated));
10000 }
10001 }
10002}
10003
10004fn position_fills_pending(
10005 identity: &OrderIdentity,
10006 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
10007 dispatch: &WsDispatchState,
10008) -> bool {
10009 position_modify_pending(identity, dispatch)
10010 || events.iter().any(|(id, deferred)| {
10011 !deferred.fills.is_empty()
10012 && dispatch.order_identity(id).is_some_and(|other| {
10013 other.strategy_id == identity.strategy_id
10014 && other.instrument_id == identity.instrument_id
10015 })
10016 })
10017}
10018
10019fn position_modify_pending(identity: &OrderIdentity, dispatch: &WsDispatchState) -> bool {
10020 dispatch.order_identities.iter().any(|entry| {
10021 entry.value().strategy_id == identity.strategy_id
10022 && entry.value().instrument_id == identity.instrument_id
10023 && dispatch.has_pending_modify(entry.key())
10024 })
10025}
10026
10027fn child_recovery_report(
10028 cloid: ClientOrderId,
10029 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
10030 dispatch: &WsDispatchState,
10031 emitter: &ExecutionEventEmitter,
10032) -> Option<OrderStatusReport> {
10033 let identity = dispatch.order_identity(&cloid)?;
10034 if identity.has_child_publication() || identity.has_inferred_fills() {
10035 return None;
10036 }
10037
10038 let baseline = identity.child_fill_baseline()?;
10039 let deferred = events.get(&cloid)?;
10040 let order = deferred.order.as_ref()?;
10041 if !identity.matches_venue_order_id(VenueOrderId::new(&order.order_id)) {
10042 return None;
10043 }
10044
10045 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id)?;
10046 let report = parse_ws_order_status_report(
10047 order,
10048 &instrument,
10049 emitter.account_id(),
10050 get_atomic_clock_realtime().get_time_ns(),
10051 )
10052 .ok()?;
10053 let report = dispatch.translate_order_cloid(dispatch.apply_order_contingencies(order, report));
10054 if !report.order_status.is_closed() || report.filled_qty > report.quantity {
10055 return None;
10056 }
10057
10058 let mut total = baseline.quantity.as_decimal();
10059 for fill in &deferred.fills {
10060 if baseline.trade_ids.contains(&fill.trade_id) {
10061 continue;
10062 }
10063
10064 if fill.venue_order_id != report.venue_order_id
10065 || fill.instrument_id != report.instrument_id
10066 || fill.account_id != report.account_id
10067 || Some(fill.order_side) != report.order_side
10068 || fill.commission.is_none()
10069 {
10070 return None;
10071 }
10072
10073 total += fill.last_qty.as_decimal();
10074 }
10075
10076 (total == report.filled_qty.as_decimal()).then_some(report)
10077}
10078
10079fn dispatch_lighter_fill(
10080 filled: OrderFilled,
10081 emitter: &ExecutionEventEmitter,
10082 dispatch: &WsDispatchState,
10083 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
10084) {
10085 let cloid = filled.client_order_id;
10086 if let Some(identity) = dispatch.order_identity(&cloid)
10087 && !identity.publish_fill(&filled)
10088 {
10089 return;
10090 }
10091
10092 ensure_fill_triggered(&filled, emitter, dispatch);
10093 emitter.send_order_event(OrderEventAny::Filled(filled));
10094 release_terminal(cloid, false, dispatch, emitter, events);
10095}
10096
10097fn defer_terminal(
10098 order: &LighterOrder,
10099 event: ParsedOrderEvent,
10100 identity: &OrderIdentity,
10101 cloid: ClientOrderId,
10102 dispatch: &WsDispatchState,
10103) -> Option<ParsedOrderEvent> {
10104 if !matches!(
10105 event,
10106 ParsedOrderEvent::Canceled(_) | ParsedOrderEvent::Expired(_)
10107 ) {
10108 return Some(event);
10109 }
10110
10111 let mut pending = dispatch.terminals_pending.lock();
10112 if pending.contains_key(&cloid) {
10113 return None;
10114 }
10115
10116 if order.filled_base_amount <= identity.filled_qty() {
10117 return Some(event);
10118 }
10119
10120 pending.insert(
10121 cloid,
10122 PendingTerminal {
10123 order: order.clone(),
10124 event,
10125 retry_at: tokio::time::Instant::now() + TERMINAL_FILL_DELAY,
10126 attempts: 0,
10127 },
10128 );
10129
10130 None
10131}
10132
10133fn release_terminal(
10134 cloid: ClientOrderId,
10135 force: bool,
10136 dispatch: &WsDispatchState,
10137 emitter: &ExecutionEventEmitter,
10138 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
10139) {
10140 let Some(identity) = dispatch.order_identity(&cloid) else {
10141 return;
10142 };
10143
10144 let terminal = {
10145 let mut pending = dispatch.terminals_pending.lock();
10146
10147 let Some(terminal) = pending.get(&cloid) else {
10148 return;
10149 };
10150
10151 if (identity.child_fill_baseline().is_some() && !identity.has_inferred_fills())
10152 || dispatch.has_pending_modify(&cloid)
10153 || position_fills_pending(&identity, events, dispatch)
10154 {
10155 return;
10156 }
10157
10158 if !force && identity.filled_qty() < terminal.order.filled_base_amount {
10159 return;
10160 }
10161
10162 pending.remove(&cloid).expect("pending terminal exists")
10163 };
10164
10165 let missing_qty = terminal.order.filled_base_amount - identity.filled_qty();
10166 if missing_qty > Decimal::ZERO {
10167 log::error!(
10168 "Lighter terminal fill recovery exhausted for {cloid}, venue_order_id={}, missing quantity={missing_qty}; releasing terminal for reconciliation",
10169 terminal.order.order_id,
10170 );
10171 }
10172
10173 dispatch_tracked_order_event(
10174 terminal.event,
10175 cloid,
10176 VenueOrderId::new(&terminal.order.order_id),
10177 &identity,
10178 emitter.account_id(),
10179 emitter.trader_id(),
10180 emitter,
10181 dispatch,
10182 get_atomic_clock_realtime().get_time_ns(),
10183 );
10184}
10185
10186async fn run_terminal_recovery(
10187 context: AckedOrderProbeContext,
10188 producers_stopped: CancellationToken,
10189) {
10190 let mut interval = tokio::time::interval(Duration::from_millis(100));
10191 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
10192 let mut immediate = false;
10193
10194 loop {
10195 tokio::select! {
10196 biased;
10197 () = context.cancellation_token.cancelled() => break,
10198 () = context.dispatch.terminals_recover.notified() => immediate = true,
10199 _ = interval.tick() => {},
10200 }
10201
10202 tokio::select! {
10203 biased;
10204 () = context.cancellation_token.cancelled() => break,
10205 () = context.dispatch.terminals_recover.notified() => {
10206 immediate = true;
10207 continue;
10208 }
10209 () = recover_terminals(&context, immediate) => {},
10210 }
10211 immediate = false;
10212 }
10213
10214 producers_stopped.cancelled().await;
10215
10216 let drain = async {
10217 for _ in 0..ACKED_ORDER_PROBE_ATTEMPTS {
10218 recover_terminals(&context, true).await;
10219 if context.dispatch.terminals_pending.lock().is_empty() {
10220 break;
10221 }
10222 }
10223 };
10224
10225 let _ = tokio::time::timeout(TERMINAL_SHUTDOWN_TIMEOUT, drain).await;
10226 let events = context.dispatch.order_events.lock();
10227 let remaining = context
10228 .dispatch
10229 .terminals_pending
10230 .lock()
10231 .keys()
10232 .copied()
10233 .collect::<Vec<_>>();
10234
10235 for cloid in remaining {
10236 release_terminal(cloid, true, &context.dispatch, &context.emitter, &events);
10237 }
10238}
10239
10240async fn recover_terminals(context: &AckedOrderProbeContext, immediate: bool) {
10241 let now = tokio::time::Instant::now();
10242 let candidates = context
10243 .dispatch
10244 .terminals_pending
10245 .lock()
10246 .iter()
10247 .filter(|(_, terminal)| immediate || terminal.retry_at <= now)
10248 .map(|(cloid, terminal)| {
10249 (
10250 *cloid,
10251 terminal.order.clone(),
10252 terminal.attempts >= ACKED_ORDER_PROBE_ATTEMPTS,
10253 )
10254 })
10255 .collect::<Vec<_>>();
10256
10257 stream::iter(candidates)
10258 .for_each_concurrent(4, |(cloid, order, exhausted)| async move {
10259 {
10260 let events = context.dispatch.order_events.lock();
10261 release_terminal(
10262 cloid,
10263 exhausted,
10264 &context.dispatch,
10265 &context.emitter,
10266 &events,
10267 );
10268
10269 if exhausted {
10270 return;
10271 }
10272
10273 let mut pending = context.dispatch.terminals_pending.lock();
10274
10275 let Some(terminal) = pending.get_mut(&cloid) else {
10276 return;
10277 };
10278
10279 terminal.attempts += 1;
10280 }
10281
10282 let result = tokio::time::timeout(
10283 TERMINAL_RECOVERY_TIMEOUT,
10284 recover_terminal_trades(order.market_index, order.order_index, context),
10285 )
10286 .await;
10287
10288 match result {
10289 Ok(Ok(())) => {}
10290 Ok(Err(e)) => log::warn!(
10291 "Lighter terminal trade lookup failed for {cloid}: {}",
10292 scrub_auth(&format!("{e:#}"))
10293 ),
10294 Err(_) => log::warn!("Lighter terminal trade lookup timed out for {cloid}"),
10295 }
10296
10297 let events = context.dispatch.order_events.lock();
10298 release_terminal(cloid, false, &context.dispatch, &context.emitter, &events);
10299
10300 let exhausted = {
10301 let mut pending = context.dispatch.terminals_pending.lock();
10302
10303 let Some(terminal) = pending.get_mut(&cloid) else {
10304 return;
10305 };
10306
10307 terminal.retry_at = tokio::time::Instant::now() + TERMINAL_FILL_DELAY;
10308 terminal.attempts >= ACKED_ORDER_PROBE_ATTEMPTS
10309 };
10310
10311 if exhausted {
10312 release_terminal(cloid, true, &context.dispatch, &context.emitter, &events);
10313 }
10314 })
10315 .await;
10316}
10317
10318async fn recover_filled_reports(sweep: &OrderReportSweep, context: &AckedOrderProbeContext) {
10319 for (index, report) in &sweep.orders {
10320 if report.order_status != OrderStatus::Filled
10321 || !context.dispatch.restore_cached_order(
10322 report.venue_order_id,
10323 *index,
10324 report.instrument_id,
10325 )
10326 {
10327 continue;
10328 }
10329
10330 let Some(cloid) = context
10331 .dispatch
10332 .resolve_client_order_index_for_venue(*index, report.venue_order_id)
10333 else {
10334 continue;
10335 };
10336
10337 let Some(identity) = context.dispatch.order_identity(&cloid) else {
10338 continue;
10339 };
10340
10341 if identity.child_fill_baseline().is_some()
10342 || identity.has_inferred_fills()
10343 || report.filled_qty.as_decimal() <= identity.filled_qty()
10344 {
10345 continue;
10346 }
10347
10348 let Some(market_index) = context.registry.market_index(&report.instrument_id) else {
10349 continue;
10350 };
10351
10352 let result = async {
10353 let order_index = report
10354 .venue_order_id
10355 .as_str()
10356 .parse::<i64>()
10357 .context("invalid Lighter filled order index")?;
10358 tokio::time::timeout(
10359 TERMINAL_RECOVERY_TIMEOUT,
10360 recover_terminal_trades(market_index, order_index, context),
10361 )
10362 .await
10363 .context("Lighter filled-order trade lookup timed out")?
10364 }
10365 .await;
10366
10367 if let Err(e) = result {
10368 log::warn!(
10369 "Lighter filled-order recovery unavailable for {cloid}: {}",
10370 scrub_auth(&format!("{e:#}"))
10371 );
10372 }
10373 }
10374}
10375
10376async fn recover_report_fills(
10377 order: &LighterOrder,
10378 context: &AckedOrderProbeContext,
10379) -> anyhow::Result<()> {
10380 if !order.status.is_terminal() {
10381 return Ok(());
10382 }
10383
10384 let venue = VenueOrderId::new(&order.order_id);
10385
10386 let Some(instrument_id) = context.registry.instrument_id(order.market_index) else {
10387 return Ok(());
10388 };
10389
10390 if !context
10391 .dispatch
10392 .restore_cached_order(venue, order.client_order_index, instrument_id)
10393 {
10394 return Ok(());
10395 }
10396
10397 let Some(cloid) = context
10398 .dispatch
10399 .resolve_client_order_index_for_venue(order.client_order_index, venue)
10400 else {
10401 return Ok(());
10402 };
10403
10404 let missing = (order.status == LighterOrderStatus::Filled
10405 || context.dispatch.order_identities.contains_key(&cloid))
10406 && context
10407 .dispatch
10408 .order_identity(&cloid)
10409 .is_some_and(|identity| {
10410 order.filled_base_amount > identity.filled_qty()
10411 && (order.status != LighterOrderStatus::Filled
10412 || !identity.has_inferred_fills())
10413 });
10414
10415 if !missing {
10416 return Ok(());
10417 }
10418
10419 if order.status == LighterOrderStatus::Filled {
10420 tokio::time::timeout(
10421 TERMINAL_RECOVERY_TIMEOUT,
10422 recover_terminal_trades(order.market_index, order.order_index, context),
10423 )
10424 .await
10425 .context("Lighter filled-order trade lookup timed out")??;
10426 } else {
10427 dispatch_lighter_order(
10428 order,
10429 &context.dispatch,
10430 &context.emitter,
10431 &context.registry,
10432 context.account_id,
10433 context.emitter.trader_id(),
10434 context.clock.get_time_ns(),
10435 );
10436 }
10437
10438 Ok(())
10439}
10440
10441async fn recover_terminal_trades(
10442 market_index: i64,
10443 order_index: i64,
10444 context: &AckedOrderProbeContext,
10445) -> anyhow::Result<()> {
10446 let auth = build_auth_token_for(&context.credential)?;
10447
10448 let mut query = Zeroizing::new(LighterTradesQuery {
10449 authorization: None,
10450 auth: Some(auth),
10451 market_id: Some(market_index),
10452 account_index: Some(context.credential.account_index()),
10453 order_index: Some(order_index),
10454 order_index_str: None,
10455 integrator_account_index: None,
10456 sort_by: LighterTradeSortBy::Timestamp,
10457 sort_dir: Some(LighterSortDirection::Desc),
10458 cursor: None,
10459 from_timestamp: None,
10460 ask_filter: None,
10461 role: None,
10462 trade_type: None,
10463 limit: LIGHTER_REST_PAGE_SIZE,
10464 aggregate: None,
10465 });
10466
10467 let mut cursors = AHashSet::new();
10468 let mut trades = Vec::new();
10469
10470 for page in 0..MAX_RECONCILIATION_PAGES {
10471 let response = context.http_client.get_trades(&query).await?;
10472 for trade in response.trades {
10473 let account = context.credential.account_index();
10474 anyhow::ensure!(
10475 trade.market_id == market_index
10476 && ((trade.bid_account_id == account && trade.bid_id == order_index)
10477 || (trade.ask_account_id == account && trade.ask_id == order_index)),
10478 "Lighter terminal recovery returned a trade outside the requested order"
10479 );
10480 trades.push(trade);
10481 }
10482
10483 match response.next_cursor.filter(|cursor| !cursor.is_empty()) {
10484 Some(cursor) => {
10485 anyhow::ensure!(
10486 cursors.insert(cursor.clone()),
10487 "Lighter terminal recovery repeated cursor"
10488 );
10489 anyhow::ensure!(
10490 page + 1 < MAX_RECONCILIATION_PAGES,
10491 "Lighter terminal recovery exceeded page limit"
10492 );
10493 query.cursor = Some(cursor);
10494 }
10495 None => break,
10496 }
10497 }
10498
10499 trades.sort_by_key(|trade| (trade.timestamp, trade.trade_id));
10500 for trade in trades {
10501 dispatch_lighter_trade(
10502 &trade,
10503 &context.dispatch,
10504 &context.emitter,
10505 &context.registry,
10506 context.account_id,
10507 context.emitter.trader_id(),
10508 Some(context.credential.account_index()),
10509 context.clock.get_time_ns(),
10510 );
10511 }
10512
10513 Ok(())
10514}
10515
10516fn ensure_fill_triggered(
10517 filled: &OrderFilled,
10518 emitter: &ExecutionEventEmitter,
10519 dispatch: &WsDispatchState,
10520) {
10521 if TRIGGERABLE_ORDER_TYPES.contains(&filled.order_type)
10522 && dispatch.mark_triggered_emitted(filled.client_order_id)
10523 {
10524 let triggered = OrderTriggered::new(
10525 filled.trader_id,
10526 filled.strategy_id,
10527 filled.instrument_id,
10528 filled.client_order_id,
10529 UUID4::new(),
10530 filled.ts_event,
10531 filled.ts_init,
10532 false,
10533 Some(filled.venue_order_id),
10534 Some(filled.account_id),
10535 );
10536 emitter.send_order_event(OrderEventAny::Triggered(triggered));
10537 }
10538}
10539
10540#[expect(
10544 clippy::too_many_arguments,
10545 reason = "shared cleanup point across the typed-event variants"
10546)]
10547#[expect(
10548 clippy::needless_pass_by_value,
10549 reason = "event is destructured into typed OrderEventAny variants that consume the payload"
10550)]
10551fn dispatch_tracked_order_event(
10552 event: ParsedOrderEvent,
10553 cloid: ClientOrderId,
10554 venue_order_id: VenueOrderId,
10555 identity: &OrderIdentity,
10556 account_id: AccountId,
10557 trader_id: TraderId,
10558 emitter: &ExecutionEventEmitter,
10559 dispatch: &WsDispatchState,
10560 ts_init: UnixNanos,
10561) {
10562 let is_terminal;
10563
10564 match event {
10565 ParsedOrderEvent::Accepted(e) => {
10566 if !dispatch.claim_accepted_emission(&cloid) {
10567 log::debug!("Skipping duplicate OrderAccepted for {cloid}");
10568 return;
10569 }
10570
10571 is_terminal = false;
10572 emitter.send_order_event(OrderEventAny::Accepted(e));
10573 }
10574 ParsedOrderEvent::Triggered(e) => {
10575 if !dispatch.mark_triggered_emitted(cloid) {
10576 log::debug!("Skipping duplicate OrderTriggered for {cloid}");
10577 return;
10578 }
10579
10580 ensure_accepted_emitted(
10581 cloid,
10582 venue_order_id,
10583 identity,
10584 account_id,
10585 trader_id,
10586 emitter,
10587 dispatch,
10588 ts_init,
10589 );
10590 is_terminal = false;
10591 emitter.send_order_event(OrderEventAny::Triggered(e));
10592 }
10593 ParsedOrderEvent::Updated(e) => {
10594 dispatch.clear_pending_modify_if_update(&cloid, e.quantity, e.price, e.trigger_price);
10599 is_terminal = false;
10600 identity.publish_update(e.event_id);
10601 emitter.send_order_event(OrderEventAny::Updated(e));
10602 }
10603 ParsedOrderEvent::UpdatedThenTriggered { updated, triggered } => {
10604 if !dispatch.mark_triggered_emitted(cloid) {
10605 log::debug!("Skipping duplicate OrderTriggered for {cloid}");
10606 return;
10607 }
10608
10609 ensure_accepted_emitted(
10610 cloid,
10611 venue_order_id,
10612 identity,
10613 account_id,
10614 trader_id,
10615 emitter,
10616 dispatch,
10617 ts_init,
10618 );
10619 dispatch.clear_pending_modify_if_update(
10620 &cloid,
10621 updated.quantity,
10622 updated.price,
10623 updated.trigger_price,
10624 );
10625 is_terminal = false;
10626 identity.publish_update(updated.event_id);
10627 emitter.send_order_event(OrderEventAny::Updated(updated));
10628 emitter.send_order_event(OrderEventAny::Triggered(triggered));
10629 }
10630 ParsedOrderEvent::Canceled(e) => {
10631 ensure_accepted_emitted(
10632 cloid,
10633 venue_order_id,
10634 identity,
10635 account_id,
10636 trader_id,
10637 emitter,
10638 dispatch,
10639 ts_init,
10640 );
10641 is_terminal = true;
10642 emitter.send_order_event(OrderEventAny::Canceled(e));
10643 }
10644 ParsedOrderEvent::Expired(e) => {
10645 ensure_accepted_emitted(
10646 cloid,
10647 venue_order_id,
10648 identity,
10649 account_id,
10650 trader_id,
10651 emitter,
10652 dispatch,
10653 ts_init,
10654 );
10655 is_terminal = true;
10656 emitter.send_order_event(OrderEventAny::Expired(e));
10657 }
10658 ParsedOrderEvent::Rejected(e) => {
10659 is_terminal = true;
10660 emitter.send_order_event(OrderEventAny::Rejected(e));
10661 }
10662 }
10663
10664 if is_terminal {
10665 dispatch.clear_pending_order_action(&cloid);
10666 dispatch.retire_order_identity(&cloid);
10667 }
10668}
10669
10670#[expect(
10673 clippy::too_many_arguments,
10674 reason = "synthesized events need the full identity context to populate the event"
10675)]
10676fn ensure_accepted_emitted(
10677 cloid: ClientOrderId,
10678 venue_order_id: VenueOrderId,
10679 identity: &OrderIdentity,
10680 account_id: AccountId,
10681 trader_id: TraderId,
10682 emitter: &ExecutionEventEmitter,
10683 dispatch: &WsDispatchState,
10684 ts_init: UnixNanos,
10685) {
10686 if !dispatch.claim_accepted_emission(&cloid) {
10687 return;
10688 }
10689
10690 let accepted = OrderAccepted::new(
10691 trader_id,
10692 identity.strategy_id,
10693 identity.instrument_id,
10694 cloid,
10695 venue_order_id,
10696 account_id,
10697 UUID4::new(),
10698 ts_init,
10699 ts_init,
10700 false,
10701 );
10702 emitter.send_order_event(OrderEventAny::Accepted(accepted));
10703}
10704
10705#[cfg(test)]
10706#[path = "../tests/integration/exec_client.rs"]
10707mod exec_client_tests;
10708
10709#[cfg(test)]
10710mod tests {
10711 use std::{
10712 cell::RefCell,
10713 rc::Rc,
10714 sync::{Arc, atomic::AtomicUsize},
10715 };
10716
10717 use axum::{
10718 Router,
10719 extract::Query,
10720 response::IntoResponse,
10721 routing::{get, post},
10722 };
10723 use nautilus_common::{
10724 cache::Cache,
10725 clock::{Clock, VirtualClock},
10726 factories::OrderFactory,
10727 live::runner::replace_exec_event_sender,
10728 messages::{ExecutionEvent, ExecutionReport as EngineExecutionReport},
10729 testing::wait_until_async,
10730 };
10731 use nautilus_execution::engine::ExecutionEngine;
10732 use nautilus_live::manager::{ExecutionManager, ExecutionManagerConfig};
10733 use nautilus_model::{
10734 accounts::MarginAccount,
10735 data::QuoteTick,
10736 enums::{ContingencyType, LiquiditySide, OrderSide, OrderStatus, TimeInForce},
10737 events::{
10738 AccountState, OrderCanceled, OrderEventAny, OrderPendingCancel, OrderSubmitted,
10739 OrderTriggered,
10740 },
10741 identifiers::{
10742 InstrumentId, OrderListId, PositionId, StrategyId, Symbol, TradeId, TraderId,
10743 VenueOrderId,
10744 },
10745 instruments::CryptoPerpetual,
10746 orders::{LimitOrder, OrderList, OrderTestBuilder, stubs::TestOrderEventStubs},
10747 position::Position,
10748 types::{Currency, Money, Price},
10749 };
10750 use rstest::rstest;
10751
10752 use super::*;
10753 use crate::{
10754 common::{
10755 consts::{
10756 LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
10757 LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX, LIGHTER_VENUE,
10758 },
10759 enums::{LighterDeployment, LighterEnvironment, LighterProductType},
10760 },
10761 http::models::{LighterNextNonce, LighterOrders, LighterTx},
10762 signing::tx::TX_HASH_BYTES,
10763 };
10764
10765 const TEST_PRIVATE_KEY: &str =
10766 "0b8e0f63c24d8baacd9d29ad4e9a4b73c4a8d2bb8b16dc4fa9d7c2e1d3a8b1f0e8d3a4c5b6e7f001";
10767 const TEST_ACCOUNT_INDEX: u64 = 12345;
10768 const TEST_ACCOUNT_INDEX_I64: i64 = 12345;
10769 const TEST_API_KEY_INDEX: u8 = 5;
10770 const TEST_NEXT_NONCE: i64 = 42;
10771 const TEST_MARKET_INDEX: i64 = 0;
10772 const TEST_ORDER_NONCE: i64 = 281_474_720_725_346;
10773 const TEST_SUBMISSION_NONCE: i64 = 2_042;
10774
10775 fn handle_send_tx_ack(
10776 dispatch: &WsDispatchState,
10777 account_index: Option<i64>,
10778 code: i64,
10779 tx_hash: Option<&str>,
10780 ) -> Option<PendingSendTx> {
10781 handle_send_tx_ack_for_connection(dispatch, account_index, 0, code, tx_hash)
10782 }
10783
10784 #[expect(
10785 clippy::too_many_arguments,
10786 reason = "test wrapper preserves the established rejection call shape"
10787 )]
10788 fn handle_send_tx_rejection(
10789 dispatch: &WsDispatchState,
10790 emitter: &ExecutionEventEmitter,
10791 account_index: Option<i64>,
10792 now: UnixNanos,
10793 source: SendTxRejectionSource,
10794 code: Option<i64>,
10795 message: &str,
10796 tx_hash: Option<&str>,
10797 ) -> bool {
10798 handle_send_tx_rejection_for_connection(
10799 dispatch,
10800 emitter,
10801 account_index,
10802 0,
10803 now,
10804 source,
10805 code,
10806 message,
10807 tx_hash,
10808 )
10809 }
10810
10811 fn trader_id() -> TraderId {
10812 TraderId::from("TRADER-001")
10813 }
10814
10815 fn client_id() -> ClientId {
10816 ClientId::from("LIGHTER")
10817 }
10818
10819 fn account_id() -> AccountId {
10820 AccountId::from("LIGHTER-001")
10821 }
10822
10823 fn strategy_id() -> StrategyId {
10824 StrategyId::from("S-001")
10825 }
10826
10827 fn test_credential() -> Credential {
10828 Credential::new(TEST_API_KEY_INDEX, TEST_PRIVATE_KEY, TEST_ACCOUNT_INDEX).unwrap()
10829 }
10830
10831 fn test_config() -> LighterExecutionClientConfig {
10832 test_config_with_use_gtd(true)
10833 }
10834
10835 fn test_config_with_use_gtd(use_gtd: bool) -> LighterExecutionClientConfig {
10836 LighterExecutionClientConfig {
10837 account_id: account_id(),
10838 account_index: Some(TEST_ACCOUNT_INDEX),
10839 api_key_index: Some(TEST_API_KEY_INDEX),
10840 private_key: Some(TEST_PRIVATE_KEY.into()),
10841 base_url_http: Some("http://127.0.0.1:1".to_string()),
10842 base_url_ws: Some("ws://127.0.0.1:1/stream".to_string()),
10843 proxy_url: None,
10844 environment: LighterEnvironment::Testnet,
10845 deployment: LighterDeployment::Lighter,
10846 venue: None,
10847 http_timeout_secs: 1,
10848 ws_timeout_secs: 1,
10849 market_order_slippage_bps: 50,
10850 rest_quota_per_min: None,
10851 sendtx_quota_per_min: None,
10852 transport_backend: Default::default(),
10853 use_gtd,
10854 scheduled_cancel_timeout_secs: None,
10855 }
10856 }
10857
10858 #[rstest]
10859 fn format_between_timestamps_uses_lighter_seconds_range() {
10860 let start = UnixNanos::from(1_700_000_000_123_456_789);
10861 let end = UnixNanos::from(1_700_003_600_987_654_321);
10862 let now = UnixNanos::from(1_700_007_200_000_000_000);
10863
10864 assert_eq!(
10865 format_between_timestamps(Some(start), Some(end), now),
10866 Some("1700000000-1700003600".to_string()),
10867 );
10868 assert_eq!(
10869 format_between_timestamps(Some(start), None, now),
10870 Some("1700000000-1700007200".to_string()),
10871 );
10872 assert_eq!(
10873 format_between_timestamps(None, Some(end), now),
10874 Some("0-1700003600".to_string()),
10875 );
10876 assert_eq!(format_between_timestamps(None, None, now), None);
10877 }
10878
10879 #[rstest]
10880 #[case::write_timeout(LighterWsError::Transport(SendError::WriteTimeout), "Ambiguous")]
10881 #[case::broken_pipe(
10882 LighterWsError::Transport(SendError::BrokenPipe("writer closed".to_string())),
10883 "Ambiguous",
10884 )]
10885 #[case::handler_result_lost(
10886 LighterWsError::SendTxOutcomeUnknown("result sender dropped".to_string()),
10887 "Ambiguous",
10888 )]
10889 #[case::network(LighterWsError::Network("disconnected".to_string()), "Ambiguous")]
10890 #[case::parse(LighterWsError::Parse("invalid ack".to_string()), "Ambiguous")]
10891 #[case::invalid_input(
10892 LighterWsError::Transport(SendError::InvalidInput("invalid payload".to_string())),
10893 "NotSent",
10894 )]
10895 #[case::buffer_full(LighterWsError::Transport(SendError::BufferFull), "NotSent")]
10896 #[case::closed(LighterWsError::Transport(SendError::Closed), "NotSent")]
10897 #[case::connection_changed(LighterWsError::Transport(SendError::ConnectionChanged), "NotSent")]
10898 #[case::wait_timeout(LighterWsError::Transport(SendError::Timeout), "NotSent")]
10899 #[case::authentication(
10900 LighterWsError::Authentication("invalid token".to_string()),
10901 "NotSent",
10902 )]
10903 #[case::client_unavailable(
10904 LighterWsError::Client("handler unavailable".to_string()),
10905 "NotSent",
10906 )]
10907 fn ws_command_failure_classifies_delivery_evidence(
10908 #[case] error: LighterWsError,
10909 #[case] expected: &str,
10910 ) {
10911 let failure = classify_lighter_ws_command_failure("submit_order", &error);
10912
10913 let actual = match &failure {
10914 CommandFailure::NotSent(_) => "NotSent",
10915 CommandFailure::Ambiguous(_) => "Ambiguous",
10916 CommandFailure::VenueRejected(_) => "VenueRejected",
10917 };
10918
10919 assert_eq!(actual, expected);
10920 assert!(command_failure_reason(&failure).contains("submit_order"));
10921 }
10922
10923 #[rstest]
10924 fn venue_rejection_reason_is_clean_and_bounded() {
10925 let oversized = "x".repeat(STRATEGY_REASON_MAX_CHARS + 100);
10926 let message = format!("<html>\n rejected\0 because {oversized}</html>");
10927
10928 let failure = venue_rejection_failure(Some(20_001), &message);
10929 let reason = command_failure_reason(&failure);
10930
10931 assert!(matches!(&failure, CommandFailure::VenueRejected(_)));
10932 assert!(!reason.contains('<'));
10933 assert!(!reason.contains('\0'));
10934 assert!(!reason.contains(" "));
10935 assert_eq!(reason.chars().count(), STRATEGY_REASON_MAX_CHARS);
10936 assert!(reason.starts_with("LIGHTER_20001: rejected because "));
10937 }
10938
10939 #[rstest]
10940 fn venue_rejection_reason_uses_stable_fallback_for_empty_markup() {
10941 let failure = venue_rejection_failure(Some(20_001), "<html></html>\n");
10942
10943 assert_eq!(
10944 failure,
10945 CommandFailure::VenueRejected("LIGHTER_20001".to_string()),
10946 );
10947 }
10948
10949 #[rstest]
10950 fn venue_rejection_reason_preserves_comparison_and_removes_format_controls() {
10951 let failure = venue_rejection_failure(
10952 Some(20_001),
10953 "price must be < 100\u{202e}\u{2066}\u{200b}<strong>now</strong>",
10954 );
10955
10956 assert_eq!(
10957 failure,
10958 CommandFailure::VenueRejected("LIGHTER_20001: price must be < 100 now".to_string(),),
10959 );
10960 }
10961
10962 #[rstest]
10963 fn unsupported_order_type_reason_keeps_lighter_capabilities() {
10964 assert_eq!(
10965 unsupported_lighter_order_type_reason(OrderType::MarketToLimit),
10966 "UNSUPPORTED_ORDER_TYPE: MARKET_TO_LIMIT; Lighter supports MARKET, LIMIT, STOP_MARKET, STOP_LIMIT, MARKET_IF_TOUCHED, and LIMIT_IF_TOUCHED",
10967 );
10968 }
10969
10970 #[rstest]
10971 fn mass_status_propagates_commission_error_and_downgrades_other_fill_errors() {
10972 let e = anyhow::Error::new(LighterCommissionError::new("invalid precision"))
10973 .context("failed to parse Lighter fill report");
10974
10975 let commission_error = fill_reports_for_mass_status(Err(e))
10976 .expect_err("commission construction must fail mass status");
10977 let ordinary_error = fill_reports_for_mass_status(Err(anyhow::anyhow!("transport failed")))
10978 .expect("ordinary source errors make mass status incomplete");
10979
10980 let complete = fill_reports_for_mass_status(Ok(FillSweep {
10981 reports: Vec::new(),
10982 covers_window: true,
10983 withheld_instruments: AHashSet::new(),
10984 }))
10985 .expect("complete fill sweep");
10986
10987 assert!(is_commission_error(&commission_error));
10988 assert!(ordinary_error.0.is_empty());
10989 assert!(!ordinary_error.1);
10990 assert!(complete.0.is_empty());
10991 assert!(complete.1);
10992 }
10993
10994 fn create_execution_client() -> (
10995 LighterExecutionClient,
10996 Rc<RefCell<Cache>>,
10997 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
10998 ) {
10999 create_execution_client_with_config(test_config())
11000 }
11001
11002 #[rstest]
11003 #[case(
11004 "proxy_url",
11005 "127.0.0.1:8080",
11006 "Lighter proxy_url must be an absolute http:// or https:// URL with a host"
11007 )]
11008 #[case(
11009 "base_url_ws",
11010 "wss://example.com/stream?readonly=true",
11011 "Lighter execution WebSocket URL must not set readonly=true; it does not disable trading"
11012 )]
11013 #[case(
11014 "venue",
11015 "LIGHTER-ALT",
11016 "Lighter execution venue must not contain '-'; use '_' so the account ID issuer can match"
11017 )]
11018 fn execution_constructor_rejects_config_footguns(
11019 #[case] field: &str,
11020 #[case] value: &str,
11021 #[case] expected: &str,
11022 ) {
11023 let (client, _, _) = create_execution_client();
11024 let mut config = test_config();
11025 match field {
11026 "proxy_url" => config.proxy_url = Some(value.into()),
11027 "base_url_ws" => config.base_url_ws = Some(value.to_string()),
11028 "venue" => config.venue = Some(Venue::from(value)),
11029 _ => unreachable!(),
11030 }
11031
11032 assert_eq!(
11033 LighterExecutionClient::new(client.core, config)
11034 .unwrap_err()
11035 .to_string(),
11036 expected
11037 );
11038 }
11039
11040 #[rstest]
11041 #[case(0, true)]
11042 #[case(9_999, true)]
11043 #[case(10_000, false)]
11044 #[case(u32::MAX, false)]
11045 fn market_order_slippage_constructor_validates_range(#[case] value: u32, #[case] valid: bool) {
11046 let (client, _, _) = create_execution_client();
11047 let mut config = test_config();
11048 config.market_order_slippage_bps = value;
11049 let result = LighterExecutionClient::new(client.core, config);
11050
11051 if valid {
11052 assert_eq!(result.unwrap().config().market_order_slippage_bps, value);
11053 } else {
11054 assert_eq!(
11055 result.unwrap_err().to_string(),
11056 "Lighter market_order_slippage_bps must be in [0, 10000)"
11057 );
11058 }
11059 }
11060
11061 #[rstest]
11062 #[case(419)]
11063 #[case(1_295_941)]
11064 #[case(u64::MAX)]
11065 fn scheduled_cancel_constructor_rejects_invalid_timeout(#[case] timeout: u64) {
11066 let (client, _, _) = create_execution_client();
11067 let mut config = test_config();
11068 config.scheduled_cancel_timeout_secs = Some(timeout);
11069 let error = LighterExecutionClient::new(client.core, config).unwrap_err();
11070 assert_eq!(
11071 error.to_string(),
11072 "scheduled_cancel_timeout_secs must be in [420, 1295940]"
11073 );
11074 }
11075
11076 #[tokio::test]
11077 async fn scheduled_cancel_timeout_preserves_non_tail_nonce() {
11078 let (mut client, _, _) = create_execution_client();
11079 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
11080 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
11081 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
11082 let context = client
11083 .fanout_dispatch_context(client.credential.as_ref().unwrap())
11084 .unwrap();
11085
11086 let later = {
11087 let renewal = context.refresh_scheduled_cancel(420);
11088 tokio::pin!(renewal);
11089 tokio::select! {
11090 result = &mut renewal => panic!("renewal must wait for quota: {result:?}"),
11091 () = tokio::time::sleep(Duration::from_millis(20)) => {},
11092 }
11093 let later = context.build_tx_context().unwrap();
11094 assert_eq!(later.context.nonce, TEST_NEXT_NONCE + 1);
11095 assert!(
11096 tokio::time::timeout(Duration::from_millis(50), &mut renewal)
11097 .await
11098 .is_err()
11099 );
11100 later
11101 };
11102
11103 assert!(
11104 tokio::time::timeout(
11105 Duration::from_millis(150),
11106 later.send_reservation.wait_for_turn()
11107 )
11108 .await
11109 .is_err()
11110 );
11111 assert!(!client.pending_tasks.all_finished());
11112 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11113 assert_eq!(
11114 client
11115 .dispatch
11116 .nonce_manager
11117 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
11118 Some(TEST_NEXT_NONCE + 1)
11119 );
11120 drop(later);
11121 client.begin_session_shutdown();
11122 client.finish_session_shutdown().await.unwrap();
11123 assert!(client.pending_tasks.is_empty());
11124 }
11125
11126 #[tokio::test]
11127 async fn scheduled_cancel_timeout_cancels_quota_wait_and_releases_nonce() {
11128 let (mut client, _, _) = create_execution_client();
11129 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
11130 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
11131 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
11132 let context = client
11133 .fanout_dispatch_context(client.credential.as_ref().unwrap())
11134 .unwrap();
11135 assert!(
11136 tokio::time::timeout(
11137 Duration::from_millis(50),
11138 context.refresh_scheduled_cancel(420)
11139 )
11140 .await
11141 .is_err()
11142 );
11143 tokio::time::timeout(Duration::from_secs(2), async {
11144 while !client.pending_tasks.all_finished() {
11145 tokio::task::yield_now().await;
11146 }
11147 })
11148 .await
11149 .unwrap();
11150
11151 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11152 assert_eq!(
11153 client
11154 .dispatch
11155 .nonce_manager
11156 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11157 .unwrap(),
11158 TEST_NEXT_NONCE
11159 );
11160 client.begin_session_shutdown();
11161 client.finish_session_shutdown().await.unwrap();
11162 assert!(client.pending_tasks.is_empty());
11163 }
11164
11165 #[rstest]
11166 #[tokio::test]
11167 async fn account_cancellation_dropped_during_quota_wait_is_retained_until_shutdown() {
11168 let (mut client, _, _) = create_execution_client();
11169 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
11170 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
11171 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
11172 {
11173 let pending = client.cancel_all_orders_account();
11174 tokio::pin!(pending);
11175 tokio::select! {
11176 result = &mut pending => panic!("control must wait for quota: {result:?}"),
11177 () = tokio::time::sleep(Duration::from_millis(50)) => {}
11178 }
11179 }
11180
11181 assert!(!client.pending_tasks.is_empty());
11182 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11183
11184 client.begin_session_shutdown();
11185 client.finish_session_shutdown().await.unwrap();
11186
11187 assert!(client.pending_tasks.is_empty());
11188 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11189 assert_eq!(
11190 client
11191 .dispatch
11192 .nonce_manager
11193 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11194 .unwrap(),
11195 TEST_NEXT_NONCE,
11196 );
11197 }
11198
11199 #[rstest]
11200 #[tokio::test]
11201 async fn account_cancellation_spawn_rejection_releases_latest_nonce() {
11202 let (client, _, _) = create_execution_client();
11203 client.pending_tasks.begin_shutdown();
11204
11205 let error = client.cancel_all_orders_account().await.unwrap_err();
11206
11207 assert_eq!(
11208 error.to_string(),
11209 "failed to spawn Lighter account cancellation"
11210 );
11211 assert!(client.pending_tasks.is_empty());
11212 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11213 assert_eq!(
11214 client
11215 .dispatch
11216 .nonce_manager
11217 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11218 .unwrap(),
11219 TEST_NEXT_NONCE,
11220 );
11221 }
11222
11223 #[rstest]
11224 #[tokio::test]
11225 async fn account_cancellation_ambiguous_error_identifies_pending_transaction() {
11226 let (mut client, _, _) = create_execution_client();
11227 client.ws_client.drop_next_send_tx_result_for_test().await;
11228
11229 let error = client.cancel_all_orders_account().await.unwrap_err();
11230 let pending = client.dispatch.take_pending_sendtx();
11231
11232 assert_eq!(pending.len(), 1);
11233 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
11234 assert_eq!(pending[0].api_key_index, TEST_API_KEY_INDEX);
11235 assert_eq!(
11236 error.to_string(),
11237 format!(
11238 "failed to submit Lighter account cancellation transaction {}",
11239 pending[0].tx_hash
11240 ),
11241 );
11242 assert!(matches!(
11243 error.downcast_ref::<LighterWsError>(),
11244 Some(LighterWsError::SendTxOutcomeUnknown(_)),
11245 ));
11246 assert_eq!(
11247 client
11248 .dispatch
11249 .nonce_manager
11250 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11251 .unwrap(),
11252 TEST_NEXT_NONCE + 1,
11253 );
11254 client.begin_session_shutdown();
11255 client.finish_session_shutdown().await.unwrap();
11256 }
11257
11258 #[rstest]
11259 #[case::definite_failure(false, 0, TEST_NEXT_NONCE + 1)]
11260 #[case::ambiguous_failure(true, 1, TEST_NEXT_NONCE + 2)]
11261 #[tokio::test]
11262 async fn account_cancellation_dropped_caller_preserves_dispatch_failure_handling(
11263 #[case] ambiguous: bool,
11264 #[case] pending_count: usize,
11265 #[case] next_nonce: i64,
11266 ) {
11267 let (mut client, _, _) = create_execution_client();
11268 if ambiguous {
11269 client.ws_client.drop_next_send_tx_result_for_test().await;
11270 }
11271
11272 let mut blocker = client.build_tx_context(&test_credential()).unwrap();
11273 {
11274 let pending = client.cancel_all_orders_account();
11275 tokio::pin!(pending);
11276 tokio::select! {
11277 result = &mut pending => panic!("control must wait for its turn: {result:?}"),
11278 () = tokio::time::sleep(Duration::from_millis(50)) => {}
11279 }
11280 }
11281
11282 assert!(!client.pending_tasks.is_empty());
11283 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11284 blocker.send_reservation.release();
11285 wait_until_async(
11286 || async { client.pending_tasks.is_empty() },
11287 Duration::from_secs(2),
11288 )
11289 .await;
11290
11291 assert_eq!(client.dispatch.pending_sendtx_len(), pending_count);
11292 assert_eq!(
11293 client
11294 .dispatch
11295 .nonce_manager
11296 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11297 .unwrap(),
11298 next_nonce,
11299 );
11300 client.begin_session_shutdown();
11301 client.finish_session_shutdown().await.unwrap();
11302 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11303 }
11304
11305 #[rstest]
11306 #[case::first_start(false)]
11307 #[case::already_started(true)]
11308 fn execution_sender_is_available_before_start_and_replaced_on_start(#[case] started: bool) {
11309 let (client, _cache, _rx) = create_execution_client();
11310 let (first_sender, mut first_receiver) = tokio::sync::mpsc::unbounded_channel();
11311 replace_exec_event_sender(first_sender);
11312 let mut client = LighterExecutionClient::new(client.core, test_config()).unwrap();
11313 let emitter = client.emitter.clone();
11314
11315 let submitted = OrderSubmitted::new(
11316 trader_id(),
11317 strategy_id(),
11318 InstrumentId::from("ETH-PERP.LIGHTER"),
11319 ClientOrderId::from("BEFORE-START"),
11320 account_id(),
11321 UUID4::new(),
11322 UnixNanos::from(17),
11323 UnixNanos::from(23),
11324 );
11325 emitter
11326 .try_send_order_event(OrderEventAny::Submitted(submitted.clone()))
11327 .unwrap();
11328
11329 if started {
11330 client.start().unwrap();
11331 }
11332
11333 let (second_sender, mut second_receiver) = tokio::sync::mpsc::unbounded_channel();
11334 replace_exec_event_sender(second_sender);
11335 client.start().unwrap();
11336
11337 let after_start = OrderSubmitted::new(
11338 trader_id(),
11339 strategy_id(),
11340 InstrumentId::from("BTC-PERP.LIGHTER"),
11341 ClientOrderId::from("AFTER-START"),
11342 account_id(),
11343 UUID4::new(),
11344 UnixNanos::from(29),
11345 UnixNanos::from(31),
11346 );
11347 emitter
11348 .try_send_order_event(OrderEventAny::Submitted(after_start.clone()))
11349 .unwrap();
11350
11351 for (receiver, expected) in [
11352 (&mut first_receiver, submitted),
11353 (&mut second_receiver, after_start),
11354 ] {
11355 let ExecutionEvent::Order(OrderEventAny::Submitted(actual)) =
11356 receiver.try_recv().unwrap()
11357 else {
11358 panic!("expected submitted event");
11359 };
11360
11361 assert_eq!(actual, expected);
11362 assert!(receiver.try_recv().is_err());
11363 }
11364 }
11365
11366 fn create_execution_client_with_config(
11367 config: LighterExecutionClientConfig,
11368 ) -> (
11369 LighterExecutionClient,
11370 Rc<RefCell<Cache>>,
11371 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
11372 ) {
11373 let cache = Rc::new(RefCell::new(Cache::default()));
11374 let venue = config.resolved_venue();
11375
11376 let core = ExecutionClientCore::new(
11377 trader_id(),
11378 client_id(),
11379 venue,
11380 OmsType::Netting,
11381 account_id(),
11382 AccountType::Margin,
11383 None,
11384 cache.clone(),
11385 );
11386
11387 let mut client = LighterExecutionClient::new(core, config).unwrap();
11388 client.dispatch.nonce_manager.refresh(
11389 TEST_ACCOUNT_INDEX_I64,
11390 TEST_API_KEY_INDEX,
11391 TEST_NEXT_NONCE,
11392 );
11393
11394 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
11395 client.emitter.set_sender(sender);
11396
11397 (client, cache, receiver)
11398 }
11399
11400 #[rstest]
11401 #[case::inline_deferred(false, true, false, false)]
11402 #[case::worker_deferred(true, true, false, false)]
11403 #[case::inline_absent(false, false, false, false)]
11404 #[case::worker_absent(true, false, false, false)]
11405 #[case::inline_account_orders_failure(false, true, true, false)]
11406 #[case::worker_account_orders_failure(true, true, true, false)]
11407 #[case::inline_ambiguous_account_orders(false, true, false, true)]
11408 #[case::worker_ambiguous_account_orders(true, true, false, true)]
11409 #[tokio::test]
11410 async fn single_order_lookup_distinguishes_deferred_report_from_absence(
11411 #[case] worker: bool,
11412 #[case] present: bool,
11413 #[case] account_orders_fails: bool,
11414 #[case] duplicate: bool,
11415 ) {
11416 let rig = dispatcher_rig("DEFERRED-PROBE");
11417 register_identity(&rig);
11418 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
11419 let venue_order_id = VenueOrderId::from(order.order_id.as_str());
11420
11421 if !duplicate {
11422 rig.dispatch.observe_create_submission(
11423 &rig.cloid,
11424 order.client_order_index,
11425 TEST_SUBMISSION_NONCE,
11426 venue_order_id,
11427 );
11428 }
11429
11430 rig.dispatch
11431 .order_identity(&rig.cloid)
11432 .unwrap()
11433 .publish_update(UUID4::new());
11434
11435 let response = LighterOrders {
11436 code: 200,
11437 message: None,
11438 next_cursor: None,
11439 orders: if duplicate {
11440 vec![order.clone(), order]
11441 } else if present {
11442 vec![order]
11443 } else {
11444 Vec::new()
11445 },
11446 };
11447
11448 let body = serde_json::to_string(&response).unwrap();
11449 let account_orders_calls = Arc::new(AtomicUsize::new(0));
11450 let active_calls = Arc::new(AtomicUsize::new(0));
11451 let inactive_calls = Arc::new(AtomicUsize::new(0));
11452
11453 let app = Router::new()
11454 .route(
11455 "/api/v1/accountOrders",
11456 get({
11457 let body = body.clone();
11458 let calls = Arc::clone(&account_orders_calls);
11459 move || {
11460 calls.fetch_add(1, Ordering::SeqCst);
11461 let body = body.clone();
11462 async move {
11463 if account_orders_fails {
11464 (axum::http::StatusCode::SERVICE_UNAVAILABLE, body).into_response()
11465 } else {
11466 (axum::http::StatusCode::OK, body).into_response()
11467 }
11468 }
11469 }
11470 }),
11471 )
11472 .route(
11473 "/api/v1/accountActiveOrders",
11474 get({
11475 let body = body.clone();
11476 let calls = Arc::clone(&active_calls);
11477 move || {
11478 calls.fetch_add(1, Ordering::SeqCst);
11479 let body = body.clone();
11480 async move { body }
11481 }
11482 }),
11483 )
11484 .route(
11485 "/api/v1/accountInactiveOrders",
11486 get({
11487 let calls = Arc::clone(&inactive_calls);
11488 move || {
11489 calls.fetch_add(1, Ordering::SeqCst);
11490 async move {
11491 axum::Json(LighterOrders {
11492 code: 200,
11493 message: None,
11494 next_cursor: None,
11495 orders: Vec::new(),
11496 })
11497 }
11498 }
11499 }),
11500 );
11501
11502 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
11503 let addr = listener.local_addr().unwrap();
11504 let server = tokio::spawn(async move {
11505 axum::serve(listener, app).await.unwrap();
11506 });
11507
11508 let config = LighterExecutionClientConfig {
11509 base_url_http: Some(format!("http://{addr}")),
11510 ..test_config()
11511 };
11512
11513 let (mut client, _cache, mut rx) = create_execution_client_with_config(config);
11514 client.registry = rig.registry;
11515 client.dispatch = rig.dispatch;
11516
11517 let command = GenerateOrderStatusReport::new(
11518 UUID4::new(),
11519 UnixNanos::from(1),
11520 Some(rig.instrument_id),
11521 Some(rig.cloid),
11522 if duplicate {
11523 None
11524 } else {
11525 Some(venue_order_id)
11526 },
11527 None,
11528 None,
11529 );
11530
11531 let result = if worker {
11532 let task = client.generate_order_status_report_task(&command).unwrap();
11533 task.collection.await;
11534 (task.finish)()
11535 } else {
11536 client.generate_order_status_report(&command).await
11537 };
11538
11539 server.abort();
11540 let _ = server.await;
11541
11542 assert_eq!(
11543 account_orders_calls.load(Ordering::SeqCst),
11544 if account_orders_fails { 4 } else { 1 }
11545 );
11546 assert_eq!(
11547 active_calls.load(Ordering::SeqCst),
11548 usize::from(!present || account_orders_fails)
11549 );
11550 assert_eq!(inactive_calls.load(Ordering::SeqCst), usize::from(!present));
11551
11552 if duplicate {
11553 let error = result.unwrap_err();
11554 assert!(
11555 error
11556 .to_string()
11557 .contains("ambiguous Lighter account-order lookup"),
11558 "unexpected error: {error:#}"
11559 );
11560 } else if present {
11561 assert_eq!(
11562 result.unwrap_err().to_string(),
11563 "Lighter order report deferred until execution state is applied"
11564 );
11565 } else {
11566 assert_eq!(result.unwrap(), None);
11567 }
11568
11569 assert!(
11570 client
11571 .dispatch
11572 .order_identity(&rig.cloid)
11573 .unwrap()
11574 .has_publication()
11575 );
11576 assert!(rx.try_recv().is_err());
11577 }
11578
11579 #[rstest]
11580 #[case::other_market(LighterOrderStatus::Open, 999, false, false, false, 0)]
11581 #[case::terminal(
11582 LighterOrderStatus::Canceled,
11583 TEST_MARKET_INDEX,
11584 false,
11585 false,
11586 false,
11587 0
11588 )]
11589 #[case::pending_flip(LighterOrderStatus::Open, TEST_MARKET_INDEX, true, false, false, 0)]
11590 #[case::direct_open(LighterOrderStatus::Open, TEST_MARKET_INDEX, false, true, false, 0)]
11591 #[case::known_terminal(LighterOrderStatus::Canceled, TEST_MARKET_INDEX, false, true, true, 0)]
11592 #[case::known_other_market(LighterOrderStatus::Open, 999, false, false, true, 1)]
11593 #[tokio::test]
11594 async fn client_order_lookup_filters_market_status_and_pending(
11595 #[case] status: LighterOrderStatus,
11596 #[case] market_index: i64,
11597 #[case] stop_pending_after_response: bool,
11598 #[case] expect_present: bool,
11599 #[case] known_venue: bool,
11600 #[case] expected_inactive_calls: usize,
11601 ) {
11602 let rig = dispatcher_rig("FILTERED-PROBE");
11603 register_identity(&rig);
11604 let mut order = dispatcher_test_order(&rig, status);
11605 order.market_index = market_index;
11606 let venue_order_id = known_venue.then(|| VenueOrderId::from(order.order_id.as_str()));
11607 rig.dispatch
11608 .order_identity(&rig.cloid)
11609 .unwrap()
11610 .publish_update(UUID4::new());
11611
11612 let account_orders_body = serde_json::to_string(&LighterOrders {
11613 code: 200,
11614 message: None,
11615 next_cursor: None,
11616 orders: vec![order],
11617 })
11618 .unwrap();
11619
11620 let empty_body = serde_json::to_string(&LighterOrders {
11621 code: 200,
11622 message: None,
11623 next_cursor: None,
11624 orders: Vec::new(),
11625 })
11626 .unwrap();
11627
11628 let account_orders_calls = Arc::new(AtomicUsize::new(0));
11629 let active_calls = Arc::new(AtomicUsize::new(0));
11630 let inactive_calls = Arc::new(AtomicUsize::new(0));
11631 let pending = Arc::new(AtomicBool::new(true));
11632
11633 let app = Router::new()
11634 .route(
11635 "/api/v1/accountOrders",
11636 get({
11637 let body = account_orders_body.clone();
11638 let calls = Arc::clone(&account_orders_calls);
11639 let pending = Arc::clone(&pending);
11640 move || {
11641 calls.fetch_add(1, Ordering::SeqCst);
11642 let body = body.clone();
11643 let pending = Arc::clone(&pending);
11644 async move {
11645 if stop_pending_after_response {
11646 pending.store(false, Ordering::Release);
11647 }
11648
11649 body
11650 }
11651 }
11652 }),
11653 )
11654 .route(
11655 "/api/v1/accountActiveOrders",
11656 get({
11657 let body = empty_body.clone();
11658 let calls = Arc::clone(&active_calls);
11659 move || {
11660 calls.fetch_add(1, Ordering::SeqCst);
11661 let body = body.clone();
11662 async move { body }
11663 }
11664 }),
11665 )
11666 .route(
11667 "/api/v1/accountInactiveOrders",
11668 get({
11669 let calls = Arc::clone(&inactive_calls);
11670 move || {
11671 calls.fetch_add(1, Ordering::SeqCst);
11672 let body = empty_body.clone();
11673 async move { body }
11674 }
11675 }),
11676 );
11677
11678 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
11679 let addr = listener.local_addr().unwrap();
11680 let server = tokio::spawn(async move {
11681 axum::serve(listener, app).await.unwrap();
11682 });
11683
11684 let config = LighterExecutionClientConfig {
11685 base_url_http: Some(format!("http://{addr}")),
11686 ..test_config()
11687 };
11688
11689 let (mut client, _cache, _rx) = create_execution_client_with_config(config);
11690 client.registry = rig.registry;
11691 client.dispatch = rig.dispatch;
11692 let credential = test_credential();
11693 let lookup = OrderReportLookup::new(
11694 &client.registry,
11695 &client.dispatch,
11696 Some(rig.instrument_id),
11697 Some(&rig.cloid),
11698 venue_order_id.as_ref(),
11699 )
11700 .unwrap();
11701 let result = collect_order_while_pending(
11702 &client.http_client,
11703 &client.registry,
11704 &credential,
11705 &lookup,
11706 || pending.load(Ordering::Acquire),
11707 )
11708 .await;
11709 server.abort();
11710 let _ = server.await;
11711
11712 assert_eq!(account_orders_calls.load(Ordering::SeqCst), 1);
11713 assert_eq!(
11714 active_calls.load(Ordering::SeqCst),
11715 usize::from(!expect_present && !stop_pending_after_response)
11716 );
11717 assert_eq!(
11718 inactive_calls.load(Ordering::SeqCst),
11719 expected_inactive_calls
11720 );
11721
11722 if expect_present {
11723 assert!(result.unwrap().is_some());
11724 } else {
11725 assert_eq!(result.unwrap(), None);
11726 }
11727 }
11728
11729 fn register_test_instrument(
11730 client: &LighterExecutionClient,
11731 cache: &Rc<RefCell<Cache>>,
11732 ) -> InstrumentId {
11733 let instrument_id =
11734 client
11735 .registry
11736 .insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
11737 let instrument = InstrumentAny::CryptoPerpetual(
11738 CryptoPerpetual::builder()
11739 .instrument_id(instrument_id)
11740 .raw_symbol(Symbol::new("ETH-PERP"))
11741 .base_currency(Currency::from("ETH"))
11742 .quote_currency(Currency::from("USDC"))
11743 .settlement_currency(Currency::from("USDC"))
11744 .is_inverse(false)
11745 .price_precision(2)
11746 .size_precision(4)
11747 .price_increment(Price::from("0.01"))
11748 .size_increment(Quantity::from("0.0001"))
11749 .min_notional(Money::from("10.000000 USDC"))
11750 .ts_event(UnixNanos::default())
11751 .ts_init(UnixNanos::default())
11752 .build()
11753 .unwrap(),
11754 );
11755
11756 cache.borrow_mut().add_instrument(instrument).unwrap();
11757
11758 instrument_id
11759 }
11760
11761 fn test_order_factory() -> OrderFactory {
11762 let clock = Rc::new(RefCell::new(VirtualClock::new()));
11763
11764 OrderFactory::new(
11765 trader_id(),
11766 strategy_id(),
11767 Some(0),
11768 Some(0),
11769 clock,
11770 false,
11771 false,
11772 )
11773 }
11774
11775 fn test_limit_order(
11776 factory: &mut OrderFactory,
11777 instrument_id: InstrumentId,
11778 client_order_id: &str,
11779 ) -> OrderAny {
11780 test_limit_order_with(
11781 factory,
11782 instrument_id,
11783 client_order_id,
11784 OrderSide::Buy,
11785 TimeInForce::Gtc,
11786 None,
11787 false,
11788 )
11789 }
11790
11791 fn test_limit_order_with(
11792 factory: &mut OrderFactory,
11793 instrument_id: InstrumentId,
11794 client_order_id: &str,
11795 side: OrderSide,
11796 tif: TimeInForce,
11797 expire_time: Option<UnixNanos>,
11798 reduce_only: bool,
11799 ) -> OrderAny {
11800 factory.limit(
11801 instrument_id,
11802 side,
11803 Quantity::from("0.1000"),
11804 Price::from("2361.31"),
11805 Some(tif),
11806 expire_time,
11807 Some(false),
11808 Some(reduce_only),
11809 None,
11810 None,
11811 None,
11812 None,
11813 None,
11814 None,
11815 None,
11816 Some(ClientOrderId::from(client_order_id)),
11817 )
11818 }
11819
11820 fn cache_order(cache: &Rc<RefCell<Cache>>, order: OrderAny) {
11821 cache
11822 .borrow_mut()
11823 .add_order(order, None, Some(client_id()), false)
11824 .unwrap();
11825 }
11826
11827 fn cache_accepted_order(
11828 cache: &Rc<RefCell<Cache>>,
11829 order: OrderAny,
11830 venue_order_id: VenueOrderId,
11831 client_id: Option<ClientId>,
11832 ) -> (InstrumentId, ClientOrderId) {
11833 let instrument_id = order.instrument_id();
11834 let client_order_id = order.client_order_id();
11835 cache
11836 .borrow_mut()
11837 .add_order(order, None, client_id, false)
11838 .unwrap();
11839
11840 let accepted = OrderEventAny::Accepted(OrderAccepted::new(
11841 trader_id(),
11842 strategy_id(),
11843 instrument_id,
11844 client_order_id,
11845 venue_order_id,
11846 account_id(),
11847 UUID4::new(),
11848 UnixNanos::default(),
11849 UnixNanos::default(),
11850 false,
11851 ));
11852 cache.borrow_mut().update_order(&accepted).unwrap();
11853
11854 (instrument_id, client_order_id)
11855 }
11856
11857 fn cache_pending_cancel_order(
11858 cache: &Rc<RefCell<Cache>>,
11859 order: OrderAny,
11860 venue_order_id: VenueOrderId,
11861 causation_id: Option<UUID4>,
11862 ) {
11863 let (instrument_id, client_order_id) =
11864 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
11865
11866 apply_pending_cancel(
11867 cache,
11868 instrument_id,
11869 client_order_id,
11870 venue_order_id,
11871 causation_id,
11872 );
11873 }
11874
11875 fn apply_pending_cancel(
11876 cache: &Rc<RefCell<Cache>>,
11877 instrument_id: InstrumentId,
11878 client_order_id: ClientOrderId,
11879 venue_order_id: VenueOrderId,
11880 causation_id: Option<UUID4>,
11881 ) {
11882 let mut pending_cancel = OrderPendingCancel::new(
11883 trader_id(),
11884 strategy_id(),
11885 instrument_id,
11886 client_order_id,
11887 Some(account_id()),
11888 UUID4::new(),
11889 UnixNanos::default(),
11890 UnixNanos::default(),
11891 false,
11892 Some(venue_order_id),
11893 );
11894 pending_cancel.causation_id = causation_id;
11895 cache
11896 .borrow_mut()
11897 .update_order(&OrderEventAny::PendingCancel(pending_cancel))
11898 .unwrap();
11899 }
11900
11901 fn submit_order_list_command(orders: &[OrderAny], order_list_id: &str) -> SubmitOrderList {
11902 let order_list = OrderList::new(
11903 OrderListId::from(order_list_id),
11904 orders[0].instrument_id(),
11905 strategy_id(),
11906 orders.iter().map(|order| order.client_order_id()).collect(),
11907 UnixNanos::default(),
11908 );
11909 let order_inits = orders
11910 .iter()
11911 .map(|order| order.init_event().clone())
11912 .collect();
11913
11914 SubmitOrderList::new(
11915 trader_id(),
11916 Some(client_id()),
11917 strategy_id(),
11918 order_list,
11919 order_inits,
11920 None,
11921 None,
11922 None,
11923 UUID4::new(),
11924 UnixNanos::default(),
11925 None,
11926 )
11927 }
11928
11929 fn test_contingent_limit_order(
11930 instrument_id: InstrumentId,
11931 client_order_id: &str,
11932 order_list_id: &str,
11933 linked_order_id: &str,
11934 ) -> OrderAny {
11935 OrderAny::Limit(LimitOrder::new(
11936 trader_id(),
11937 strategy_id(),
11938 instrument_id,
11939 ClientOrderId::from(client_order_id),
11940 OrderSide::Buy,
11941 Quantity::from("0.1000"),
11942 Price::from("2361.31"),
11943 TimeInForce::Gtc,
11944 None,
11945 false,
11946 false,
11947 false,
11948 None,
11949 None,
11950 None,
11951 Some(ContingencyType::Oco),
11952 Some(OrderListId::from(order_list_id)),
11953 Some(vec![ClientOrderId::from(linked_order_id)]),
11954 None,
11955 None,
11956 None,
11957 None,
11958 None,
11959 UUID4::new(),
11960 UnixNanos::default(),
11961 ))
11962 }
11963
11964 async fn recv_order_event(
11965 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
11966 ) -> OrderEventAny {
11967 let event = tokio::time::timeout(Duration::from_secs(2), rx.recv())
11968 .await
11969 .expect("timed out waiting for execution event")
11970 .expect("execution event channel closed");
11971
11972 match event {
11973 ExecutionEvent::Order(event) => event,
11974 event => panic!("expected order event, was {event:?}"),
11975 }
11976 }
11977
11978 async fn assert_modify_denied_reason(
11979 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
11980 reason_part: &str,
11981 ) {
11982 match recv_order_event(rx).await {
11983 OrderEventAny::ModifyDenied(event) => {
11984 assert!(
11985 event.reason.contains(reason_part),
11986 "expected modify denial containing `{reason_part}`, was `{}`",
11987 event.reason,
11988 );
11989 }
11990 event => panic!("expected modify denied event, was {event:?}"),
11991 }
11992 }
11993
11994 fn assert_nonce_reusable(dispatch: &WsDispatchState) {
11995 assert_eq!(
11996 dispatch
11997 .nonce_manager
11998 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
11999 Some(TEST_NEXT_NONCE - 1),
12000 );
12001 assert_eq!(
12002 dispatch
12003 .nonce_manager
12004 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12005 .unwrap(),
12006 TEST_NEXT_NONCE,
12007 );
12008 }
12009
12010 #[tokio::test]
12011 async fn auth_token_rotation_retries_failed_mint() {
12012 let credential = test_credential();
12013 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
12014 let cancellation_token = CancellationToken::new();
12015 let mint_attempts = Arc::new(AtomicUsize::new(0));
12016 let subscribe_attempts = Arc::new(AtomicUsize::new(0));
12017
12018 let outcome = tokio::time::timeout(
12019 Duration::from_secs(3),
12020 refresh_auth_token_until_rotated(
12021 &credential,
12022 &channels,
12023 &cancellation_token,
12024 AuthTokenRefreshBackoff {
12025 initial_delay: Duration::from_millis(10),
12026 max_delay: Duration::from_millis(20),
12027 window: Duration::from_secs(1),
12028 },
12029 {
12030 let mint_attempts = Arc::clone(&mint_attempts);
12031 move |_| {
12032 let attempt = mint_attempts.fetch_add(1, Ordering::AcqRel);
12033 if attempt == 0 {
12034 Err(anyhow::anyhow!("mint unavailable"))
12035 } else {
12036 Ok(format!("token-{attempt}").into())
12037 }
12038 }
12039 },
12040 {
12041 let subscribe_attempts = Arc::clone(&subscribe_attempts);
12042 move |_channel, _token| {
12043 let subscribe_attempts = Arc::clone(&subscribe_attempts);
12044 async move {
12045 subscribe_attempts.fetch_add(1, Ordering::AcqRel);
12046 Ok::<(), crate::websocket::error::LighterWsError>(())
12047 }
12048 }
12049 },
12050 ),
12051 )
12052 .await
12053 .expect("rotation retry must complete within the test window");
12054
12055 assert_eq!(outcome, AuthTokenRefreshOutcome::Rotated);
12056 assert_eq!(
12057 mint_attempts.load(Ordering::Acquire),
12058 2,
12059 "failed mint must retry before the next refresh interval",
12060 );
12061 assert_eq!(
12062 subscribe_attempts.load(Ordering::Acquire),
12063 channels.len(),
12064 "subscriptions must wait until token mint succeeds",
12065 );
12066 }
12067
12068 #[tokio::test]
12069 async fn auth_token_rotation_retries_failed_resubscribe() {
12070 let credential = test_credential();
12071 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
12072 let cancellation_token = CancellationToken::new();
12073 let mint_attempts = Arc::new(AtomicUsize::new(0));
12074 let subscribe_attempts = Arc::new(AtomicUsize::new(0));
12075
12076 let outcome = tokio::time::timeout(
12077 Duration::from_secs(3),
12078 refresh_auth_token_until_rotated(
12079 &credential,
12080 &channels,
12081 &cancellation_token,
12082 AuthTokenRefreshBackoff {
12083 initial_delay: Duration::from_millis(10),
12084 max_delay: Duration::from_millis(20),
12085 window: Duration::from_secs(1),
12086 },
12087 {
12088 let mint_attempts = Arc::clone(&mint_attempts);
12089 move |_| {
12090 let attempt = mint_attempts.fetch_add(1, Ordering::AcqRel);
12091 Ok(format!("token-{attempt}").into())
12092 }
12093 },
12094 {
12095 let subscribe_attempts = Arc::clone(&subscribe_attempts);
12096 move |_channel, _token| {
12097 let subscribe_attempts = Arc::clone(&subscribe_attempts);
12098 async move {
12099 let attempt = subscribe_attempts.fetch_add(1, Ordering::AcqRel);
12100 if attempt == 0 {
12101 Err(crate::websocket::error::LighterWsError::Client(
12102 "handler unavailable".to_string(),
12103 ))
12104 } else {
12105 Ok(())
12106 }
12107 }
12108 }
12109 },
12110 ),
12111 )
12112 .await
12113 .expect("rotation retry must complete within the test window");
12114
12115 assert_eq!(outcome, AuthTokenRefreshOutcome::Rotated);
12116 assert_eq!(
12117 mint_attempts.load(Ordering::Acquire),
12118 2,
12119 "failed resubscribe must trigger a fresh auth-token mint",
12120 );
12121 assert_eq!(
12122 subscribe_attempts.load(Ordering::Acquire),
12123 channels.len() * 2,
12124 "failed resubscribe must retry the private account-channel set",
12125 );
12126 }
12127
12128 #[tokio::test]
12129 async fn auth_token_rotation_exhausts_retry_window() {
12130 let credential = test_credential();
12131 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
12132 let cancellation_token = CancellationToken::new();
12133 let mint_attempts = Arc::new(AtomicUsize::new(0));
12134
12135 let refresh = refresh_auth_token_until_rotated(
12136 &credential,
12137 &channels,
12138 &cancellation_token,
12139 AuthTokenRefreshBackoff {
12140 initial_delay: Duration::from_millis(100),
12141 max_delay: Duration::from_millis(100),
12142 window: Duration::from_secs(1),
12143 },
12144 {
12145 let mint_attempts = Arc::clone(&mint_attempts);
12146 move |_| {
12147 mint_attempts.fetch_add(1, Ordering::AcqRel);
12148 Err(anyhow::anyhow!("mint unavailable"))
12149 }
12150 },
12151 |_channel, _token| async { Ok::<(), crate::websocket::error::LighterWsError>(()) },
12152 );
12153 let observe_retry = wait_until_async(
12154 || async { mint_attempts.load(Ordering::Acquire) > 1 },
12155 Duration::from_secs(2),
12156 );
12157
12158 let (outcome, ()) = tokio::time::timeout(Duration::from_secs(3), async {
12159 tokio::join!(refresh, observe_retry)
12160 })
12161 .await
12162 .expect("rotation retry exhaustion must complete within the test window");
12163
12164 assert_eq!(outcome, AuthTokenRefreshOutcome::Exhausted);
12165 assert!(
12166 mint_attempts.load(Ordering::Acquire) > 1,
12167 "persistent mint failure must retry until the window is exhausted",
12168 );
12169 }
12170
12171 #[tokio::test]
12172 async fn auth_token_rotation_cancels_during_retry_backoff() {
12173 let credential = test_credential();
12174 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
12175 let cancellation_token = CancellationToken::new();
12176 let mint_attempts = Arc::new(AtomicUsize::new(0));
12177
12178 let cancel = cancellation_token.clone();
12179 let cancel_after_first_attempt = wait_until_async(
12180 || async { mint_attempts.load(Ordering::Acquire) > 0 },
12181 Duration::from_secs(2),
12182 );
12183
12184 let refresh = refresh_auth_token_until_rotated(
12185 &credential,
12186 &channels,
12187 &cancellation_token,
12188 AuthTokenRefreshBackoff {
12189 initial_delay: Duration::from_secs(2),
12190 max_delay: Duration::from_secs(2),
12191 window: Duration::from_secs(5),
12192 },
12193 {
12194 let mint_attempts = Arc::clone(&mint_attempts);
12195 move |_| {
12196 mint_attempts.fetch_add(1, Ordering::AcqRel);
12197 Err(anyhow::anyhow!("mint unavailable"))
12198 }
12199 },
12200 |_channel, _token| async { Ok::<(), crate::websocket::error::LighterWsError>(()) },
12201 );
12202
12203 let cancel_task = async move {
12204 cancel_after_first_attempt.await;
12205 cancel.cancel();
12206 };
12207
12208 let (outcome, ()) = tokio::time::timeout(Duration::from_secs(6), async {
12209 tokio::join!(refresh, cancel_task)
12210 })
12211 .await
12212 .expect("rotation cancellation must complete within the test window");
12213
12214 assert_eq!(outcome, AuthTokenRefreshOutcome::Cancelled);
12215 assert_eq!(
12216 mint_attempts.load(Ordering::Acquire),
12217 1,
12218 "cancellation during backoff must stop before the next retry",
12219 );
12220 }
12221
12222 #[rstest]
12223 #[case::rotated(AuthTokenRefreshOutcome::Rotated, Some(AUTH_TOKEN_REFRESH_INTERVAL))]
12224 #[case::cancelled(AuthTokenRefreshOutcome::Cancelled, None)]
12225 #[case::exhausted(
12226 AuthTokenRefreshOutcome::Exhausted,
12227 Some(AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY)
12228 )]
12229 fn auth_token_refresh_next_delay_matches_outcome(
12230 #[case] outcome: AuthTokenRefreshOutcome,
12231 #[case] expected: Option<Duration>,
12232 ) {
12233 assert_eq!(auth_token_refresh_next_delay(outcome), expected);
12234 }
12235
12236 #[rstest]
12237 fn auth_token_rotation_channels_match_private_account_streams() {
12238 assert_eq!(
12239 auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64),
12240 [
12241 LighterWsChannel::AccountAllOrders(TEST_ACCOUNT_INDEX_I64),
12242 LighterWsChannel::AccountAllTrades(TEST_ACCOUNT_INDEX_I64),
12243 LighterWsChannel::AccountAllPositions(TEST_ACCOUNT_INDEX_I64),
12244 LighterWsChannel::AccountAllAssets(TEST_ACCOUNT_INDEX_I64),
12245 LighterWsChannel::AccountTx(TEST_ACCOUNT_INDEX_I64),
12246 LighterWsChannel::UserStats(TEST_ACCOUNT_INDEX_I64),
12247 ],
12248 );
12249 }
12250
12251 #[rstest]
12252 #[case::below_max(
12253 Duration::from_millis(10),
12254 Duration::from_millis(100),
12255 Duration::from_millis(20)
12256 )]
12257 #[case::at_max(
12258 Duration::from_millis(100),
12259 Duration::from_millis(100),
12260 Duration::from_millis(100)
12261 )]
12262 #[case::overflow_clamps(Duration::MAX, Duration::from_secs(300), Duration::from_secs(300))]
12263 fn next_auth_token_refresh_retry_delay_doubles_and_caps(
12264 #[case] current: Duration,
12265 #[case] max: Duration,
12266 #[case] expected: Duration,
12267 ) {
12268 assert_eq!(next_auth_token_refresh_retry_delay(current, max), expected);
12269 }
12270
12271 #[rstest]
12272 #[tokio::test]
12273 async fn auth_token_refresh_is_retained_until_session_shutdown() {
12274 let (mut client, _cache, _rx) = create_execution_client();
12275
12276 client
12277 .spawn_auth_token_refresh(test_credential())
12278 .expect("spawn auth token refresh");
12279 client
12280 .nonce_recovery_inflight
12281 .store(true, Ordering::Release);
12282
12283 assert!(
12284 client
12285 .auth_refresh_handle
12286 .as_ref()
12287 .is_some_and(|handle| !handle.is_finished()),
12288 );
12289
12290 client.begin_session_shutdown();
12291 client
12292 .finish_session_shutdown()
12293 .await
12294 .expect("session shutdown");
12295
12296 assert!(client.auth_refresh_handle.is_none());
12297 assert!(!client.nonce_recovery_inflight.load(Ordering::Acquire));
12298 }
12299
12300 #[rstest]
12301 #[tokio::test]
12302 async fn session_shutdown_rejects_tasks_registered_during_abort() {
12303 struct SpawnOnDrop {
12304 pending_tasks: TaskSpawner,
12305 rejected: Arc<AtomicBool>,
12306 polled: Arc<AtomicBool>,
12307 }
12308
12309 impl Drop for SpawnOnDrop {
12310 fn drop(&mut self) {
12311 let polled = Arc::clone(&self.polled);
12312 let result = self.pending_tasks.spawn(async move {
12313 polled.store(true, Ordering::Release);
12314 });
12315 self.rejected.store(result.is_err(), Ordering::Release);
12316 }
12317 }
12318
12319 let (mut client, _cache, _rx) = create_execution_client();
12320 let rejected = Arc::new(AtomicBool::new(false));
12321 let polled = Arc::new(AtomicBool::new(false));
12322 let pending_spawner = client.pending_tasks.spawner().expect("task spawner");
12323 let guard = SpawnOnDrop {
12324 pending_tasks: pending_spawner.clone(),
12325 rejected: Arc::clone(&rejected),
12326 polled: Arc::clone(&polled),
12327 };
12328
12329 pending_spawner
12330 .spawn(async move {
12331 let _guard = guard;
12332 std::future::pending::<()>().await;
12333 })
12334 .expect("parent task spawn");
12335
12336 client
12337 .finish_session_shutdown()
12338 .await
12339 .expect("session shutdown");
12340
12341 assert!(client.pending_tasks.is_empty());
12342 assert!(rejected.load(Ordering::Acquire));
12343 assert!(!polled.load(Ordering::Acquire));
12344 }
12345
12346 #[rstest]
12347 #[tokio::test]
12348 async fn session_shutdown_discards_pending_sendtx_before_epoch_reuse() {
12349 struct EnqueueOnDrop {
12350 dispatch: WsDispatchState,
12351 pending: Option<PendingSendTx>,
12352 }
12353
12354 impl Drop for EnqueueOnDrop {
12355 fn drop(&mut self) {
12356 self.dispatch
12357 .enqueue_pending_sendtx(self.pending.take().expect("pending sendTx"));
12358 }
12359 }
12360
12361 let (mut client, _cache, _rx) = create_execution_client();
12362 let old_epoch = client.ws_client.connection_epoch();
12363 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12364 batch_id: None,
12365 connection_epoch: old_epoch,
12366 kind: PendingSendTxKind::Other,
12367 submitted_at: UnixNanos::default(),
12368 nonce: TEST_NEXT_NONCE,
12369 api_key_index: TEST_API_KEY_INDEX,
12370 tx_hash: "shutdown-pending".to_string(),
12371 });
12372 let guard = EnqueueOnDrop {
12373 dispatch: client.dispatch.clone(),
12374 pending: Some(PendingSendTx {
12375 batch_id: None,
12376 connection_epoch: old_epoch,
12377 kind: PendingSendTxKind::Other,
12378 submitted_at: UnixNanos::default(),
12379 nonce: TEST_NEXT_NONCE + 1,
12380 api_key_index: TEST_API_KEY_INDEX,
12381 tx_hash: "shutdown-late".to_string(),
12382 }),
12383 };
12384
12385 client
12386 .pending_tasks
12387 .spawn(async move {
12388 let _guard = guard;
12389 std::future::pending::<()>().await;
12390 })
12391 .expect("pending sendTx producer spawn");
12392 client
12393 .ws_stream_handle
12394 .insert(get_runtime().spawn(std::future::pending()));
12395
12396 client.begin_session_shutdown();
12397 client
12398 .finish_session_shutdown()
12399 .await
12400 .expect("session shutdown");
12401
12402 assert_eq!(client.ws_client.connection_epoch(), old_epoch);
12403 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12404 }
12405
12406 #[rstest]
12407 fn replacing_ws_client_preserves_cached_instruments() {
12408 let (mut client, cache, _rx) = create_execution_client();
12409 let instrument_id = register_test_instrument(&client, &cache);
12410 let instrument = cache
12411 .borrow()
12412 .instrument(&instrument_id)
12413 .expect("instrument")
12414 .clone();
12415 client
12416 .ws_client
12417 .cache_instrument(TEST_MARKET_INDEX, instrument.clone());
12418
12419 let _old_ws_client = client.take_ws_client();
12420
12421 let cached_instruments = client.ws_client.instruments_cache();
12422 assert_eq!(
12423 cached_instruments
12424 .get(&TEST_MARKET_INDEX)
12425 .expect("cached instrument")
12426 .value(),
12427 &instrument,
12428 );
12429 }
12430
12431 #[rstest]
12432 fn tx_send_sequencer_allocates_under_registration_lock() {
12433 let sequencer = TxSendSequencer::new();
12434
12435 let mut lower = sequencer
12436 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12437 assert!(
12438 sequencer.state.try_lock().is_none(),
12439 "allocation must be serialized with registration"
12440 );
12441 Ok(TEST_NEXT_NONCE)
12442 })
12443 .unwrap();
12444
12445 let higher = sequencer
12446 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12447 Ok(TEST_NEXT_NONCE + 1)
12448 })
12449 .unwrap();
12450
12451 assert!(!sequencer.ready_to_send(higher.key, higher.nonce));
12452 lower.release();
12453 assert!(sequencer.ready_to_send(higher.key, higher.nonce));
12454 }
12455
12456 #[rstest]
12457 fn tx_send_sequencer_retains_reused_nonce_reservation() {
12458 let (client, _cache, _rx) = create_execution_client();
12459 let context = client
12460 .fanout_dispatch_context(client.credential.as_ref().unwrap())
12461 .unwrap();
12462 let mut original = context.build_tx_context().unwrap();
12463 let nonce = original.context.nonce;
12464 rollback_tx_dispatch(
12465 &client.dispatch,
12466 client.credential.as_ref().unwrap(),
12467 None,
12468 nonce,
12469 );
12470 let mut replacement = context.build_tx_context().unwrap();
12471 let higher = context.build_tx_context().unwrap();
12472 original.send_reservation.release();
12473 assert_eq!(replacement.context.nonce, nonce);
12474 assert_eq!(higher.context.nonce, nonce + 1);
12475 assert!(
12476 !context
12477 .tx_send_sequencer
12478 .ready_to_send(higher.send_reservation.key, higher.context.nonce)
12479 );
12480 replacement.send_reservation.release();
12481 assert!(
12482 context
12483 .tx_send_sequencer
12484 .ready_to_send(higher.send_reservation.key, higher.context.nonce)
12485 );
12486 }
12487
12488 #[tokio::test]
12489 async fn tx_send_sequencer_blocks_higher_nonce_until_lower_batch_releases() {
12490 let sequencer = TxSendSequencer::new();
12491
12492 let mut lower_a = sequencer
12493 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12494 Ok(TEST_NEXT_NONCE)
12495 })
12496 .unwrap();
12497
12498 let mut lower_b = sequencer
12499 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12500 Ok(TEST_NEXT_NONCE + 1)
12501 })
12502 .unwrap();
12503
12504 let mut higher = sequencer
12505 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12506 Ok(TEST_NEXT_NONCE + 2)
12507 })
12508 .unwrap();
12509
12510 let (sent_tx, mut sent_rx) = tokio::sync::mpsc::unbounded_channel();
12511
12512 let higher_task = tokio::spawn(async move {
12513 higher.wait_for_turn().await;
12514 sent_tx.send(higher.nonce).unwrap();
12515 higher.release();
12516 });
12517
12518 assert!(
12519 tokio::time::timeout(Duration::from_millis(50), sent_rx.recv())
12520 .await
12521 .is_err(),
12522 "higher nonce must wait while the lower batch is pending",
12523 );
12524
12525 {
12526 let lower_reservations = [&lower_a, &lower_b];
12527 tokio::time::timeout(
12528 Duration::from_millis(50),
12529 wait_for_tx_send_reservations(&lower_reservations),
12530 )
12531 .await
12532 .expect("lower batch must already have the send turn");
12533 }
12534
12535 lower_a.release();
12536 lower_b.release();
12537
12538 let sent = tokio::time::timeout(Duration::from_secs(2), sent_rx.recv())
12539 .await
12540 .expect("higher nonce must proceed after lower batch releases")
12541 .expect("send channel must stay open");
12542 higher_task.await.unwrap();
12543
12544 assert_eq!(
12545 sent,
12546 TEST_NEXT_NONCE + 2,
12547 "higher nonce must send after the lower batch releases",
12548 );
12549 }
12550
12551 #[tokio::test]
12552 async fn tx_send_reservation_drop_releases_lower_nonce() {
12553 let sequencer = TxSendSequencer::new();
12554
12555 let lower = sequencer
12556 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12557 Ok(TEST_NEXT_NONCE)
12558 })
12559 .unwrap();
12560
12561 let mut higher = sequencer
12562 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12563 Ok(TEST_NEXT_NONCE + 1)
12564 })
12565 .unwrap();
12566
12567 let (sent_tx, mut sent_rx) = tokio::sync::mpsc::unbounded_channel();
12568
12569 let higher_task = tokio::spawn(async move {
12570 higher.wait_for_turn().await;
12571 sent_tx.send(higher.nonce).unwrap();
12572 higher.release();
12573 });
12574
12575 assert!(
12576 tokio::time::timeout(Duration::from_millis(50), sent_rx.recv())
12577 .await
12578 .is_err(),
12579 "higher nonce must wait while the lower reservation is pending",
12580 );
12581
12582 drop(lower);
12583
12584 let sent = tokio::time::timeout(Duration::from_secs(2), sent_rx.recv())
12585 .await
12586 .expect("higher nonce must proceed after lower reservation drops")
12587 .expect("send channel must stay open");
12588 higher_task.await.unwrap();
12589
12590 assert_eq!(
12591 sent,
12592 TEST_NEXT_NONCE + 1,
12593 "higher nonce must send after the lower reservation drops",
12594 );
12595 }
12596
12597 #[tokio::test]
12598 async fn tx_send_sequencer_keeps_nonce_streams_independent() {
12599 let sequencer = TxSendSequencer::new();
12600
12601 let _other_account = sequencer
12602 .reserve(TEST_ACCOUNT_INDEX_I64 + 1, TEST_API_KEY_INDEX, || {
12603 Ok(TEST_NEXT_NONCE)
12604 })
12605 .unwrap();
12606
12607 let _other_api_key = sequencer
12608 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX + 1, || {
12609 Ok(TEST_NEXT_NONCE)
12610 })
12611 .unwrap();
12612
12613 let mut current = sequencer
12614 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12615 Ok(TEST_NEXT_NONCE + 10)
12616 })
12617 .unwrap();
12618
12619 tokio::time::timeout(Duration::from_millis(50), current.wait_for_turn())
12620 .await
12621 .expect("lower nonces for other keys must not block this key");
12622 current.release();
12623 }
12624
12625 #[rstest]
12626 fn tx_dispatch_guard_rolls_back_nonce_and_cloid_when_armed() {
12627 let dispatch = WsDispatchState::new();
12628 let credential = test_credential();
12629 dispatch
12630 .nonce_manager
12631 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
12632
12633 let cloid = ClientOrderId::from("O-GUARD-ARMED");
12634 let client_order_index = dispatch.derive_client_order_index(&cloid);
12635 dispatch.register_cloid(client_order_index, cloid).unwrap();
12636 let nonce = dispatch
12637 .nonce_manager
12638 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12639 .unwrap();
12640
12641 {
12642 let _guard = TxDispatchGuard::new(
12643 dispatch.clone(),
12644 &credential,
12645 Some(client_order_index),
12646 nonce,
12647 );
12648 }
12649
12650 assert_nonce_reusable(&dispatch);
12651 assert!(dispatch.cloid_map.get(&client_order_index).is_none());
12652 }
12653
12654 #[rstest]
12655 fn tx_dispatch_guard_rolls_back_nonce_without_cloid_when_armed() {
12656 let dispatch = WsDispatchState::new();
12657 let credential = test_credential();
12658 dispatch
12659 .nonce_manager
12660 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
12661 let nonce = dispatch
12662 .nonce_manager
12663 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12664 .unwrap();
12665
12666 {
12667 let _guard = TxDispatchGuard::new(dispatch.clone(), &credential, None, nonce);
12668 }
12669
12670 assert_nonce_reusable(&dispatch);
12671 assert!(dispatch.cloid_map.is_empty());
12672 }
12673
12674 #[rstest]
12675 fn tx_dispatch_guard_preserves_nonce_and_cloid_when_disarmed() {
12676 let dispatch = WsDispatchState::new();
12677 let credential = test_credential();
12678 dispatch
12679 .nonce_manager
12680 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
12681
12682 let cloid = ClientOrderId::from("O-GUARD-DISARMED");
12683 let client_order_index = dispatch.derive_client_order_index(&cloid);
12684 dispatch.register_cloid(client_order_index, cloid).unwrap();
12685 let nonce = dispatch
12686 .nonce_manager
12687 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12688 .unwrap();
12689
12690 {
12691 let mut guard = TxDispatchGuard::new(
12692 dispatch.clone(),
12693 &credential,
12694 Some(client_order_index),
12695 nonce,
12696 );
12697 guard.disarm();
12698 }
12699
12700 assert_eq!(
12701 dispatch
12702 .nonce_manager
12703 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12704 Some(TEST_NEXT_NONCE),
12705 );
12706 assert_eq!(
12707 dispatch
12708 .cloid_map
12709 .get(&client_order_index)
12710 .map(|entry| *entry.value()),
12711 Some(cloid),
12712 );
12713 }
12714
12715 #[tokio::test]
12716 async fn submit_order_send_failure_emits_submitted_then_rejected_and_rolls_back() {
12717 let (client, cache, mut rx) = create_execution_client();
12718 let instrument_id = register_test_instrument(&client, &cache);
12719 let mut factory = test_order_factory();
12720 let order = test_limit_order(&mut factory, instrument_id, "O-SUBMIT-FAIL");
12721 let client_order_index = client
12722 .dispatch
12723 .derive_client_order_index(&order.client_order_id());
12724 cache_order(&cache, order.clone());
12725
12726 let command = SubmitOrder::from_order(
12727 &order,
12728 trader_id(),
12729 Some(client_id()),
12730 None,
12731 UUID4::new(),
12732 UnixNanos::default(),
12733 );
12734 client.submit_order(command).unwrap();
12735
12736 let submitted = recv_order_event(&mut rx).await;
12737 let rejected = recv_order_event(&mut rx).await;
12738
12739 match submitted {
12740 OrderEventAny::Submitted(event) => {
12741 assert_eq!(event.client_order_id, order.client_order_id());
12742 assert_eq!(event.instrument_id, instrument_id);
12743 }
12744 event => panic!("expected submitted event, was {event:?}"),
12745 }
12746
12747 match rejected {
12748 OrderEventAny::Rejected(event) => {
12749 assert_eq!(event.client_order_id, order.client_order_id());
12750 assert_eq!(event.instrument_id, instrument_id);
12751 assert!(
12752 event
12753 .reason
12754 .contains("Lighter submit_order dispatch failed"),
12755 );
12756 assert!(event.reason.contains("handler unavailable"));
12757 }
12758 event => panic!("expected rejected event, was {event:?}"),
12759 }
12760
12761 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12762 assert_nonce_reusable(&client.dispatch);
12763 assert_eq!(
12764 client.dispatch.pending_sendtx_len(),
12765 0,
12766 "local-send-failure must remove the pending entry by nonce",
12767 );
12768 }
12769
12770 #[tokio::test]
12771 async fn submit_order_unknown_send_outcome_retains_pending_state() {
12772 let (client, cache, mut rx) = create_execution_client();
12773 let instrument_id = register_test_instrument(&client, &cache);
12774 let mut factory = test_order_factory();
12775 let order = test_limit_order(&mut factory, instrument_id, "O-SUBMIT-UNKNOWN");
12776 let client_order_index = client
12777 .dispatch
12778 .derive_client_order_index(&order.client_order_id());
12779 cache_order(&cache, order.clone());
12780 client.ws_client.drop_next_send_tx_result_for_test().await;
12781
12782 client
12783 .submit_order(SubmitOrder::from_order(
12784 &order,
12785 trader_id(),
12786 Some(client_id()),
12787 None,
12788 UUID4::new(),
12789 UnixNanos::default(),
12790 ))
12791 .unwrap();
12792
12793 assert!(matches!(
12794 recv_order_event(&mut rx).await,
12795 OrderEventAny::Submitted(_)
12796 ));
12797 assert!(
12798 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12799 .await
12800 .is_err(),
12801 "unknown outcome must not emit OrderRejected",
12802 );
12803 assert!(client.dispatch.cloid_map.contains_key(&client_order_index));
12804 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
12805 assert_eq!(
12806 client
12807 .dispatch
12808 .nonce_manager
12809 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12810 Some(TEST_NEXT_NONCE),
12811 );
12812
12813 let stale = client.dispatch.drain_pending_sendtx(0);
12814 assert_eq!(stale.len(), 1);
12815 assert!(matches!(stale[0].kind, PendingSendTxKind::Create { .. }));
12816 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12817 assert!(client.dispatch.cloid_map.contains_key(&client_order_index));
12818 assert!(
12819 client
12820 .dispatch
12821 .order_identity(&order.client_order_id())
12822 .is_some(),
12823 "reconnect draining must retain identity for reconciliation",
12824 );
12825 assert_eq!(
12826 client
12827 .dispatch
12828 .nonce_manager
12829 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12830 Some(TEST_NEXT_NONCE),
12831 );
12832 }
12833
12834 #[tokio::test]
12835 async fn submit_order_cloid_probe_exhaustion_denies_without_overwrite() {
12836 let (client, cache, mut rx) = create_execution_client();
12837 let instrument_id = register_test_instrument(&client, &cache);
12838 let mut factory = test_order_factory();
12839 let order = test_limit_order(&mut factory, instrument_id, "O-PROBE-EXHAUSTED");
12840 let cloid = order.client_order_id();
12841 let mut index = client.dispatch.derive_client_order_index(&cloid);
12842 let mut existing = Vec::new();
12843
12844 for attempt in 0..=crate::websocket::dispatch::CLOID_INDEX_PROBE_LIMIT {
12845 let existing_cloid = ClientOrderId::from(format!("EXISTING-{attempt}"));
12846 client.dispatch.cloid_map.insert(index, existing_cloid);
12847 existing.push((index, existing_cloid));
12848 index = if index == i64::from(i32::MAX) {
12849 0
12850 } else {
12851 index + 1
12852 };
12853 }
12854
12855 cache_order(&cache, order.clone());
12856
12857 client
12858 .submit_order(SubmitOrder::from_order(
12859 &order,
12860 trader_id(),
12861 Some(client_id()),
12862 None,
12863 UUID4::new(),
12864 UnixNanos::default(),
12865 ))
12866 .unwrap();
12867
12868 let event = recv_order_event(&mut rx).await;
12869 assert!(matches!(event, OrderEventAny::Denied(_)));
12870 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12871 assert!(client.dispatch.order_identity(&cloid).is_none());
12872
12873 for (existing_index, existing_cloid) in existing {
12874 assert_eq!(
12875 client
12876 .dispatch
12877 .cloid_map
12878 .get(&existing_index)
12879 .map(|entry| *entry.value()),
12880 Some(existing_cloid),
12881 );
12882 }
12883 }
12884
12885 #[tokio::test]
12886 async fn submit_sell_order_send_failure_dispatches_and_rolls_back() {
12887 let (client, cache, mut rx) = create_execution_client();
12890 let instrument_id = register_test_instrument(&client, &cache);
12891 let mut factory = test_order_factory();
12892 let order = test_limit_order_with(
12893 &mut factory,
12894 instrument_id,
12895 "O-SUBMIT-FAIL-SELL",
12896 OrderSide::Sell,
12897 TimeInForce::Gtc,
12898 None,
12899 false,
12900 );
12901 let client_order_index = client
12902 .dispatch
12903 .derive_client_order_index(&order.client_order_id());
12904 cache_order(&cache, order.clone());
12905
12906 let command = SubmitOrder::from_order(
12907 &order,
12908 trader_id(),
12909 Some(client_id()),
12910 None,
12911 UUID4::new(),
12912 UnixNanos::default(),
12913 );
12914 client.submit_order(command).unwrap();
12915
12916 let _submitted = recv_order_event(&mut rx).await;
12917 let rejected = recv_order_event(&mut rx).await;
12918
12919 match rejected {
12920 OrderEventAny::Rejected(event) => {
12921 assert_eq!(event.client_order_id, order.client_order_id());
12922 assert!(
12923 event
12924 .reason
12925 .contains("Lighter submit_order dispatch failed"),
12926 );
12927 }
12928 event => panic!("expected rejected event, was {event:?}"),
12929 }
12930
12931 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12932 assert_nonce_reusable(&client.dispatch);
12933 }
12934
12935 #[tokio::test]
12936 async fn submit_gtd_order_with_explicit_expiry_dispatches_and_rolls_back() {
12937 let (client, cache, mut rx) = create_execution_client();
12942 let instrument_id = register_test_instrument(&client, &cache);
12943 let mut factory = test_order_factory();
12944 let expiry = client.clock.get_time_ns() + DurationNanos::from_mins(10);
12945 let order = test_limit_order_with(
12946 &mut factory,
12947 instrument_id,
12948 "O-SUBMIT-FAIL-GTD",
12949 OrderSide::Buy,
12950 TimeInForce::Gtd,
12951 Some(expiry),
12952 false,
12953 );
12954 let client_order_index = client
12955 .dispatch
12956 .derive_client_order_index(&order.client_order_id());
12957 cache_order(&cache, order.clone());
12958
12959 let command = SubmitOrder::from_order(
12960 &order,
12961 trader_id(),
12962 Some(client_id()),
12963 None,
12964 UUID4::new(),
12965 UnixNanos::default(),
12966 );
12967 client.submit_order(command).unwrap();
12968
12969 let _submitted = recv_order_event(&mut rx).await;
12970 let rejected = recv_order_event(&mut rx).await;
12971
12972 match rejected {
12973 OrderEventAny::Rejected(event) => {
12974 assert!(
12975 event
12976 .reason
12977 .contains("Lighter submit_order dispatch failed"),
12978 );
12979 }
12980 event => panic!("expected rejected event, was {event:?}"),
12981 }
12982
12983 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12984 assert_nonce_reusable(&client.dispatch);
12985 }
12986
12987 #[tokio::test]
12988 async fn submit_gtd_order_with_short_expiry_is_denied_before_nonce_allocation() {
12989 let (client, cache, mut rx) = create_execution_client();
12990 let instrument_id = register_test_instrument(&client, &cache);
12991 let mut factory = test_order_factory();
12992 let expiry = UnixNanos::from(
12993 client
12994 .clock
12995 .get_time_ns()
12996 .as_u64()
12997 .saturating_add(60 * 1_000_000_000),
12998 );
12999 let order = test_limit_order_with(
13000 &mut factory,
13001 instrument_id,
13002 "O-SHORT-GTD",
13003 OrderSide::Buy,
13004 TimeInForce::Gtd,
13005 Some(expiry),
13006 false,
13007 );
13008 cache_order(&cache, order.clone());
13009
13010 let command = SubmitOrder::from_order(
13011 &order,
13012 trader_id(),
13013 Some(client_id()),
13014 None,
13015 UUID4::new(),
13016 UnixNanos::default(),
13017 );
13018 client.submit_order(command).unwrap();
13019
13020 let denied = recv_order_event(&mut rx).await;
13021 match denied {
13022 OrderEventAny::Denied(event) => {
13023 assert!(event.reason.contains("at least 5 minutes"));
13024 }
13025 event => panic!("expected denied event, was {event:?}"),
13026 }
13027
13028 assert_nonce_reusable(&client.dispatch);
13029 }
13030
13031 #[tokio::test]
13032 async fn submit_gtd_order_with_short_expiry_dispatches_when_use_gtd_false() {
13033 let (client, cache, mut rx) =
13038 create_execution_client_with_config(test_config_with_use_gtd(false));
13039 let instrument_id = register_test_instrument(&client, &cache);
13040 let mut factory = test_order_factory();
13041 let expiry = UnixNanos::from(
13042 client
13043 .clock
13044 .get_time_ns()
13045 .as_u64()
13046 .saturating_add(60 * 1_000_000_000),
13047 );
13048 let order = test_limit_order_with(
13049 &mut factory,
13050 instrument_id,
13051 "O-SHORT-GTD-MANAGED",
13052 OrderSide::Buy,
13053 TimeInForce::Gtd,
13054 Some(expiry),
13055 false,
13056 );
13057 let client_order_index = client
13058 .dispatch
13059 .derive_client_order_index(&order.client_order_id());
13060 cache_order(&cache, order.clone());
13061
13062 let command = SubmitOrder::from_order(
13063 &order,
13064 trader_id(),
13065 Some(client_id()),
13066 None,
13067 UUID4::new(),
13068 UnixNanos::default(),
13069 );
13070 client.submit_order(command).unwrap();
13071
13072 let _submitted = recv_order_event(&mut rx).await;
13073 let rejected = recv_order_event(&mut rx).await;
13074
13075 match rejected {
13076 OrderEventAny::Rejected(event) => {
13077 assert!(
13078 event
13079 .reason
13080 .contains("Lighter submit_order dispatch failed"),
13081 );
13082 }
13083 event => panic!("expected rejected event, was {event:?}"),
13084 }
13085
13086 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
13087 assert_nonce_reusable(&client.dispatch);
13088 }
13089
13090 #[rstest]
13091 fn prepare_managed_gtd_plan_uses_safe_venue_expiry() {
13092 let (client, cache, _rx) =
13093 create_execution_client_with_config(test_config_with_use_gtd(false));
13094 let instrument_id = register_test_instrument(&client, &cache);
13095 let mut factory = test_order_factory();
13096 let expiry = UnixNanos::from(
13097 client
13098 .clock
13099 .get_time_ns()
13100 .as_u64()
13101 .saturating_add(60 * 1_000_000_000),
13102 );
13103 let order = test_limit_order_with(
13104 &mut factory,
13105 instrument_id,
13106 "O-SHORT-GTD-PLAN",
13107 OrderSide::Buy,
13108 TimeInForce::Gtd,
13109 Some(expiry),
13110 false,
13111 );
13112
13113 let before_ms = (client.clock.get_time_ns().as_u64() / 1_000_000) as i64;
13114 let plan = client
13115 .prepare_create_order_plan(&order, None)
13116 .expect("managed GTD plan must validate");
13117 let after_ms = (client.clock.get_time_ns().as_u64() / 1_000_000) as i64;
13118
13119 assert!(
13120 (before_ms + crate::websocket::dispatch::ORDER_EXPIRY_DEFAULT_GTC_MS
13121 ..=after_ms + crate::websocket::dispatch::ORDER_EXPIRY_DEFAULT_GTC_MS)
13122 .contains(&plan.order_expiry)
13123 );
13124 assert_eq!(
13125 plan.time_in_force,
13126 crate::common::enums::LighterTimeInForce::GoodTillTime as u8,
13127 );
13128 }
13129
13130 #[tokio::test]
13131 async fn submit_reduce_only_order_dispatches_and_rolls_back() {
13132 let (client, cache, mut rx) = create_execution_client();
13134 let instrument_id = register_test_instrument(&client, &cache);
13135 let mut factory = test_order_factory();
13136 let order = test_limit_order_with(
13137 &mut factory,
13138 instrument_id,
13139 "O-SUBMIT-FAIL-REDUCE",
13140 OrderSide::Sell,
13141 TimeInForce::Gtc,
13142 None,
13143 true,
13144 );
13145 assert!(order.is_reduce_only());
13146 let client_order_index = client
13147 .dispatch
13148 .derive_client_order_index(&order.client_order_id());
13149 cache_order(&cache, order.clone());
13150
13151 let command = SubmitOrder::from_order(
13152 &order,
13153 trader_id(),
13154 Some(client_id()),
13155 None,
13156 UUID4::new(),
13157 UnixNanos::default(),
13158 );
13159 client.submit_order(command).unwrap();
13160
13161 let _submitted = recv_order_event(&mut rx).await;
13162 let rejected = recv_order_event(&mut rx).await;
13163
13164 match rejected {
13165 OrderEventAny::Rejected(event) => {
13166 assert!(
13167 event
13168 .reason
13169 .contains("Lighter submit_order dispatch failed"),
13170 );
13171 }
13172 event => panic!("expected rejected event, was {event:?}"),
13173 }
13174
13175 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
13176 assert_nonce_reusable(&client.dispatch);
13177 }
13178
13179 #[rstest]
13180 #[case::nonce_refresh(false)]
13181 #[case::submission_gate(true)]
13182 #[tokio::test]
13183 async fn batch_signing_retries_after_nonce_readiness_recovers(#[case] lock_submission: bool) {
13184 let (client, _cache, _rx) = create_execution_client();
13185 let credential = test_credential();
13186 let context = client.fanout_dispatch_context(&credential).unwrap();
13187 let connection_epoch = context.ws_client.connection_epoch();
13188 let guard = if lock_submission {
13189 Some(context.nonce_submission_gate.write().await)
13190 } else {
13191 context
13192 .nonce_ready_connection_epoch
13193 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
13194 None
13195 };
13196
13197 let error = context.build_tx_context().unwrap_err();
13198 drop(guard);
13199 context
13200 .nonce_ready_connection_epoch
13201 .store(connection_epoch, Ordering::Release);
13202 let reserved = context.build_tx_context().unwrap();
13203
13204 assert_eq!(reserved.context.nonce, TEST_NEXT_NONCE);
13205 assert!(should_retry_batch_signing(&error));
13206 assert_eq!(
13207 error.to_string(),
13208 if lock_submission {
13209 "Lighter nonce refresh is in progress".to_string()
13210 } else {
13211 format!("Lighter nonce refresh is pending for connection epoch {connection_epoch}")
13212 }
13213 );
13214 }
13215
13216 #[rstest]
13217 #[case::state_changed(TxPreparationError::StateChanged.into(), true)]
13218 #[case::skip_window(NonceError::SkipWindowExhausted {
13219 account_index: TEST_ACCOUNT_INDEX_I64,
13220 api_key_index: TEST_API_KEY_INDEX,
13221 outstanding: 17,
13222 skip_window: 16,
13223 }.into(), true)]
13224 #[case::uninitialized(NonceError::NotInitialized {
13225 account_index: TEST_ACCOUNT_INDEX_I64,
13226 api_key_index: TEST_API_KEY_INDEX,
13227 }.into(), false)]
13228 #[case::signing(anyhow::anyhow!("invalid signing key"), false)]
13229 fn batch_signing_retry_classifies_preparation_errors(
13230 #[case] error: anyhow::Error,
13231 #[case] expected: bool,
13232 ) {
13233 assert_eq!(should_retry_batch_signing(&error), expected);
13234 }
13235
13236 #[rstest]
13237 #[case::submit(false)]
13238 #[case::modify(true)]
13239 #[tokio::test]
13240 async fn batch_orders_wait_for_nonce_readiness(#[case] modify: bool) {
13241 let (client, cache, mut rx) = create_execution_client();
13242 let instrument_id = register_test_instrument(&client, &cache);
13243 let mut factory = test_order_factory();
13244 let orders: Vec<_> = ["O-READY-A", "O-READY-B"]
13245 .into_iter()
13246 .map(|id| test_limit_order(&mut factory, instrument_id, id))
13247 .collect();
13248
13249 for order in &orders {
13250 cache_order(&cache, order.clone());
13251 }
13252
13253 client.ws_client.drop_next_send_tx_result_for_test().await;
13254 client
13255 .nonce_ready_connection_epoch
13256 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
13257
13258 if modify {
13259 let modifies = orders
13260 .iter()
13261 .enumerate()
13262 .map(|(i, order)| {
13263 ModifyOrder::new(
13264 trader_id(),
13265 Some(client_id()),
13266 strategy_id(),
13267 instrument_id,
13268 order.client_order_id(),
13269 Some(VenueOrderId::from(format!("{}", 123 + i))),
13270 Some(Quantity::from("0.2000")),
13271 Some(Price::from("2362.00")),
13272 None,
13273 UUID4::new(),
13274 UnixNanos::default(),
13275 None,
13276 None,
13277 )
13278 })
13279 .collect();
13280
13281 client
13282 .batch_modify_orders(BatchModifyOrders::new(
13283 trader_id(),
13284 Some(client_id()),
13285 strategy_id(),
13286 instrument_id,
13287 modifies,
13288 UUID4::new(),
13289 UnixNanos::default(),
13290 None,
13291 None,
13292 ))
13293 .unwrap();
13294 } else {
13295 client
13296 .submit_order_list(submit_order_list_command(&orders, "OL-READY"))
13297 .unwrap();
13298 }
13299
13300 assert!(
13301 tokio::time::timeout(Duration::from_millis(100), rx.recv())
13302 .await
13303 .is_err()
13304 );
13305 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13306 assert_eq!(
13307 client
13308 .dispatch
13309 .nonce_manager
13310 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13311 Some(TEST_NEXT_NONCE - 1)
13312 );
13313 client
13314 .nonce_ready_connection_epoch
13315 .store(0, Ordering::Release);
13316 wait_until_async(
13317 || async { client.dispatch.pending_sendtx_len() == 2 },
13318 Duration::from_secs(2),
13319 )
13320 .await;
13321
13322 let pending = client.dispatch.pending_sendtx.lock().clone();
13323 assert_eq!(pending.len(), 2);
13324 assert_eq!(pending[0].batch_id, pending[1].batch_id);
13325 assert!(
13326 pending[0]
13327 .batch_id
13328 .as_ref()
13329 .unwrap()
13330 .starts_with(TX_BATCH_ID_PREFIX)
13331 );
13332
13333 for (i, (pending, order)) in pending.iter().zip(&orders).enumerate() {
13334 assert_eq!(pending.nonce, TEST_NEXT_NONCE + i as i64);
13335 assert_eq!(pending.api_key_index, TEST_API_KEY_INDEX);
13336 assert_eq!(pending.connection_epoch, 0);
13337
13338 if modify {
13339 assert!(
13340 matches!(&pending.kind, PendingSendTxKind::Modify { client_order_id, .. } if *client_order_id == order.client_order_id())
13341 );
13342 assert_eq!(
13343 client
13344 .dispatch
13345 .pending_order_action(&order.client_order_id()),
13346 Some(PendingOrderAction::Modify)
13347 );
13348 } else {
13349 assert!(
13350 matches!(&pending.kind, PendingSendTxKind::Create { order: pending_order, .. } if pending_order.client_order_id() == order.client_order_id())
13351 );
13352 let event = recv_order_event(&mut rx).await;
13353
13354 let OrderEventAny::Submitted(event) = event else {
13355 panic!("expected Submitted after nonce readiness")
13356 };
13357
13358 assert_eq!(event.client_order_id, order.client_order_id());
13359 }
13360 }
13361
13362 assert!(
13363 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13364 .await
13365 .is_err()
13366 );
13367 assert_eq!(
13368 client
13369 .dispatch
13370 .nonce_manager
13371 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13372 Some(TEST_NEXT_NONCE + 1)
13373 );
13374 }
13375
13376 #[tokio::test]
13377 async fn submit_order_list_send_failure_emits_submitted_then_rejected_and_rolls_back() {
13378 let (client, cache, mut rx) = create_execution_client();
13379 let instrument_id = register_test_instrument(&client, &cache);
13380 let mut factory = test_order_factory();
13381 let order_a = test_limit_order(&mut factory, instrument_id, "O-LIST-FAIL-A");
13382 let order_b = test_limit_order(&mut factory, instrument_id, "O-LIST-FAIL-B");
13383 let index_a = client
13384 .dispatch
13385 .derive_client_order_index(&order_a.client_order_id());
13386 let index_b = client
13387 .dispatch
13388 .derive_client_order_index(&order_b.client_order_id());
13389 cache_order(&cache, order_a.clone());
13390 cache_order(&cache, order_b.clone());
13391
13392 let command = submit_order_list_command(&[order_a.clone(), order_b.clone()], "OL-FAIL");
13393 client.submit_order_list(command).unwrap();
13394
13395 for expected in [order_a.client_order_id(), order_b.client_order_id()] {
13396 match recv_order_event(&mut rx).await {
13397 OrderEventAny::Submitted(e) => assert_eq!(e.client_order_id, expected),
13398 other => panic!("expected Submitted, was {other:?}"),
13399 }
13400 }
13401
13402 for expected in [order_b.client_order_id(), order_a.client_order_id()] {
13403 match recv_order_event(&mut rx).await {
13404 OrderEventAny::Rejected(e) => {
13405 assert_eq!(e.client_order_id, expected);
13406 assert!(e.reason.contains("Lighter submit_order dispatch failed"));
13407 }
13408 other => panic!("expected Rejected, was {other:?}"),
13409 }
13410 }
13411
13412 assert!(client.dispatch.cloid_map.get(&index_a).is_none());
13413 assert!(client.dispatch.cloid_map.get(&index_b).is_none());
13414 assert_nonce_reusable(&client.dispatch);
13415 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13416 }
13417
13418 #[rstest]
13419 #[case::spill(16)]
13420 #[case::three(31)]
13421 #[tokio::test]
13422 async fn submit_order_list_chunks_large_lists_on_send_failure(#[case] count: usize) {
13423 let (client, cache, mut rx) = create_execution_client();
13424 let instrument_id = register_test_instrument(&client, &cache);
13425 let mut factory = test_order_factory();
13426 let mut orders = Vec::new();
13427
13428 for i in 0..count {
13429 let order = test_limit_order(&mut factory, instrument_id, &format!("O-LIST-MAX-{i}"));
13430 cache_order(&cache, order.clone());
13431 orders.push(order);
13432 }
13433
13434 let command = submit_order_list_command(&orders, "OL-MAX");
13435 client.submit_order_list(command).unwrap();
13436
13437 for chunk in orders.chunks(LIGHTER_MAX_BATCH_TX) {
13438 for order in chunk {
13439 match recv_order_event(&mut rx).await {
13440 OrderEventAny::Submitted(e) => {
13441 assert_eq!(e.client_order_id, order.client_order_id());
13442 }
13443 other => panic!("expected Submitted, was {other:?}"),
13444 }
13445 }
13446
13447 for order in chunk.iter().rev() {
13448 match recv_order_event(&mut rx).await {
13449 OrderEventAny::Rejected(e) => {
13450 assert_eq!(e.client_order_id, order.client_order_id());
13451 assert!(e.reason.contains("Lighter submit_order dispatch failed"));
13452 }
13453 other => panic!("expected Rejected, was {other:?}"),
13454 }
13455 }
13456 }
13457
13458 assert!(
13459 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13460 .await
13461 .is_err(),
13462 "chunked send failure must not emit extra events",
13463 );
13464
13465 for order in &orders {
13466 let index = client
13467 .dispatch
13468 .derive_client_order_index(&order.client_order_id());
13469 assert!(client.dispatch.cloid_map.get(&index).is_none());
13470 }
13471
13472 assert_nonce_reusable(&client.dispatch);
13473 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13474 }
13475
13476 #[tokio::test]
13477 async fn submit_order_list_denies_unsupported_order_and_dispatches_supported() {
13478 let (client, cache, mut rx) = create_execution_client();
13479 let instrument_id = register_test_instrument(&client, &cache);
13480 let mut factory = test_order_factory();
13481 let valid = test_limit_order(&mut factory, instrument_id, "O-LIST-VALID");
13482 let unsupported = factory.limit(
13483 instrument_id,
13484 OrderSide::Buy,
13485 Quantity::from("0.1000"),
13486 Price::from("2361.31"),
13487 Some(TimeInForce::Gtc),
13488 None,
13489 Some(false),
13490 Some(false),
13491 None,
13492 Some(Quantity::from("0.0500")),
13493 None,
13494 None,
13495 None,
13496 None,
13497 None,
13498 Some(ClientOrderId::from("O-LIST-ICEBERG")),
13499 );
13500 let unsupported_index = client
13501 .dispatch
13502 .derive_client_order_index(&unsupported.client_order_id());
13503 cache_order(&cache, valid.clone());
13504 cache_order(&cache, unsupported.clone());
13505
13506 let command =
13507 submit_order_list_command(&[unsupported.clone(), valid.clone()], "OL-PARTIAL");
13508 client.submit_order_list(command).unwrap();
13509
13510 match recv_order_event(&mut rx).await {
13511 OrderEventAny::Denied(e) => {
13512 assert_eq!(e.client_order_id, unsupported.client_order_id());
13513 assert!(e.reason.contains("display_qty"));
13514 }
13515 other => panic!("expected Denied, was {other:?}"),
13516 }
13517
13518 match recv_order_event(&mut rx).await {
13519 OrderEventAny::Submitted(e) => assert_eq!(e.client_order_id, valid.client_order_id()),
13520 other => panic!("expected Submitted, was {other:?}"),
13521 }
13522
13523 match recv_order_event(&mut rx).await {
13524 OrderEventAny::Rejected(e) => {
13525 assert_eq!(e.client_order_id, valid.client_order_id());
13526 assert!(e.reason.contains("Lighter submit_order dispatch failed"),);
13527 }
13528 other => panic!("expected Rejected, was {other:?}"),
13529 }
13530
13531 assert!(client.dispatch.cloid_map.get(&unsupported_index).is_none());
13532 assert_nonce_reusable(&client.dispatch);
13533 }
13534
13535 #[rstest]
13536 #[case(LighterGroupingType::OneTriggersTheOther)]
13537 #[case(LighterGroupingType::OneCancelsTheOther)]
13538 #[case(LighterGroupingType::OneTriggersOneCancelsTheOther)]
13539 fn prepare_native_group_preserves_relationships_and_sizes_children(
13540 #[case] grouping: LighterGroupingType,
13541 ) {
13542 let (client, cache, _rx) = create_execution_client();
13543 let instrument = register_test_instrument(&client, &cache);
13544 let mut orders = test_native_group(instrument, grouping);
13545 orders.reverse();
13546 let plan = client.prepare_create_group_plan(&orders, None).unwrap();
13547 let context = client
13548 .fanout_dispatch_context(client.credential.as_ref().unwrap())
13549 .unwrap();
13550 let signed = context.sign_create_group(&plan).unwrap();
13551 let json: serde_json::Value = serde_json::from_str(signed.tx_info.get()).unwrap();
13552 let bodies = json["Orders"].as_array().unwrap();
13553 let has_parent = grouping != LighterGroupingType::OneCancelsTheOther;
13554 assert_eq!(plan.grouping_type, grouping);
13555 assert_eq!(signed.tx_type, LighterTxType::CreateGroupedOrders as u8);
13556 assert_eq!(bodies.len(), orders.len());
13557 assert_eq!(bodies[0]["BaseAmount"], 1_000);
13558
13559 for (index, (body, leg)) in bodies.iter().zip(&plan.orders).enumerate() {
13560 assert_eq!(body["MarketIndex"], TEST_MARKET_INDEX);
13561 assert_eq!(
13562 body["ClientOrderIndex"],
13563 client
13564 .dispatch
13565 .client_order_index(&leg.order.client_order_id())
13566 .unwrap()
13567 );
13568 assert_eq!(body["Price"], leg.price);
13569 assert_eq!(body["Type"], leg.order_type);
13570 assert_eq!(body["IsAsk"], u8::from(index > 0 || !has_parent));
13571 assert_eq!(body["ReduceOnly"], u8::from(index > 0 || !has_parent));
13572 assert_eq!(body["TimeInForce"], leg.time_in_force);
13573 assert_eq!(body["TriggerPrice"], leg.trigger_price);
13574 assert_eq!(body["OrderExpiry"], leg.order_expiry);
13575 assert_eq!(
13576 body["BaseAmount"],
13577 if has_parent && index > 0 { 0 } else { 1_000 }
13578 );
13579 assert_eq!(leg.order.quantity(), Quantity::from("0.1000"));
13580 }
13581
13582 assert_eq!(signed.pending.kind.creates().count(), orders.len());
13583 assert_eq!(
13584 client
13585 .dispatch
13586 .nonce_manager
13587 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13588 Some(TEST_NEXT_NONCE)
13589 );
13590 }
13591
13592 #[tokio::test]
13593 async fn submit_native_group_send_failure_rejects_every_leg_and_releases_one_nonce() {
13594 let (client, cache, mut rx) = create_execution_client();
13595 let instrument = register_test_instrument(&client, &cache);
13596 let orders = test_native_group(
13597 instrument,
13598 LighterGroupingType::OneTriggersOneCancelsTheOther,
13599 );
13600
13601 for order in &orders {
13602 cache_order(&cache, order.clone());
13603 }
13604
13605 client
13606 .submit_order_list(submit_order_list_command(&orders, "OL-GROUP"))
13607 .unwrap();
13608
13609 for order in &orders {
13610 match recv_order_event(&mut rx).await {
13611 OrderEventAny::Submitted(event) => {
13612 assert_eq!(event.client_order_id, order.client_order_id());
13613 }
13614 event => panic!("expected submitted event, was {event:?}"),
13615 }
13616 }
13617
13618 for order in &orders {
13619 match recv_order_event(&mut rx).await {
13620 OrderEventAny::Rejected(event) => {
13621 assert_eq!(event.client_order_id, order.client_order_id());
13622 assert!(
13623 event
13624 .reason
13625 .contains("Lighter submit_order dispatch failed")
13626 );
13627 }
13628 event => panic!("expected rejected event, was {event:?}"),
13629 }
13630
13631 assert!(
13632 client
13633 .dispatch
13634 .order_identity(&order.client_order_id())
13635 .is_none()
13636 );
13637 }
13638
13639 assert_eq!(client.dispatch.cloid_map.len(), 0);
13640 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13641 assert_nonce_reusable(&client.dispatch);
13642 }
13643
13644 #[tokio::test]
13645 async fn native_group_denies_default_bracket_without_dispatch() {
13646 let (client, cache, mut rx) = create_execution_client();
13647 let instrument = register_test_instrument(&client, &cache);
13648 let orders = test_order_factory()
13649 .bracket()
13650 .instrument_id(instrument)
13651 .order_side(OrderSide::Buy)
13652 .quantity(Quantity::from("0.1000"))
13653 .entry_order_type(OrderType::Limit)
13654 .entry_price(Price::from("2300.00"))
13655 .tp_price(Price::from("2400.00"))
13656 .sl_trigger_price(Price::from("2200.00"))
13657 .call();
13658
13659 for order in &orders {
13660 cache_order(&cache, order.clone());
13661 }
13662
13663 client
13664 .submit_order_list(submit_order_list_command(
13665 &orders,
13666 orders[0].order_list_id().unwrap().as_str(),
13667 ))
13668 .unwrap();
13669
13670 for order in &orders {
13671 match recv_order_event(&mut rx).await {
13672 OrderEventAny::Denied(event) => {
13673 assert_eq!(event.client_order_id, order.client_order_id());
13674 assert_eq!(
13675 event.reason.as_str(),
13676 "UNSUPPORTED_ORDER_LIST: Lighter grouped submit failed: Lighter native groups do not support Nautilus OUO quantity updates"
13677 );
13678 }
13679 event => panic!("expected denied event, was {event:?}"),
13680 }
13681 }
13682
13683 assert_eq!(client.dispatch.cloid_map.len(), 0);
13684 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13685 assert_nonce_reusable(&client.dispatch);
13686 }
13687
13688 #[rstest]
13689 #[case(LighterGroupingType::OneTriggersTheOther)]
13690 #[case(LighterGroupingType::OneCancelsTheOther)]
13691 #[case(LighterGroupingType::OneTriggersOneCancelsTheOther)]
13692 #[tokio::test]
13693 async fn native_group_denies_quote_quantity_without_dispatch(
13694 #[case] grouping: LighterGroupingType,
13695 #[values(false, true)] first_leg: bool,
13696 ) {
13697 let (client, cache, mut rx) = create_execution_client();
13698 let instrument = register_test_instrument(&client, &cache);
13699 let mut orders = test_native_group(instrument, grouping);
13700
13701 let index = if first_leg { 0 } else { orders.len() - 1 };
13702 let order = &orders[index];
13703 let mut builder = OrderTestBuilder::new(order.order_type());
13704 builder
13705 .trader_id(order.trader_id())
13706 .strategy_id(order.strategy_id())
13707 .instrument_id(instrument)
13708 .client_order_id(order.client_order_id())
13709 .side(order.order_side())
13710 .quantity(order.quantity())
13711 .price(order.price().unwrap())
13712 .reduce_only(order.is_reduce_only())
13713 .quote_quantity(true)
13714 .order_list_id(order.order_list_id().unwrap());
13715
13716 if let Some(price) = order.trigger_price() {
13717 builder.trigger_price(price);
13718 }
13719
13720 if let Some(parent) = order.parent_order_id() {
13721 builder.parent_order_id(parent);
13722 }
13723
13724 if let Some(contingency) = order.contingency_type() {
13725 builder.contingency_type(contingency);
13726 }
13727
13728 if let Some(linked) = order.linked_order_ids() {
13729 builder.linked_order_ids(linked.to_vec());
13730 }
13731
13732 orders[index] = builder.build();
13733
13734 for order in &orders {
13735 cache_order(&cache, order.clone());
13736 }
13737
13738 client
13739 .submit_order_list(submit_order_list_command(&orders, "OL-GROUP"))
13740 .unwrap();
13741
13742 for order in &orders {
13743 match recv_order_event(&mut rx).await {
13744 OrderEventAny::Denied(event) => {
13745 assert_eq!(event.client_order_id, order.client_order_id());
13746 assert_eq!(event.reason.as_str(), "UNSUPPORTED_QUOTE_QUANTITY");
13747 }
13748 event => panic!("expected denied event, was {event:?}"),
13749 }
13750 }
13751
13752 assert!(matches!(
13753 rx.try_recv(),
13754 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
13755 ));
13756 assert_eq!(client.dispatch.cloid_map.len(), 0);
13757 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13758 assert_nonce_reusable(&client.dispatch);
13759 }
13760
13761 #[rstest]
13762 #[case(true)]
13763 #[case(false)]
13764 fn native_group_preserves_explicit_gtd_expiries(#[case] matching: bool) {
13765 let (client, cache, _rx) = create_execution_client();
13766 let instrument = register_test_instrument(&client, &cache);
13767 let mut orders = test_native_group(instrument, LighterGroupingType::OneCancelsTheOther);
13768 let expiry = client.clock.get_time_ns() + DurationNanos::from_mins(10);
13769
13770 for (index, order) in orders.iter_mut().enumerate() {
13771 *order = OrderTestBuilder::new(order.order_type())
13772 .trader_id(order.trader_id())
13773 .strategy_id(order.strategy_id())
13774 .instrument_id(instrument)
13775 .client_order_id(order.client_order_id())
13776 .side(order.order_side())
13777 .quantity(order.quantity())
13778 .price(order.price().unwrap())
13779 .trigger_price(order.trigger_price().unwrap())
13780 .reduce_only(true)
13781 .order_list_id(order.order_list_id().unwrap())
13782 .contingency_type(ContingencyType::Oco)
13783 .linked_order_ids(order.linked_order_ids().unwrap().to_vec())
13784 .time_in_force(TimeInForce::Gtd)
13785 .expire_time(UnixNanos::from(
13786 expiry.as_u64() + u64::from(!matching) * index as u64 * 1_000_000_000,
13787 ))
13788 .build();
13789 }
13790
13791 let result = client.prepare_create_group_plan(&orders, None);
13792 if matching {
13793 let plan = result.unwrap();
13794 assert_eq!(
13795 plan.orders
13796 .iter()
13797 .map(|leg| leg.order_expiry)
13798 .collect::<Vec<_>>(),
13799 vec![expiry.as_u64() as i64 / 1_000_000; 2]
13800 );
13801 } else {
13802 assert_eq!(
13803 result.err().unwrap().to_string(),
13804 "Lighter group children require matching nonzero expiries"
13805 );
13806 }
13807
13808 assert_eq!(client.dispatch.cloid_map.len(), 0);
13809 }
13810
13811 #[rstest]
13812 fn native_group_denies_quantity_mismatch_without_registering_orders() {
13813 let (client, cache, _rx) = create_execution_client();
13814 let instrument = register_test_instrument(&client, &cache);
13815 let mut orders = test_native_group(
13816 instrument,
13817 LighterGroupingType::OneTriggersOneCancelsTheOther,
13818 );
13819 orders[1].set_quantity(Quantity::from("0.2000"));
13820 let error = client
13821 .prepare_create_group_plan(&orders, None)
13822 .err()
13823 .unwrap();
13824 assert_eq!(
13825 error.to_string(),
13826 "Lighter group quantities must match before parent-fill sizing"
13827 );
13828 assert_eq!(client.dispatch.cloid_map.len(), 0);
13829 }
13830
13831 #[rstest]
13832 fn native_group_denies_duplicate_client_ids_without_registering_orders() {
13833 let (client, cache, _rx) = create_execution_client();
13834 let instrument = register_test_instrument(&client, &cache);
13835 let mut orders = test_native_group(
13836 instrument,
13837 LighterGroupingType::OneTriggersOneCancelsTheOther,
13838 );
13839 orders[1] = orders[2].clone();
13840 let error = client
13841 .prepare_create_group_plan(&orders, None)
13842 .err()
13843 .unwrap();
13844 assert_eq!(
13845 error.to_string(),
13846 "Lighter groups require unique client order IDs"
13847 );
13848 assert_eq!(client.dispatch.cloid_map.len(), 0);
13849 }
13850
13851 #[rstest]
13852 fn native_group_reports_keep_client_links_and_held_quantities() {
13853 let (client, cache, _rx) = create_execution_client();
13854 let instrument = register_test_instrument(&client, &cache);
13855 let orders = test_native_group(
13856 instrument,
13857 LighterGroupingType::OneTriggersOneCancelsTheOther,
13858 );
13859 let plan = client.prepare_create_group_plan(&orders, None).unwrap();
13860 let context = client
13861 .fanout_dispatch_context(client.credential.as_ref().unwrap())
13862 .unwrap();
13863 let signed = context.sign_create_group(&plan).unwrap();
13864 context.enqueue_batch_tx(&signed);
13865 let venues = [
13866 VenueOrderId::from("281477001"),
13867 VenueOrderId::from("281477002"),
13868 VenueOrderId::from("281477003"),
13869 ];
13870 let indices: Vec<_> = signed
13871 .pending
13872 .kind
13873 .creates()
13874 .map(|(_, index)| index)
13875 .collect();
13876
13877 for (index, order) in orders.iter().enumerate() {
13878 assert!(client.dispatch.observe_create_submission(
13879 &order.client_order_id(),
13880 indices[index],
13881 signed.pending.nonce,
13882 venues[index]
13883 ));
13884 }
13885
13886 let mut parent = reconciliation_raw_order(indices[0], venues[0], LighterOrderStatus::Open);
13887 parent.to_trigger_order_id_0 = venues[1].to_string();
13888 parent.to_trigger_order_id_1 = venues[2].to_string();
13889 let parent_report = parse_http_order_to_report(
13890 &parent,
13891 &client.registry,
13892 account_id(),
13893 UnixNanos::from(7),
13894 &client.dispatch,
13895 )
13896 .unwrap();
13897 let parent_report = client.dispatch.translate_order_cloid(parent_report);
13898 let mut child =
13899 reconciliation_raw_order(indices[1], venues[1], LighterOrderStatus::Pending);
13900 child.order_type = LighterOrderKind::StopLossLimit;
13901 child.parent_order_id = venues[0].to_string();
13902 child.to_cancel_order_id_0 = venues[2].to_string();
13903 child.initial_base_amount = Decimal::ZERO;
13904 child.filled_base_amount = Decimal::ZERO;
13905 child.trigger_price = Decimal::from(2200);
13906 let child_report = parse_http_order_to_report(
13907 &child,
13908 &client.registry,
13909 account_id(),
13910 UnixNanos::from(8),
13911 &client.dispatch,
13912 )
13913 .unwrap();
13914 let child_report = client.dispatch.translate_order_cloid(child_report);
13915 client
13916 .dispatch
13917 .retire_order_identity(&orders[0].client_order_id());
13918 child.parent_order_id = "0".to_string();
13919 child.to_cancel_order_id_0 = "0".to_string();
13920 child.status = LighterOrderStatus::Open;
13921 child.trigger_status = LighterTriggerStatus::Na;
13922 child.trigger_price = Decimal::ZERO;
13923 child.initial_base_amount = Decimal::from_str("0.0400").unwrap();
13924 child.remaining_base_amount = child.initial_base_amount;
13925 let activated = parse_http_order_to_report(
13926 &child,
13927 &client.registry,
13928 account_id(),
13929 UnixNanos::from(9),
13930 &client.dispatch,
13931 )
13932 .unwrap();
13933 let activated = client.dispatch.translate_order_cloid(activated);
13934 assert_eq!(
13935 parent_report.client_order_id,
13936 Some(orders[0].client_order_id())
13937 );
13938 assert_eq!(parent_report.contingency_type, Some(ContingencyType::Oto));
13939 assert_eq!(
13940 parent_report.linked_order_ids,
13941 Some(vec![
13942 orders[1].client_order_id(),
13943 orders[2].client_order_id()
13944 ])
13945 );
13946 assert_eq!(parent_report.parent_order_id, None);
13947 assert_eq!(
13948 child_report.client_order_id,
13949 Some(orders[1].client_order_id())
13950 );
13951 assert_eq!(child_report.contingency_type, Some(ContingencyType::Oco));
13952 assert_eq!(
13953 child_report.linked_order_ids,
13954 Some(vec![orders[2].client_order_id()])
13955 );
13956 assert_eq!(
13957 child_report.parent_order_id,
13958 Some(orders[0].client_order_id())
13959 );
13960 assert_eq!(child_report.quantity, Quantity::from("0.1000"));
13961 assert_eq!(activated.order_type, OrderType::StopLimit);
13962 assert_eq!(activated.order_status, OrderStatus::Triggered);
13963 assert_eq!(activated.trigger_price, Some(Price::from("2200.00")));
13964 assert_eq!(activated.quantity, Quantity::from("0.0400"));
13965 assert_eq!(activated.parent_order_id, Some(orders[0].client_order_id()));
13966 assert_eq!(
13967 activated.linked_order_ids,
13968 Some(vec![orders[2].client_order_id()])
13969 );
13970 }
13971
13972 #[rstest]
13973 #[case::unactivated(LighterOrderStatus::CanceledChild, "0.0000", "0.0000", true, true)]
13974 #[case::activated(LighterOrderStatus::CanceledChild, "0.0400", "0.0000", true, false)]
13975 #[case::filled(LighterOrderStatus::CanceledChild, "0.0000", "0.0200", true, false)]
13976 #[case::no_parent(LighterOrderStatus::CanceledChild, "0.0000", "0.0000", false, false)]
13977 #[case::other_cancel(LighterOrderStatus::CanceledOco, "0.0000", "0.0000", true, false)]
13978 fn unowned_child_cancel_filters_only_unactivated_children(
13979 #[case] status: LighterOrderStatus,
13980 #[case] initial: &str,
13981 #[case] filled: &str,
13982 #[case] has_parent: bool,
13983 #[case] filtered: bool,
13984 ) {
13985 let (client, cache, mut rx) = create_execution_client();
13986 register_test_instrument(&client, &cache);
13987 let mut raw = reconciliation_raw_order(42, VenueOrderId::from("281477004"), status);
13988 raw.order_type = LighterOrderKind::StopLossLimit;
13989 raw.initial_base_amount = Decimal::from_str(initial).unwrap();
13990 raw.remaining_base_amount = Decimal::ZERO;
13991 raw.filled_base_amount = Decimal::from_str(filled).unwrap();
13992 raw.parent_order_id = if has_parent { "281477001" } else { "0" }.to_string();
13993 raw.to_cancel_order_id_0 = "281477005".to_string();
13994 let report = parse_http_order_to_report(
13995 &raw,
13996 &client.registry,
13997 account_id(),
13998 UnixNanos::from(2),
13999 &client.dispatch,
14000 )
14001 .unwrap();
14002
14003 let cmd = GenerateOrderStatusReports::new(
14004 UUID4::new(),
14005 UnixNanos::from(3),
14006 false,
14007 None,
14008 None,
14009 None,
14010 None,
14011 None,
14012 );
14013
14014 let reports = finish_order_reports(
14015 &client.core,
14016 &client.dispatch,
14017 OrderReportSweep {
14018 orders: vec![(raw.client_order_index, report)],
14019 ..Default::default()
14020 },
14021 &cmd,
14022 Some(&[]),
14023 )
14024 .unwrap();
14025
14026 dispatch_lighter_order(
14027 &raw,
14028 &client.dispatch,
14029 &client.emitter,
14030 &client.registry,
14031 account_id(),
14032 trader_id(),
14033 UnixNanos::from(4),
14034 );
14035 let events = drain_events(&mut rx);
14036 assert_eq!(reports.len(), usize::from(!filtered));
14037 assert_eq!(events.len(), usize::from(!filtered));
14038
14039 if !filtered {
14040 let report = &reports[0];
14041 assert_eq!(
14042 report.quantity.as_decimal(),
14043 Decimal::from_str(initial).unwrap()
14044 );
14045 assert_eq!(
14046 report.filled_qty.as_decimal(),
14047 Decimal::from_str(filled).unwrap()
14048 );
14049 assert_eq!(report.order_status, OrderStatus::Canceled);
14050 assert_eq!(report.cancel_reason.as_deref(), status.as_cancel_reason());
14051 assert_eq!(
14052 report.parent_order_id,
14053 has_parent.then(|| ClientOrderId::from("281477001"))
14054 );
14055 assert_eq!(
14056 report.linked_order_ids,
14057 Some(vec![ClientOrderId::from("281477005")])
14058 );
14059 }
14060 }
14061
14062 #[rstest]
14063 #[case::normal_index(42)]
14064 #[case::zero_index(0)]
14065 fn unowned_child_cancel_preserves_known_children(#[case] index: i64) {
14066 let (client, cache, _rx) = create_execution_client();
14067 let instrument = register_test_instrument(&client, &cache);
14068 let orders = test_native_group(
14069 instrument,
14070 LighterGroupingType::OneTriggersOneCancelsTheOther,
14071 );
14072 let child = orders[1].clone();
14073 let venue = VenueOrderId::from("281477004");
14074 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
14075 let mut raw = reconciliation_raw_order(index, venue, LighterOrderStatus::CanceledChild);
14076 raw.order_type = LighterOrderKind::StopLossLimit;
14077 raw.initial_base_amount = Decimal::ZERO;
14078 raw.remaining_base_amount = Decimal::ZERO;
14079 raw.parent_order_id = "281477001".to_string();
14080 raw.to_cancel_order_id_0 = "281477005".to_string();
14081 let report = parse_http_order_to_report(
14082 &raw,
14083 &client.registry,
14084 account_id(),
14085 UnixNanos::from(2),
14086 &client.dispatch,
14087 )
14088 .unwrap();
14089
14090 let cmd = GenerateOrderStatusReports::new(
14091 UUID4::new(),
14092 UnixNanos::from(3),
14093 false,
14094 None,
14095 None,
14096 None,
14097 None,
14098 None,
14099 );
14100
14101 let reports = finish_order_reports(
14102 &client.core,
14103 &client.dispatch,
14104 OrderReportSweep {
14105 orders: vec![(index, report)],
14106 ..Default::default()
14107 },
14108 &cmd,
14109 Some(&[]),
14110 )
14111 .unwrap();
14112
14113 assert_eq!(reports.len(), 1);
14114 assert_eq!(reports[0].venue_order_id, venue);
14115 assert_eq!(reports[0].order_status, OrderStatus::Canceled);
14116 assert_eq!(
14117 client
14118 .dispatch
14119 .resolve_client_order_index_for_venue(index, venue),
14120 Some(child.client_order_id()),
14121 );
14122
14123 if index != 0 {
14124 assert_eq!(reports[0].quantity, child.quantity());
14125 assert_eq!(reports[0].client_order_id, Some(child.client_order_id()));
14126 assert_eq!(
14127 reports[0].parent_order_id,
14128 Some(orders[0].client_order_id())
14129 );
14130 assert_eq!(
14131 reports[0].linked_order_ids,
14132 Some(vec![orders[2].client_order_id()])
14133 );
14134 }
14135 }
14136
14137 #[tokio::test]
14138 async fn native_group_canceled_children_do_not_reappear_after_restart() {
14139 let (client, cache, mut rx) = create_execution_client();
14140 let instrument_id = register_test_instrument(&client, &cache);
14141 let orders = test_native_group(
14142 instrument_id,
14143 LighterGroupingType::OneTriggersOneCancelsTheOther,
14144 );
14145 let venues = [
14146 VenueOrderId::from("281477001"),
14147 VenueOrderId::from("281477002"),
14148 VenueOrderId::from("281477003"),
14149 ];
14150 let history = native_group_canceled_orders(&orders, &venues);
14151 for (i, (order, raw)) in orders.iter().zip(&history).enumerate() {
14152 cache_accepted_order(&cache, order.clone(), venues[i], Some(client_id()));
14153 let cached = cache
14154 .borrow()
14155 .order_owned(&order.client_order_id())
14156 .unwrap();
14157 client
14158 .dispatch
14159 .restore_reconciled_order(&cached, raw.client_order_index, venues[i], false)
14160 .unwrap();
14161
14162 for _ in 0..2 {
14163 dispatch_lighter_order(
14164 raw,
14165 &client.dispatch,
14166 &client.emitter,
14167 &client.registry,
14168 account_id(),
14169 trader_id(),
14170 UnixNanos::from(2),
14171 );
14172 }
14173 }
14174
14175 let events = drain_events(&mut rx);
14176 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
14177 let mut engine = ExecutionEngine::new(clock, cache.clone(), None);
14178 let applied = Rc::new(RefCell::new(Vec::new()));
14179 let captured = applied.clone();
14180 let handler = TypedHandler::from(move |event: &OrderEventAny| {
14181 captured.borrow_mut().push(event.clone());
14182 });
14183
14184 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
14185 let subscription = AppliedFillSubscription(handler);
14186
14187 for event in events {
14188 match event {
14189 ExecutionEvent::Order(event) => engine.process(&event),
14190 ExecutionEvent::Report(report) => engine.reconcile_execution_report(&report),
14191 other => panic!("unexpected cancellation event: {other:?}"),
14192 }
14193 }
14194
14195 let applied = applied.take();
14196 assert_eq!(applied.len(), 3, "applied cancellation events: {applied:?}");
14197
14198 for (event, order) in applied.iter().zip(&orders) {
14199 assert_eq!(event.client_order_id(), order.client_order_id());
14200 assert!(matches!(event, OrderEventAny::Canceled(_)));
14201 assert_eq!(
14202 cache
14203 .borrow()
14204 .order(&order.client_order_id())
14205 .unwrap()
14206 .status(),
14207 OrderStatus::Canceled
14208 );
14209 }
14210
14211 drop(subscription);
14212
14213 let (mut fresh, fresh_cache, mut fresh_rx) = create_execution_client();
14214 register_test_instrument(&fresh, &fresh_cache);
14215 fresh.dispatch.note_active_market(TEST_MARKET_INDEX);
14216 fresh.dispatch.replace_position_snapshot(&[], &[], &[]);
14217
14218 for raw in &history[1..] {
14219 dispatch_lighter_order(
14220 raw,
14221 &fresh.dispatch,
14222 &fresh.emitter,
14223 &fresh.registry,
14224 account_id(),
14225 trader_id(),
14226 UnixNanos::from(3),
14227 );
14228 }
14229
14230 let app = Router::new()
14231 .route(
14232 "/api/v1/accountActiveOrders",
14233 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
14234 )
14235 .route(
14236 "/api/v1/accountInactiveOrders",
14237 get(move || {
14238 let history = history.clone();
14239 async move { axum::Json(serde_json::json!({"code":200,"orders":history})) }
14240 }),
14241 )
14242 .route(
14243 "/api/v1/trades",
14244 get(|| async { axum::Json(serde_json::json!({"code":200,"trades":[]})) }),
14245 );
14246
14247 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
14248 let addr = listener.local_addr().unwrap();
14249 let server = tokio::spawn(async move {
14250 axum::serve(listener, app).await.unwrap();
14251 });
14252
14253 fresh.http_client = LighterHttpClient::new(
14254 LighterEnvironment::Testnet,
14255 Some(format!("http://{addr}")),
14256 1,
14257 None,
14258 )
14259 .unwrap();
14260 let first = fresh.generate_mass_status(None).await.unwrap().unwrap();
14261 let second = fresh.generate_mass_status(None).await.unwrap().unwrap();
14262
14263 let parent_cmd = GenerateOrderStatusReport::new(
14264 UUID4::new(),
14265 UnixNanos::from(4),
14266 Some(instrument_id),
14267 None,
14268 Some(venues[0]),
14269 None,
14270 None,
14271 );
14272 let parent_report = fresh
14273 .generate_order_status_report(&parent_cmd)
14274 .await
14275 .unwrap()
14276 .unwrap();
14277
14278 for venue in &venues[1..] {
14279 let cmd = GenerateOrderStatusReport::new(
14280 UUID4::new(),
14281 UnixNanos::from(4),
14282 Some(instrument_id),
14283 None,
14284 Some(*venue),
14285 None,
14286 None,
14287 );
14288 assert_eq!(
14289 fresh.generate_order_status_report(&cmd).await.unwrap(),
14290 None
14291 );
14292 let task = fresh.generate_order_status_report_task(&cmd).unwrap();
14293 task.collection.await;
14294 assert_eq!((task.finish)().unwrap(), None);
14295 fresh
14296 .query_order(QueryOrder::new(
14297 trader_id(),
14298 Some(client_id()),
14299 strategy_id(),
14300 instrument_id,
14301 ClientOrderId::from(venue.as_str()),
14302 Some(*venue),
14303 UUID4::new(),
14304 UnixNanos::from(5),
14305 None,
14306 None,
14307 ))
14308 .unwrap();
14309 }
14310
14311 fresh.pending_tasks.begin_shutdown();
14312 fresh
14313 .pending_tasks
14314 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
14315 .await
14316 .unwrap();
14317 server.abort();
14318 let _ = server.await;
14319 let mut expected_parent = first.order_reports()[&venues[0]].clone();
14320 expected_parent.report_id = parent_report.report_id;
14321 expected_parent.ts_init = parent_report.ts_init;
14322 assert_eq!(parent_report, expected_parent);
14323
14324 for mass in [&first, &second] {
14325 assert!(mass.reports_complete());
14326 assert_eq!(mass.order_reports().len(), 1);
14327 let parent = &mass.order_reports()[&venues[0]];
14328 assert_eq!(parent.order_status, OrderStatus::Canceled);
14329 assert_eq!(parent.quantity, orders[0].quantity());
14330 assert_eq!(parent.filled_qty, Quantity::from("0.0000"));
14331 assert_eq!(
14332 parent.linked_order_ids,
14333 Some(vec![
14334 ClientOrderId::from("281477002"),
14335 ClientOrderId::from("281477003")
14336 ])
14337 );
14338 assert!(mass.fill_reports().is_empty());
14339 assert!(mass.position_reports().is_empty());
14340 }
14341
14342 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
14343 let mut engine = ExecutionEngine::new(clock.clone(), fresh_cache.clone(), None);
14344 engine.register_client(Box::new(fresh)).unwrap();
14345 let engine = Rc::new(RefCell::new(engine));
14346 let mut manager = ExecutionManager::new(
14347 clock,
14348 fresh_cache.clone(),
14349 ExecutionManagerConfig::default(),
14350 )
14351 .unwrap();
14352 let applied = Rc::new(RefCell::new(Vec::new()));
14353 let captured = applied.clone();
14354
14355 let handler = TypedHandler::from(move |event: &OrderEventAny| {
14356 if matches!(event, OrderEventAny::Canceled(_) | OrderEventAny::Filled(_)) {
14357 captured.borrow_mut().push(event.clone());
14358 }
14359 });
14360
14361 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
14362 let subscription = AppliedFillSubscription(handler);
14363 manager.reconcile_execution_mass_status(&first, &engine);
14364 manager.reconcile_execution_mass_status(&second, &engine);
14365 drop(subscription);
14366 let applied = applied.take();
14367 assert_eq!(applied.len(), 1, "fresh reconciliation events: {applied:?}");
14368
14369 let OrderEventAny::Canceled(canceled) = &applied[0] else {
14370 panic!("expected the parent cancellation");
14371 };
14372
14373 assert_eq!(
14374 canceled.client_order_id,
14375 ClientOrderId::from(venues[0].as_str())
14376 );
14377 assert_eq!(canceled.venue_order_id, Some(venues[0]));
14378 assert_eq!(
14379 fresh_cache
14380 .borrow()
14381 .orders_total_count(None, None, None, None, None),
14382 1
14383 );
14384 assert!(drain_events(&mut fresh_rx).is_empty());
14385 }
14386
14387 fn native_group_canceled_orders(
14388 orders: &[OrderAny],
14389 venues: &[VenueOrderId; 3],
14390 ) -> Vec<LighterOrder> {
14391 let mut history = Vec::new();
14392
14393 for (i, order) in orders.iter().enumerate() {
14394 let mut raw = reconciliation_raw_order(
14395 42 + i as i64,
14396 venues[i],
14397 LighterOrderStatus::CanceledChild,
14398 );
14399
14400 raw.initial_base_amount = Decimal::ZERO;
14401 raw.remaining_base_amount = Decimal::ZERO;
14402 raw.price = order.price().unwrap().as_decimal();
14403 raw.trigger_price = order
14404 .trigger_price()
14405 .map_or(Decimal::ZERO, |price| price.as_decimal());
14406 raw.is_ask = i > 0;
14407 raw.side = Some(if i == 0 {
14408 LighterOrderSide::Buy
14409 } else {
14410 LighterOrderSide::Sell
14411 });
14412
14413 raw.reduce_only = i > 0;
14414 if i == 0 {
14415 raw.status = LighterOrderStatus::Canceled;
14416 raw.initial_base_amount = order.quantity().as_decimal();
14417 raw.to_trigger_order_id_0 = venues[1].to_string();
14418 raw.to_trigger_order_id_1 = venues[2].to_string();
14419 } else {
14420 raw.order_type = if i == 1 {
14421 LighterOrderKind::StopLossLimit
14422 } else {
14423 LighterOrderKind::TakeProfitLimit
14424 };
14425
14426 raw.parent_order_id = venues[0].to_string();
14427 raw.to_cancel_order_id_0 = venues[3 - i].to_string();
14428 }
14429
14430 history.push(raw);
14431 }
14432
14433 history
14434 }
14435
14436 #[rstest]
14437 fn native_group_restores_held_child_before_first_stream_frame() {
14438 let (client, cache, mut rx) = create_execution_client();
14439 let instrument = register_test_instrument(&client, &cache);
14440 let orders = test_native_group(
14441 instrument,
14442 LighterGroupingType::OneTriggersOneCancelsTheOther,
14443 );
14444 let child = orders[1].clone();
14445 let venue = VenueOrderId::from("281477004");
14446 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
14447 let cached = cache
14448 .borrow()
14449 .order_owned(&child.client_order_id())
14450 .unwrap();
14451 client.dispatch.seed_order_restorations(vec![cached]);
14452 let (_, index) = forced_probed_index(child.client_order_id());
14453 let mut raw = reconciliation_raw_order(index, venue, LighterOrderStatus::Pending);
14454 raw.order_type = LighterOrderKind::StopLossLimit;
14455 raw.initial_base_amount = Decimal::ZERO;
14456 raw.filled_base_amount = Decimal::ZERO;
14457 raw.is_ask = true;
14458 raw.side = Some(LighterOrderSide::Sell);
14459 raw.reduce_only = true;
14460 raw.price = Decimal::from(2190);
14461 raw.trigger_price = Decimal::from(2200);
14462 dispatch_lighter_order(
14463 &raw,
14464 &client.dispatch,
14465 &client.emitter,
14466 &client.registry,
14467 account_id(),
14468 trader_id(),
14469 UnixNanos::from(1),
14470 );
14471 let events = drain_events(&mut rx);
14472 let report = parse_http_order_to_report(
14473 &raw,
14474 &client.registry,
14475 account_id(),
14476 UnixNanos::from(2),
14477 &client.dispatch,
14478 )
14479 .unwrap();
14480 let report = client.dispatch.translate_order_cloid(report);
14481 assert!(
14482 events.is_empty(),
14483 "restored pending frame must be a tracked replay: {events:?}"
14484 );
14485 assert_eq!(
14486 cache
14487 .borrow()
14488 .order(&child.client_order_id())
14489 .unwrap()
14490 .quantity(),
14491 Quantity::from("0.1000")
14492 );
14493 assert_eq!(
14494 client.dispatch.client_order_index(&child.client_order_id()),
14495 Some(index)
14496 );
14497 assert_eq!(report.client_order_id, Some(child.client_order_id()));
14498 assert_eq!(report.quantity, Quantity::from("0.1000"));
14499 assert_eq!(report.parent_order_id, Some(orders[0].client_order_id()));
14500 assert_eq!(
14501 report.linked_order_ids,
14502 Some(vec![orders[2].client_order_id()])
14503 );
14504 assert_eq!(report.contingency_type, Some(ContingencyType::Oco));
14505 assert_eq!(report.order_type, OrderType::StopLimit);
14506 assert_eq!(report.trigger_price, Some(Price::from("2200.00")));
14507 assert_eq!(report.price, Some(Price::from("2190.00")));
14508 assert_eq!(report.order_side, Some(OrderSide::Sell));
14509 assert_eq!(report.order_status, OrderStatus::Accepted);
14510 assert!(report.reduce_only);
14511 }
14512
14513 #[rstest]
14514 #[case(true)]
14515 #[case(false)]
14516 #[tokio::test]
14517 async fn native_oco_cancel_targets_peer_with_venue_or_shared_submission_index(
14518 #[case] acknowledged: bool,
14519 #[values(false, true)] reverse: bool,
14520 ) {
14521 let (client, cache, mut rx) = create_execution_client();
14522 let instrument = register_test_instrument(&client, &cache);
14523 let orders = test_native_group(
14524 instrument,
14525 LighterGroupingType::OneTriggersOneCancelsTheOther,
14526 );
14527 let group = client.prepare_create_group_plan(&orders, None).unwrap();
14528 let context = client
14529 .fanout_dispatch_context(client.credential.as_ref().unwrap())
14530 .unwrap();
14531 let mut signed = context.sign_create_group(&group).unwrap();
14532 client
14533 .dispatch
14534 .enqueue_pending_sendtx(signed.pending.clone());
14535 let first = VenueOrderId::from("281477101");
14536 let second = VenueOrderId::from("281477102");
14537 cache_accepted_order(&cache, orders[1].clone(), first, Some(client_id()));
14538 let first_index = client
14539 .dispatch
14540 .client_order_index(&orders[1].client_order_id())
14541 .unwrap();
14542 let second_index = client
14543 .dispatch
14544 .client_order_index(&orders[2].client_order_id())
14545 .unwrap();
14546 assert!(client.dispatch.observe_create_submission(
14547 &orders[1].client_order_id(),
14548 first_index,
14549 signed.pending.nonce,
14550 first
14551 ));
14552
14553 if acknowledged {
14554 cache_accepted_order(&cache, orders[2].clone(), second, Some(client_id()));
14555 assert!(client.dispatch.observe_create_submission(
14556 &orders[2].client_order_id(),
14557 second_index,
14558 signed.pending.nonce,
14559 second
14560 ));
14561 } else {
14562 cache_order(&cache, orders[2].clone());
14563 cache
14564 .borrow_mut()
14565 .update_order(&TestOrderEventStubs::submitted(&orders[2], account_id()))
14566 .unwrap();
14567 }
14568
14569 let command = CancelOrder::new(
14570 trader_id(),
14571 Some(client_id()),
14572 strategy_id(),
14573 instrument,
14574 orders[1].client_order_id(),
14575 None,
14576 UUID4::new(),
14577 UnixNanos::default(),
14578 None,
14579 None,
14580 );
14581 let plans = client.prepare_oco_cancel_plans(&command).unwrap();
14582 assert_eq!(plans.len(), 2);
14583
14584 for (index, plan) in plans.iter().enumerate() {
14585 assert_eq!(plan.client_order_id, orders[index + 1].client_order_id());
14586 assert_eq!(plan.strategy_id, strategy_id());
14587 assert_eq!(plan.instrument_id, instrument);
14588 assert_eq!(plan.market_index, TEST_MARKET_INDEX);
14589 }
14590
14591 assert_eq!(plans[0].command_id, Some(command.command_id));
14592 assert_eq!(plans[1].command_id, None);
14593 assert_eq!(plans[0].venue_order_id, Some(first));
14594 assert_eq!(plans[0].venue_index, 281_477_101);
14595 assert_eq!(plans[1].venue_order_id, acknowledged.then_some(second));
14596 assert_eq!(
14597 plans[1].venue_index,
14598 if acknowledged {
14599 281_477_102
14600 } else {
14601 second_index
14602 }
14603 );
14604
14605 for plan in plans {
14606 context.reject_batch_plan(&BatchOrderPlan::Cancel(plan, false), "signing failed");
14607 }
14608
14609 let OrderEventAny::CancelDenied(denied) = recv_order_event(&mut rx).await else {
14610 panic!("expected cancel denial")
14611 };
14612
14613 assert_eq!(denied.client_order_id, command.client_order_id);
14614 assert_eq!(denied.causation_id, Some(command.command_id));
14615 assert!(rx.try_recv().is_err());
14616
14617 let mut peer_command = command.clone();
14618 peer_command.client_order_id = orders[2].client_order_id();
14619 peer_command.command_id = UUID4::new();
14620 let mut cancels = vec![command.clone(), peer_command.clone()];
14621 if reverse {
14622 cancels.reverse();
14623 }
14624
14625 let batch = BatchCancelOrders::new(
14626 trader_id(),
14627 Some(client_id()),
14628 strategy_id(),
14629 instrument,
14630 cancels,
14631 UUID4::new(),
14632 UnixNanos::default(),
14633 None,
14634 None,
14635 );
14636 client.dispatch.remove_pending_sendtx_by_hash(
14637 signed.pending.connection_epoch,
14638 &signed.pending.tx_hash,
14639 );
14640 signed.send_reservation.release();
14641 client.batch_cancel_orders(batch).unwrap();
14642 let mut denied_ids = AHashSet::new();
14643
14644 for _ in 0..2 {
14645 let OrderEventAny::CancelDenied(denied) = recv_order_event(&mut rx).await else {
14646 panic!("expected cancel denial")
14647 };
14648
14649 let expected = if denied.client_order_id == command.client_order_id {
14650 command.command_id
14651 } else {
14652 assert_eq!(denied.client_order_id, peer_command.client_order_id);
14653 peer_command.command_id
14654 };
14655
14656 assert!(denied_ids.insert(denied.client_order_id));
14657 assert_eq!(denied.causation_id, Some(expected));
14658 }
14659
14660 assert_eq!(
14661 denied_ids,
14662 AHashSet::from_iter([command.client_order_id, peer_command.client_order_id])
14663 );
14664 assert!(rx.try_recv().is_err());
14665 let mut override_venue = command.clone();
14666 override_venue.venue_order_id = Some(VenueOrderId::from("281477103"));
14667 assert!(client.prepare_oco_cancel_plans(&override_venue).is_none());
14668
14669 if !acknowledged {
14670 client
14671 .dispatch
14672 .mark_order_submission(&orders[2].client_order_id(), signed.pending.nonce + 1);
14673 assert!(client.prepare_oco_cancel_plans(&command).is_none());
14674 }
14675 }
14676
14677 #[rstest]
14678 fn native_group_denies_held_child_modification() {
14679 let (client, cache, _rx) = create_execution_client();
14680 let instrument = register_test_instrument(&client, &cache);
14681 let orders = test_native_group(instrument, LighterGroupingType::OneTriggersTheOther);
14682 let child = orders[1].clone();
14683 let venue = VenueOrderId::from("281477004");
14684 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
14685 client.dispatch.store_snapshot(
14686 child.client_order_id(),
14687 OrderShapeSnapshot {
14688 quantity: Quantity::from("0.0000"),
14689 price: child.price(),
14690 trigger_price: child.trigger_price(),
14691 },
14692 );
14693
14694 let command = ModifyOrder::new(
14695 trader_id(),
14696 Some(client_id()),
14697 strategy_id(),
14698 instrument,
14699 child.client_order_id(),
14700 Some(venue),
14701 None,
14702 Some(Price::from("2180.00")),
14703 None,
14704 UUID4::new(),
14705 UnixNanos::default(),
14706 None,
14707 None,
14708 );
14709 let error = client.prepare_modify_order_plan(&command).err().unwrap();
14710 assert_eq!(
14711 error.to_string(),
14712 "Lighter cannot modify an OTO child while it waits for parent execution"
14713 );
14714 }
14715
14716 #[rstest]
14717 fn native_group_modifies_triggered_child_without_rearming_trigger() {
14718 let (client, cache, _rx) = create_execution_client();
14719 let instrument = register_test_instrument(&client, &cache);
14720 let orders = test_native_group(
14721 instrument,
14722 LighterGroupingType::OneTriggersOneCancelsTheOther,
14723 );
14724 let child = orders[2].clone();
14725 let venue = VenueOrderId::from("281477004");
14726 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
14727 client
14728 .dispatch
14729 .mark_triggered_emitted(child.client_order_id());
14730
14731 let command = ModifyOrder::new(
14732 trader_id(),
14733 Some(client_id()),
14734 strategy_id(),
14735 instrument,
14736 child.client_order_id(),
14737 Some(venue),
14738 None,
14739 Some(Price::from("2380.00")),
14740 None,
14741 UUID4::new(),
14742 UnixNanos::default(),
14743 None,
14744 None,
14745 );
14746 let plan = client.prepare_modify_order_plan(&command).unwrap();
14747 let mut rearm = command;
14748 rearm.trigger_price = Some(Price::from("2450.00"));
14749 let error = client.prepare_modify_order_plan(&rearm).err().unwrap();
14750 assert_eq!(plan.trigger_price, 0);
14751 assert_eq!(plan.price, 238_000);
14752 assert_eq!(plan.base_amount, 1_000);
14753 assert_eq!(
14754 error.to_string(),
14755 "Lighter cannot modify a trigger price after the order has triggered"
14756 );
14757 }
14758
14759 fn test_native_group(
14760 instrument_id: InstrumentId,
14761 grouping: LighterGroupingType,
14762 ) -> Vec<OrderAny> {
14763 let has_parent = grouping != LighterGroupingType::OneCancelsTheOther;
14764 let ids = [
14765 ClientOrderId::from("O-GROUP-ENTRY"),
14766 ClientOrderId::from("O-GROUP-SL"),
14767 ClientOrderId::from("O-GROUP-TP"),
14768 ];
14769 let mut orders = Vec::new();
14770 if has_parent {
14771 orders.push(
14772 OrderTestBuilder::new(OrderType::Limit)
14773 .trader_id(trader_id())
14774 .strategy_id(strategy_id())
14775 .instrument_id(instrument_id)
14776 .client_order_id(ids[0])
14777 .side(OrderSide::Buy)
14778 .quantity(Quantity::from("0.1000"))
14779 .price(Price::from("2300.00"))
14780 .order_list_id(OrderListId::from("OL-GROUP"))
14781 .contingency_type(ContingencyType::Oto)
14782 .linked_order_ids(if grouping == LighterGroupingType::OneTriggersTheOther {
14783 vec![ids[1]]
14784 } else {
14785 vec![ids[1], ids[2]]
14786 })
14787 .build(),
14788 );
14789 }
14790
14791 let count = if grouping == LighterGroupingType::OneTriggersTheOther {
14792 1
14793 } else {
14794 2
14795 };
14796
14797 let children = [
14798 (
14799 OrderType::StopLimit,
14800 Price::from("2190.00"),
14801 Price::from("2200.00"),
14802 ),
14803 (
14804 OrderType::LimitIfTouched,
14805 Price::from("2390.00"),
14806 Price::from("2400.00"),
14807 ),
14808 ];
14809
14810 for (index, (kind, price, trigger)) in children.into_iter().take(count).enumerate() {
14811 let mut builder = OrderTestBuilder::new(kind);
14812 builder
14813 .trader_id(trader_id())
14814 .strategy_id(strategy_id())
14815 .instrument_id(instrument_id)
14816 .client_order_id(ids[index + 1])
14817 .side(OrderSide::Sell)
14818 .quantity(Quantity::from("0.1000"))
14819 .price(price)
14820 .trigger_price(trigger)
14821 .reduce_only(true)
14822 .order_list_id(OrderListId::from("OL-GROUP"));
14823
14824 if has_parent {
14825 builder.parent_order_id(ids[0]);
14826 }
14827
14828 if count == 2 {
14829 builder
14830 .contingency_type(ContingencyType::Oco)
14831 .linked_order_ids(vec![ids[2 - index]]);
14832 }
14833
14834 orders.push(builder.build());
14835 }
14836
14837 orders
14838 }
14839
14840 #[tokio::test]
14841 async fn submit_order_list_plain_limit_oco_denies_all_without_dispatch() {
14842 let (client, cache, mut rx) = create_execution_client();
14843 let instrument_id = register_test_instrument(&client, &cache);
14844 let order_a_id = "O-LIST-OCO-A";
14845 let order_b_id = "O-LIST-OCO-B";
14846 let order_a = test_contingent_limit_order(instrument_id, order_a_id, "OL-OCO", order_b_id);
14847 let order_b = test_contingent_limit_order(instrument_id, order_b_id, "OL-OCO", order_a_id);
14848 cache_order(&cache, order_a.clone());
14849 cache_order(&cache, order_b.clone());
14850
14851 let command = submit_order_list_command(&[order_a.clone(), order_b.clone()], "OL-OCO");
14852 client.submit_order_list(command).unwrap();
14853
14854 for order in [&order_a, &order_b] {
14855 match recv_order_event(&mut rx).await {
14856 OrderEventAny::Denied(e) => {
14857 assert_eq!(e.client_order_id, order.client_order_id());
14858 assert!(e.reason.contains("reduce-only SL/TP orders"));
14859 }
14860 other => panic!("expected Denied, was {other:?}"),
14861 }
14862 }
14863
14864 assert!(
14865 tokio::time::timeout(Duration::from_millis(50), rx.recv())
14866 .await
14867 .is_err(),
14868 "grouped list denial must not emit extra events",
14869 );
14870 assert_nonce_reusable(&client.dispatch);
14871 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14872 }
14873
14874 #[tokio::test]
14875 async fn cancel_order_send_failure_emits_cancel_denied_and_rolls_back() {
14876 let (client, cache, mut rx) = create_execution_client();
14877 let instrument_id = register_test_instrument(&client, &cache);
14878 let mut factory = test_order_factory();
14879 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-FAIL");
14880 let client_order_id = order.client_order_id();
14881 let venue_order_id = VenueOrderId::from("123");
14882 cache_pending_cancel_order(&cache, order, venue_order_id, None);
14883
14884 let command = CancelOrder::new(
14885 trader_id(),
14886 Some(client_id()),
14887 strategy_id(),
14888 instrument_id,
14889 client_order_id,
14890 Some(venue_order_id),
14891 UUID4::new(),
14892 UnixNanos::default(),
14893 None,
14894 None,
14895 );
14896 client.cancel_order(command).unwrap();
14897
14898 let rejected = recv_order_event(&mut rx).await;
14899
14900 match rejected {
14901 OrderEventAny::CancelDenied(event) => {
14902 assert_eq!(event.client_order_id, client_order_id);
14903 assert_eq!(event.instrument_id, instrument_id);
14904 assert_eq!(event.venue_order_id, Some(venue_order_id));
14905 assert!(
14906 event
14907 .reason
14908 .contains("Lighter cancel_order dispatch failed"),
14909 );
14910 assert!(event.reason.contains("handler unavailable"));
14911 }
14912 event => panic!("expected cancel rejected event, was {event:?}"),
14913 }
14914
14915 assert_nonce_reusable(&client.dispatch);
14916 assert_eq!(
14917 client.dispatch.pending_sendtx_len(),
14918 0,
14919 "local-send-failure must remove the pending cancel entry",
14920 );
14921 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
14922 }
14923
14924 #[tokio::test]
14925 async fn cancel_order_unknown_send_outcome_retains_pending_state() {
14926 let (client, cache, mut rx) = create_execution_client();
14927 let instrument_id = register_test_instrument(&client, &cache);
14928 let mut factory = test_order_factory();
14929 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-UNKNOWN");
14930 let client_order_id = order.client_order_id();
14931 let venue_order_id = VenueOrderId::from("123");
14932 cache_pending_cancel_order(&cache, order, venue_order_id, None);
14933 client.ws_client.drop_next_send_tx_result_for_test().await;
14934
14935 client
14936 .cancel_order(CancelOrder::new(
14937 trader_id(),
14938 Some(client_id()),
14939 strategy_id(),
14940 instrument_id,
14941 client_order_id,
14942 Some(venue_order_id),
14943 UUID4::new(),
14944 UnixNanos::default(),
14945 None,
14946 None,
14947 ))
14948 .unwrap();
14949
14950 assert!(
14951 tokio::time::timeout(Duration::from_millis(50), rx.recv())
14952 .await
14953 .is_err(),
14954 "unknown outcome must not emit OrderCancelRejected",
14955 );
14956 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
14957 assert_eq!(
14958 client.dispatch.pending_order_action(&client_order_id),
14959 Some(PendingOrderAction::Cancel),
14960 );
14961 assert_eq!(
14962 client
14963 .dispatch
14964 .nonce_manager
14965 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
14966 Some(TEST_NEXT_NONCE),
14967 );
14968 }
14969
14970 #[tokio::test]
14971 async fn cancel_order_prepare_failure_emits_cancel_denied_without_dispatch() {
14972 let (client, cache, mut rx) = create_execution_client();
14973 let instrument_id = register_test_instrument(&client, &cache);
14974 let mut factory = test_order_factory();
14975 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-NO-VOI");
14976 let client_order_id = order.client_order_id();
14977 cache_pending_cancel_order(&cache, order, VenueOrderId::from("123"), None);
14978
14979 let command = CancelOrder::new(
14980 trader_id(),
14981 Some(client_id()),
14982 strategy_id(),
14983 instrument_id,
14984 client_order_id,
14985 None,
14986 UUID4::new(),
14987 UnixNanos::default(),
14988 None,
14989 None,
14990 );
14991 client.cancel_order(command).unwrap();
14992
14993 let rejected = recv_order_event(&mut rx).await;
14994 match rejected {
14995 OrderEventAny::CancelDenied(event) => {
14996 assert_eq!(event.client_order_id, client_order_id);
14997 assert_eq!(event.instrument_id, instrument_id);
14998 assert_eq!(event.venue_order_id, None);
14999 assert!(event.reason.contains("Lighter cancel_order failed"));
15000 assert!(event.reason.contains("venue order_id not yet known"));
15001 }
15002 event => panic!("expected cancel rejected event, was {event:?}"),
15003 }
15004
15005 assert_nonce_reusable(&client.dispatch);
15006 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15007 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
15008 assert!(
15009 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15010 .await
15011 .is_err(),
15012 "prepare failure must emit exactly one cancel rejection",
15013 );
15014 }
15015
15016 #[tokio::test]
15017 async fn cancel_order_prepare_failure_emits_cancel_denied_when_order_uncached() {
15018 let (client, cache, mut rx) = create_execution_client();
15019 let instrument_id = register_test_instrument(&client, &cache);
15020 let client_order_id = ClientOrderId::from("O-CANCEL-NO-CACHE");
15021 let venue_order_id = VenueOrderId::from("123");
15022
15023 let command = CancelOrder::new(
15024 trader_id(),
15025 Some(client_id()),
15026 strategy_id(),
15027 instrument_id,
15028 client_order_id,
15029 Some(venue_order_id),
15030 UUID4::new(),
15031 UnixNanos::default(),
15032 None,
15033 None,
15034 );
15035 client.cancel_order(command).unwrap();
15036
15037 let rejected = recv_order_event(&mut rx).await;
15038 match rejected {
15039 OrderEventAny::CancelDenied(event) => {
15040 assert_eq!(event.client_order_id, client_order_id);
15041 assert_eq!(event.instrument_id, instrument_id);
15042 assert_eq!(event.venue_order_id, Some(venue_order_id));
15043 assert!(event.reason.contains("Lighter cancel_order failed"));
15044 assert!(event.reason.contains("order not found in cache"));
15045 }
15046 event => panic!("expected cancel rejected event, was {event:?}"),
15047 }
15048
15049 assert_nonce_reusable(&client.dispatch);
15050 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15051 assert!(
15052 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15053 .await
15054 .is_err(),
15055 "prepare failure must emit exactly one cancel rejection",
15056 );
15057 }
15058
15059 #[rstest]
15060 #[case::buy(Some(OrderSide::Buy), false, vec![0, 2])]
15061 #[case::sell(Some(OrderSide::Sell), false, vec![1, 3])]
15062 #[case::unsided(None, false, vec![0, 1, 2, 3])]
15063 #[case::empty_buy(Some(OrderSide::Buy), true, vec![])]
15064 #[case::empty_sell(Some(OrderSide::Sell), true, vec![])]
15065 #[case::empty_unsided(None, true, vec![])]
15066 #[tokio::test]
15067 async fn cancel_all_orders_filters_orders_and_preserves_owners(
15068 #[case] order_side: Option<OrderSide>,
15069 #[case] empty: bool,
15070 #[case] expected_indices: Vec<usize>,
15071 ) {
15072 let (client, cache, mut rx) = create_execution_client();
15073 let instrument_id = register_test_instrument(&client, &cache);
15074 let other_instrument_id = InstrumentId::from("BTC-PERP.LIGHTER");
15075 let selected_account = account_id();
15076 let other_account = AccountId::from("LIGHTER-002");
15077 let mut orders = Vec::new();
15078
15079 for (index, (instrument, owner, side, open, account)) in [
15080 (
15081 instrument_id,
15082 "S-001",
15083 OrderSide::Buy,
15084 true,
15085 selected_account,
15086 ),
15087 (
15088 instrument_id,
15089 "S-001",
15090 OrderSide::Sell,
15091 true,
15092 selected_account,
15093 ),
15094 (
15095 instrument_id,
15096 "S-002",
15097 OrderSide::Buy,
15098 true,
15099 selected_account,
15100 ),
15101 (
15102 instrument_id,
15103 "S-002",
15104 OrderSide::Sell,
15105 true,
15106 selected_account,
15107 ),
15108 (
15109 other_instrument_id,
15110 "S-001",
15111 OrderSide::Buy,
15112 true,
15113 selected_account,
15114 ),
15115 (
15116 other_instrument_id,
15117 "S-002",
15118 OrderSide::Sell,
15119 true,
15120 selected_account,
15121 ),
15122 (
15123 instrument_id,
15124 "S-001",
15125 OrderSide::Buy,
15126 false,
15127 selected_account,
15128 ),
15129 (
15130 instrument_id,
15131 "S-002",
15132 OrderSide::Sell,
15133 false,
15134 selected_account,
15135 ),
15136 (instrument_id, "S-001", OrderSide::Buy, true, other_account),
15137 (instrument_id, "S-002", OrderSide::Sell, true, other_account),
15138 ]
15139 .into_iter()
15140 .enumerate()
15141 {
15142 if empty
15143 && account == selected_account
15144 && instrument == instrument_id
15145 && open
15146 && order_side.is_none_or(|filter| filter == side)
15147 {
15148 continue;
15149 }
15150
15151 let order = OrderTestBuilder::new(OrderType::Limit)
15152 .trader_id(trader_id())
15153 .strategy_id(StrategyId::from(owner))
15154 .instrument_id(instrument)
15155 .client_order_id(ClientOrderId::from(format!("O-CANCEL-ALL-{index}")))
15156 .side(side)
15157 .quantity(Quantity::from("0.1000"))
15158 .price(Price::from("2361.31"))
15159 .build();
15160 let venue_order_id = VenueOrderId::from(format!("{}", 123 + index));
15161 let accepted = TestOrderEventStubs::accepted(&order, account, venue_order_id);
15162 cache_order(&cache, order.clone());
15163 cache.borrow_mut().update_order(&accepted).unwrap();
15164 client
15165 .dispatch
15166 .venue_id_map
15167 .insert(order.client_order_id(), venue_order_id);
15168
15169 let event = if open {
15170 OrderEventAny::PendingCancel(OrderPendingCancel::new(
15171 order.trader_id(),
15172 order.strategy_id(),
15173 instrument,
15174 order.client_order_id(),
15175 Some(account),
15176 UUID4::new(),
15177 UnixNanos::default(),
15178 UnixNanos::default(),
15179 false,
15180 Some(venue_order_id),
15181 ))
15182 } else {
15183 TestOrderEventStubs::canceled(&order, account, Some(venue_order_id))
15184 };
15185
15186 cache.borrow_mut().update_order(&event).unwrap();
15187 orders.push((order, venue_order_id));
15188 }
15189
15190 client
15191 .cancel_all_orders(CancelAllOrders::new(
15192 trader_id(),
15193 Some(client_id()),
15194 strategy_id(),
15195 instrument_id,
15196 order_side,
15197 UUID4::new(),
15198 UnixNanos::default(),
15199 None,
15200 None,
15201 ))
15202 .unwrap();
15203
15204 let mut actual = Vec::new();
15207
15208 for _ in &expected_indices {
15209 match recv_order_event(&mut rx).await {
15210 OrderEventAny::CancelDenied(event) => {
15211 assert!(
15212 event
15213 .reason
15214 .contains("Lighter cancel_order dispatch failed")
15215 );
15216 assert!(event.reason.contains("handler unavailable"));
15217 assert_eq!(event.trader_id, trader_id());
15218 assert_eq!(event.account_id, Some(account_id()));
15219 actual.push((
15220 event.client_order_id,
15221 event.strategy_id,
15222 event.instrument_id,
15223 event.venue_order_id,
15224 ));
15225 }
15226 event => panic!("expected cancel rejected event, was {event:?}"),
15227 }
15228 }
15229
15230 let mut expected: Vec<_> = expected_indices
15231 .iter()
15232 .map(|&index| {
15233 let (order, venue_order_id) = &orders[index];
15234 (
15235 order.client_order_id(),
15236 order.strategy_id(),
15237 order.instrument_id(),
15238 Some(*venue_order_id),
15239 )
15240 })
15241 .collect();
15242
15243 actual.sort();
15244 expected.sort();
15245
15246 assert_eq!(actual, expected);
15247 assert!(
15248 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15249 .await
15250 .is_err()
15251 );
15252 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15253
15254 for (order, _) in orders {
15255 assert_eq!(
15256 client
15257 .dispatch
15258 .pending_order_action(&order.client_order_id()),
15259 None
15260 );
15261 }
15262
15263 if empty {
15264 assert_nonce_reusable(&client.dispatch);
15265 }
15266 }
15267
15268 #[tokio::test]
15269 async fn cancel_all_orders_prepare_failure_denies_open_order() {
15270 let (client, cache, mut rx) = create_execution_client();
15271 let instrument_id = register_test_instrument(&client, &cache);
15272 let mut factory = test_order_factory();
15273 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-ALL-NO-VOI");
15274 let client_order_id = order.client_order_id();
15275 cache_accepted_order(&cache, order, VenueOrderId::from("123"), Some(client_id()));
15276
15277 let command = CancelAllOrders::new(
15278 trader_id(),
15279 Some(client_id()),
15280 strategy_id(),
15281 instrument_id,
15282 Some(OrderSide::Buy),
15283 UUID4::new(),
15284 UnixNanos::default(),
15285 None,
15286 None,
15287 );
15288 client.cancel_all_orders(command).unwrap();
15289
15290 let OrderEventAny::CancelDenied(denied) = recv_order_event(&mut rx).await else {
15291 panic!("Expected cancel denial");
15292 };
15293
15294 assert_eq!(denied.client_order_id, client_order_id);
15295 assert_eq!(denied.instrument_id, instrument_id);
15296 assert_eq!(
15297 cache.borrow().order(&client_order_id).unwrap().status(),
15298 OrderStatus::Accepted
15299 );
15300
15301 assert_nonce_reusable(&client.dispatch);
15302 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15303 assert!(
15304 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15305 .await
15306 .is_err(),
15307 "cancel-all prepare failure must not emit an invalid cancel rejection",
15308 );
15309 }
15310
15311 #[tokio::test]
15312 async fn cancel_order_nonce_prepare_failure_emits_cancel_denied() {
15313 let mut config = test_config();
15314 config.base_url_http = Some(spawn_next_nonce_server(100).await);
15315 let (client, cache, mut rx) = create_execution_client_with_config(config);
15316 let instrument_id = register_test_instrument(&client, &cache);
15317 let mut factory = test_order_factory();
15318 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-NONCE-FAIL");
15319 let client_order_id = order.client_order_id();
15320 let venue_order_id = VenueOrderId::from("123");
15321 cache_pending_cancel_order(&cache, order, venue_order_id, None);
15322
15323 let window = i64::from(client.dispatch.nonce_manager.skip_window());
15324 for _ in 0..window {
15325 client
15326 .dispatch
15327 .nonce_manager
15328 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15329 .unwrap();
15330 }
15331
15332 let command = CancelOrder::new(
15333 trader_id(),
15334 Some(client_id()),
15335 strategy_id(),
15336 instrument_id,
15337 client_order_id,
15338 Some(venue_order_id),
15339 UUID4::new(),
15340 UnixNanos::default(),
15341 None,
15342 None,
15343 );
15344 client.cancel_order(command).unwrap();
15345
15346 let rejected = recv_order_event(&mut rx).await;
15347 match rejected {
15348 OrderEventAny::CancelDenied(event) => {
15349 assert_eq!(event.client_order_id, client_order_id);
15350 assert_eq!(event.venue_order_id, Some(venue_order_id));
15351 assert!(event.reason.contains("failed to allocate Lighter nonce"),);
15352 assert!(event.reason.contains("skip-window exhausted"));
15353 }
15354 event => panic!("expected cancel rejected event, was {event:?}"),
15355 }
15356
15357 wait_for_spawned_tasks(&client).await;
15358 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15359 assert_eq!(
15360 client
15361 .dispatch
15362 .nonce_manager
15363 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15364 .unwrap(),
15365 100,
15366 );
15367 }
15368
15369 struct CancelProbe {
15370 command: CancelOrder,
15371 identity: (u64, i64),
15372 token: CancellationToken,
15373 status: OrderStatus,
15374 }
15375
15376 impl CancelProbe {
15377 fn cache(
15378 client: &LighterExecutionClient,
15379 cache: &Rc<RefCell<Cache>>,
15380 instrument_id: InstrumentId,
15381 marked: bool,
15382 ) -> Self {
15383 let mut factory = test_order_factory();
15384 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-PROBE");
15385 let client_order_id = order.client_order_id();
15386 let venue_order_id = VenueOrderId::from("123");
15387
15388 let command = CancelOrder::new(
15389 trader_id(),
15390 Some(client_id()),
15391 strategy_id(),
15392 instrument_id,
15393 client_order_id,
15394 Some(venue_order_id),
15395 UUID4::new(),
15396 UnixNanos::default(),
15397 None,
15398 None,
15399 );
15400
15401 if marked {
15402 cache_pending_cancel_order(cache, order, venue_order_id, Some(command.command_id));
15403 } else {
15404 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
15405 }
15406
15407 let epoch = client.ws_client.connection_epoch();
15408 let nonce = client
15409 .dispatch
15410 .nonce_manager
15411 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15412 .unwrap();
15413 client
15414 .dispatch
15415 .nonce_manager
15416 .ack_success(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, nonce)
15417 .unwrap();
15418 client
15419 .dispatch
15420 .register_cancel_command(client_order_id, command.command_id, marked);
15421 client.dispatch.promote_pending_cancel(
15422 &client_order_id,
15423 command.command_id,
15424 epoch,
15425 nonce,
15426 marked,
15427 );
15428 let token = client
15429 .dispatch
15430 .pending_cancel_token(&client_order_id, epoch, nonce)
15431 .unwrap();
15432
15433 Self {
15434 command,
15435 identity: (epoch, nonce),
15436 token,
15437 status: if marked {
15438 OrderStatus::PendingCancel
15439 } else {
15440 OrderStatus::Accepted
15441 },
15442 }
15443 }
15444
15445 fn pending_command(&self, cache: &Rc<RefCell<Cache>>) -> CancelOrder {
15446 let mut command = self.command.clone();
15447 command.command_id = UUID4::new();
15448 apply_pending_cancel(
15449 cache,
15450 command.instrument_id,
15451 command.client_order_id,
15452 command.venue_order_id.unwrap(),
15453 Some(command.command_id),
15454 );
15455 command
15456 }
15457
15458 fn assert_preserved(
15459 &self,
15460 client: &LighterExecutionClient,
15461 cache: &Rc<RefCell<Cache>>,
15462 command: &CancelOrder,
15463 event: &OrderEventAny,
15464 ) {
15465 let OrderEventAny::CancelDenied(denied) = event else {
15466 panic!("Expected local cancel denial, was {event:?}");
15467 };
15468
15469 cache.borrow_mut().update_order(event).unwrap();
15470 let id = self.command.client_order_id;
15471 let cached = cache.borrow().try_order_owned(&id).unwrap();
15472 assert_eq!(denied.causation_id, Some(command.command_id));
15473 assert_eq!(denied.trader_id, command.trader_id);
15474 assert_eq!(denied.strategy_id, command.strategy_id);
15475 assert_eq!(denied.instrument_id, command.instrument_id);
15476 assert_eq!(denied.client_order_id, command.client_order_id);
15477 assert_eq!(denied.venue_order_id, command.venue_order_id);
15478 assert_eq!(cached.status(), self.status);
15479 assert_eq!(cached.quantity(), Quantity::from("0.1000"));
15480 assert_eq!(cached.price(), Some(Price::from("2361.31")));
15481 assert_eq!(cached.venue_order_id(), self.command.venue_order_id);
15482 assert_eq!(
15483 cached.has_pending_cancel_command(),
15484 self.status == OrderStatus::PendingCancel
15485 );
15486 assert_eq!(
15487 client.dispatch.pending_order_action(&id),
15488 Some(PendingOrderAction::cancel(
15489 self.status == OrderStatus::PendingCancel
15490 ))
15491 );
15492 assert_eq!(
15493 client.dispatch.pending_cancel_identity(&id),
15494 Some((self.command.command_id, self.identity))
15495 );
15496 assert!(!self.token.is_cancelled());
15497 assert!(
15498 !client
15499 .dispatch
15500 .pending_cancel_token(&id, self.identity.0, self.identity.1)
15501 .unwrap()
15502 .is_cancelled()
15503 );
15504 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15505 }
15506 }
15507
15508 #[rstest]
15509 #[case::marked(true)]
15510 #[case::unmarked(false)]
15511 #[tokio::test]
15512 async fn cancel_order_nonce_failure_preserves_existing_probe(#[case] marked: bool) {
15513 let mut config = test_config();
15514 config.base_url_http = Some(spawn_next_nonce_server(100).await);
15515 let (client, cache, mut rx) = create_execution_client_with_config(config);
15516 let instrument_id = register_test_instrument(&client, &cache);
15517 let probe = CancelProbe::cache(&client, &cache, instrument_id, marked);
15518 let command = probe.pending_command(&cache);
15519 let window = i64::from(client.dispatch.nonce_manager.skip_window());
15520 for _ in 0..window {
15521 client
15522 .dispatch
15523 .nonce_manager
15524 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15525 .unwrap();
15526 }
15527
15528 client.cancel_order(command.clone()).unwrap();
15529 let event = recv_order_event(&mut rx).await;
15530 wait_for_spawned_tasks(&client).await;
15531 probe.assert_preserved(&client, &cache, &command, &event);
15532
15533 let OrderEventAny::CancelDenied(denied) = event else {
15534 unreachable!()
15535 };
15536
15537 assert!(denied.reason.contains("skip-window exhausted"));
15538 assert!(rx.try_recv().is_err());
15539 }
15540
15541 #[rstest]
15542 #[case::marked(true)]
15543 #[case::unmarked(false)]
15544 #[tokio::test]
15545 async fn batch_cancel_validation_failure_preserves_existing_probe(#[case] marked: bool) {
15546 let (client, cache, mut rx) = create_execution_client();
15547 let instrument_id = register_test_instrument(&client, &cache);
15548 let probe = CancelProbe::cache(&client, &cache, instrument_id, marked);
15549 let mut command = probe.pending_command(&cache);
15550 command.venue_order_id = Some(VenueOrderId::from("invalid"));
15551 client
15552 .batch_cancel_orders(BatchCancelOrders::new(
15553 trader_id(),
15554 Some(client_id()),
15555 strategy_id(),
15556 instrument_id,
15557 vec![command.clone()],
15558 UUID4::new(),
15559 UnixNanos::default(),
15560 None,
15561 None,
15562 ))
15563 .unwrap();
15564 let event = recv_order_event(&mut rx).await;
15565 probe.assert_preserved(&client, &cache, &command, &event);
15566
15567 let OrderEventAny::CancelDenied(denied) = event else {
15568 unreachable!()
15569 };
15570
15571 assert!(denied.reason.contains("not an integer index"));
15572 assert!(rx.try_recv().is_err());
15573 }
15574
15575 #[rstest]
15576 #[case::single_marked(false, true)]
15577 #[case::single_unmarked(false, false)]
15578 #[case::batch_marked(true, true)]
15579 #[case::batch_unmarked(true, false)]
15580 #[tokio::test]
15581 async fn cancel_order_send_failure_preserves_existing_probe(
15582 #[case] batch: bool,
15583 #[case] marked: bool,
15584 ) {
15585 let (client, cache, mut rx) = create_execution_client();
15586 let instrument_id = register_test_instrument(&client, &cache);
15587 let probe = CancelProbe::cache(&client, &cache, instrument_id, marked);
15588 let command = probe.pending_command(&cache);
15589
15590 if batch {
15591 client
15592 .batch_cancel_orders(BatchCancelOrders::new(
15593 trader_id(),
15594 Some(client_id()),
15595 strategy_id(),
15596 instrument_id,
15597 vec![command.clone()],
15598 UUID4::new(),
15599 UnixNanos::default(),
15600 None,
15601 None,
15602 ))
15603 .unwrap();
15604 } else {
15605 client.cancel_order(command.clone()).unwrap();
15606 }
15607
15608 let event = recv_order_event(&mut rx).await;
15609 probe.assert_preserved(&client, &cache, &command, &event);
15610
15611 let OrderEventAny::CancelDenied(denied) = event else {
15612 unreachable!()
15613 };
15614
15615 assert!(denied.reason.contains("handler unavailable"));
15616 assert!(rx.try_recv().is_err());
15617 }
15618
15619 #[tokio::test]
15620 async fn batch_cancel_releases_split_reservations_before_waiting_for_ack() {
15621 let (client, cache, _rx) = create_execution_client();
15622 let instrument_id = register_test_instrument(&client, &cache);
15623 client.ws_client.drop_next_send_tx_result_for_test().await;
15624 let context = client
15625 .fanout_dispatch_context(client.credential.as_ref().unwrap())
15626 .unwrap();
15627
15628 let plan = CancelOrderPlan {
15629 command_id: Some(UUID4::new()),
15630 client_order_id: ClientOrderId::from("O-SPLIT-CANCEL"),
15631 strategy_id: strategy_id(),
15632 instrument_id,
15633 venue_order_id: Some(VenueOrderId::from("123")),
15634 market_index: TEST_MARKET_INDEX,
15635 venue_index: 123,
15636 };
15637
15638 let first = context.sign_cancel_order(&plan).unwrap();
15639 let interleaved = context.build_tx_context().unwrap();
15640 let second = context.sign_cancel_order(&plan).unwrap();
15641 let first_nonce = first.nonce;
15642 let second_nonce = second.nonce;
15643 drop(interleaved);
15644
15645 let task = tokio::spawn(async move {
15646 context
15647 .send_batch(vec![
15648 PreparedBatchTx::from((first, false)),
15649 PreparedBatchTx::from((second, false)),
15650 ])
15651 .await;
15652 });
15653
15654 wait_until_async(
15655 || async { client.dispatch.pending_sendtx_len() != 0 },
15656 Duration::from_secs(2),
15657 )
15658 .await;
15659
15660 let released = tokio::time::timeout(
15661 Duration::from_millis(250),
15662 client.nonce_submission_gate.write(),
15663 )
15664 .await
15665 .is_ok();
15666 let pending = client.dispatch.pending_sendtx.lock().clone();
15667 task.abort();
15668 let _ = task.await;
15669
15670 assert!(
15671 released,
15672 "unacknowledged group must not hold later nonce reservations"
15673 );
15674 assert_eq!(second_nonce, first_nonce + 2);
15675 assert_eq!(pending.len(), 1);
15676 assert_eq!(pending[0].nonce, first_nonce);
15677 }
15678
15679 #[rstest]
15680 #[case::not_sent(false)]
15681 #[case::ambiguous(true)]
15682 #[tokio::test]
15683 async fn batch_cancel_orders_send_failure_preserves_delivery_semantics(
15684 #[case] ambiguous: bool,
15685 ) {
15686 let (client, cache, mut rx) = create_execution_client();
15687 let instrument_id = register_test_instrument(&client, &cache);
15688 let mut factory = test_order_factory();
15689 let cancels = ["O-BATCH-CANCEL-A", "O-BATCH-CANCEL-B"]
15690 .iter()
15691 .enumerate()
15692 .map(|(i, id)| {
15693 let order = test_limit_order(&mut factory, instrument_id, id);
15694 let client_order_id = order.client_order_id();
15695 let venue_order_id = VenueOrderId::from(format!("{}", 123 + i).as_str());
15696 cache_pending_cancel_order(&cache, order, venue_order_id, None);
15697
15698 CancelOrder::new(
15699 trader_id(),
15700 Some(client_id()),
15701 strategy_id(),
15702 instrument_id,
15703 client_order_id,
15704 Some(venue_order_id),
15705 UUID4::new(),
15706 UnixNanos::default(),
15707 None,
15708 None,
15709 )
15710 })
15711 .collect::<Vec<_>>();
15712
15713 let command = BatchCancelOrders::new(
15714 trader_id(),
15715 Some(client_id()),
15716 strategy_id(),
15717 instrument_id,
15718 cancels.clone(),
15719 UUID4::new(),
15720 UnixNanos::default(),
15721 None,
15722 None,
15723 );
15724
15725 if ambiguous {
15726 client.ws_client.drop_next_send_tx_result_for_test().await;
15727 }
15728
15729 client.batch_cancel_orders(command).unwrap();
15730
15731 if ambiguous {
15732 assert!(
15733 tokio::time::timeout(Duration::from_millis(100), rx.recv())
15734 .await
15735 .is_err()
15736 );
15737 let pending = client.dispatch.pending_sendtx.lock().clone();
15738 assert_eq!(pending.len(), 2);
15739 assert_eq!(pending[0].batch_id, pending[1].batch_id);
15740 assert!(
15741 pending[0]
15742 .batch_id
15743 .as_ref()
15744 .unwrap()
15745 .starts_with("tx-batch:")
15746 );
15747 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
15748 assert_eq!(pending[1].nonce, TEST_NEXT_NONCE + 1);
15749
15750 for cancel in cancels {
15751 assert_eq!(
15752 client
15753 .dispatch
15754 .pending_order_action(&cancel.client_order_id),
15755 Some(PendingOrderAction::Cancel)
15756 );
15757 }
15758
15759 assert_eq!(
15760 client
15761 .dispatch
15762 .nonce_manager
15763 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
15764 Some(TEST_NEXT_NONCE + 1)
15765 );
15766 return;
15767 }
15768
15769 let first = recv_order_event(&mut rx).await;
15770 let second = recv_order_event(&mut rx).await;
15771
15772 let rejected_ids = [first, second].map(|event| match event {
15773 OrderEventAny::CancelDenied(e) => {
15774 assert!(e.reason.contains("Lighter cancel_order dispatch failed"),);
15775 e.client_order_id
15776 }
15777 other => panic!("expected CancelDenied, was {other:?}"),
15778 });
15779
15780 for cancel in cancels {
15781 assert!(rejected_ids.contains(&cancel.client_order_id));
15782 assert_eq!(
15783 client
15784 .dispatch
15785 .pending_order_action(&cancel.client_order_id),
15786 None,
15787 );
15788 }
15789
15790 assert_nonce_reusable(&client.dispatch);
15791 assert_eq!(
15792 client.dispatch.pending_sendtx_len(),
15793 0,
15794 "local-send-failure must remove batch cancel pending entries",
15795 );
15796 }
15797
15798 #[rstest]
15799 #[case::not_sent(false)]
15800 #[case::ambiguous(true)]
15801 #[tokio::test]
15802 async fn batch_cancel_duplicate_target_resolves_each_command(#[case] ambiguous: bool) {
15803 let (client, cache, mut rx) = create_execution_client();
15804 let instrument_id = register_test_instrument(&client, &cache);
15805 let mut factory = test_order_factory();
15806 let order = test_limit_order(&mut factory, instrument_id, "O-DUPLICATE-CANCEL");
15807 let client_order_id = order.client_order_id();
15808 let venue_order_id = VenueOrderId::from("123");
15809 let first_id = UUID4::new();
15810 let second_id = UUID4::new();
15811 cache_pending_cancel_order(&cache, order, venue_order_id, Some(first_id));
15812
15813 let mut pending = OrderPendingCancel::new(
15814 trader_id(),
15815 strategy_id(),
15816 instrument_id,
15817 client_order_id,
15818 Some(account_id()),
15819 UUID4::new(),
15820 UnixNanos::default(),
15821 UnixNanos::default(),
15822 false,
15823 Some(venue_order_id),
15824 );
15825 pending.causation_id = Some(second_id);
15826 cache
15827 .borrow_mut()
15828 .update_order(&OrderEventAny::PendingCancel(pending))
15829 .unwrap();
15830
15831 let mut cancels = [first_id, second_id].map(|command_id| {
15832 CancelOrder::new(
15833 trader_id(),
15834 Some(client_id()),
15835 strategy_id(),
15836 instrument_id,
15837 client_order_id,
15838 Some(venue_order_id),
15839 command_id,
15840 UnixNanos::default(),
15841 None,
15842 None,
15843 )
15844 });
15845
15846 cancels[0].strategy_id = StrategyId::from("REQUESTER-031");
15847
15848 if ambiguous {
15849 client.ws_client.drop_next_send_tx_result_for_test().await;
15850 }
15851
15852 client
15853 .batch_cancel_orders(BatchCancelOrders::new(
15854 trader_id(),
15855 Some(client_id()),
15856 strategy_id(),
15857 instrument_id,
15858 cancels.to_vec(),
15859 UUID4::new(),
15860 UnixNanos::default(),
15861 None,
15862 None,
15863 ))
15864 .unwrap();
15865 wait_until_async(
15866 || async { client.pending_tasks.is_empty() },
15867 BATCH_ACK_TIMEOUT + Duration::from_secs(2),
15868 )
15869 .await;
15870 let first = recv_order_event(&mut rx).await;
15871
15872 let OrderEventAny::CancelDenied(denied) = &first else {
15873 panic!("expected duplicate command denial, was {first:?}");
15874 };
15875
15876 assert_eq!(denied.causation_id, Some(first_id));
15877 assert_eq!(denied.client_order_id, client_order_id);
15878 assert_eq!(denied.strategy_id, strategy_id());
15879 assert_eq!(denied.instrument_id, instrument_id);
15880 assert_eq!(denied.venue_order_id, Some(venue_order_id));
15881 assert_eq!(denied.reason.as_str(), "Duplicate cancel target in batch");
15882 cache.borrow_mut().update_order(&first).unwrap();
15883 assert_eq!(
15884 cache.borrow().order_ref(&client_order_id).unwrap().status(),
15885 OrderStatus::PendingCancel
15886 );
15887
15888 if ambiguous {
15889 let pending = client.dispatch.pending_sendtx.lock().clone();
15890 assert_eq!(pending.len(), 1);
15891
15892 let PendingSendTxKind::Cancel { command_id, .. } = pending[0].kind else {
15893 panic!("expected one cancel transaction");
15894 };
15895
15896 assert_eq!(command_id, Some(second_id));
15897 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
15898 assert_eq!(
15899 client.dispatch.pending_order_action(&client_order_id),
15900 Some(PendingOrderAction::Cancel)
15901 );
15902 } else {
15903 let second = recv_order_event(&mut rx).await;
15904
15905 let OrderEventAny::CancelDenied(denied) = &second else {
15906 panic!("expected unsent command denial, was {second:?}");
15907 };
15908
15909 assert_eq!(denied.causation_id, Some(second_id));
15910 cache.borrow_mut().update_order(&second).unwrap();
15911 let cached = cache.borrow().try_order_owned(&client_order_id).unwrap();
15912 assert_eq!(cached.status(), OrderStatus::Accepted);
15913 assert_eq!(cached.quantity(), Quantity::from("0.1000"));
15914 assert!(!cached.has_pending_cancel_command());
15915 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15916 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
15917 assert_nonce_reusable(&client.dispatch);
15918 }
15919
15920 assert!(rx.try_recv().is_err());
15921 }
15922
15923 #[rstest]
15924 #[case::spill(16)]
15925 #[case::three(31)]
15926 #[tokio::test]
15927 async fn batch_cancel_orders_chunks_large_commands_on_send_failure(#[case] count: usize) {
15928 let (client, cache, mut rx) = create_execution_client();
15929 let instrument_id = register_test_instrument(&client, &cache);
15930 let mut factory = test_order_factory();
15931
15932 let cancels = (0..count)
15933 .map(|i| {
15934 let order = test_limit_order(
15935 &mut factory,
15936 instrument_id,
15937 &format!("O-BATCH-CANCEL-MAX-{i}"),
15938 );
15939 let id = order.client_order_id();
15940 let venue = VenueOrderId::from(format!("{}", 1_000 + i));
15941 cache_pending_cancel_order(&cache, order, venue, None);
15942
15943 CancelOrder::new(
15944 trader_id(),
15945 Some(client_id()),
15946 strategy_id(),
15947 instrument_id,
15948 id,
15949 Some(venue),
15950 UUID4::new(),
15951 UnixNanos::default(),
15952 None,
15953 None,
15954 )
15955 })
15956 .collect::<Vec<_>>();
15957
15958 let command = BatchCancelOrders::new(
15959 trader_id(),
15960 Some(client_id()),
15961 strategy_id(),
15962 instrument_id,
15963 cancels.clone(),
15964 UUID4::new(),
15965 UnixNanos::default(),
15966 None,
15967 None,
15968 );
15969 client.batch_cancel_orders(command).unwrap();
15970
15971 let mut rejected = Vec::new();
15972
15973 for _ in &cancels {
15974 match recv_order_event(&mut rx).await {
15975 OrderEventAny::CancelDenied(e) => {
15976 assert!(e.reason.contains("Lighter cancel_order dispatch failed"));
15977 rejected.push((e.client_order_id, e.venue_order_id));
15978 }
15979 other => panic!("expected CancelDenied, was {other:?}"),
15980 }
15981 }
15982
15983 let mut expected: Vec<_> = cancels
15984 .iter()
15985 .map(|cancel| (cancel.client_order_id, cancel.venue_order_id))
15986 .collect();
15987 expected.sort();
15988 rejected.sort();
15989 assert_eq!(rejected, expected);
15990
15991 for cancel in &cancels {
15992 assert_eq!(
15993 client
15994 .dispatch
15995 .pending_order_action(&cancel.client_order_id),
15996 None
15997 );
15998 assert_eq!(
15999 client
16000 .dispatch
16001 .pending_cancel_nonce(&cancel.client_order_id),
16002 None
16003 );
16004 }
16005
16006 assert!(
16007 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16008 .await
16009 .is_err(),
16010 );
16011 assert_nonce_reusable(&client.dispatch);
16012 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16013 }
16014
16015 #[rstest]
16016 #[case::success(200, "batch-a", 0, vec!["hash-1", "hash-2"], 2, false)]
16017 #[case::partial_ack(200, "batch-a", 0, vec!["hash-2"], 1, false)]
16018 #[case::missing_hashes(200, "batch-a", 0, vec![], 0, false)]
16019 #[case::wrong_hashes(200, "batch-a", 0, vec!["unknown"], 0, false)]
16020 #[case::wrong_id(200, "unknown", 0, vec!["hash-1", "hash-2"], 0, false)]
16021 #[case::stale(200, "batch-a", 1, vec!["hash-1", "hash-2"], 0, false)]
16022 #[case::invalid_nonce(21104, "batch-a", 0, vec![], 0, true)]
16023 #[case::rejected(21727, "batch-a", 0, vec![], 0, false)]
16024 #[tokio::test]
16025 async fn batch_cancel_response_correlates_members_and_preserves_nonces(
16026 #[case] code: i64,
16027 #[case] id: &str,
16028 #[case] epoch: u64,
16029 #[case] hashes: Vec<&str>,
16030 #[case] acked_count: usize,
16031 #[case] resync_expected: bool,
16032 ) {
16033 let (client, cache, mut rx) = create_execution_client();
16034 let instrument_id = register_test_instrument(&client, &cache);
16035 for i in 0..3 {
16036 let nonce = client
16037 .dispatch
16038 .nonce_manager
16039 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
16040 .unwrap();
16041 let client_order_id = ClientOrderId::from(format!("O-BATCH-{i}"));
16042 let command_id = UUID4::new();
16043 client
16044 .dispatch
16045 .register_cancel_command(client_order_id, command_id, true);
16046 client
16047 .dispatch
16048 .promote_pending_cancel(&client_order_id, command_id, 0, nonce, true);
16049 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
16050 batch_id: Some(if i == 0 { "batch-b" } else { "batch-a" }.to_string()),
16051 connection_epoch: 0,
16052 kind: PendingSendTxKind::Cancel {
16053 command_id: Some(command_id),
16054 emit_cancel_rejected: true,
16055 strategy_id: strategy_id(),
16056 instrument_id,
16057 client_order_id,
16058 venue_order_id: Some(VenueOrderId::from(format!("{}", 100 + i))),
16059 },
16060 submitted_at: UnixNanos::default(),
16061 nonce,
16062 api_key_index: TEST_API_KEY_INDEX,
16063 tx_hash: format!("hash-{i}"),
16064 });
16065 }
16066
16067 assert!(client.dispatch.pop_pending_sendtx_if_only(0).is_none());
16068 let hashes: Vec<_> = hashes.into_iter().map(str::to_string).collect();
16069 let (acked, resync) = handle_send_tx_batch_result(
16070 &client.dispatch,
16071 &client.emitter,
16072 Some(TEST_ACCOUNT_INDEX_I64),
16073 epoch,
16074 UnixNanos::default(),
16075 id,
16076 code,
16077 "batch failure",
16078 &hashes,
16079 );
16080 assert_eq!(resync, resync_expected);
16081 assert_eq!(acked.len(), acked_count);
16082 let rejected = code != 200;
16083 assert_eq!(
16084 client.dispatch.pending_sendtx_len(),
16085 if rejected { 1 } else { 3 - acked_count }
16086 );
16087
16088 if rejected {
16089 for i in [2, 1] {
16090 match recv_order_event(&mut rx).await {
16091 OrderEventAny::CancelRejected(event) => {
16092 assert_eq!(
16093 event.client_order_id,
16094 ClientOrderId::from(format!("O-BATCH-{i}"))
16095 );
16096 assert_eq!(event.strategy_id, strategy_id());
16097 assert_eq!(event.instrument_id, instrument_id);
16098 assert_eq!(
16099 event.venue_order_id,
16100 Some(VenueOrderId::from(format!("{}", 100 + i)))
16101 );
16102 }
16103 event => panic!("expected rejection, was {event:?}"),
16104 }
16105
16106 assert_eq!(
16107 client
16108 .dispatch
16109 .pending_order_action(&ClientOrderId::from(format!("O-BATCH-{i}"))),
16110 None
16111 );
16112 }
16113
16114 assert_eq!(
16115 client
16116 .dispatch
16117 .nonce_manager
16118 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
16119 Some(TEST_NEXT_NONCE)
16120 );
16121 } else {
16122 for pending in &acked {
16123 assert!(hashes.contains(&pending.tx_hash));
16124 }
16125
16126 assert_eq!(
16127 client
16128 .dispatch
16129 .nonce_manager
16130 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
16131 Some(TEST_NEXT_NONCE + 2)
16132 );
16133 }
16134
16135 assert_eq!(
16136 client
16137 .dispatch
16138 .pending_order_action(&ClientOrderId::from("O-BATCH-0")),
16139 Some(PendingOrderAction::Cancel)
16140 );
16141 assert!(
16142 tokio::time::timeout(Duration::from_millis(20), rx.recv())
16143 .await
16144 .is_err()
16145 );
16146 let remaining = client.dispatch.drain_pending_sendtx(0);
16147 assert_eq!(remaining.len(), if rejected { 1 } else { 3 - acked_count });
16148 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16149 }
16150
16151 #[rstest]
16152 fn cancel_order_from_cancel_all_preserves_tracing_ids() {
16153 let instrument_id = InstrumentId::from("ETH-PERP.LIGHTER");
16154 let client_order_id = ClientOrderId::from("O-CANCEL-ALL-CHILD");
16155 let command_id = UUID4::new();
16156 let correlation_id = UUID4::new();
16157 let causation_id = UUID4::new();
16158 let ts_init = UnixNanos::default();
16159 let mut cmd = CancelAllOrders::new(
16160 trader_id(),
16161 Some(client_id()),
16162 strategy_id(),
16163 instrument_id,
16164 Some(OrderSide::Buy),
16165 command_id,
16166 ts_init,
16167 None,
16168 Some(correlation_id),
16169 );
16170 cmd.causation_id = Some(causation_id);
16171
16172 let owner = StrategyId::from("S-002");
16173 let order_cmd = cancel_order_from_cancel_all(&cmd, client_order_id, owner);
16174
16175 assert_eq!(order_cmd.trader_id, trader_id());
16176 assert_eq!(order_cmd.client_id, Some(client_id()));
16177 assert_eq!(order_cmd.strategy_id, owner);
16178 assert_eq!(order_cmd.instrument_id, instrument_id);
16179 assert_eq!(order_cmd.client_order_id, client_order_id);
16180 assert_eq!(order_cmd.venue_order_id, None);
16181 assert_eq!(order_cmd.command_id, command_id);
16182 assert_eq!(order_cmd.ts_init, ts_init);
16183 assert_eq!(order_cmd.params, None);
16184 assert_eq!(order_cmd.correlation_id, Some(correlation_id));
16185 assert_eq!(order_cmd.causation_id, Some(causation_id));
16186 }
16187
16188 #[rstest]
16189 #[case::not_sent(false)]
16190 #[case::ambiguous(true)]
16191 #[tokio::test]
16192 async fn batch_modify_orders_send_failure_preserves_delivery_semantics(
16193 #[case] ambiguous: bool,
16194 ) {
16195 let (client, cache, mut rx) = create_execution_client();
16196 let instrument_id = register_test_instrument(&client, &cache);
16197 let mut factory = test_order_factory();
16198
16199 let modifies: Vec<_> = (0..2)
16200 .map(|i| {
16201 let order =
16202 test_limit_order(&mut factory, instrument_id, &format!("O-BATCH-MODIFY-{i}"));
16203 let id = order.client_order_id();
16204 cache_order(&cache, order);
16205
16206 ModifyOrder::new(
16207 trader_id(),
16208 Some(client_id()),
16209 strategy_id(),
16210 instrument_id,
16211 id,
16212 Some(VenueOrderId::from(format!("{}", 123 + i))),
16213 Some(Quantity::from("0.2000")),
16214 Some(Price::from("2362.00")),
16215 None,
16216 UUID4::new(),
16217 UnixNanos::default(),
16218 None,
16219 None,
16220 )
16221 })
16222 .collect();
16223
16224 if ambiguous {
16225 client.ws_client.drop_next_send_tx_result_for_test().await;
16226 }
16227
16228 client
16229 .batch_modify_orders(BatchModifyOrders::new(
16230 trader_id(),
16231 Some(client_id()),
16232 strategy_id(),
16233 instrument_id,
16234 modifies.clone(),
16235 UUID4::new(),
16236 UnixNanos::default(),
16237 None,
16238 None,
16239 ))
16240 .unwrap();
16241
16242 if ambiguous {
16243 wait_until_async(
16244 || async { client.dispatch.pending_sendtx_len() == 2 },
16245 Duration::from_secs(2),
16246 )
16247 .await;
16248 assert!(
16249 tokio::time::timeout(Duration::from_millis(100), rx.recv())
16250 .await
16251 .is_err()
16252 );
16253 let pending = client.dispatch.pending_sendtx.lock().clone();
16254 assert_eq!(pending.len(), 2);
16255 assert_eq!(pending[0].batch_id, pending[1].batch_id);
16256 assert!(
16257 pending[0]
16258 .batch_id
16259 .as_ref()
16260 .unwrap()
16261 .starts_with(TX_BATCH_ID_PREFIX)
16262 );
16263 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
16264 assert_eq!(pending[1].nonce, TEST_NEXT_NONCE + 1);
16265
16266 for modify in &modifies {
16267 assert_eq!(
16268 client
16269 .dispatch
16270 .pending_order_action(&modify.client_order_id),
16271 Some(PendingOrderAction::Modify)
16272 );
16273 }
16274 } else {
16275 let mut ids = Vec::new();
16276
16277 for _ in 0..2 {
16278 match recv_order_event(&mut rx).await {
16279 OrderEventAny::ModifyDenied(e) => {
16280 assert!(e.reason.contains("Lighter modify_order dispatch failed"));
16281 assert_eq!(e.instrument_id, instrument_id);
16282 assert_eq!(e.strategy_id, strategy_id());
16283 ids.push((e.client_order_id, e.venue_order_id));
16284 }
16285 other => panic!("expected ModifyDenied, was {other:?}"),
16286 }
16287 }
16288
16289 ids.sort();
16290 let mut expected: Vec<_> = modifies
16291 .iter()
16292 .map(|modify| (modify.client_order_id, modify.venue_order_id))
16293 .collect();
16294 expected.sort();
16295 assert_eq!(ids, expected);
16296 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16297
16298 for modify in &modifies {
16299 assert_eq!(
16300 client
16301 .dispatch
16302 .pending_order_action(&modify.client_order_id),
16303 None
16304 );
16305 }
16306
16307 assert_nonce_reusable(&client.dispatch);
16308 }
16309 }
16310
16311 #[tokio::test]
16312 async fn batch_modify_orders_rejects_invalid_child_and_dispatches_supported() {
16313 let (client, cache, mut rx) = create_execution_client();
16314 let instrument_id = register_test_instrument(&client, &cache);
16315 let mut factory = test_order_factory();
16316 let mut modifies = Vec::new();
16317
16318 for (id, venue_id) in [("O-MODIFY-INVALID", "invalid"), ("O-MODIFY-VALID", "456")] {
16319 let order = test_limit_order(&mut factory, instrument_id, id);
16320 let client_order_id = order.client_order_id();
16321 cache_order(&cache, order);
16322 modifies.push(ModifyOrder::new(
16323 trader_id(),
16324 Some(client_id()),
16325 strategy_id(),
16326 instrument_id,
16327 client_order_id,
16328 Some(VenueOrderId::from(venue_id)),
16329 Some(Quantity::from("0.2000")),
16330 Some(Price::from("2362.00")),
16331 None,
16332 UUID4::new(),
16333 UnixNanos::default(),
16334 None,
16335 None,
16336 ));
16337 }
16338
16339 client.ws_client.drop_next_send_tx_result_for_test().await;
16340 client
16341 .batch_modify_orders(BatchModifyOrders::new(
16342 trader_id(),
16343 Some(client_id()),
16344 strategy_id(),
16345 instrument_id,
16346 modifies.clone(),
16347 UUID4::new(),
16348 UnixNanos::default(),
16349 None,
16350 None,
16351 ))
16352 .unwrap();
16353 let event = recv_order_event(&mut rx).await;
16354
16355 let OrderEventAny::ModifyDenied(event) = event else {
16356 panic!("expected invalid-child rejection")
16357 };
16358
16359 assert_eq!(event.client_order_id, modifies[0].client_order_id);
16360 assert_eq!(event.venue_order_id, modifies[0].venue_order_id);
16361 assert!(event.reason.contains("invalid"));
16362 wait_until_async(
16363 || async { client.dispatch.pending_sendtx_len() == 1 },
16364 Duration::from_secs(2),
16365 )
16366 .await;
16367 let pending = client.dispatch.pending_sendtx.lock().clone();
16368 assert_eq!(pending.len(), 1);
16369 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
16370 assert!(
16371 matches!(&pending[0].kind, PendingSendTxKind::Modify { client_order_id, .. } if *client_order_id == modifies[1].client_order_id)
16372 );
16373 assert_eq!(
16374 client
16375 .dispatch
16376 .pending_order_action(&modifies[0].client_order_id),
16377 None
16378 );
16379 assert_eq!(
16380 client
16381 .dispatch
16382 .pending_order_action(&modifies[1].client_order_id),
16383 Some(PendingOrderAction::Modify)
16384 );
16385 assert!(rx.try_recv().is_err());
16386 }
16387
16388 #[rstest]
16389 #[case::empty(0)]
16390 #[case::full(15)]
16391 #[case::spill(16)]
16392 #[case::three(31)]
16393 #[tokio::test]
16394 async fn batch_modify_orders_chunks_large_commands_on_send_failure(#[case] count: usize) {
16395 let (client, cache, mut rx) = create_execution_client();
16396 let instrument_id = register_test_instrument(&client, &cache);
16397 let mut factory = test_order_factory();
16398
16399 let modifies: Vec<_> = (0..count)
16400 .map(|i| {
16401 let order =
16402 test_limit_order(&mut factory, instrument_id, &format!("O-BATCH-SIZE-{i}"));
16403 let id = order.client_order_id();
16404 cache_order(&cache, order);
16405
16406 ModifyOrder::new(
16407 trader_id(),
16408 Some(client_id()),
16409 strategy_id(),
16410 instrument_id,
16411 id,
16412 Some(VenueOrderId::from(format!("{}", 123 + i))),
16413 Some(Quantity::from("0.2000")),
16414 Some(Price::from("2362.00")),
16415 None,
16416 UUID4::new(),
16417 UnixNanos::default(),
16418 None,
16419 None,
16420 )
16421 })
16422 .collect();
16423
16424 client
16425 .batch_modify_orders(BatchModifyOrders::new(
16426 trader_id(),
16427 Some(client_id()),
16428 strategy_id(),
16429 instrument_id,
16430 modifies.clone(),
16431 UUID4::new(),
16432 UnixNanos::default(),
16433 None,
16434 None,
16435 ))
16436 .unwrap();
16437
16438 let mut rejected = Vec::new();
16439
16440 for _ in &modifies {
16441 match recv_order_event(&mut rx).await {
16442 OrderEventAny::ModifyDenied(e) => {
16443 assert!(e.reason.contains("Lighter modify_order dispatch failed"));
16444 rejected.push((e.client_order_id, e.venue_order_id));
16445 }
16446 other => panic!("expected ModifyDenied, was {other:?}"),
16447 }
16448 }
16449
16450 let mut expected: Vec<_> = modifies
16451 .iter()
16452 .map(|modify| (modify.client_order_id, modify.venue_order_id))
16453 .collect();
16454 expected.sort();
16455 rejected.sort();
16456 assert_eq!(rejected, expected);
16457
16458 for modify in &modifies {
16459 assert_eq!(
16460 client
16461 .dispatch
16462 .pending_order_action(&modify.client_order_id),
16463 None
16464 );
16465 assert_eq!(
16466 client
16467 .dispatch
16468 .pending_cancel_nonce(&modify.client_order_id),
16469 None
16470 );
16471 }
16472
16473 assert!(
16474 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16475 .await
16476 .is_err(),
16477 );
16478 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16479 assert_nonce_reusable(&client.dispatch);
16480 }
16481
16482 #[rstest]
16483 #[case::modify(false, false)]
16484 #[case::batch_modify(false, true)]
16485 #[case::cancel(true, false)]
16486 #[case::batch_cancel(true, true)]
16487 #[tokio::test]
16488 async fn command_admission_failure_denies_each_unsent_child(
16489 #[case] cancel: bool,
16490 #[case] batch: bool,
16491 #[values(false, true)] shutdown: bool,
16492 ) {
16493 let (mut client, cache, mut receiver) = create_execution_client();
16494 let instrument = register_test_instrument(&client, &cache);
16495 let mut factory = test_order_factory();
16496
16497 let count = if batch { 2 } else { 1 };
16498 let mut modifies = Vec::new();
16499 let mut cancels = Vec::new();
16500 let mut expected = Vec::new();
16501
16502 for index in 0..count {
16503 let order = test_limit_order(&mut factory, instrument, &format!("O-ADMISSION-{index}"));
16504 let id = order.client_order_id();
16505 let venue = VenueOrderId::from(format!("{}", 123 + index));
16506 cache_accepted_order(&cache, order, venue, Some(client_id()));
16507 let cause = UUID4::new();
16508 expected.push((id, venue, cause));
16509 modifies.push(ModifyOrder::new(
16510 trader_id(),
16511 Some(client_id()),
16512 strategy_id(),
16513 instrument,
16514 id,
16515 Some(venue),
16516 Some(Quantity::from("0.2000")),
16517 Some(Price::from("2362.00")),
16518 None,
16519 cause,
16520 17.into(),
16521 None,
16522 None,
16523 ));
16524 cancels.push(CancelOrder::new(
16525 trader_id(),
16526 Some(client_id()),
16527 strategy_id(),
16528 instrument,
16529 id,
16530 Some(venue),
16531 cause,
16532 19.into(),
16533 None,
16534 None,
16535 ));
16536 }
16537
16538 let before = expected
16539 .iter()
16540 .map(|(id, _, _)| cache.borrow().order(id).unwrap().clone())
16541 .collect::<Vec<_>>();
16542
16543 if shutdown {
16544 client.begin_session_shutdown();
16545 } else {
16546 client.credential = None;
16547 }
16548
16549 let result = match (cancel, batch) {
16550 (false, false) => client.modify_order(modifies.remove(0)),
16551 (false, true) => client.batch_modify_orders(BatchModifyOrders::new(
16552 trader_id(),
16553 Some(client_id()),
16554 strategy_id(),
16555 instrument,
16556 modifies,
16557 UUID4::new(),
16558 23.into(),
16559 None,
16560 None,
16561 )),
16562 (true, false) => client.cancel_order(cancels.remove(0)),
16563 (true, true) => client.batch_cancel_orders(BatchCancelOrders::new(
16564 trader_id(),
16565 Some(client_id()),
16566 strategy_id(),
16567 instrument,
16568 cancels,
16569 UUID4::new(),
16570 29.into(),
16571 None,
16572 None,
16573 )),
16574 };
16575
16576 assert_eq!(result.is_err(), !shutdown || batch);
16577
16578 for ((id, venue, cause), original) in expected.into_iter().zip(before) {
16579 let event = match receiver.try_recv().unwrap() {
16580 ExecutionEvent::Order(event) => event,
16581 other => panic!("expected command denial, was {other:?}"),
16582 };
16583
16584 let (actual_id, actual_instrument, actual_strategy, actual_venue, actual_cause, reason) =
16585 match event {
16586 OrderEventAny::ModifyDenied(event) if !cancel => (
16587 event.client_order_id,
16588 event.instrument_id,
16589 event.strategy_id,
16590 event.venue_order_id,
16591 event.causation_id,
16592 event.reason,
16593 ),
16594 OrderEventAny::CancelDenied(event) if cancel => (
16595 event.client_order_id,
16596 event.instrument_id,
16597 event.strategy_id,
16598 event.venue_order_id,
16599 event.causation_id,
16600 event.reason,
16601 ),
16602 other => panic!("unexpected command outcome {other:?}"),
16603 };
16604
16605 assert_eq!(
16606 (
16607 actual_id,
16608 actual_instrument,
16609 actual_strategy,
16610 actual_venue,
16611 actual_cause
16612 ),
16613 (id, instrument, strategy_id(), Some(venue), Some(cause))
16614 );
16615 assert_eq!(
16616 reason.as_str(),
16617 if shutdown {
16618 "Lighter task admission is closed"
16619 } else if cancel {
16620 "Lighter execution client cannot cancel without credentials"
16621 } else {
16622 "Lighter execution client cannot modify without credentials"
16623 }
16624 );
16625 assert_eq!(
16626 serde_json::to_value(cache.borrow().order(&id).unwrap().clone()).unwrap(),
16627 serde_json::to_value(&original).unwrap()
16628 );
16629 assert_eq!(client.dispatch.pending_order_action(&id), None);
16630 }
16631
16632 assert!(matches!(
16633 receiver.try_recv(),
16634 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
16635 ));
16636 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16637 assert_nonce_reusable(&client.dispatch);
16638 }
16639
16640 #[tokio::test]
16641 async fn modify_order_send_failure_emits_modify_denied_and_rolls_back() {
16642 let (client, cache, mut rx) = create_execution_client();
16643 let instrument_id = register_test_instrument(&client, &cache);
16644 let mut factory = test_order_factory();
16645 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-FAIL");
16646 let client_order_id = order.client_order_id();
16647 let venue_order_id = VenueOrderId::from("123");
16648 cache_order(&cache, order);
16649
16650 let command = ModifyOrder::new(
16651 trader_id(),
16652 Some(client_id()),
16653 strategy_id(),
16654 instrument_id,
16655 client_order_id,
16656 Some(venue_order_id),
16657 Some(Quantity::from("0.2000")),
16658 Some(Price::from("2362.00")),
16659 None,
16660 UUID4::new(),
16661 UnixNanos::default(),
16662 None,
16663 None,
16664 );
16665 client.modify_order(command).unwrap();
16666
16667 let rejected = recv_order_event(&mut rx).await;
16668
16669 match rejected {
16670 OrderEventAny::ModifyDenied(event) => {
16671 assert_eq!(event.client_order_id, client_order_id);
16672 assert_eq!(event.instrument_id, instrument_id);
16673 assert_eq!(event.venue_order_id, Some(venue_order_id));
16674 assert!(
16675 event
16676 .reason
16677 .contains("Lighter modify_order dispatch failed"),
16678 );
16679 assert!(event.reason.contains("handler unavailable"));
16680 }
16681 event => panic!("expected modify rejected event, was {event:?}"),
16682 }
16683
16684 assert_nonce_reusable(&client.dispatch);
16685 assert_eq!(
16686 client.dispatch.pending_sendtx_len(),
16687 0,
16688 "local-send-failure must remove the pending modify entry",
16689 );
16690 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
16691 }
16692
16693 #[tokio::test]
16694 async fn modify_order_unknown_send_outcome_retains_pending_state() {
16695 let (client, cache, mut rx) = create_execution_client();
16696 let instrument_id = register_test_instrument(&client, &cache);
16697 let mut factory = test_order_factory();
16698 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-UNKNOWN");
16699 let client_order_id = order.client_order_id();
16700 let venue_order_id = VenueOrderId::from("123");
16701 cache_order(&cache, order);
16702 client.ws_client.drop_next_send_tx_result_for_test().await;
16703
16704 client
16705 .modify_order(ModifyOrder::new(
16706 trader_id(),
16707 Some(client_id()),
16708 strategy_id(),
16709 instrument_id,
16710 client_order_id,
16711 Some(venue_order_id),
16712 Some(Quantity::from("0.2000")),
16713 Some(Price::from("2362.00")),
16714 None,
16715 UUID4::new(),
16716 UnixNanos::default(),
16717 None,
16718 None,
16719 ))
16720 .unwrap();
16721
16722 assert!(
16723 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16724 .await
16725 .is_err(),
16726 "unknown outcome must not emit OrderModifyRejected",
16727 );
16728 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
16729 assert_eq!(
16730 client.dispatch.pending_order_action(&client_order_id),
16731 Some(PendingOrderAction::Modify),
16732 );
16733 assert_eq!(
16734 client
16735 .dispatch
16736 .nonce_manager
16737 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
16738 Some(TEST_NEXT_NONCE),
16739 );
16740 }
16741
16742 #[tokio::test]
16743 async fn modify_order_prepare_failure_emits_modify_denied_without_dispatch() {
16744 let (client, cache, mut rx) = create_execution_client();
16745 let instrument_id = register_test_instrument(&client, &cache);
16746 let client_order_id = ClientOrderId::from("O-MODIFY-NO-CACHE");
16747 let venue_order_id = VenueOrderId::from("123");
16748
16749 let command = ModifyOrder::new(
16750 trader_id(),
16751 Some(client_id()),
16752 strategy_id(),
16753 instrument_id,
16754 client_order_id,
16755 Some(venue_order_id),
16756 Some(Quantity::from("0.2000")),
16757 Some(Price::from("2362.00")),
16758 None,
16759 UUID4::new(),
16760 UnixNanos::default(),
16761 None,
16762 None,
16763 );
16764 client.modify_order(command).unwrap();
16765
16766 let rejected = recv_order_event(&mut rx).await;
16767 match rejected {
16768 OrderEventAny::ModifyDenied(event) => {
16769 assert_eq!(event.client_order_id, client_order_id);
16770 assert_eq!(event.instrument_id, instrument_id);
16771 assert_eq!(event.venue_order_id, Some(venue_order_id));
16772 assert!(event.reason.contains("Lighter modify_order failed"));
16773 assert!(event.reason.contains("order not found in cache"));
16774 }
16775 event => panic!("expected modify rejected event, was {event:?}"),
16776 }
16777
16778 assert_nonce_reusable(&client.dispatch);
16779 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16780 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
16781 assert!(
16782 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16783 .await
16784 .is_err(),
16785 "prepare failure must emit exactly one modify rejection",
16786 );
16787 }
16788
16789 #[tokio::test]
16790 async fn modify_order_prepare_failure_emits_modify_denied_when_instrument_uncached() {
16791 let (client, cache, mut rx) = create_execution_client();
16792 let instrument_id =
16793 client
16794 .registry
16795 .insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
16796 let mut factory = test_order_factory();
16797 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-NO-INSTRUMENT");
16798 let client_order_id = order.client_order_id();
16799 let venue_order_id = VenueOrderId::from("123");
16800 cache_order(&cache, order);
16801
16802 let command = ModifyOrder::new(
16803 trader_id(),
16804 Some(client_id()),
16805 strategy_id(),
16806 instrument_id,
16807 client_order_id,
16808 Some(venue_order_id),
16809 Some(Quantity::from("0.2000")),
16810 Some(Price::from("2362.00")),
16811 None,
16812 UUID4::new(),
16813 UnixNanos::default(),
16814 None,
16815 None,
16816 );
16817 client.modify_order(command).unwrap();
16818
16819 let rejected = recv_order_event(&mut rx).await;
16820 match rejected {
16821 OrderEventAny::ModifyDenied(event) => {
16822 assert_eq!(event.client_order_id, client_order_id);
16823 assert_eq!(event.instrument_id, instrument_id);
16824 assert_eq!(event.venue_order_id, Some(venue_order_id));
16825 assert!(event.reason.contains("Lighter modify_order failed"));
16826 assert!(event.reason.contains("instrument not found in cache"));
16827 }
16828 event => panic!("expected modify rejected event, was {event:?}"),
16829 }
16830
16831 assert_nonce_reusable(&client.dispatch);
16832 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16833 assert!(
16834 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16835 .await
16836 .is_err(),
16837 "prepare failure must emit exactly one modify rejection",
16838 );
16839 }
16840
16841 #[rstest]
16842 #[case::default(None, "dispatch failed")]
16843 #[case::invalid(Some(serde_json::json!("0")), "market_order_slippage_bps must be an unsigned integer")]
16844 #[case::out_of_range(Some(serde_json::json!(10_000)), "market_order_slippage_bps must be in [0, 10000)")]
16845 #[tokio::test]
16846 async fn modify_stop_market_validates_slippage_and_derives_price_from_trigger(
16847 #[case] slippage: Option<serde_json::Value>,
16848 #[case] expected_reason: &str,
16849 ) {
16850 let (client, cache, mut rx) = create_execution_client();
16851 let instrument_id = register_test_instrument(&client, &cache);
16852 let mut factory = test_order_factory();
16853 let order = factory.stop_market(
16854 instrument_id,
16855 OrderSide::Sell,
16856 Quantity::from("0.1000"),
16857 Price::from("2300.00"), None,
16859 Some(TimeInForce::Gtc),
16860 None,
16861 Some(false),
16862 Some(false),
16863 None,
16864 None,
16865 None,
16866 None,
16867 None,
16868 None,
16869 Some(ClientOrderId::from("O-MODIFY-STOP-MARKET")),
16870 );
16871 let client_order_id = order.client_order_id();
16872 let venue_order_id = VenueOrderId::from("123");
16873 cache_order(&cache, order);
16874
16875 let params = slippage.map(|value| {
16876 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": value})).unwrap()
16877 });
16878
16879 let command = ModifyOrder::new(
16880 trader_id(),
16881 Some(client_id()),
16882 strategy_id(),
16883 instrument_id,
16884 client_order_id,
16885 Some(venue_order_id),
16886 None,
16887 None,
16888 Some(Price::from("2310.00")),
16889 UUID4::new(),
16890 UnixNanos::default(),
16891 params,
16892 None,
16893 );
16894 client.modify_order(command).unwrap();
16895
16896 let rejected = recv_order_event(&mut rx).await;
16897 match rejected {
16898 OrderEventAny::ModifyDenied(event) => {
16899 assert_eq!(event.client_order_id, client_order_id);
16900 assert_eq!(event.instrument_id, instrument_id);
16901 assert_eq!(event.venue_order_id, Some(venue_order_id));
16902 assert!(
16903 event.reason.contains(expected_reason),
16904 "expected {expected_reason}, was: {}",
16905 event.reason,
16906 );
16907 }
16908 event => panic!("expected modify rejected event, was {event:?}"),
16909 }
16910
16911 assert!(rx.try_recv().is_err());
16912 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16913 assert_nonce_reusable(&client.dispatch);
16914 }
16915
16916 #[tokio::test]
16917 async fn modify_order_rejects_sub_tick_quantity_before_nonce_allocation() {
16918 let (client, cache, mut rx) = create_execution_client();
16919 let instrument_id = register_test_instrument(&client, &cache);
16920 let mut factory = test_order_factory();
16921 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-SUB-TICK");
16922 let client_order_id = order.client_order_id();
16923 cache_order(&cache, order);
16924
16925 client
16926 .modify_order(ModifyOrder::new(
16927 trader_id(),
16928 Some(client_id()),
16929 strategy_id(),
16930 instrument_id,
16931 client_order_id,
16932 Some(VenueOrderId::from("123")),
16933 Some(Quantity::from("0.00001")),
16934 Some(Price::from("2362.00")),
16935 None,
16936 UUID4::new(),
16937 UnixNanos::default(),
16938 None,
16939 None,
16940 ))
16941 .unwrap();
16942
16943 assert_modify_denied_reason(&mut rx, "rounds to 0 ticks").await;
16944 assert_nonce_reusable(&client.dispatch);
16945 }
16946
16947 #[tokio::test]
16948 async fn modify_order_rejects_below_min_notional_before_nonce_allocation() {
16949 let (client, cache, mut rx) = create_execution_client();
16950 let instrument_id = register_test_instrument(&client, &cache);
16951 let mut factory = test_order_factory();
16952 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-MIN-NOTIONAL");
16953 let client_order_id = order.client_order_id();
16954 cache_order(&cache, order);
16955
16956 client
16957 .modify_order(ModifyOrder::new(
16958 trader_id(),
16959 Some(client_id()),
16960 strategy_id(),
16961 instrument_id,
16962 client_order_id,
16963 Some(VenueOrderId::from("123")),
16964 Some(Quantity::from("0.0001")),
16965 Some(Price::from("2362.00")),
16966 None,
16967 UUID4::new(),
16968 UnixNanos::default(),
16969 None,
16970 None,
16971 ))
16972 .unwrap();
16973
16974 assert_modify_denied_reason(&mut rx, "below Lighter min_quote_amount").await;
16975 assert_nonce_reusable(&client.dispatch);
16976 }
16977
16978 #[tokio::test]
16979 async fn modify_conditional_order_rejects_sub_tick_trigger_before_nonce_allocation() {
16980 let (client, cache, mut rx) = create_execution_client();
16981 let instrument_id = register_test_instrument(&client, &cache);
16982 let mut factory = test_order_factory();
16983 let order = factory.stop_market(
16984 instrument_id,
16985 OrderSide::Sell,
16986 Quantity::from("0.1000"),
16987 Price::from("2300.00"),
16988 None,
16989 Some(TimeInForce::Gtc),
16990 None,
16991 Some(false),
16992 Some(false),
16993 None,
16994 None,
16995 None,
16996 None,
16997 None,
16998 None,
16999 Some(ClientOrderId::from("O-MODIFY-SUB-TICK-TRIGGER")),
17000 );
17001 let client_order_id = order.client_order_id();
17002 cache_order(&cache, order);
17003
17004 client
17005 .modify_order(ModifyOrder::new(
17006 trader_id(),
17007 Some(client_id()),
17008 strategy_id(),
17009 instrument_id,
17010 client_order_id,
17011 Some(VenueOrderId::from("123")),
17012 None,
17013 None,
17014 Some(Price::from("0.001")),
17015 UUID4::new(),
17016 UnixNanos::default(),
17017 None,
17018 None,
17019 ))
17020 .unwrap();
17021
17022 assert_modify_denied_reason(&mut rx, "rounds to 0 ticks").await;
17023 assert_nonce_reusable(&client.dispatch);
17024 }
17025
17026 #[tokio::test]
17027 async fn modify_order_nonce_prepare_failure_emits_modify_denied() {
17028 let mut config = test_config();
17029 config.base_url_http = Some(spawn_next_nonce_server(101).await);
17030 let (client, cache, mut rx) = create_execution_client_with_config(config);
17031 let instrument_id = register_test_instrument(&client, &cache);
17032 let mut factory = test_order_factory();
17033 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-NONCE-FAIL");
17034 let client_order_id = order.client_order_id();
17035 let venue_order_id = VenueOrderId::from("123");
17036 cache_order(&cache, order);
17037
17038 let window = i64::from(client.dispatch.nonce_manager.skip_window());
17039 for _ in 0..window {
17040 client
17041 .dispatch
17042 .nonce_manager
17043 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
17044 .unwrap();
17045 }
17046
17047 let command = ModifyOrder::new(
17048 trader_id(),
17049 Some(client_id()),
17050 strategy_id(),
17051 instrument_id,
17052 client_order_id,
17053 Some(venue_order_id),
17054 Some(Quantity::from("0.2000")),
17055 Some(Price::from("2362.00")),
17056 None,
17057 UUID4::new(),
17058 UnixNanos::default(),
17059 None,
17060 None,
17061 );
17062 client.modify_order(command).unwrap();
17063
17064 let rejected = recv_order_event(&mut rx).await;
17065 match rejected {
17066 OrderEventAny::ModifyDenied(event) => {
17067 assert_eq!(event.client_order_id, client_order_id);
17068 assert_eq!(event.venue_order_id, Some(venue_order_id));
17069 assert!(event.reason.contains("failed to allocate Lighter nonce"),);
17070 assert!(event.reason.contains("skip-window exhausted"));
17071 }
17072 event => panic!("expected modify rejected event, was {event:?}"),
17073 }
17074
17075 wait_for_spawned_tasks(&client).await;
17076 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
17077 assert_eq!(
17078 client
17079 .dispatch
17080 .nonce_manager
17081 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
17082 .unwrap(),
17083 101,
17084 );
17085 }
17086
17087 #[tokio::test]
17088 async fn update_leverage_requires_credentials() {
17089 let (mut client, _cache, _rx) = create_execution_client();
17090 client.credential = None;
17091 let instrument_id = InstrumentId::from("ETH-PERP.LIGHTER");
17092
17093 let err = client
17094 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
17095 .unwrap_err();
17096
17097 assert!(
17098 err.to_string()
17099 .contains("cannot update leverage without credentials"),
17100 );
17101 }
17102
17103 #[tokio::test]
17104 async fn update_leverage_requires_registered_instrument() {
17105 let (client, _cache, _rx) = create_execution_client();
17106 let unknown = InstrumentId::from("DOGE-PERP.LIGHTER");
17107
17108 let err = client
17109 .update_leverage(unknown, 500, LighterPositionMarginMode::Isolated)
17110 .unwrap_err();
17111
17112 assert!(
17113 err.to_string()
17114 .contains("no Lighter market_index registered")
17115 );
17116 assert_nonce_reusable(&client.dispatch);
17119 }
17120
17121 #[tokio::test]
17122 async fn update_leverage_dispatches_and_rolls_back_on_send_failure() {
17123 let (client, cache, _rx) = create_execution_client();
17124 let instrument_id = register_test_instrument(&client, &cache);
17125
17126 client
17127 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
17128 .unwrap();
17129
17130 wait_for_spawned_tasks(&client).await;
17131 assert_nonce_reusable(&client.dispatch);
17132 }
17133
17134 #[tokio::test]
17135 async fn update_leverage_unknown_send_outcome_retains_pending_nonce() {
17136 let (client, cache, _rx) = create_execution_client();
17137 let instrument_id = register_test_instrument(&client, &cache);
17138 client.ws_client.drop_next_send_tx_result_for_test().await;
17139
17140 client
17141 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
17142 .unwrap();
17143
17144 wait_for_spawned_tasks(&client).await;
17145 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
17146 assert_eq!(
17147 client
17148 .dispatch
17149 .nonce_manager
17150 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
17151 Some(TEST_NEXT_NONCE),
17152 );
17153 }
17154
17155 #[tokio::test]
17156 async fn update_leverage_rejects_zero_margin_fraction() {
17157 let (client, cache, _rx) = create_execution_client();
17158 let instrument_id = register_test_instrument(&client, &cache);
17159
17160 let err = client
17161 .update_leverage(instrument_id, 0, LighterPositionMarginMode::Cross)
17162 .unwrap_err();
17163 assert!(err.to_string().contains("must be in 1..=10_000"));
17164 }
17165
17166 #[tokio::test]
17167 async fn update_leverage_rejects_above_margin_fraction_tick() {
17168 let (client, cache, _rx) = create_execution_client();
17169 let instrument_id = register_test_instrument(&client, &cache);
17170
17171 let err = client
17172 .update_leverage(instrument_id, 10_001, LighterPositionMarginMode::Cross)
17173 .unwrap_err();
17174 assert!(err.to_string().contains("must be in 1..=10_000"));
17175 }
17176
17177 #[tokio::test]
17178 async fn update_leverage_accepts_minimum_margin_fraction() {
17179 let (client, cache, _rx) = create_execution_client();
17182 let instrument_id = register_test_instrument(&client, &cache);
17183
17184 client
17185 .update_leverage(instrument_id, 1, LighterPositionMarginMode::Cross)
17186 .unwrap();
17187
17188 wait_for_spawned_tasks(&client).await;
17189 assert_nonce_reusable(&client.dispatch);
17190 }
17191
17192 #[tokio::test]
17193 async fn update_leverage_accepts_maximum_margin_fraction() {
17194 let (client, cache, _rx) = create_execution_client();
17197 let instrument_id = register_test_instrument(&client, &cache);
17198
17199 client
17200 .update_leverage(instrument_id, 10_000, LighterPositionMarginMode::Isolated)
17201 .unwrap();
17202
17203 wait_for_spawned_tasks(&client).await;
17204 assert_nonce_reusable(&client.dispatch);
17205 }
17206
17207 async fn wait_for_spawned_tasks(client: &LighterExecutionClient) {
17208 wait_until_async(
17209 || async { client.pending_tasks_all_finished() },
17210 Duration::from_secs(2),
17211 )
17212 .await;
17213 }
17214
17215 fn mark_all_streams_ready(client: &LighterExecutionClient) {
17216 let ready = &client.dispatch.account_streams_ready;
17217 ready.mark_orders();
17218 ready.mark_trades();
17219 ready.mark_positions();
17220 ready.mark_assets();
17221 ready.mark_user_stats();
17222 }
17223
17224 #[tokio::test]
17225 async fn await_account_streams_ready_times_out_when_no_frame_arrives() {
17226 let (client, _cache, _rx) = create_execution_client();
17229
17230 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
17231
17232 assert!(
17233 err.to_string().contains("Timeout")
17234 && err.to_string().contains("Lighter account streams"),
17235 "unexpected error message, was {err}",
17236 );
17237 }
17238
17239 #[tokio::test]
17240 async fn await_account_streams_ready_returns_when_all_streams_marked() {
17241 let (client, _cache, _rx) = create_execution_client();
17242 mark_all_streams_ready(&client);
17243
17244 client.await_account_streams_ready(0.05).await.unwrap();
17245 }
17246
17247 #[tokio::test]
17248 async fn await_account_streams_ready_returns_when_streams_arrive_mid_wait() {
17249 let (client, _cache, _rx) = create_execution_client();
17250 let ready = Arc::clone(&client.dispatch.account_streams_ready);
17251 let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
17252 let mut pending_tx = Some(pending_tx);
17253 let wait = client.await_account_streams_ready(30.0);
17254 tokio::pin!(wait);
17255
17256 let producer = tokio::spawn(async move {
17257 pending_rx.await.expect("readiness waiter parked");
17258 ready.mark_orders();
17259 ready.mark_trades();
17260 ready.mark_positions();
17261 ready.mark_assets();
17262 ready.mark_user_stats();
17263 });
17264
17265 tokio::time::timeout(
17267 Duration::from_secs(2),
17268 std::future::poll_fn(|cx| {
17269 let result = wait.as_mut().poll(cx);
17270 if result.is_pending()
17271 && let Some(tx) = pending_tx.take()
17272 {
17273 tx.send(()).expect("readiness producer alive");
17274 }
17275
17276 result
17277 }),
17278 )
17279 .await
17280 .expect("marks must wake the waiter before its deadline")
17281 .unwrap();
17282
17283 assert!(pending_tx.is_none(), "waiter must initially be pending");
17284 producer.await.unwrap();
17285 }
17286
17287 #[tokio::test]
17288 async fn await_account_streams_ready_times_out_with_partial_marks() {
17289 let (client, _cache, _rx) = create_execution_client();
17293 let ready = &client.dispatch.account_streams_ready;
17294 ready.mark_orders();
17295 ready.mark_trades();
17296 ready.mark_positions();
17297
17298 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
17299 assert!(
17300 err.to_string().contains("assets"),
17301 "pending list should call out the missing stream, was {err}",
17302 );
17303 }
17304
17305 #[tokio::test]
17306 async fn await_account_streams_ready_after_reset_requires_new_marks() {
17307 let (client, _cache, _rx) = create_execution_client();
17312 mark_all_streams_ready(&client);
17313 client.await_account_streams_ready(0.05).await.unwrap();
17314
17315 client.dispatch.account_streams_ready.reset();
17316
17317 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
17318 let msg = err.to_string();
17319 assert!(msg.contains("orders"), "pending list missing orders: {msg}");
17320 assert!(msg.contains("trades"), "pending list missing trades: {msg}");
17321 assert!(
17322 msg.contains("positions"),
17323 "pending list missing positions: {msg}",
17324 );
17325 assert!(msg.contains("assets"), "pending list missing assets: {msg}");
17326 assert!(
17327 msg.contains("user_stats"),
17328 "pending list missing user_stats: {msg}",
17329 );
17330 }
17331
17332 fn test_market_order(
17333 factory: &mut OrderFactory,
17334 instrument_id: InstrumentId,
17335 client_order_id: &str,
17336 side: OrderSide,
17337 ) -> OrderAny {
17338 factory.market(
17339 instrument_id,
17340 side,
17341 Quantity::from("0.1000"),
17342 Some(TimeInForce::Ioc),
17343 Some(false),
17344 Some(false),
17345 None,
17346 None,
17347 None,
17348 Some(ClientOrderId::from(client_order_id)),
17349 )
17350 }
17351
17352 fn add_test_quote(
17353 cache: &Rc<RefCell<Cache>>,
17354 instrument_id: InstrumentId,
17355 bid: &str,
17356 ask: &str,
17357 ) {
17358 let quote = QuoteTick::new(
17359 instrument_id,
17360 Price::from(bid),
17361 Price::from(ask),
17362 Quantity::from("1.0000"),
17363 Quantity::from("1.0000"),
17364 UnixNanos::default(),
17365 UnixNanos::default(),
17366 );
17367 cache.borrow_mut().add_quote(quote).unwrap();
17368 }
17369
17370 #[tokio::test]
17371 async fn submit_market_order_without_cached_quote_emits_denied() {
17372 let (client, cache, mut rx) = create_execution_client();
17373 let instrument_id = register_test_instrument(&client, &cache);
17374 let mut factory = test_order_factory();
17375 let order = test_market_order(
17376 &mut factory,
17377 instrument_id,
17378 "O-MARKET-NO-QUOTE",
17379 OrderSide::Buy,
17380 );
17381 cache_order(&cache, order.clone());
17382
17383 let command = SubmitOrder::from_order(
17384 &order,
17385 trader_id(),
17386 Some(client_id()),
17387 None,
17388 UUID4::new(),
17389 UnixNanos::default(),
17390 );
17391 let _ = client.submit_order(command);
17393
17394 let event = recv_order_event(&mut rx).await;
17395 match event {
17396 OrderEventAny::Denied(event) => {
17397 assert!(
17398 event.reason.contains("no cached quote"),
17399 "expected no-cached-quote in reason, was {:?}",
17400 event.reason,
17401 );
17402 }
17403 event => panic!("expected denied event, was {event:?}"),
17404 }
17405
17406 assert_nonce_reusable(&client.dispatch);
17407 }
17408
17409 #[tokio::test]
17410 async fn submit_market_buy_with_quote_uses_ask_widened_by_slippage() {
17411 let (client, cache, mut rx) = create_execution_client();
17412 let instrument_id = register_test_instrument(&client, &cache);
17413 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
17414
17415 let mut factory = test_order_factory();
17416 let order = test_market_order(
17417 &mut factory,
17418 instrument_id,
17419 "O-MARKET-QUOTED-BUY",
17420 OrderSide::Buy,
17421 );
17422 cache_order(&cache, order.clone());
17423
17424 let command = SubmitOrder::from_order(
17425 &order,
17426 trader_id(),
17427 Some(client_id()),
17428 None,
17429 UUID4::new(),
17430 UnixNanos::default(),
17431 );
17432 let _ = client.submit_order(command);
17433
17434 let submitted = recv_order_event(&mut rx).await;
17435 assert!(
17436 matches!(submitted, OrderEventAny::Submitted(_)),
17437 "expected submitted, was {submitted:?}",
17438 );
17439 let rejected = recv_order_event(&mut rx).await;
17440 match rejected {
17441 OrderEventAny::Rejected(event) => {
17442 assert!(
17443 event
17444 .reason
17445 .contains("Lighter submit_order dispatch failed"),
17446 );
17447 }
17448 event => panic!("expected rejected event, was {event:?}"),
17449 }
17450
17451 assert_nonce_reusable(&client.dispatch);
17452 }
17453
17454 #[tokio::test]
17455 async fn submit_order_with_sub_tick_quantity_emits_denied() {
17456 let (client, cache, mut rx) = create_execution_client();
17457 let instrument_id = register_test_instrument(&client, &cache);
17458 let mut factory = test_order_factory();
17459 let order = factory.limit(
17461 instrument_id,
17462 OrderSide::Buy,
17463 Quantity::from("0.00001"),
17464 Price::from("2361.31"),
17465 Some(TimeInForce::Gtc),
17466 None,
17467 Some(false),
17468 Some(false),
17469 None,
17470 None,
17471 None,
17472 None,
17473 None,
17474 None,
17475 None,
17476 Some(ClientOrderId::from("O-SUB-TICK-QTY")),
17477 );
17478 cache_order(&cache, order.clone());
17479
17480 let command = SubmitOrder::from_order(
17481 &order,
17482 trader_id(),
17483 Some(client_id()),
17484 None,
17485 UUID4::new(),
17486 UnixNanos::default(),
17487 );
17488 let _ = client.submit_order(command);
17489
17490 let event = recv_order_event(&mut rx).await;
17491 match event {
17492 OrderEventAny::Denied(event) => {
17493 assert!(
17494 event.reason.contains("rounds to 0 ticks"),
17495 "expected rounds-to-0 in reason, was {:?}",
17496 event.reason,
17497 );
17498 }
17499 event => panic!("expected denied event, was {event:?}"),
17500 }
17501
17502 assert_nonce_reusable(&client.dispatch);
17503 }
17504
17505 #[tokio::test]
17506 async fn submit_order_below_min_notional_emits_denied() {
17507 let (client, cache, mut rx) = create_execution_client();
17508 let instrument_id = register_test_instrument(&client, &cache);
17509 let mut factory = test_order_factory();
17510 let order = factory.limit(
17511 instrument_id,
17512 OrderSide::Buy,
17513 Quantity::from("0.0010"),
17514 Price::from("2361.31"),
17515 Some(TimeInForce::Gtc),
17516 None,
17517 Some(false),
17518 Some(false),
17519 None,
17520 None,
17521 None,
17522 None,
17523 None,
17524 None,
17525 None,
17526 Some(ClientOrderId::from("O-BELOW-MIN-NOTIONAL")),
17527 );
17528 cache_order(&cache, order.clone());
17529
17530 let command = SubmitOrder::from_order(
17531 &order,
17532 trader_id(),
17533 Some(client_id()),
17534 None,
17535 UUID4::new(),
17536 UnixNanos::default(),
17537 );
17538 let _ = client.submit_order(command);
17539
17540 let event = recv_order_event(&mut rx).await;
17541 match event {
17542 OrderEventAny::Denied(event) => {
17543 assert!(
17544 event.reason.contains("min_quote_amount"),
17545 "expected min_quote_amount in reason, was {:?}",
17546 event.reason,
17547 );
17548 }
17549 event => panic!("expected denied event, was {event:?}"),
17550 }
17551
17552 assert_nonce_reusable(&client.dispatch);
17553 }
17554
17555 #[tokio::test]
17556 async fn submit_stop_market_with_sub_tick_trigger_emits_denied() {
17557 let (client, cache, mut rx) = create_execution_client();
17558 let instrument_id = register_test_instrument(&client, &cache);
17559 let mut factory = test_order_factory();
17560 let order = factory.stop_market(
17562 instrument_id,
17563 OrderSide::Buy,
17564 Quantity::from("0.1000"),
17565 Price::from("0.001"),
17566 None,
17567 Some(TimeInForce::Gtc),
17568 None,
17569 Some(false),
17570 Some(false),
17571 None,
17572 None,
17573 None,
17574 None,
17575 None,
17576 None,
17577 Some(ClientOrderId::from("O-STOP-SUB-TICK")),
17578 );
17579 cache_order(&cache, order.clone());
17580
17581 let command = SubmitOrder::from_order(
17582 &order,
17583 trader_id(),
17584 Some(client_id()),
17585 None,
17586 UUID4::new(),
17587 UnixNanos::default(),
17588 );
17589 let _ = client.submit_order(command);
17590
17591 let event = recv_order_event(&mut rx).await;
17592 match event {
17593 OrderEventAny::Denied(event) => {
17594 assert!(
17595 event.reason.contains("rounds to 0 ticks"),
17596 "expected rounds-to-0 in reason, was {:?}",
17597 event.reason,
17598 );
17599 }
17600 event => panic!("expected denied event, was {event:?}"),
17601 }
17602
17603 assert_nonce_reusable(&client.dispatch);
17604 }
17605
17606 #[tokio::test]
17607 async fn submit_stop_market_dispatches_using_trigger_widened_by_slippage() {
17608 let (client, cache, mut rx) = create_execution_client();
17609 let instrument_id = register_test_instrument(&client, &cache);
17610 let mut factory = test_order_factory();
17611 let order = factory.stop_market(
17612 instrument_id,
17613 OrderSide::Sell,
17614 Quantity::from("0.1000"),
17615 Price::from("2300.00"), None,
17617 Some(TimeInForce::Gtc),
17618 None,
17619 Some(false),
17620 Some(false),
17621 None,
17622 None,
17623 None,
17624 None,
17625 None,
17626 None,
17627 Some(ClientOrderId::from("O-STOP-MARKET")),
17628 );
17629 cache_order(&cache, order.clone());
17630
17631 let command = SubmitOrder::from_order(
17632 &order,
17633 trader_id(),
17634 Some(client_id()),
17635 None,
17636 UUID4::new(),
17637 UnixNanos::default(),
17638 );
17639 let _ = client.submit_order(command);
17640
17641 let submitted = recv_order_event(&mut rx).await;
17642 assert!(matches!(submitted, OrderEventAny::Submitted(_)));
17643 let rejected = recv_order_event(&mut rx).await;
17644 assert!(matches!(rejected, OrderEventAny::Rejected(_)));
17645 assert_nonce_reusable(&client.dispatch);
17646 }
17647
17648 #[tokio::test]
17649 async fn submit_market_order_respects_per_order_slippage_override() {
17650 let (client, cache, mut rx) = create_execution_client();
17653 let instrument_id = register_test_instrument(&client, &cache);
17654 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
17655
17656 let mut factory = test_order_factory();
17657 let order = test_market_order(
17658 &mut factory,
17659 instrument_id,
17660 "O-MARKET-ZERO-SLIP",
17661 OrderSide::Buy,
17662 );
17663 cache_order(&cache, order.clone());
17664
17665 let params: Params =
17666 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 0})).unwrap();
17667 let mut command = SubmitOrder::from_order(
17668 &order,
17669 trader_id(),
17670 Some(client_id()),
17671 None,
17672 UUID4::new(),
17673 UnixNanos::default(),
17674 );
17675 command.params = Some(params);
17676 let _ = client.submit_order(command);
17677
17678 let submitted = recv_order_event(&mut rx).await;
17679 assert!(matches!(submitted, OrderEventAny::Submitted(_)));
17680 let rejected = recv_order_event(&mut rx).await;
17681 assert!(matches!(rejected, OrderEventAny::Rejected(_)));
17682 assert_nonce_reusable(&client.dispatch);
17683 }
17684
17685 #[tokio::test]
17686 async fn resolve_slippage_bps_prefers_params_over_config_default() {
17687 let config = LighterExecutionClientConfig {
17688 market_order_slippage_bps: 73,
17689 ..test_config()
17690 };
17691
17692 let (client, _cache, _rx) = create_execution_client_with_config(config);
17693 assert_eq!(client.resolve_slippage_bps(None).unwrap(), 73);
17694
17695 let override_params: Params =
17696 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 100})).unwrap();
17697 assert_eq!(
17698 client.resolve_slippage_bps(Some(&override_params)).unwrap(),
17699 100
17700 );
17701
17702 let unrelated_params: Params =
17703 serde_json::from_value(serde_json::json!({"other_key": 999})).unwrap();
17704 assert_eq!(
17705 client
17706 .resolve_slippage_bps(Some(&unrelated_params))
17707 .unwrap(),
17708 73
17709 );
17710 }
17711
17712 #[rstest]
17713 #[case::zero(serde_json::json!(0), Ok(0))]
17714 #[case::integer(serde_json::json!(37), Ok(37))]
17715 #[case::maximum(serde_json::json!(9_999), Ok(9_999))]
17716 #[case::out_of_range(serde_json::json!(10_000), Err("must be in [0, 10000)"))]
17717 #[case::maximum_u32(serde_json::json!(u32::MAX), Err("must be in [0, 10000)"))]
17718 #[case::float(serde_json::json!(0.0), Err("must be an unsigned integer"))]
17719 #[case::negative(serde_json::json!(-1), Err("must be an unsigned integer"))]
17720 #[case::string(serde_json::json!("0"), Err("must be an unsigned integer"))]
17721 #[case::null(serde_json::Value::Null, Err("must be an unsigned integer"))]
17722 #[case::boolean(serde_json::json!(false), Err("must be an unsigned integer"))]
17723 #[case::overflow(serde_json::json!(u64::from(u32::MAX) + 1), Err("exceeds u32 range"))]
17724 #[case::maximum_u64(serde_json::json!(u64::MAX), Err("exceeds u32 range"))]
17725 fn resolve_slippage_bps_validates_present_override(
17726 #[case] value: serde_json::Value,
17727 #[case] expected: Result<u32, &str>,
17728 ) {
17729 let (client, _cache, _rx) = create_execution_client();
17730 let params = serde_json::from_value(serde_json::json!({
17731 "market_order_slippage_bps": value,
17732 }))
17733 .unwrap();
17734
17735 let result = client.resolve_slippage_bps(Some(¶ms));
17736
17737 assert_eq!(
17738 result.map_err(|e| e.to_string()),
17739 expected.map_err(|reason| format!("Lighter market_order_slippage_bps {reason}")),
17740 );
17741 }
17742
17743 #[tokio::test]
17744 async fn submit_order_denies_invalid_slippage_before_dispatch() {
17745 let (client, cache, mut rx) = create_execution_client();
17746 let instrument_id = register_test_instrument(&client, &cache);
17747 let mut factory = test_order_factory();
17748 let order = test_limit_order(&mut factory, instrument_id, "O-SLIPPAGE-INVALID");
17749 cache_order(&cache, order.clone());
17750 let mut command = SubmitOrder::from_order(
17751 &order,
17752 trader_id(),
17753 Some(client_id()),
17754 None,
17755 UUID4::new(),
17756 UnixNanos::default(),
17757 );
17758 command.params = Some(
17759 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 0.0})).unwrap(),
17760 );
17761
17762 client.submit_order(command).unwrap();
17763
17764 match recv_order_event(&mut rx).await {
17765 OrderEventAny::Denied(event) => {
17766 assert_eq!(event.client_order_id, order.client_order_id());
17767 assert_eq!(
17768 event.reason,
17769 "VALIDATION_FAILED: Lighter submit_order failed: Lighter market_order_slippage_bps must be an unsigned integer",
17770 );
17771 }
17772 event => panic!("expected denied event, was {event:?}"),
17773 }
17774
17775 assert!(rx.try_recv().is_err());
17776 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
17777 assert_nonce_reusable(&client.dispatch);
17778 }
17779
17780 #[rstest]
17781 #[case::out_of_range(10_000, "market_order_slippage_bps must be in [0, 10000)")]
17782 #[case::overflow(u64::from(u32::MAX) + 1, "market_order_slippage_bps exceeds u32 range")]
17783 #[tokio::test]
17784 async fn submit_order_list_denies_invalid_slippage_before_dispatch(
17785 #[values(false, true)] grouped: bool,
17786 #[case] value: u64,
17787 #[case] expected_reason: &str,
17788 ) {
17789 let (client, cache, mut rx) = create_execution_client();
17790 let instrument_id = register_test_instrument(&client, &cache);
17791 let mut factory = test_order_factory();
17792
17793 let orders = if grouped {
17794 test_native_group(instrument_id, LighterGroupingType::OneCancelsTheOther)
17795 } else {
17796 vec![
17797 test_limit_order(&mut factory, instrument_id, "O-SLIPPAGE-LIST-A"),
17798 test_limit_order(&mut factory, instrument_id, "O-SLIPPAGE-LIST-B"),
17799 ]
17800 };
17801
17802 for order in &orders {
17803 cache_order(&cache, order.clone());
17804 }
17805
17806 let mut command = submit_order_list_command(&orders, "OL-SLIPPAGE-INVALID");
17807 command.params = Some(
17808 serde_json::from_value(serde_json::json!({
17809 "market_order_slippage_bps": value,
17810 }))
17811 .unwrap(),
17812 );
17813
17814 client.submit_order_list(command).unwrap();
17815
17816 for order in &orders {
17817 match recv_order_event(&mut rx).await {
17818 OrderEventAny::Denied(event) => {
17819 assert_eq!(event.client_order_id, order.client_order_id());
17820 assert!(event.reason.contains(expected_reason));
17821 }
17822 event => panic!("expected denied event, was {event:?}"),
17823 }
17824 }
17825
17826 assert!(rx.try_recv().is_err());
17827 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
17828 assert_nonce_reusable(&client.dispatch);
17829 }
17830
17831 #[tokio::test]
17832 async fn submit_market_sell_with_quote_uses_bid_widened_by_slippage() {
17833 let (client, cache, mut rx) = create_execution_client();
17834 let instrument_id = register_test_instrument(&client, &cache);
17835 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
17836
17837 let mut factory = test_order_factory();
17838 let order = test_market_order(
17839 &mut factory,
17840 instrument_id,
17841 "O-MARKET-QUOTED-SELL",
17842 OrderSide::Sell,
17843 );
17844 cache_order(&cache, order.clone());
17845
17846 let command = SubmitOrder::from_order(
17847 &order,
17848 trader_id(),
17849 Some(client_id()),
17850 None,
17851 UUID4::new(),
17852 UnixNanos::default(),
17853 );
17854 let _ = client.submit_order(command);
17855
17856 let submitted = recv_order_event(&mut rx).await;
17857 assert!(
17858 matches!(submitted, OrderEventAny::Submitted(_)),
17859 "expected submitted, was {submitted:?}",
17860 );
17861 let rejected = recv_order_event(&mut rx).await;
17862 match rejected {
17863 OrderEventAny::Rejected(event) => {
17864 assert!(
17865 event
17866 .reason
17867 .contains("Lighter submit_order dispatch failed"),
17868 );
17869 }
17870 event => panic!("expected rejected event, was {event:?}"),
17871 }
17872
17873 assert_nonce_reusable(&client.dispatch);
17874 }
17875
17876 #[rstest]
17877 fn integrator_attributes_tag_mainnet_orders() {
17878 assert_eq!(
17879 integrator_attributes(Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)),
17880 L2TxAttributes {
17881 integrator_account_index: LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
17882 ..Default::default()
17883 },
17884 );
17885 }
17886
17887 #[rstest]
17888 #[case("expire_maker", "account_index", 0, 0)]
17889 #[case("expire_maker", "master_account_index", 0, 1)]
17890 #[case("expire_taker", "account_index", 1, 0)]
17891 #[case("EXPIRE_TAKER", "MASTER_ACCOUNT_INDEX", 1, 1)]
17892 #[case("cancel_both", "account_index", 2, 0)]
17893 #[case("cancel_both", "master_account_index", 2, 1)]
17894 #[case("reduce", "account_index", 3, 0)]
17895 fn resolve_self_trade_params(
17896 #[case] behavior: &str,
17897 #[case] equality: &str,
17898 #[case] expected_behavior: u8,
17899 #[case] expected_equality: u8,
17900 ) {
17901 let (client, _cache, _rx) = create_execution_client();
17902 let params: Params = serde_json::from_value(serde_json::json!({
17903 "self_trade_behavior_mode": behavior,
17904 "self_trade_equality_mode": equality,
17905 }))
17906 .unwrap();
17907
17908 let attrs = client.resolve_create_attributes(Some(¶ms)).unwrap();
17909
17910 assert_eq!(
17911 attrs,
17912 L2TxAttributes {
17913 self_trade_behavior_mode: Some(
17914 serde_json::from_value(serde_json::json!(expected_behavior)).unwrap()
17915 ),
17916 self_trade_equality_mode: Some(
17917 serde_json::from_value(serde_json::json!(expected_equality)).unwrap()
17918 ),
17919 ..Default::default()
17920 }
17921 );
17922 }
17923
17924 #[rstest]
17925 fn resolve_self_trade_defaults_are_explicit() {
17926 let (client, _cache, _rx) = create_execution_client();
17927 let attrs = client.resolve_create_attributes(None).unwrap();
17928
17929 assert_eq!(
17930 attrs,
17931 L2TxAttributes {
17932 self_trade_behavior_mode: Some(LighterSelfTradeBehaviorMode::ExpireMaker),
17933 self_trade_equality_mode: Some(LighterSelfTradeEqualityMode::AccountIndex),
17934 ..Default::default()
17935 }
17936 );
17937 }
17938
17939 #[rstest]
17940 #[case(serde_json::json!({"self_trade_behavior_mode": "unknown"}), "invalid Lighter self_trade_behavior_mode")]
17941 #[case(serde_json::json!({"self_trade_equality_mode": "unknown"}), "invalid Lighter self_trade_equality_mode")]
17942 #[case(serde_json::json!({"self_trade_behavior_mode": 1}), "Lighter self_trade_behavior_mode must be a string")]
17943 #[case(serde_json::json!({"self_trade_equality_mode": null}), "Lighter self_trade_equality_mode must be a string")]
17944 #[case(serde_json::json!({"self_trade_behavior_mode": "reduce", "self_trade_equality_mode": "master_account_index"}), "Lighter reduce self-trade behavior cannot combine with master-account equality")]
17945 fn resolve_self_trade_rejects_invalid_params(
17946 #[case] value: serde_json::Value,
17947 #[case] expected: &str,
17948 ) {
17949 let (client, _cache, _rx) = create_execution_client();
17950 let params: Params = serde_json::from_value(value).unwrap();
17951
17952 let error = client.resolve_create_attributes(Some(¶ms)).unwrap_err();
17953
17954 assert_eq!(error.to_string(), expected);
17955 assert_nonce_reusable(&client.dispatch);
17956 }
17957
17958 #[rstest]
17959 #[case::single(false, false)]
17960 #[case::list(true, false)]
17961 #[case::group(true, true)]
17962 #[tokio::test]
17963 async fn submit_self_trade_invalid_combination_denies_before_dispatch(
17964 #[case] list: bool,
17965 #[case] grouped: bool,
17966 ) {
17967 let (client, cache, mut rx) = create_execution_client();
17968 let instrument = register_test_instrument(&client, &cache);
17969 let mut factory = test_order_factory();
17970
17971 let orders = if grouped {
17972 test_native_group(
17973 instrument,
17974 LighterGroupingType::OneTriggersOneCancelsTheOther,
17975 )
17976 } else {
17977 vec![test_limit_order(
17978 &mut factory,
17979 instrument,
17980 "O-SELF-TRADE-INVALID",
17981 )]
17982 };
17983
17984 for order in &orders {
17985 cache_order(&cache, order.clone());
17986 }
17987
17988 let params: Params = serde_json::from_value(serde_json::json!({
17989 "self_trade_behavior_mode": "reduce",
17990 "self_trade_equality_mode": "master_account_index",
17991 }))
17992 .unwrap();
17993
17994 if list {
17995 let mut command = submit_order_list_command(&orders, "OL-SELF-TRADE-INVALID");
17996 command.params = Some(params);
17997 client.submit_order_list(command).unwrap();
17998 } else {
17999 let mut command = SubmitOrder::from_order(
18000 &orders[0],
18001 trader_id(),
18002 Some(client_id()),
18003 None,
18004 UUID4::new(),
18005 UnixNanos::default(),
18006 );
18007 command.params = Some(params);
18008 client.submit_order(command).unwrap();
18009 }
18010
18011 for order in &orders {
18012 match recv_order_event(&mut rx).await {
18013 OrderEventAny::Denied(event) => {
18014 assert_eq!(event.client_order_id, order.client_order_id());
18015 assert!(event.reason.contains("Lighter reduce self-trade behavior cannot combine with master-account equality"));
18016 }
18017 event => panic!("expected OrderDenied, was {event:?}"),
18018 }
18019 }
18020
18021 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
18022 assert_nonce_reusable(&client.dispatch);
18023 }
18024
18025 #[rstest]
18026 fn integrator_attributes_leave_testnet_orders_unattributed() {
18027 assert_eq!(integrator_attributes(None), L2TxAttributes::default());
18028 }
18029
18030 #[rstest]
18031 #[case::lighter_mainnet(
18032 LighterDeployment::Lighter,
18033 LighterEnvironment::Mainnet,
18034 Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)
18035 )]
18036 #[case::lighter_testnet(LighterDeployment::Lighter, LighterEnvironment::Testnet, None)]
18037 #[case::robinhood_mainnet(
18038 LighterDeployment::Robinhood,
18039 LighterEnvironment::Mainnet,
18040 Some(LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)
18041 )]
18042 #[case::robinhood_testnet(LighterDeployment::Robinhood, LighterEnvironment::Testnet, None)]
18043 fn integrator_account_index_follows_deployment(
18044 #[case] deployment: LighterDeployment,
18045 #[case] environment: LighterEnvironment,
18046 #[case] expected: Option<u64>,
18047 ) {
18048 let mut config = test_config();
18049 config.deployment = deployment;
18050 config.environment = environment;
18051 config.venue = Some(*LIGHTER_VENUE);
18052 let (client, _cache, _rx) = create_execution_client_with_config(config);
18053
18054 assert_eq!(client.integrator_account_index(), expected);
18055 }
18056
18057 #[rstest]
18058 fn robinhood_orders_carry_integrator_attributes() {
18059 let mut config = test_config();
18060 config.deployment = LighterDeployment::Robinhood;
18061 config.environment = LighterEnvironment::Mainnet;
18062
18063 assert_eq!(
18064 integrator_attributes(deployment::integrator_account_index(
18065 config.deployment,
18066 config.environment,
18067 )),
18068 L2TxAttributes {
18069 integrator_account_index: LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
18070 ..Default::default()
18071 },
18072 );
18073 }
18074
18075 use std::str::FromStr;
18076
18077 use nautilus_live::ExecutionEventEmitter;
18078 use rust_decimal::Decimal;
18079
18080 use crate::{
18081 common::enums::{
18082 LighterOrderKind, LighterOrderSide, LighterOrderStatus, LighterOrderTimeInForce,
18083 LighterTradeType, LighterTriggerStatus,
18084 },
18085 http::models::{LighterOrder, LighterTrade},
18086 };
18087
18088 fn dispatcher_emitter() -> (
18089 ExecutionEventEmitter,
18090 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
18091 ) {
18092 let mut emitter = ExecutionEventEmitter::new(
18093 get_atomic_clock_realtime(),
18094 trader_id(),
18095 account_id(),
18096 AccountType::Margin,
18097 None,
18098 );
18099 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
18100 emitter.set_sender(sender);
18101 (emitter, receiver)
18102 }
18103
18104 struct DispatcherRig {
18108 dispatch: WsDispatchState,
18109 registry: Arc<MarketRegistry>,
18110 emitter: ExecutionEventEmitter,
18111 rx: tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
18112 instrument_id: InstrumentId,
18113 cloid: ClientOrderId,
18114 }
18115
18116 fn dispatcher_rig(cloid_suffix: &str) -> DispatcherRig {
18117 let registry = Arc::new(MarketRegistry::new());
18118 let instrument_id = registry.insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
18123 let instrument = InstrumentAny::CryptoPerpetual(
18124 CryptoPerpetual::builder()
18125 .instrument_id(instrument_id)
18126 .raw_symbol(Symbol::new("ETH-PERP"))
18127 .base_currency(Currency::from("ETH"))
18128 .quote_currency(Currency::from("USDC"))
18129 .settlement_currency(Currency::from("USDC"))
18130 .is_inverse(false)
18131 .price_precision(2)
18132 .size_precision(4)
18133 .price_increment(Price::from("0.01"))
18134 .size_increment(Quantity::from("0.0001"))
18135 .min_notional(Money::from("10.000000 USDC"))
18136 .ts_event(UnixNanos::default())
18137 .ts_init(UnixNanos::default())
18138 .build()
18139 .unwrap(),
18140 );
18141 LIGHTER_INSTRUMENT_CACHE.insert(instrument_id, instrument);
18142 let (emitter, rx) = dispatcher_emitter();
18143
18144 DispatcherRig {
18145 dispatch: WsDispatchState::new(),
18146 registry,
18147 emitter,
18148 rx,
18149 instrument_id,
18150 cloid: ClientOrderId::new(format!("CLOID-{cloid_suffix}")),
18151 }
18152 }
18153
18154 fn register_identity(rig: &DispatcherRig) {
18155 register_identity_for(rig, OrderType::Limit);
18156 }
18157
18158 fn register_identity_for(rig: &DispatcherRig, order_type: OrderType) {
18159 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
18160 let client_order_index = rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
18161 rig.dispatch.register_order_identity(
18162 rig.cloid,
18163 OrderIdentity::new(
18164 rig.instrument_id,
18165 strategy_id(),
18166 OrderSide::Buy,
18167 order_type,
18168 client_order_index,
18169 ),
18170 );
18171 rig.dispatch
18172 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
18173 }
18174
18175 fn register_unowned_identity(rig: &DispatcherRig) {
18176 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
18177 let client_order_index = rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
18178 rig.dispatch.register_order_identity(
18179 rig.cloid,
18180 OrderIdentity::new(
18181 rig.instrument_id,
18182 strategy_id(),
18183 OrderSide::Buy,
18184 OrderType::Limit,
18185 client_order_index,
18186 ),
18187 );
18188 }
18189
18190 fn dispatcher_test_order(rig: &DispatcherRig, status: LighterOrderStatus) -> LighterOrder {
18191 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
18192 rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
18193
18194 LighterOrder {
18195 order_index: 281_476_929_510_110,
18196 client_order_index: derived,
18197 order_id: "281476929510110".to_string(),
18198 client_order_id: derived.to_string(),
18199 market_index: TEST_MARKET_INDEX,
18200 owner_account_index: TEST_ACCOUNT_INDEX_I64,
18201 initial_base_amount: Decimal::from_str("0.0050").unwrap(),
18202 price: Decimal::from_str("2352.74").unwrap(),
18203 nonce: TEST_ORDER_NONCE,
18204 remaining_base_amount: Decimal::from_str("0.0050").unwrap(),
18205 is_ask: false,
18206 base_size: 50,
18207 base_price: 235_274,
18208 filled_base_amount: Decimal::ZERO,
18209 filled_quote_amount: Decimal::ZERO,
18210 side: Some(LighterOrderSide::Buy),
18211 order_type: LighterOrderKind::Limit,
18212 time_in_force: LighterOrderTimeInForce::GoodTillTime,
18213 reduce_only: false,
18214 trigger_price: Decimal::ZERO,
18215 order_expiry: 1_780_360_584_479,
18216 status,
18217 trigger_status: LighterTriggerStatus::Na,
18218 trigger_time: 0,
18219 parent_order_index: 0,
18220 parent_order_id: "0".to_string(),
18221 to_trigger_order_id_0: "0".to_string(),
18222 to_trigger_order_id_1: "0".to_string(),
18223 to_cancel_order_id_0: "0".to_string(),
18224 integrator_fee_collector_index: "0".to_string(),
18225 integrator_taker_fee: Decimal::ZERO,
18226 integrator_maker_fee: Decimal::ZERO,
18227 block_height: 227_535_532,
18228 timestamp: 1_777_941_383_576,
18229 created_at: 1_777_941_383_576,
18230 updated_at: 1_777_941_383_900,
18231 transaction_time: 1_777_941_383_576_735,
18232 }
18233 }
18234
18235 fn dispatcher_test_trade(rig: &DispatcherRig, user_is_bidder: bool) -> LighterTrade {
18236 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
18237 rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
18238
18239 LighterTrade {
18240 trade_id: 19_209_006_902,
18241 trade_id_str: Some("19209006902".to_string()),
18242 tx_hash: "000000128b1ee814".to_string(),
18243 trade_type: LighterTradeType::Trade,
18244 market_id: TEST_MARKET_INDEX,
18245 size: Decimal::from_str("0.1336").unwrap(),
18246 price: Decimal::from_str("2352.73").unwrap(),
18247 usd_amount: Decimal::from_str("314.324728").unwrap(),
18248 ask_id: 281_476_929_510_102,
18249 ask_id_str: Some("281476929510102".to_string()),
18250 bid_id: 562_947_905_631_053,
18251 bid_id_str: Some("562947905631053".to_string()),
18252 ask_client_id: if user_is_bidder { 0 } else { derived },
18253 ask_client_id_str: Some(if user_is_bidder {
18254 "0".to_string()
18255 } else {
18256 derived.to_string()
18257 }),
18258 bid_client_id: if user_is_bidder { derived } else { 0 },
18259 bid_client_id_str: Some(if user_is_bidder {
18260 derived.to_string()
18261 } else {
18262 "0".to_string()
18263 }),
18264 ask_account_id: if user_is_bidder {
18265 91_249
18266 } else {
18267 TEST_ACCOUNT_INDEX_I64
18268 },
18269 bid_account_id: if user_is_bidder {
18270 TEST_ACCOUNT_INDEX_I64
18271 } else {
18272 91_249
18273 },
18274 is_maker_ask: false,
18275 block_height: 227_535_535,
18276 timestamp: 1_777_941_384_181,
18277 taker_fee: Some(196),
18278 taker_position_size_before: None,
18279 taker_entry_quote_before: None,
18280 taker_initial_margin_fraction_before: None,
18281 taker_position_sign_changed: None,
18282 maker_fee: Some(28),
18283 maker_position_size_before: None,
18284 maker_entry_quote_before: None,
18285 maker_initial_margin_fraction_before: None,
18286 maker_position_sign_changed: None,
18287 transaction_time: 1_777_941_384_181_586,
18288 ask_account_pnl: None,
18289 bid_account_pnl: None,
18290 }
18291 }
18292
18293 fn drain_events(
18297 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
18298 ) -> Vec<ExecutionEvent> {
18299 let mut events = Vec::new();
18300 while let Ok(event) = rx.try_recv() {
18301 events.push(event);
18302 }
18303
18304 events
18305 }
18306
18307 #[rstest]
18308 fn dispatch_lighter_order_tracked_emits_accepted_then_silent_repeat() {
18309 let mut rig = dispatcher_rig("1");
18310 register_identity(&rig);
18311 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18312
18313 dispatch_lighter_order(
18314 &order,
18315 &rig.dispatch,
18316 &rig.emitter,
18317 &rig.registry,
18318 account_id(),
18319 trader_id(),
18320 UnixNanos::from(1),
18321 );
18322 dispatch_lighter_order(
18323 &order,
18324 &rig.dispatch,
18325 &rig.emitter,
18326 &rig.registry,
18327 account_id(),
18328 trader_id(),
18329 UnixNanos::from(2),
18330 );
18331
18332 let events = drain_events(&mut rig.rx);
18333 assert_eq!(
18334 events.len(),
18335 1,
18336 "exactly one event expected, was {events:?}",
18337 );
18338
18339 match &events[0] {
18340 ExecutionEvent::Order(OrderEventAny::Accepted(e)) => {
18341 assert_eq!(e.client_order_id, rig.cloid);
18342 assert_eq!(e.venue_order_id.to_string(), "281476929510110");
18343 }
18344 other => panic!("expected Accepted, was {other:?}"),
18345 }
18346
18347 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
18348 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_some());
18349 }
18350
18351 #[rstest]
18352 fn dispatch_lighter_order_tracked_emits_updated_on_shape_change() {
18353 let mut rig = dispatcher_rig("2");
18354 register_identity(&rig);
18355 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18356
18357 dispatch_lighter_order(
18358 &order,
18359 &rig.dispatch,
18360 &rig.emitter,
18361 &rig.registry,
18362 account_id(),
18363 trader_id(),
18364 UnixNanos::from(1),
18365 );
18366 assert_eq!(drain_events(&mut rig.rx).len(), 1);
18367
18368 let mut modified = order;
18369 modified.price = Decimal::from_str("2400.00").unwrap();
18370
18371 dispatch_lighter_order(
18372 &modified,
18373 &rig.dispatch,
18374 &rig.emitter,
18375 &rig.registry,
18376 account_id(),
18377 trader_id(),
18378 UnixNanos::from(2),
18379 );
18380
18381 let events = drain_events(&mut rig.rx);
18382 assert_eq!(
18383 events.len(),
18384 1,
18385 "expected one Updated event, was {events:?}",
18386 );
18387
18388 match &events[0] {
18389 ExecutionEvent::Order(OrderEventAny::Updated(e)) => {
18390 assert_eq!(e.client_order_id, rig.cloid);
18391 assert_eq!(e.price, Some(Price::from("2400.00")));
18392 }
18393 other => panic!("expected Updated, was {other:?}"),
18394 }
18395
18396 let snapshot = rig.dispatch.snapshot_for(&rig.cloid).expect("snapshot");
18397 assert_eq!(snapshot.price, Some(Price::from("2400.00")));
18398 }
18399
18400 #[rstest]
18401 #[case(LighterOrderStatus::Pending)]
18402 #[case(LighterOrderStatus::Open)]
18403 fn dispatch_lighter_order_defers_live_frame_while_cancel_pending(
18404 #[case] status: LighterOrderStatus,
18405 ) {
18406 let mut rig = dispatcher_rig(match status {
18407 LighterOrderStatus::Pending => "24-PENDING",
18408 LighterOrderStatus::Open => "24-OPEN",
18409 other => panic!("unexpected case {other:?}"),
18410 });
18411
18412 register_identity(&rig);
18413 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18414
18415 dispatch_lighter_order(
18416 &order,
18417 &rig.dispatch,
18418 &rig.emitter,
18419 &rig.registry,
18420 account_id(),
18421 trader_id(),
18422 UnixNanos::from(1),
18423 );
18424 assert_eq!(drain_events(&mut rig.rx).len(), 1);
18425
18426 rig.dispatch
18427 .set_pending_order_action(rig.cloid, PendingOrderAction::Cancel);
18428 let mut late = order;
18429 late.status = status;
18430 late.initial_base_amount = Decimal::from_str("0.0075").unwrap();
18431 late.price = Decimal::from_str("2400.00").unwrap();
18432
18433 dispatch_lighter_order(
18434 &late,
18435 &rig.dispatch,
18436 &rig.emitter,
18437 &rig.registry,
18438 account_id(),
18439 trader_id(),
18440 UnixNanos::from(2),
18441 );
18442
18443 let events = drain_events(&mut rig.rx);
18444 assert_eq!(events.len(), 0, "late frame must be silent: {events:?}");
18445 assert_eq!(
18446 rig.dispatch.pending_order_action(&rig.cloid),
18447 Some(PendingOrderAction::Cancel),
18448 );
18449 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
18450 assert!(!rig.dispatch.triggered_was_emitted(&rig.cloid));
18451 assert_eq!(
18452 rig.dispatch.lookup_venue_order_id(&rig.cloid),
18453 Some(VenueOrderId::new("281476929510110")),
18454 );
18455 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
18456 assert_eq!(
18457 rig.dispatch.snapshot_for(&rig.cloid),
18458 Some(crate::websocket::dispatch::OrderShapeSnapshot {
18459 quantity: Quantity::from("0.0050"),
18460 price: Some(Price::from("2352.74")),
18461 trigger_price: None,
18462 }),
18463 );
18464 }
18465
18466 #[rstest]
18467 #[case::standalone(false)]
18468 #[case::native_child(true)]
18469 fn dispatch_lighter_order_emits_updated_then_triggered_for_one_frame(
18470 #[case] native_child: bool,
18471 ) {
18472 let mut rig = dispatcher_rig("25");
18473
18474 if native_child {
18475 let child = OrderTestBuilder::new(OrderType::StopLimit)
18476 .trader_id(trader_id())
18477 .strategy_id(strategy_id())
18478 .instrument_id(rig.instrument_id)
18479 .client_order_id(rig.cloid)
18480 .side(OrderSide::Buy)
18481 .quantity(Quantity::from("0.0050"))
18482 .price(Price::from("2352.74"))
18483 .trigger_price(Price::from("2300.00"))
18484 .parent_order_id(ClientOrderId::new("TRIGGER-PARENT"))
18485 .build();
18486 rig.dispatch.register_create_identity(&child).unwrap();
18487 rig.dispatch
18488 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
18489 } else {
18490 register_identity_for(&rig, OrderType::StopLimit);
18491 }
18492
18493 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
18494 order.order_type = LighterOrderKind::StopLossLimit;
18495 order.trigger_price = Decimal::from_str("2300.00").unwrap();
18496
18497 dispatch_lighter_order(
18498 &order,
18499 &rig.dispatch,
18500 &rig.emitter,
18501 &rig.registry,
18502 account_id(),
18503 trader_id(),
18504 UnixNanos::from(1),
18505 );
18506 let accepted = drain_events(&mut rig.rx);
18507 assert_eq!(accepted.len(), 1, "expected one acceptance: {accepted:?}");
18508 assert!(matches!(
18509 accepted[0],
18510 ExecutionEvent::Order(OrderEventAny::Accepted(_)),
18511 ));
18512
18513 let command = ModifyOrder::new(
18514 trader_id(),
18515 Some(client_id()),
18516 strategy_id(),
18517 rig.instrument_id,
18518 rig.cloid,
18519 Some(VenueOrderId::new("281476929510110")),
18520 Some(Quantity::from("0.0075")),
18521 Some(Price::from("2400.00")),
18522 Some(Price::from("2390.00")),
18523 UUID4::new(),
18524 UnixNanos::from(2),
18525 None,
18526 None,
18527 );
18528 rig.dispatch.set_pending_modify_action(&command);
18529 rig.dispatch.set_pending_modify_shape(
18530 &command,
18531 OrderShapeSnapshot {
18532 quantity: command.quantity.unwrap(),
18533 price: command.price,
18534 trigger_price: command.trigger_price,
18535 },
18536 );
18537
18538 rig.dispatch
18539 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 29);
18540 let mut triggered = order;
18541 triggered.status = LighterOrderStatus::Open;
18542 triggered.trigger_status = LighterTriggerStatus::Ready;
18543 triggered.initial_base_amount = Decimal::from_str("0.0075").unwrap();
18544 triggered.remaining_base_amount = triggered.initial_base_amount;
18545 triggered.price = Decimal::from_str("2400.00").unwrap();
18546 triggered.trigger_price = Decimal::from_str("2390.00").unwrap();
18547
18548 dispatch_lighter_order(
18549 &triggered,
18550 &rig.dispatch,
18551 &rig.emitter,
18552 &rig.registry,
18553 account_id(),
18554 trader_id(),
18555 UnixNanos::from(2),
18556 );
18557
18558 assert!(drain_events(&mut rig.rx).is_empty());
18559 let (client, _, _) = create_execution_client();
18560 let mut context = acked_probe_context(&client);
18561 context.dispatch = rig.dispatch.clone();
18562 context.registry = rig.registry.clone();
18563 context.emitter = rig.emitter.clone();
18564
18565 let pending = PendingSendTx {
18566 batch_id: None,
18567 connection_epoch: 0,
18568 kind: PendingSendTxKind::Modify {
18569 command_id: UUID4::default(),
18570 strategy_id: strategy_id(),
18571 instrument_id: rig.instrument_id,
18572 client_order_id: rig.cloid,
18573 venue_order_id: command.venue_order_id,
18574 },
18575 submitted_at: UnixNanos::from(2),
18576 nonce: 29,
18577 api_key_index: TEST_API_KEY_INDEX,
18578 tx_hash: "confirmed-trigger".to_string(),
18579 };
18580
18581 confirm_modify_order(&pending, &context, &triggered).unwrap();
18582 let events = drain_events(&mut rig.rx);
18583 assert_eq!(
18584 events.len(),
18585 2,
18586 "expected Updated then Triggered: {events:?}",
18587 );
18588
18589 match &events[0] {
18590 ExecutionEvent::Order(OrderEventAny::Updated(event)) => {
18591 assert_eq!(event.trader_id, trader_id());
18592 assert_eq!(event.strategy_id, strategy_id());
18593 assert_eq!(event.instrument_id, rig.instrument_id);
18594 assert_eq!(event.client_order_id, rig.cloid);
18595 assert_eq!(
18596 event.venue_order_id,
18597 Some(VenueOrderId::new("281476929510110")),
18598 );
18599 assert_eq!(event.account_id, Some(account_id()));
18600 assert_eq!(event.quantity, Quantity::from("0.0075"));
18601 assert_eq!(event.price, Some(Price::from("2400.00")));
18602 assert_eq!(event.trigger_price, Some(Price::from("2390.00")));
18603 }
18604 other => panic!("first event should be Updated, was {other:?}"),
18605 }
18606
18607 match &events[1] {
18608 ExecutionEvent::Order(OrderEventAny::Triggered(event)) => {
18609 assert_eq!(event.trader_id, trader_id());
18610 assert_eq!(event.strategy_id, strategy_id());
18611 assert_eq!(event.instrument_id, rig.instrument_id);
18612 assert_eq!(event.client_order_id, rig.cloid);
18613 assert_eq!(
18614 event.venue_order_id,
18615 Some(VenueOrderId::new("281476929510110")),
18616 );
18617 assert_eq!(event.account_id, Some(account_id()));
18618 }
18619 other => panic!("second event should be Triggered, was {other:?}"),
18620 }
18621
18622 assert_eq!(rig.dispatch.pending_order_action(&rig.cloid), None);
18623 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
18624 assert!(rig.dispatch.triggered_was_emitted(&rig.cloid));
18625 assert_eq!(
18626 rig.dispatch.snapshot_for(&rig.cloid),
18627 Some(crate::websocket::dispatch::OrderShapeSnapshot {
18628 quantity: Quantity::from("0.0075"),
18629 price: Some(Price::from("2400.00")),
18630 trigger_price: Some(Price::from("2390.00")),
18631 }),
18632 );
18633
18634 dispatch_lighter_order(
18635 &triggered,
18636 &rig.dispatch,
18637 &rig.emitter,
18638 &rig.registry,
18639 account_id(),
18640 trader_id(),
18641 UnixNanos::from(3),
18642 );
18643 assert!(
18644 drain_events(&mut rig.rx).is_empty(),
18645 "replayed frame must be silent",
18646 );
18647 }
18648
18649 #[rstest]
18650 #[case(LighterTriggerStatus::Ready)]
18651 #[case(LighterTriggerStatus::Na)]
18652 fn dispatch_lighter_order_defers_trigger_until_modify_resolves(
18653 #[case] trigger_status: LighterTriggerStatus,
18654 ) {
18655 let mut rig = dispatcher_rig("26");
18656 register_identity_for(&rig, OrderType::StopLimit);
18657 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
18658 order.order_type = LighterOrderKind::StopLossLimit;
18659 order.trigger_price = Decimal::from_str("2300.00").unwrap();
18660
18661 dispatch_lighter_order(
18662 &order,
18663 &rig.dispatch,
18664 &rig.emitter,
18665 &rig.registry,
18666 account_id(),
18667 trader_id(),
18668 UnixNanos::from(1),
18669 );
18670 assert_eq!(drain_events(&mut rig.rx).len(), 1);
18671
18672 let command = ModifyOrder::new(
18673 trader_id(),
18674 Some(client_id()),
18675 strategy_id(),
18676 rig.instrument_id,
18677 rig.cloid,
18678 Some(VenueOrderId::new("281476929510110")),
18679 Some(Quantity::from("0.0075")),
18680 Some(Price::from("2400.00")),
18681 Some(Price::from("2390.00")),
18682 UUID4::new(),
18683 UnixNanos::from(2),
18684 None,
18685 None,
18686 );
18687 rig.dispatch.set_pending_modify_action(&command);
18688 rig.dispatch.set_pending_modify_shape(
18689 &command,
18690 OrderShapeSnapshot {
18691 quantity: command.quantity.unwrap(),
18692 price: command.price,
18693 trigger_price: command.trigger_price,
18694 },
18695 );
18696
18697 order.status = LighterOrderStatus::Open;
18698 order.trigger_status = trigger_status;
18699 dispatch_lighter_order(
18700 &order,
18701 &rig.dispatch,
18702 &rig.emitter,
18703 &rig.registry,
18704 account_id(),
18705 trader_id(),
18706 UnixNanos::from(2),
18707 );
18708
18709 assert!(drain_events(&mut rig.rx).is_empty());
18710 assert_eq!(
18711 rig.dispatch.pending_order_action(&rig.cloid),
18712 Some(PendingOrderAction::Modify),
18713 );
18714 assert!(!rig.dispatch.triggered_was_emitted(&rig.cloid));
18715
18716 assert!(
18717 rig.dispatch
18718 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
18719 );
18720 dispatch_lighter_order(
18721 &order,
18722 &rig.dispatch,
18723 &rig.emitter,
18724 &rig.registry,
18725 account_id(),
18726 trader_id(),
18727 UnixNanos::from(3),
18728 );
18729
18730 let events = drain_events(&mut rig.rx);
18731 assert_eq!(events.len(), 1, "expected deferred trigger: {events:?}");
18732
18733 match &events[0] {
18734 ExecutionEvent::Order(OrderEventAny::Triggered(event)) => {
18735 assert_eq!(event.client_order_id, rig.cloid);
18736 assert_eq!(
18737 event.venue_order_id,
18738 Some(VenueOrderId::new("281476929510110")),
18739 );
18740 assert_eq!(event.account_id, Some(account_id()));
18741 }
18742 other => panic!("expected Triggered, was {other:?}"),
18743 }
18744
18745 assert!(rig.dispatch.triggered_was_emitted(&rig.cloid));
18746 }
18747
18748 #[rstest]
18749 fn dispatch_lighter_order_pending_emits_accepted_then_updated() {
18750 let mut rig = dispatcher_rig("23");
18751 register_identity(&rig);
18752 let order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
18753
18754 dispatch_lighter_order(
18755 &order,
18756 &rig.dispatch,
18757 &rig.emitter,
18758 &rig.registry,
18759 account_id(),
18760 trader_id(),
18761 UnixNanos::from(1),
18762 );
18763 let accepted = drain_events(&mut rig.rx);
18764
18765 let mut modified = order;
18766 modified.price = Decimal::from_str("2400.00").unwrap();
18767 dispatch_lighter_order(
18768 &modified,
18769 &rig.dispatch,
18770 &rig.emitter,
18771 &rig.registry,
18772 account_id(),
18773 trader_id(),
18774 UnixNanos::from(2),
18775 );
18776 let updated = drain_events(&mut rig.rx);
18777
18778 assert_eq!(
18779 accepted.len(),
18780 1,
18781 "expected one Accepted event: {accepted:?}"
18782 );
18783 assert!(matches!(
18784 accepted[0],
18785 ExecutionEvent::Order(OrderEventAny::Accepted(_))
18786 ));
18787 assert_eq!(updated.len(), 1, "expected one Updated event: {updated:?}");
18788 assert!(matches!(
18789 updated[0],
18790 ExecutionEvent::Order(OrderEventAny::Updated(_))
18791 ));
18792 assert_eq!(
18793 rig.dispatch
18794 .snapshot_for(&rig.cloid)
18795 .expect("snapshot")
18796 .price,
18797 Some(Price::from("2400.00")),
18798 );
18799 }
18800
18801 #[rstest]
18802 fn dispatch_lighter_order_untracked_emits_report() {
18803 let mut rig = dispatcher_rig("3");
18804 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18806 order.client_order_id = "external-1".to_string();
18807 order.client_order_index = 0;
18808
18809 dispatch_lighter_order(
18810 &order,
18811 &rig.dispatch,
18812 &rig.emitter,
18813 &rig.registry,
18814 account_id(),
18815 trader_id(),
18816 UnixNanos::from(1),
18817 );
18818
18819 let events = drain_events(&mut rig.rx);
18820 assert_eq!(events.len(), 1);
18821
18822 match &events[0] {
18823 ExecutionEvent::Report(report) => match report {
18824 EngineExecutionReport::Order(r) => {
18825 assert_eq!(r.venue_order_id.to_string(), "281476929510110");
18826 }
18827 other => panic!("expected order report, was {other:?}"),
18828 },
18829 other => panic!("expected report, was {other:?}"),
18830 }
18831
18832 assert!(
18833 !rig.dispatch
18834 .accepted_was_emitted(&ClientOrderId::new("external-1"))
18835 );
18836 }
18837
18838 #[rstest]
18839 fn same_index_external_order_waits_for_create_submission_ownership() {
18840 let mut rig = dispatcher_rig("EPOCH-ORDER");
18841 register_unowned_identity(&rig);
18842 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18843 let venue_order_id = VenueOrderId::new(order.order_id.as_str());
18844
18845 dispatch_lighter_order(
18846 &order,
18847 &rig.dispatch,
18848 &rig.emitter,
18849 &rig.registry,
18850 account_id(),
18851 trader_id(),
18852 UnixNanos::from(1),
18853 );
18854
18855 let external = drain_events(&mut rig.rx);
18856 assert_eq!(external.len(), 1);
18857
18858 match &external[0] {
18859 ExecutionEvent::Report(EngineExecutionReport::Order(report)) => {
18860 assert_eq!(report.venue_order_id, venue_order_id);
18861 assert_eq!(
18862 report.client_order_id,
18863 Some(ClientOrderId::new(venue_order_id.as_str())),
18864 );
18865 }
18866 other => panic!("expected one external order report, was {other:?}"),
18867 }
18868
18869 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
18870 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
18871
18872 rig.dispatch
18873 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
18874 dispatch_lighter_order(
18875 &order,
18876 &rig.dispatch,
18877 &rig.emitter,
18878 &rig.registry,
18879 account_id(),
18880 trader_id(),
18881 UnixNanos::from(2),
18882 );
18883
18884 let replayed_external = drain_events(&mut rig.rx);
18885 assert_eq!(replayed_external.len(), 1);
18886
18887 match &replayed_external[0] {
18888 ExecutionEvent::Report(EngineExecutionReport::Order(report)) => {
18889 assert_eq!(report.venue_order_id, venue_order_id);
18890 assert_eq!(
18891 report.client_order_id,
18892 Some(ClientOrderId::new(venue_order_id.as_str())),
18893 );
18894 }
18895 other => panic!("expected replayed external order report, was {other:?}"),
18896 }
18897
18898 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
18899 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
18900
18901 let mut local_order = order.clone();
18902 local_order.order_id = "281476929510111".to_string();
18903 let local_venue_order_id = VenueOrderId::new(local_order.order_id.as_str());
18904 dispatch_lighter_order(
18905 &local_order,
18906 &rig.dispatch,
18907 &rig.emitter,
18908 &rig.registry,
18909 account_id(),
18910 trader_id(),
18911 UnixNanos::from(3),
18912 );
18913
18914 let local = drain_events(&mut rig.rx);
18915 assert_eq!(local.len(), 1);
18916
18917 match &local[0] {
18918 ExecutionEvent::Order(OrderEventAny::Accepted(event)) => {
18919 assert_eq!(event.client_order_id, rig.cloid);
18920 assert_eq!(event.venue_order_id, local_venue_order_id);
18921 assert_eq!(event.account_id, account_id());
18922 }
18923 other => panic!("expected one local OrderAccepted, was {other:?}"),
18924 }
18925
18926 assert_eq!(
18927 rig.dispatch
18928 .venue_id_map
18929 .get(&rig.cloid)
18930 .map(|entry| *entry),
18931 Some(local_venue_order_id),
18932 );
18933 assert_eq!(
18934 rig.dispatch
18935 .resolve_live_order_cloid(order.client_order_id.as_str(), local_venue_order_id,),
18936 Some(rig.cloid),
18937 "a bound identity remains stable across replay epochs",
18938 );
18939 }
18940
18941 #[rstest]
18942 #[case::order_first(false)]
18943 #[case::trade_first(true)]
18944 fn quantity_amend_immediate_fill_orders_update_before_fill(
18945 #[case] trade_first: bool,
18946 #[values(false, true)] reduced: bool,
18947 ) {
18948 let mut rig = dispatcher_rig("AMEND-FILL");
18949 let mut cached = OrderTestBuilder::new(OrderType::Limit)
18950 .trader_id(trader_id())
18951 .strategy_id(strategy_id())
18952 .instrument_id(rig.instrument_id)
18953 .client_order_id(rig.cloid)
18954 .side(OrderSide::Buy)
18955 .quantity(Quantity::from("0.0050"))
18956 .price(Price::from("2352.74"))
18957 .build();
18958 rig.dispatch.register_create_identity(&cached).unwrap();
18959 rig.dispatch
18960 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
18961 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18962 dispatch_lighter_order(
18963 &frame,
18964 &rig.dispatch,
18965 &rig.emitter,
18966 &rig.registry,
18967 account_id(),
18968 trader_id(),
18969 UnixNanos::from(1),
18970 );
18971
18972 for event in drain_events(&mut rig.rx) {
18973 let ExecutionEvent::Order(event) = event else {
18974 panic!("expected order event");
18975 };
18976
18977 cached.apply(event).unwrap();
18978 }
18979
18980 let command = ModifyOrder::new(
18981 trader_id(),
18982 Some(client_id()),
18983 strategy_id(),
18984 rig.instrument_id,
18985 rig.cloid,
18986 cached.venue_order_id(),
18987 Some(Quantity::from("0.0100")),
18988 Some(Price::from("2400.00")),
18989 None,
18990 UUID4::new(),
18991 UnixNanos::from(2),
18992 None,
18993 None,
18994 );
18995 rig.dispatch.set_pending_modify_action(&command);
18996 rig.dispatch.set_pending_modify_shape(
18997 &command,
18998 OrderShapeSnapshot {
18999 quantity: Quantity::from("0.0100"),
19000 price: Some(Price::from("2400.00")),
19001 trigger_price: None,
19002 },
19003 );
19004
19005 rig.dispatch
19006 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 19);
19007
19008 let pending = PendingSendTx {
19009 batch_id: None,
19010 connection_epoch: 0,
19011 kind: PendingSendTxKind::Modify {
19012 command_id: UUID4::default(),
19013 strategy_id: strategy_id(),
19014 instrument_id: rig.instrument_id,
19015 client_order_id: rig.cloid,
19016 venue_order_id: cached.venue_order_id(),
19017 },
19018 submitted_at: UnixNanos::from(2),
19019 nonce: 19,
19020 api_key_index: TEST_API_KEY_INDEX,
19021 tx_hash: "confirmed-amend".to_string(),
19022 };
19023
19024 let mut trade = dispatcher_test_trade(&rig, true);
19025 trade.bid_id = frame.order_index;
19026 trade.bid_id_str = Some(frame.order_id.clone());
19027 let expected_qty = Quantity::from(if reduced { "0.0040" } else { "0.0100" });
19028 trade.size = expected_qty.as_decimal();
19029 trade.usd_amount = trade.size * trade.price;
19030
19031 if trade_first {
19032 dispatch_lighter_trade(
19033 &trade,
19034 &rig.dispatch,
19035 &rig.emitter,
19036 &rig.registry,
19037 account_id(),
19038 trader_id(),
19039 Some(TEST_ACCOUNT_INDEX_I64),
19040 UnixNanos::from(3),
19041 );
19042 assert!(
19043 drain_events(&mut rig.rx).is_empty(),
19044 "unconfirmed amend fill must wait"
19045 );
19046 assert!(
19047 !rig.dispatch
19048 .seen_trade_ids
19049 .contains(&parse_lighter_trade_id(&trade).unwrap())
19050 );
19051 }
19052
19053 frame.status = LighterOrderStatus::Filled;
19054 frame.initial_base_amount = Decimal::from_str("0.0100").unwrap();
19055 frame.remaining_base_amount = Decimal::ZERO;
19056 frame.filled_base_amount = trade.size;
19057 frame.price = Decimal::from_str("2400.00").unwrap();
19058 dispatch_lighter_order(
19059 &frame,
19060 &rig.dispatch,
19061 &rig.emitter,
19062 &rig.registry,
19063 account_id(),
19064 trader_id(),
19065 UnixNanos::from(4),
19066 );
19067
19068 if !trade_first {
19069 dispatch_lighter_trade(
19070 &trade,
19071 &rig.dispatch,
19072 &rig.emitter,
19073 &rig.registry,
19074 account_id(),
19075 trader_id(),
19076 Some(TEST_ACCOUNT_INDEX_I64),
19077 UnixNanos::from(5),
19078 );
19079 }
19080
19081 assert!(
19082 drain_events(&mut rig.rx).is_empty(),
19083 "order echo must wait for transaction confirmation"
19084 );
19085 let (client, _, _) = create_execution_client();
19086 let mut context = acked_probe_context(&client);
19087 context.dispatch = rig.dispatch.clone();
19088 context.registry = rig.registry.clone();
19089 context.emitter = rig.emitter.clone();
19090 confirm_modify_order(&pending, &context, &frame).unwrap();
19091 let events = drain_events(&mut rig.rx);
19092 assert_eq!(events.len(), 2, "expected update then fill, was {events:?}");
19093
19094 let ExecutionEvent::Order(OrderEventAny::Updated(update)) = &events[0] else {
19095 panic!("expected update before fill, was {:?}", events[0]);
19096 };
19097
19098 assert_eq!(update.client_order_id, rig.cloid);
19099 assert_eq!(update.quantity, expected_qty);
19100 assert_eq!(update.price, Some(Price::from("2400.00")));
19101 assert_eq!(update.trigger_price, None);
19102
19103 let ExecutionEvent::Order(OrderEventAny::Filled(fill)) = &events[1] else {
19104 panic!("expected real fill, was {:?}", events[1]);
19105 };
19106
19107 assert_eq!(fill.trade_id, parse_lighter_trade_id(&trade).unwrap());
19108 assert_eq!(fill.last_qty, expected_qty);
19109 assert_eq!(fill.last_px, Price::from("2352.73"));
19110
19111 for event in events {
19112 let ExecutionEvent::Order(event) = event else {
19113 panic!("expected order event");
19114 };
19115
19116 cached.apply(event).unwrap();
19117 }
19118
19119 assert_eq!(cached.quantity(), expected_qty);
19120 assert_eq!(cached.filled_qty(), expected_qty);
19121 assert_eq!(cached.price(), Some(Price::from("2400.00")));
19122 assert_eq!(cached.status(), OrderStatus::Filled);
19123 dispatch_lighter_order(
19124 &frame,
19125 &rig.dispatch,
19126 &rig.emitter,
19127 &rig.registry,
19128 account_id(),
19129 trader_id(),
19130 UnixNanos::from(6),
19131 );
19132 dispatch_lighter_trade(
19133 &trade,
19134 &rig.dispatch,
19135 &rig.emitter,
19136 &rig.registry,
19137 account_id(),
19138 trader_id(),
19139 Some(TEST_ACCOUNT_INDEX_I64),
19140 UnixNanos::from(7),
19141 );
19142 assert!(
19143 drain_events(&mut rig.rx)
19144 .iter()
19145 .all(|event| matches!(event, ExecutionEvent::Report(_))),
19146 "terminal replay must not emit duplicate order events"
19147 );
19148 }
19149
19150 #[rstest]
19151 #[case(OrderType::StopMarket)]
19152 #[case(OrderType::StopLimit)]
19153 #[case(OrderType::MarketIfTouched)]
19154 #[case(OrderType::LimitIfTouched)]
19155 fn conditional_fill_synthesizes_trigger_once_across_retirement(#[case] kind: OrderType) {
19156 let mut rig = dispatcher_rig("CONDITIONAL-FILL");
19157 register_identity_for(&rig, kind);
19158 let trade = dispatcher_test_trade(&rig, true);
19159 dispatch_lighter_trade(
19160 &trade,
19161 &rig.dispatch,
19162 &rig.emitter,
19163 &rig.registry,
19164 account_id(),
19165 trader_id(),
19166 Some(TEST_ACCOUNT_INDEX_I64),
19167 UnixNanos::from(1),
19168 );
19169 let first = drain_events(&mut rig.rx);
19170 let triggerable = TRIGGERABLE_ORDER_TYPES.contains(&kind);
19171 let mut builder = OrderTestBuilder::new(kind);
19172 builder
19173 .trader_id(trader_id())
19174 .strategy_id(strategy_id())
19175 .instrument_id(rig.instrument_id)
19176 .client_order_id(rig.cloid)
19177 .side(OrderSide::Buy)
19178 .quantity(Quantity::from("0.2672"))
19179 .trigger_price(Price::from("2300.00"));
19180
19181 if LIMIT_ORDER_TYPES.contains(&kind) {
19182 builder.price(Price::from("2352.74"));
19183 }
19184
19185 let mut order = builder.build();
19186
19187 for event in &first {
19188 let ExecutionEvent::Order(event) = event else {
19189 panic!("expected order event");
19190 };
19191
19192 order.apply(event.clone()).unwrap();
19193 }
19194
19195 rig.dispatch.retire_order_identity(&rig.cloid);
19196 let mut subsequent = trade;
19197 subsequent.trade_id += 1;
19198 subsequent.trade_id_str = Some(subsequent.trade_id.to_string());
19199 dispatch_lighter_trade(
19200 &subsequent,
19201 &rig.dispatch,
19202 &rig.emitter,
19203 &rig.registry,
19204 account_id(),
19205 trader_id(),
19206 Some(TEST_ACCOUNT_INDEX_I64),
19207 UnixNanos::from(2),
19208 );
19209 let next = drain_events(&mut rig.rx);
19210 assert_eq!(first.len(), 2 + usize::from(triggerable));
19211 assert!(
19212 matches!(&first[0], ExecutionEvent::Order(OrderEventAny::Accepted(event)) if event.client_order_id == rig.cloid)
19213 );
19214
19215 if triggerable {
19216 assert!(
19217 matches!(&first[1], ExecutionEvent::Order(OrderEventAny::Triggered(event)) if event.client_order_id == rig.cloid)
19218 );
19219 }
19220
19221 assert!(
19222 matches!(first.last().unwrap(), ExecutionEvent::Order(OrderEventAny::Filled(event)) if event.client_order_id == rig.cloid && event.order_type == kind && event.last_qty == Quantity::from("0.1336"))
19223 );
19224 assert_eq!(next.len(), 1);
19225 assert!(
19226 matches!(&next[0], ExecutionEvent::Order(OrderEventAny::Filled(event)) if event.client_order_id == rig.cloid && event.trade_id.as_str() == subsequent.trade_id.to_string())
19227 );
19228
19229 let ExecutionEvent::Order(event) = &next[0] else {
19230 panic!("expected trailing fill");
19231 };
19232
19233 order.apply(event.clone()).unwrap();
19234 assert_eq!(order.status(), OrderStatus::Filled);
19235 assert_eq!(order.filled_qty(), Quantity::from("0.2672"));
19236 assert_eq!(order.is_triggered(), Some(triggerable));
19237 assert_eq!(rig.dispatch.triggered_was_emitted(&rig.cloid), triggerable);
19238 }
19239
19240 #[rstest]
19241 #[case(OrderType::Limit)]
19242 #[case(OrderType::StopLimit)]
19243 #[case(OrderType::StopMarket)]
19244 fn cached_order_restores_before_first_fill(
19245 #[case] kind: OrderType,
19246 #[values(false, true)] retired: bool,
19247 ) {
19248 let mut rig = dispatcher_rig("CACHED-FIRST-FILL");
19249 let trade = dispatcher_test_trade(&rig, true);
19250 rig.dispatch.forget_cloid(trade.bid_client_id);
19251 let venue = VenueOrderId::new(trade.bid_id_str.as_deref().unwrap());
19252 let mut builder = OrderTestBuilder::new(kind);
19253 builder
19254 .trader_id(trader_id())
19255 .strategy_id(strategy_id())
19256 .instrument_id(rig.instrument_id)
19257 .client_order_id(rig.cloid)
19258 .side(OrderSide::Buy)
19259 .quantity(Quantity::from("0.1336"));
19260
19261 if LIMIT_ORDER_TYPES.contains(&kind) {
19262 builder.price(Price::from("2352.74"));
19263 }
19264
19265 if STOP_ORDER_TYPES.contains(&kind) {
19266 builder.trigger_price(Price::from("2300.00"));
19267 }
19268
19269 let mut order = builder.build();
19270 order
19271 .apply(OrderEventAny::Accepted(OrderAccepted::new(
19272 trader_id(),
19273 strategy_id(),
19274 rig.instrument_id,
19275 rig.cloid,
19276 venue,
19277 account_id(),
19278 UUID4::new(),
19279 UnixNanos::from(1),
19280 UnixNanos::from(1),
19281 false,
19282 )))
19283 .unwrap();
19284 rig.dispatch.seed_order_restorations(vec![order.clone()]);
19285 if retired {
19286 rig.dispatch
19287 .restore_reconciled_order(&order, trade.bid_client_id, venue, true)
19288 .unwrap();
19289 }
19290
19291 dispatch_lighter_trade(
19292 &trade,
19293 &rig.dispatch,
19294 &rig.emitter,
19295 &rig.registry,
19296 account_id(),
19297 trader_id(),
19298 Some(TEST_ACCOUNT_INDEX_I64),
19299 UnixNanos::from(2),
19300 );
19301 let events = drain_events(&mut rig.rx);
19302 let triggerable = TRIGGERABLE_ORDER_TYPES.contains(&kind);
19303 assert_eq!(events.len(), 1 + usize::from(triggerable));
19304
19305 for event in events {
19306 let ExecutionEvent::Order(event) = event else {
19307 panic!("cached fill must remain tracked");
19308 };
19309
19310 assert_eq!(event.client_order_id(), rig.cloid);
19311 order.apply(event).unwrap();
19312 }
19313
19314 assert_eq!(order.status(), OrderStatus::Filled);
19315 assert_eq!(order.filled_qty(), Quantity::from("0.1336"));
19316 assert_eq!(order.venue_order_id(), Some(venue));
19317 assert_eq!(
19318 rig.dispatch.client_order_index(&rig.cloid),
19319 Some(trade.bid_client_id)
19320 );
19321 }
19322
19323 #[rstest]
19324 fn dispatch_lighter_trade_tracked_synthesizes_accepted_before_filled() {
19325 let mut rig = dispatcher_rig("4");
19329 register_identity(&rig);
19330
19331 let trade = dispatcher_test_trade(&rig, true);
19332
19333 dispatch_lighter_trade(
19334 &trade,
19335 &rig.dispatch,
19336 &rig.emitter,
19337 &rig.registry,
19338 account_id(),
19339 trader_id(),
19340 Some(TEST_ACCOUNT_INDEX_I64),
19341 UnixNanos::from(1),
19342 );
19343
19344 let events = drain_events(&mut rig.rx);
19345 assert_eq!(
19346 events.len(),
19347 2,
19348 "expected Accepted then Filled, was {events:?}",
19349 );
19350
19351 match &events[0] {
19352 ExecutionEvent::Order(OrderEventAny::Accepted(_)) => {}
19353 other => panic!("first event should be Accepted, was {other:?}"),
19354 }
19355
19356 match &events[1] {
19357 ExecutionEvent::Order(OrderEventAny::Filled(e)) => {
19358 assert_eq!(e.client_order_id, rig.cloid);
19359 assert_eq!(e.last_qty, Quantity::from("0.1336"));
19360 assert_eq!(e.last_px, Price::from("2352.73"));
19361 }
19362 other => panic!("second event should be Filled, was {other:?}"),
19363 }
19364
19365 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
19366 }
19367
19368 #[rstest]
19369 fn dispatch_lighter_trade_dedupes_repeated_trade_ids() {
19370 let mut rig = dispatcher_rig("5");
19371 register_identity(&rig);
19372 let trade = dispatcher_test_trade(&rig, true);
19373
19374 for _ in 0..3 {
19375 dispatch_lighter_trade(
19376 &trade,
19377 &rig.dispatch,
19378 &rig.emitter,
19379 &rig.registry,
19380 account_id(),
19381 trader_id(),
19382 Some(TEST_ACCOUNT_INDEX_I64),
19383 UnixNanos::from(1),
19384 );
19385 }
19386
19387 let events = drain_events(&mut rig.rx);
19388 assert_eq!(
19390 events.len(),
19391 2,
19392 "expected dedup after first dispatch, was {events:?}"
19393 );
19394 }
19395
19396 #[rstest]
19397 #[case::commission_overflow(true)]
19398 #[case::invalid_timestamp(false)]
19399 fn invalid_tracked_fill_preserves_unaccepted_order_for_replay(#[case] overflow: bool) {
19400 let mut rig = dispatcher_rig("INVALID-FILL-REPLAY");
19401 register_identity(&rig);
19402 let trade = dispatcher_test_trade(&rig, true);
19403 let mut invalid = trade.clone();
19404 if overflow {
19405 invalid.price = Decimal::from(1_000_000_000);
19406 invalid.size = Decimal::from(1_000_000_000);
19407 } else {
19408 invalid.timestamp = -1;
19409 }
19410
19411 let trade_id = parse_lighter_trade_id(&trade).unwrap();
19412
19413 if overflow {
19414 let error = parse_lighter_order_filled(
19415 &invalid,
19416 &LIGHTER_INSTRUMENT_CACHE.get(&rig.instrument_id).unwrap(),
19417 &rig.dispatch.order_identity(&rig.cloid).unwrap(),
19418 rig.cloid,
19419 account_id(),
19420 trader_id(),
19421 TEST_ACCOUNT_INDEX_I64,
19422 UnixNanos::from(1),
19423 )
19424 .unwrap_err();
19425 assert!(is_commission_error(&error));
19426 }
19427
19428 dispatch_lighter_trade(
19429 &invalid,
19430 &rig.dispatch,
19431 &rig.emitter,
19432 &rig.registry,
19433 account_id(),
19434 trader_id(),
19435 Some(TEST_ACCOUNT_INDEX_I64),
19436 UnixNanos::from(1),
19437 );
19438
19439 assert!(drain_events(&mut rig.rx).is_empty());
19440 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
19441 assert!(!rig.dispatch.seen_trade_ids.contains(&trade_id));
19442 assert!(
19443 !rig.dispatch
19444 .order_identity(&rig.cloid)
19445 .unwrap()
19446 .has_publication()
19447 );
19448 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
19449
19450 dispatch_lighter_trade(
19451 &trade,
19452 &rig.dispatch,
19453 &rig.emitter,
19454 &rig.registry,
19455 account_id(),
19456 trader_id(),
19457 Some(TEST_ACCOUNT_INDEX_I64),
19458 UnixNanos::from(2),
19459 );
19460
19461 let events = drain_events(&mut rig.rx);
19462 assert_eq!(events.len(), 2);
19463 assert!(
19464 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Accepted(event)) if event.client_order_id == rig.cloid)
19465 );
19466
19467 let ExecutionEvent::Order(OrderEventAny::Filled(actual)) = &events[1] else {
19468 panic!("expected typed fill");
19469 };
19470
19471 assert_eq!(actual.trade_id, trade_id);
19472 assert_eq!(actual.trader_id, trader_id());
19473 assert_eq!(actual.strategy_id, strategy_id());
19474 assert_eq!(actual.instrument_id, rig.instrument_id);
19475 assert_eq!(actual.client_order_id, rig.cloid);
19476 assert_eq!(actual.venue_order_id, VenueOrderId::from("562947905631053"));
19477 assert_eq!(actual.account_id, account_id());
19478 assert_eq!(actual.order_side, OrderSide::Buy);
19479 assert_eq!(actual.order_type, OrderType::Limit);
19480 assert_eq!(actual.last_qty, Quantity::from("0.1336"));
19481 assert_eq!(actual.last_px, Price::from("2352.73"));
19482 assert_eq!(actual.currency, Currency::USDC());
19483 assert_eq!(actual.liquidity_side, LiquiditySide::Maker);
19484 assert_eq!(actual.ts_event, UnixNanos::from(1_777_941_384_181_000_000));
19485 assert_eq!(actual.ts_init, UnixNanos::from(2));
19486 assert!(!actual.reconciliation);
19487 assert_eq!(actual.position_id, None);
19488 assert_eq!(actual.commission, Some(Money::from("0.00880109 USDC")));
19489 assert_eq!(actual.info, None);
19490 assert_eq!(actual.causation_id, None);
19491 assert!(rig.dispatch.seen_trade_ids.contains(&trade_id));
19492 }
19493
19494 #[rstest]
19495 #[case::tracked(true)]
19496 #[case::external(false)]
19497 fn malformed_duplicate_preserves_delivered_fill(#[case] tracked: bool) {
19498 let mut rig = dispatcher_rig("DUPLICATE-MALFORMED");
19499 if tracked {
19500 register_identity(&rig);
19501 }
19502
19503 let original = dispatcher_test_trade(&rig, true);
19504 let mut malformed = original.clone();
19505 malformed.timestamp = -1;
19506 for trade in [&original, &malformed, &original] {
19507 dispatch_lighter_trade(
19508 trade,
19509 &rig.dispatch,
19510 &rig.emitter,
19511 &rig.registry,
19512 account_id(),
19513 trader_id(),
19514 Some(TEST_ACCOUNT_INDEX_I64),
19515 UnixNanos::from(1),
19516 );
19517 }
19518
19519 let events = drain_events(&mut rig.rx);
19520 assert_eq!(events.len(), if tracked { 2 } else { 1 });
19521
19522 let fills = events
19523 .iter()
19524 .filter(|event| {
19525 matches!(
19526 event,
19527 ExecutionEvent::Order(OrderEventAny::Filled(_)) | ExecutionEvent::Report(_)
19528 )
19529 })
19530 .count();
19531
19532 assert_eq!(fills, 1);
19533 assert!(
19534 rig.dispatch
19535 .seen_trade_ids
19536 .contains(&parse_lighter_trade_id(&original).unwrap())
19537 );
19538 }
19539
19540 #[rstest]
19541 #[case::confirmed(true, false, false)]
19542 #[case::confirmed_stale_http(true, true, false)]
19543 #[case::rejected(false, false, false)]
19544 #[case::confirmed_stream(true, false, true)]
19545 #[case::rejected_stream(false, false, true)]
19546 fn cancel_overlap_preserves_amendment_and_real_fill(
19547 #[case] confirmed: bool,
19548 #[case] stale_http: bool,
19549 #[case] via_stream: bool,
19550 ) {
19551 let mut rig = dispatcher_rig("AMEND-CANCEL-FILL");
19552 let mut cached = OrderTestBuilder::new(OrderType::Limit)
19553 .trader_id(trader_id())
19554 .strategy_id(strategy_id())
19555 .instrument_id(rig.instrument_id)
19556 .client_order_id(rig.cloid)
19557 .side(OrderSide::Buy)
19558 .quantity(Quantity::from("0.0050"))
19559 .price(Price::from("2352.74"))
19560 .build();
19561 rig.dispatch.register_create_identity(&cached).unwrap();
19562 rig.dispatch
19563 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
19564 let mut raw = dispatcher_test_order(&rig, LighterOrderStatus::Open);
19565 dispatch_lighter_order(
19566 &raw,
19567 &rig.dispatch,
19568 &rig.emitter,
19569 &rig.registry,
19570 account_id(),
19571 trader_id(),
19572 UnixNanos::from(1),
19573 );
19574
19575 for event in drain_events(&mut rig.rx) {
19576 let ExecutionEvent::Order(event) = event else {
19577 panic!("expected acceptance");
19578 };
19579
19580 cached.apply(event).unwrap();
19581 }
19582
19583 let command = ModifyOrder::new(
19584 trader_id(),
19585 Some(client_id()),
19586 strategy_id(),
19587 rig.instrument_id,
19588 rig.cloid,
19589 cached.venue_order_id(),
19590 Some(Quantity::from("0.0100")),
19591 Some(Price::from("2400.00")),
19592 None,
19593 UUID4::new(),
19594 UnixNanos::from(2),
19595 None,
19596 None,
19597 );
19598 assert!(rig.dispatch.set_pending_modify_action(&command));
19599 rig.dispatch.set_pending_modify_shape(
19600 &command,
19601 OrderShapeSnapshot {
19602 quantity: command.quantity.unwrap(),
19603 price: command.price,
19604 trigger_price: None,
19605 },
19606 );
19607
19608 rig.dispatch
19609 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 19);
19610 rig.dispatch
19611 .set_pending_order_action(rig.cloid, PendingOrderAction::Cancel);
19612 rig.dispatch
19613 .set_pending_cancel_nonce(&rig.cloid, 0, 20, true);
19614 assert_eq!(rig.dispatch.pending_modify_nonce(&rig.cloid), Some((0, 19)));
19615 assert_eq!(rig.dispatch.pending_cancel_nonce(&rig.cloid), Some((0, 20)));
19616 let mut trade = dispatcher_test_trade(&rig, true);
19617 trade.bid_id = raw.order_index;
19618 trade.bid_id_str = Some(raw.order_id.clone());
19619 trade.size = Decimal::from_str("0.0030").unwrap();
19620 trade.usd_amount = trade.size * trade.price;
19621 dispatch_lighter_trade(
19622 &trade,
19623 &rig.dispatch,
19624 &rig.emitter,
19625 &rig.registry,
19626 account_id(),
19627 trader_id(),
19628 Some(TEST_ACCOUNT_INDEX_I64),
19629 UnixNanos::from(3),
19630 );
19631 raw.status = LighterOrderStatus::Canceled;
19632 raw.filled_base_amount = trade.size;
19633 if confirmed {
19634 raw.initial_base_amount = Decimal::from_str("0.0100").unwrap();
19635 raw.price = Decimal::from_str("2400.00").unwrap();
19636 }
19637
19638 raw.remaining_base_amount = raw.initial_base_amount - trade.size;
19639 dispatch_lighter_order(
19640 &raw,
19641 &rig.dispatch,
19642 &rig.emitter,
19643 &rig.registry,
19644 account_id(),
19645 trader_id(),
19646 UnixNanos::from(4),
19647 );
19648 assert!(drain_events(&mut rig.rx).is_empty());
19649 assert!(
19650 !rig.dispatch
19651 .seen_trade_ids
19652 .contains(&parse_lighter_trade_id(&trade).unwrap())
19653 );
19654
19655 let pending = PendingSendTx {
19656 batch_id: None,
19657 connection_epoch: 0,
19658 kind: PendingSendTxKind::Modify {
19659 command_id: command.command_id,
19660 strategy_id: strategy_id(),
19661 instrument_id: rig.instrument_id,
19662 client_order_id: rig.cloid,
19663 venue_order_id: cached.venue_order_id(),
19664 },
19665 submitted_at: UnixNanos::from(2),
19666 nonce: 19,
19667 api_key_index: TEST_API_KEY_INDEX,
19668 tx_hash: "amend-cancel".to_string(),
19669 };
19670
19671 let (client, _, _) = create_execution_client();
19672 let mut context = acked_probe_context(&client);
19673 context.dispatch = rig.dispatch.clone();
19674 context.registry = rig.registry.clone();
19675 context.emitter = rig.emitter.clone();
19676
19677 match (via_stream, confirmed) {
19678 (true, _) => {
19679 rig.dispatch.enqueue_pending_sendtx(pending.clone());
19680 handle_account_tx(
19681 0,
19682 &[LighterWsTx {
19683 hash: pending.tx_hash,
19684 status: if confirmed {
19685 LighterTxStatus::Executed
19686 } else {
19687 LighterTxStatus::Failed
19688 },
19689 event_info: String::new(),
19690 }],
19691 &context,
19692 );
19693 }
19694 (false, true) => {
19695 let mut http = raw;
19696 if stale_http {
19697 http.status = LighterOrderStatus::Open;
19698 }
19699
19700 confirm_modify_order(&pending, &context, &http).unwrap();
19701 }
19702 (false, false) => {
19703 reject_modify_order(
19704 &rig.dispatch,
19705 &rig.emitter,
19706 &pending,
19707 &CommandFailure::venue_rejected("amendment failed"),
19708 UnixNanos::from(5),
19709 false,
19710 );
19711 }
19712 }
19713
19714 let events = drain_events(&mut rig.rx);
19715 assert_eq!(events.len(), 3);
19716
19717 match &events[0] {
19718 ExecutionEvent::Order(OrderEventAny::Updated(update)) if confirmed => {
19719 assert_eq!(update.client_order_id, rig.cloid);
19720 assert_eq!(update.quantity, Quantity::from("0.0100"));
19721 assert_eq!(update.price, Some(Price::from("2400.00")));
19722 }
19723 ExecutionEvent::Order(OrderEventAny::ModifyRejected(rejected)) if !confirmed => {
19724 assert_eq!(rejected.client_order_id, rig.cloid);
19725 assert_eq!(
19726 rejected.reason.as_str(),
19727 if via_stream {
19728 "Lighter modify transaction failed: transaction failed without an application error"
19729 } else {
19730 "amendment failed"
19731 }
19732 );
19733 }
19734 other => panic!("expected amendment resolution before fill, was {other:?}"),
19735 }
19736
19737 let ExecutionEvent::Order(OrderEventAny::Filled(fill)) = &events[1] else {
19738 panic!("expected held real fill");
19739 };
19740
19741 assert_eq!(fill.client_order_id, rig.cloid);
19742 assert_eq!(fill.trade_id, parse_lighter_trade_id(&trade).unwrap());
19743 assert_eq!(fill.last_qty, Quantity::from("0.0030"));
19744 assert_eq!(fill.last_px, Price::from("2352.73"));
19745
19746 let ExecutionEvent::Order(OrderEventAny::Canceled(cancel)) = &events[2] else {
19747 panic!("expected cancellation after fill");
19748 };
19749
19750 assert_eq!(cancel.client_order_id, rig.cloid);
19751 assert_eq!(cancel.venue_order_id, cached.venue_order_id());
19752
19753 for event in events {
19754 let ExecutionEvent::Order(event) = event else {
19755 panic!("expected order event");
19756 };
19757
19758 cached.apply(event).unwrap();
19759 }
19760
19761 assert_eq!(cached.status(), OrderStatus::Canceled);
19762 assert_eq!(
19763 cached.quantity(),
19764 Quantity::from(if confirmed { "0.0100" } else { "0.0050" })
19765 );
19766 assert_eq!(cached.filled_qty(), Quantity::from("0.0030"));
19767 assert_eq!(
19768 cached.leaves_qty(),
19769 Quantity::from(if confirmed { "0.0070" } else { "0.0020" })
19770 );
19771 assert_eq!(rig.dispatch.pending_modify_nonce(&rig.cloid), None);
19772 assert_eq!(rig.dispatch.pending_cancel_nonce(&rig.cloid), None);
19773 assert!(
19774 rig.dispatch
19775 .seen_trade_ids
19776 .contains(&parse_lighter_trade_id(&trade).unwrap())
19777 );
19778 }
19779
19780 #[rstest]
19781 fn overlapping_modify_rejection_preserves_first_submission() {
19782 let (client, cache, mut rx) = create_execution_client();
19783 let instrument_id = register_test_instrument(&client, &cache);
19784 let cloid = ClientOrderId::from("OVERLAPPING-MODIFY");
19785 let venue = VenueOrderId::from("789");
19786 enqueue_modify(&client, instrument_id, cloid, venue, 23);
19787
19788 let command = ModifyOrder::new(
19789 trader_id(),
19790 Some(client_id()),
19791 strategy_id(),
19792 instrument_id,
19793 cloid,
19794 Some(venue),
19795 Some(Quantity::from("0.3000")),
19796 Some(Price::from("2363.00")),
19797 None,
19798 UUID4::new(),
19799 UnixNanos::from(2),
19800 None,
19801 None,
19802 );
19803 assert!(!client.admit_modify_order(&command));
19804 let events = drain_events(&mut rx);
19805 assert_eq!(events.len(), 1);
19806
19807 let ExecutionEvent::Order(OrderEventAny::ModifyDenied(event)) = &events[0] else {
19808 panic!("expected overlap rejection");
19809 };
19810
19811 assert_eq!(event.trader_id, trader_id());
19812 assert_eq!(event.strategy_id, strategy_id());
19813 assert_eq!(event.instrument_id, instrument_id);
19814 assert_eq!(event.client_order_id, cloid);
19815 assert_eq!(event.venue_order_id, Some(venue));
19816 assert_eq!(event.account_id, Some(account_id()));
19817 assert_eq!(
19818 event.reason.as_str(),
19819 "Lighter permits one unresolved modification per order; wait for its update or rejection"
19820 );
19821 assert_eq!(event.causation_id, Some(command.command_id));
19822 assert_eq!(client.dispatch.pending_modify_nonce(&cloid), Some((0, 23)));
19823 assert_eq!(
19824 client.dispatch.pending_modify_shape(&cloid),
19825 Some(OrderShapeSnapshot {
19826 quantity: Quantity::from("0.2000"),
19827 price: Some(Price::from("2362.00")),
19828 trigger_price: None
19829 })
19830 );
19831 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
19832 }
19833
19834 #[rstest]
19835 fn recovery_waits_for_confirmed_update_in_cache() {
19836 let (client, cache, mut rx) = create_execution_client();
19837 let instrument_id = register_test_instrument(&client, &cache);
19838 let rig = dispatcher_rig("HTTP-AMEND");
19839 let cloid = rig.cloid;
19840 let mut raw = dispatcher_test_order(&rig, LighterOrderStatus::Filled);
19841 let venue = VenueOrderId::new(&raw.order_id);
19842 let order = OrderTestBuilder::new(OrderType::Limit)
19843 .trader_id(trader_id())
19844 .strategy_id(strategy_id())
19845 .instrument_id(instrument_id)
19846 .client_order_id(cloid)
19847 .side(OrderSide::Buy)
19848 .quantity(Quantity::from("0.0050"))
19849 .price(Price::from("2352.74"))
19850 .build();
19851 cache_accepted_order(&cache, order, venue, Some(client_id()));
19852 let cached = cache.borrow().order_owned(&cloid).unwrap();
19853 client
19854 .dispatch
19855 .restore_reconciled_order(&cached, raw.client_order_index, venue, false)
19856 .unwrap();
19857 enqueue_modify(&client, instrument_id, cloid, venue, 23);
19858 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
19859 raw.initial_base_amount = Decimal::from_str("0.2000").unwrap();
19860 raw.filled_base_amount = raw.initial_base_amount;
19861 raw.remaining_base_amount = Decimal::ZERO;
19862 raw.price = Decimal::from_str("2362.00").unwrap();
19863 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&instrument_id).unwrap();
19864 let report =
19865 parse_ws_order_status_report(&raw, &instrument, account_id(), UnixNanos::from(3))
19866 .unwrap()
19867 .with_client_order_id(cloid);
19868 drop(instrument);
19869
19870 let command = GenerateOrderStatusReports::new(
19871 UUID4::new(),
19872 UnixNanos::default(),
19873 false,
19874 Some(instrument_id),
19875 None,
19876 None,
19877 None,
19878 None,
19879 );
19880
19881 let result = finish_order_reports(
19882 &client.core,
19883 &client.dispatch,
19884 OrderReportSweep {
19885 orders: vec![(raw.client_order_index, report)],
19886 ..Default::default()
19887 },
19888 &command,
19889 None,
19890 );
19891
19892 assert_eq!(
19893 result.unwrap_err().to_string(),
19894 "incomplete Lighter order reports: Lighter execution confirmation is pending"
19895 );
19896 assert_eq!(client.dispatch.pending_modify_nonce(&cloid), Some((0, 23)));
19897 assert!(client.dispatch.order_identities.contains_key(&cloid));
19898 let fill = reconciliation_fill_report(
19899 instrument_id,
19900 raw.client_order_index,
19901 venue,
19902 "HTTP-AMEND-TRADE",
19903 );
19904
19905 let withheld = finish_fill_reports(
19906 &client.core,
19907 &client.dispatch,
19908 BTreeSet::new(),
19909 Ok(FillSweep {
19910 reports: vec![fill.clone()],
19911 covers_window: true,
19912 withheld_instruments: AHashSet::new(),
19913 }),
19914 )
19915 .unwrap();
19916
19917 assert!(withheld.reports.is_empty());
19918 assert!(!withheld.covers_window);
19919 assert!(!client.dispatch.seen_trade_ids.contains(&fill.trade_id));
19920 confirm_modify_order(&pending, &acked_probe_context(&client), &raw).unwrap();
19921 let events = drain_events(&mut rx);
19922 assert_eq!(events.len(), 2);
19923 let mut events = events.into_iter();
19924
19925 let ExecutionEvent::Order(update @ OrderEventAny::Updated(_)) = events.next().unwrap()
19926 else {
19927 panic!("expected confirmed update before real fill");
19928 };
19929
19930 let ExecutionEvent::Order(delivered_fill @ OrderEventAny::Filled(_)) =
19931 events.next().unwrap()
19932 else {
19933 panic!("expected buffered real fill");
19934 };
19935
19936 let OrderEventAny::Filled(actual) = &delivered_fill else {
19937 unreachable!()
19938 };
19939
19940 assert_eq!(actual.trade_id, fill.trade_id);
19941 assert_eq!(actual.client_order_id, cloid);
19942 assert_eq!(actual.venue_order_id, fill.venue_order_id);
19943 assert_eq!(actual.last_qty, fill.last_qty);
19944 assert_eq!(actual.last_px, fill.last_px);
19945 assert_eq!(actual.commission, Some(fill.commission));
19946 assert_eq!(actual.liquidity_side, fill.liquidity_side);
19947 assert_eq!(actual.ts_event, fill.ts_event);
19948 assert!(!actual.reconciliation);
19949
19950 let still_withheld = finish_fill_reports(
19951 &client.core,
19952 &client.dispatch,
19953 BTreeSet::new(),
19954 Ok(FillSweep {
19955 reports: vec![fill.clone()],
19956 covers_window: true,
19957 withheld_instruments: AHashSet::new(),
19958 }),
19959 )
19960 .unwrap();
19961
19962 assert!(still_withheld.reports.is_empty());
19963 assert!(!still_withheld.covers_window);
19964 assert!(client.dispatch.seen_trade_ids.contains(&fill.trade_id));
19965 cache.borrow_mut().update_order(&update).unwrap();
19966 assert!(order_shape_pending(&client.core, &client.dispatch, &cloid));
19967 cache.borrow_mut().update_order(&delivered_fill).unwrap();
19968
19969 let delivered = finish_fill_reports(
19970 &client.core,
19971 &client.dispatch,
19972 BTreeSet::new(),
19973 Ok(FillSweep {
19974 reports: vec![fill.clone()],
19975 covers_window: true,
19976 withheld_instruments: AHashSet::new(),
19977 }),
19978 )
19979 .unwrap();
19980
19981 assert!(delivered.reports.is_empty());
19982 assert!(delivered.covers_window);
19983 assert_eq!(
19984 client.dispatch.mark_trade_reconciled(fill.trade_id),
19985 Some(TradeDedupSource::Live)
19986 );
19987 }
19988
19989 #[rstest]
19990 fn dispatch_lighter_trade_parse_failure_rolls_back_dedup() {
19991 let mut rig = dispatcher_rig("21");
19993 register_identity(&rig);
19994 let mut trade = dispatcher_test_trade(&rig, true);
19995 trade.timestamp = -1;
19996 let trade_id = parse_lighter_trade_id(&trade).expect("trade id parses");
19997
19998 dispatch_lighter_trade(
19999 &trade,
20000 &rig.dispatch,
20001 &rig.emitter,
20002 &rig.registry,
20003 account_id(),
20004 trader_id(),
20005 Some(TEST_ACCOUNT_INDEX_I64),
20006 UnixNanos::from(1),
20007 );
20008
20009 let events = drain_events(&mut rig.rx);
20010 assert!(
20011 !events
20012 .iter()
20013 .any(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Filled(_)))),
20014 "malformed trade must not emit a fill, was {events:?}",
20015 );
20016 assert!(
20017 rig.dispatch.mark_trade_seen(trade_id),
20018 "parse-failed trade must be re-markable (dedup rolled back)",
20019 );
20020 }
20021
20022 #[rstest]
20023 fn dispatch_lighter_trade_untracked_parse_failure_rolls_back_dedup() {
20024 let mut rig = dispatcher_rig("22");
20026 let mut trade = dispatcher_test_trade(&rig, true);
20027 trade.timestamp = -1;
20028 let trade_id = parse_lighter_trade_id(&trade).expect("trade id parses");
20029
20030 dispatch_lighter_trade(
20031 &trade,
20032 &rig.dispatch,
20033 &rig.emitter,
20034 &rig.registry,
20035 account_id(),
20036 trader_id(),
20037 Some(TEST_ACCOUNT_INDEX_I64),
20038 UnixNanos::from(1),
20039 );
20040
20041 let events = drain_events(&mut rig.rx);
20042 assert!(
20043 events.is_empty(),
20044 "untracked malformed trade must emit nothing, was {events:?}",
20045 );
20046 assert!(
20047 rig.dispatch.mark_trade_seen(trade_id),
20048 "untracked parse-failed trade must be re-markable (dedup rolled back)",
20049 );
20050 }
20051
20052 #[rstest]
20053 fn dispatch_tracked_order_event_terminal_cancel_retires_identity_and_removes_snapshot() {
20054 let mut rig = dispatcher_rig("6");
20055 register_identity(&rig);
20056 rig.dispatch.mark_accepted_emitted(rig.cloid);
20057 rig.dispatch.store_snapshot(
20058 rig.cloid,
20059 crate::websocket::dispatch::OrderShapeSnapshot {
20060 quantity: Quantity::from("0.0050"),
20061 price: Some(Price::from("2352.74")),
20062 trigger_price: None,
20063 },
20064 );
20065 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
20066 order.filled_base_amount = Decimal::ZERO;
20067 let mut trade = dispatcher_test_trade(&rig, true);
20068 trade.bid_id = order.order_index;
20069 trade.bid_id_str = Some(order.order_id.clone());
20070 dispatch_lighter_order(
20071 &order,
20072 &rig.dispatch,
20073 &rig.emitter,
20074 &rig.registry,
20075 account_id(),
20076 trader_id(),
20077 UnixNanos::from(1),
20078 );
20079
20080 let events = drain_events(&mut rig.rx);
20081 let canceled = events
20082 .iter()
20083 .find(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Canceled(_))))
20084 .expect("expected a Canceled event");
20085 if let ExecutionEvent::Order(OrderEventAny::Canceled(e)) = canceled {
20086 assert_eq!(e.client_order_id, rig.cloid);
20087 }
20088 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
20089 assert!(rig.dispatch.order_identity(&rig.cloid).is_some());
20090 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_none());
20091 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
20092
20093 dispatch_lighter_trade(
20094 &trade,
20095 &rig.dispatch,
20096 &rig.emitter,
20097 &rig.registry,
20098 account_id(),
20099 trader_id(),
20100 Some(TEST_ACCOUNT_INDEX_I64),
20101 UnixNanos::from(2),
20102 );
20103 let trailing = drain_events(&mut rig.rx);
20104 assert_eq!(
20105 trailing.len(),
20106 1,
20107 "expected one trailing fill: {trailing:?}"
20108 );
20109
20110 match &trailing[0] {
20111 ExecutionEvent::Order(OrderEventAny::Filled(event)) => {
20112 assert_eq!(event.client_order_id, rig.cloid);
20113 }
20114 other => panic!("expected typed trailing OrderFilled, was {other:?}"),
20115 }
20116 }
20117
20118 #[rstest]
20119 fn dispatch_tracked_cancel_after_report_seed_skips_synthesized_accept() {
20120 let mut rig = dispatcher_rig("10");
20121 register_identity(&rig);
20122
20123 let report_order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
20124 let instrument = LIGHTER_INSTRUMENT_CACHE
20125 .get(&rig.instrument_id)
20126 .expect("instrument cached");
20127 let report = parse_ws_order_status_report(
20128 &report_order,
20129 instrument.value(),
20130 account_id(),
20131 UnixNanos::from(1),
20132 )
20133 .map(|report| report.with_client_order_id(rig.cloid))
20134 .expect("report parses");
20135
20136 assert_eq!(report.order_status, OrderStatus::Accepted);
20137 rig.dispatch.seed_accepted_from_report(&report);
20138
20139 let mut cancel_order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
20140 cancel_order.filled_base_amount = Decimal::ZERO;
20141 dispatch_lighter_order(
20142 &cancel_order,
20143 &rig.dispatch,
20144 &rig.emitter,
20145 &rig.registry,
20146 account_id(),
20147 trader_id(),
20148 UnixNanos::from(2),
20149 );
20150
20151 let events = drain_events(&mut rig.rx);
20152 assert_eq!(
20153 events.len(),
20154 1,
20155 "report-seeded cancel should emit only Canceled, was {events:?}",
20156 );
20157 assert!(
20158 !events
20159 .iter()
20160 .any(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Accepted(_)))),
20161 "typed cancel must not synthesize a second Accepted",
20162 );
20163
20164 match &events[0] {
20165 ExecutionEvent::Order(OrderEventAny::Canceled(e)) => {
20166 assert_eq!(e.client_order_id, rig.cloid);
20167 assert_eq!(e.venue_order_id, Some(VenueOrderId::new("281476929510110")));
20168 }
20169 other => panic!("expected Canceled, was {other:?}"),
20170 }
20171 }
20172
20173 #[rstest]
20174 fn dispatch_tracked_cancel_after_submitted_report_seed_skips_synthesized_accept() {
20175 let mut rig = dispatcher_rig("11");
20176 register_identity(&rig);
20177
20178 let report_order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
20179 let instrument = LIGHTER_INSTRUMENT_CACHE
20180 .get(&rig.instrument_id)
20181 .expect("instrument cached");
20182 let mut report = parse_ws_order_status_report(
20183 &report_order,
20184 instrument.value(),
20185 account_id(),
20186 UnixNanos::from(1),
20187 )
20188 .map(|report| report.with_client_order_id(rig.cloid))
20189 .expect("report parses");
20190 report.order_status = OrderStatus::Submitted;
20191
20192 rig.dispatch.seed_accepted_from_report(&report);
20193 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
20194
20195 let mut cancel_order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
20196 cancel_order.filled_base_amount = Decimal::ZERO;
20197
20198 dispatch_lighter_order(
20199 &cancel_order,
20200 &rig.dispatch,
20201 &rig.emitter,
20202 &rig.registry,
20203 account_id(),
20204 trader_id(),
20205 UnixNanos::from(2),
20206 );
20207
20208 let events = drain_events(&mut rig.rx);
20209 assert_eq!(
20210 events.len(),
20211 1,
20212 "Cancel after Submitted report should emit only Canceled, was {events:?}",
20213 );
20214
20215 match &events[0] {
20216 ExecutionEvent::Order(OrderEventAny::Canceled(e)) => {
20217 assert_eq!(e.client_order_id, rig.cloid);
20218 assert_eq!(e.venue_order_id, Some(VenueOrderId::new("281476929510110")));
20219 }
20220 other => panic!("expected Canceled, was {other:?}"),
20221 }
20222 }
20223
20224 #[rstest]
20225 fn dispatch_tracked_order_event_accept_dedup_is_idempotent() {
20226 let mut rig = dispatcher_rig("7");
20227 register_identity(&rig);
20228 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
20229
20230 dispatch_lighter_order(
20233 &order,
20234 &rig.dispatch,
20235 &rig.emitter,
20236 &rig.registry,
20237 account_id(),
20238 trader_id(),
20239 UnixNanos::from(1),
20240 );
20241 dispatch_lighter_order(
20242 &order,
20243 &rig.dispatch,
20244 &rig.emitter,
20245 &rig.registry,
20246 account_id(),
20247 trader_id(),
20248 UnixNanos::from(2),
20249 );
20250
20251 let events = drain_events(&mut rig.rx);
20252 let accepted_count = events
20253 .iter()
20254 .filter(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Accepted(_))))
20255 .count();
20256 assert_eq!(accepted_count, 1, "Accepted must be emitted exactly once");
20257 }
20258
20259 #[rstest]
20260 fn dispatch_lighter_order_drops_when_instrument_uncached() {
20261 let registry = Arc::new(MarketRegistry::new());
20263 let (emitter, mut rx) = dispatcher_emitter();
20264 let dispatch = WsDispatchState::new();
20265 let cloid = ClientOrderId::new("CLOID-MISSING");
20266 dispatch.register_order_identity(
20267 cloid,
20268 OrderIdentity::new(
20269 InstrumentId::from("MISSING-PERP.LIGHTER"),
20270 strategy_id(),
20271 OrderSide::Buy,
20272 OrderType::Limit,
20273 1,
20274 ),
20275 );
20276 let mut order = LighterOrder {
20277 order_index: 1,
20278 client_order_index: 1,
20279 order_id: "1".to_string(),
20280 client_order_id: "1".to_string(),
20281 market_index: 999, owner_account_index: TEST_ACCOUNT_INDEX_I64,
20283 initial_base_amount: Decimal::ZERO,
20284 price: Decimal::ZERO,
20285 nonce: 0,
20286 remaining_base_amount: Decimal::ZERO,
20287 is_ask: false,
20288 base_size: 0,
20289 base_price: 0,
20290 filled_base_amount: Decimal::ZERO,
20291 filled_quote_amount: Decimal::ZERO,
20292 side: Some(LighterOrderSide::Buy),
20293 order_type: LighterOrderKind::Limit,
20294 time_in_force: LighterOrderTimeInForce::GoodTillTime,
20295 reduce_only: false,
20296 trigger_price: Decimal::ZERO,
20297 order_expiry: 0,
20298 status: LighterOrderStatus::Open,
20299 trigger_status: LighterTriggerStatus::Na,
20300 trigger_time: 0,
20301 parent_order_index: 0,
20302 parent_order_id: "0".to_string(),
20303 to_trigger_order_id_0: "0".to_string(),
20304 to_trigger_order_id_1: "0".to_string(),
20305 to_cancel_order_id_0: "0".to_string(),
20306 integrator_fee_collector_index: "0".to_string(),
20307 integrator_taker_fee: Decimal::ZERO,
20308 integrator_maker_fee: Decimal::ZERO,
20309 block_height: 0,
20310 timestamp: 0,
20311 created_at: 0,
20312 updated_at: 0,
20313 transaction_time: 0,
20314 };
20315 order.client_order_id = "1".to_string();
20316
20317 dispatch_lighter_order(
20318 &order,
20319 &dispatch,
20320 &emitter,
20321 ®istry,
20322 account_id(),
20323 trader_id(),
20324 UnixNanos::from(1),
20325 );
20326
20327 let events = drain_events(&mut rx);
20328 assert!(
20329 events.is_empty(),
20330 "no event for uncached instrument, was {events:?}"
20331 );
20332 assert!(dispatch.order_identities.contains_key(&cloid));
20333 }
20334
20335 #[rstest]
20336 fn dispatch_lighter_trade_filters_non_account_trades_defensively() {
20337 let mut rig = dispatcher_rig("8");
20338 let mut trade = dispatcher_test_trade(&rig, true);
20343 trade.bid_account_id = 91_249;
20344 trade.ask_account_id = 91_250;
20345
20346 dispatch_lighter_trade(
20347 &trade,
20348 &rig.dispatch,
20349 &rig.emitter,
20350 &rig.registry,
20351 account_id(),
20352 trader_id(),
20353 Some(TEST_ACCOUNT_INDEX_I64),
20354 UnixNanos::from(1),
20355 );
20356
20357 let events = drain_events(&mut rig.rx);
20358 assert!(
20359 events.is_empty(),
20360 "foreign trade must produce no event, was {events:?}"
20361 );
20362 assert!(
20363 !rig.dispatch
20364 .seen_trade_ids
20365 .contains(&TradeId::new("19209006902"),)
20366 );
20367 }
20368
20369 #[rstest]
20370 fn register_cloid_in_submit_path_uses_probed_index_on_collision() {
20371 let dispatch = WsDispatchState::new();
20376 let cloid = ClientOrderId::new("PROBE-CLOID");
20377 let derived = dispatch.derive_client_order_index(&cloid);
20378
20379 let intruder = ClientOrderId::new("INTRUDER");
20380 dispatch.cloid_map.insert(derived, intruder);
20381
20382 let chosen = dispatch.register_cloid(derived, cloid).unwrap();
20383
20384 assert_ne!(chosen, derived);
20385 assert_eq!(
20386 dispatch.cloid_map.get(&derived).map(|e| *e.value()),
20387 Some(intruder),
20388 );
20389 assert_eq!(
20390 dispatch.cloid_map.get(&chosen).map(|e| *e.value()),
20391 Some(cloid),
20392 );
20393 }
20394
20395 #[rstest]
20396 fn reconciliation_restores_active_collision_probed_identity() {
20397 let (client, cache, _rx) = create_execution_client();
20398 let instrument_id = register_test_instrument(&client, &cache);
20399 let mut factory = test_order_factory();
20400 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-ACTIVE");
20401 let cloid = order.client_order_id();
20402 let venue_order_id = VenueOrderId::from("281476929510110");
20403 cache_accepted_order(&cache, order, venue_order_id, None);
20404 let (base_index, client_order_index) = forced_probed_index(cloid);
20405 let raw =
20406 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
20407
20408 restore_reconciled_order(
20409 &client.core,
20410 &client.dispatch,
20411 VenueOrderId::new(raw.order_id.as_str()),
20412 raw.client_order_index,
20413 false,
20414 );
20415 let report = parse_http_order_to_report(
20416 &raw,
20417 &client.registry,
20418 account_id(),
20419 UnixNanos::from(1),
20420 &client.dispatch,
20421 )
20422 .unwrap();
20423 let translated = client.dispatch.translate_order_cloid(report);
20424 let replacement = ClientOrderId::from("O-RECON-ACTIVE-REPLACEMENT");
20425 let replacement_index = client
20426 .dispatch
20427 .register_cloid(client_order_index, replacement)
20428 .unwrap();
20429 let wrong_venue_order_id = VenueOrderId::from("281476929510111");
20430 let wrong_raw = reconciliation_raw_order(
20431 client_order_index,
20432 wrong_venue_order_id,
20433 LighterOrderStatus::Open,
20434 );
20435 let wrong_report = parse_http_order_to_report(
20436 &wrong_raw,
20437 &client.registry,
20438 account_id(),
20439 UnixNanos::from(2),
20440 &client.dispatch,
20441 )
20442 .unwrap();
20443 let wrong_translated = client.dispatch.translate_order_cloid(wrong_report);
20444
20445 assert_ne!(client_order_index, base_index);
20446 assert_eq!(
20447 client.dispatch.client_order_index(&cloid),
20448 Some(client_order_index),
20449 );
20450 assert_eq!(
20451 client
20452 .dispatch
20453 .resolve_client_order_index(client_order_index),
20454 Some(cloid),
20455 );
20456 assert_eq!(
20457 client.dispatch.lookup_venue_order_id(&cloid),
20458 Some(venue_order_id),
20459 );
20460 assert!(client.dispatch.order_identities.contains_key(&cloid));
20461 assert!(client.dispatch.accepted_was_emitted(&cloid));
20462 assert_eq!(translated.client_order_id, Some(cloid));
20463 assert_eq!(translated.venue_order_id, venue_order_id);
20464 assert_ne!(replacement_index, client_order_index);
20465 assert_eq!(
20466 client
20467 .dispatch
20468 .resolve_client_order_index(replacement_index),
20469 Some(replacement),
20470 );
20471 assert_eq!(
20472 wrong_translated.client_order_id,
20473 Some(ClientOrderId::new(wrong_venue_order_id.as_str())),
20474 );
20475 assert_eq!(wrong_translated.venue_order_id, wrong_venue_order_id);
20476 }
20477
20478 #[rstest]
20479 fn reconciliation_retires_stale_active_cache_from_terminal_report() {
20480 let (client, cache, _rx) = create_execution_client();
20481 let instrument_id = register_test_instrument(&client, &cache);
20482 let mut factory = test_order_factory();
20483 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-FILLED");
20484 let cloid = order.client_order_id();
20485 let venue_order_id = VenueOrderId::from("281476929510119");
20486 cache_accepted_order(&cache, order, venue_order_id, None);
20487 let (_, client_order_index) = forced_probed_index(cloid);
20488 let raw = reconciliation_raw_order(
20489 client_order_index,
20490 venue_order_id,
20491 LighterOrderStatus::Filled,
20492 );
20493
20494 restore_reconciled_order(
20495 &client.core,
20496 &client.dispatch,
20497 VenueOrderId::new(raw.order_id.as_str()),
20498 raw.client_order_index,
20499 true,
20500 );
20501 let fill = client
20502 .dispatch
20503 .translate_fill_cloid(reconciliation_fill_report(
20504 instrument_id,
20505 client_order_index,
20506 venue_order_id,
20507 "19209006919",
20508 ));
20509
20510 assert!(!client.dispatch.cloid_map.contains_key(&client_order_index));
20511 assert!(!client.dispatch.order_identities.contains_key(&cloid));
20512 assert_eq!(
20513 client.dispatch.client_order_index(&cloid),
20514 Some(client_order_index),
20515 );
20516 assert_eq!(fill.client_order_id, Some(cloid));
20517 assert_eq!(fill.venue_order_id, venue_order_id);
20518 }
20519
20520 #[rstest]
20521 fn reconciliation_restores_reused_retired_index_for_fill_translation() {
20522 let (client, cache, _rx) = create_execution_client();
20523 let instrument_id = register_test_instrument(&client, &cache);
20524 let mut factory = test_order_factory();
20525 let first_order = test_limit_order(&mut factory, instrument_id, "O-RECON-RETIRED-A");
20526 let second_order = test_limit_order(&mut factory, instrument_id, "O-RECON-RETIRED-B");
20527 let first_cloid = first_order.client_order_id();
20528 let second_cloid = second_order.client_order_id();
20529 let first_venue_order_id = VenueOrderId::from("281476929510120");
20530 let second_venue_order_id = VenueOrderId::from("281476929510121");
20531 cache_canceled_order(&cache, first_order, first_venue_order_id);
20532 cache_canceled_order(&cache, second_order, second_venue_order_id);
20533 let (_, client_order_index) = forced_probed_index(first_cloid);
20534 let first_raw = reconciliation_raw_order(
20535 client_order_index,
20536 first_venue_order_id,
20537 LighterOrderStatus::Canceled,
20538 );
20539 let second_raw = reconciliation_raw_order(
20540 client_order_index,
20541 second_venue_order_id,
20542 LighterOrderStatus::Canceled,
20543 );
20544
20545 restore_reconciled_order(
20546 &client.core,
20547 &client.dispatch,
20548 VenueOrderId::new(first_raw.order_id.as_str()),
20549 first_raw.client_order_index,
20550 true,
20551 );
20552 restore_reconciled_order(
20553 &client.core,
20554 &client.dispatch,
20555 VenueOrderId::new(second_raw.order_id.as_str()),
20556 second_raw.client_order_index,
20557 true,
20558 );
20559 let first_fill = client
20560 .dispatch
20561 .translate_fill_cloid(reconciliation_fill_report(
20562 instrument_id,
20563 client_order_index,
20564 first_venue_order_id,
20565 "19209006920",
20566 ));
20567 let second_fill = client
20568 .dispatch
20569 .translate_fill_cloid(reconciliation_fill_report(
20570 instrument_id,
20571 client_order_index,
20572 second_venue_order_id,
20573 "19209006921",
20574 ));
20575 let replacement = ClientOrderId::from("O-RECON-RETIRED-REPLACEMENT");
20576 let replacement_index = client
20577 .dispatch
20578 .register_cloid(client_order_index, replacement)
20579 .unwrap();
20580
20581 assert!(!client.dispatch.cloid_map.contains_key(&client_order_index));
20582 assert!(!client.dispatch.order_identities.contains_key(&first_cloid));
20583 assert!(!client.dispatch.order_identities.contains_key(&second_cloid));
20584 assert_eq!(
20585 client.dispatch.client_order_index(&first_cloid),
20586 Some(client_order_index),
20587 );
20588 assert_eq!(
20589 client.dispatch.client_order_index(&second_cloid),
20590 Some(client_order_index),
20591 );
20592 assert_eq!(
20593 client
20594 .dispatch
20595 .resolve_client_order_index(client_order_index),
20596 None,
20597 );
20598 assert_eq!(first_fill.client_order_id, Some(first_cloid));
20599 assert_eq!(first_fill.venue_order_id, first_venue_order_id);
20600 assert_eq!(second_fill.client_order_id, Some(second_cloid));
20601 assert_eq!(second_fill.venue_order_id, second_venue_order_id);
20602 assert_ne!(replacement_index, client_order_index);
20603 assert_eq!(
20604 client
20605 .dispatch
20606 .resolve_client_order_index(replacement_index),
20607 Some(replacement),
20608 );
20609 }
20610
20611 #[rstest]
20612 fn reconciliation_restores_triggered_marker() {
20613 let (client, cache, _rx) = create_execution_client();
20614 let instrument_id = register_test_instrument(&client, &cache);
20615 let mut factory = test_order_factory();
20616 let order = factory.stop_limit(
20617 instrument_id,
20618 OrderSide::Sell,
20619 Quantity::from("0.1000"),
20620 Price::from("2290.00"),
20621 Price::from("2300.00"),
20622 None,
20623 Some(TimeInForce::Gtc),
20624 None,
20625 Some(false),
20626 Some(false),
20627 Some(false),
20628 None,
20629 None,
20630 None,
20631 None,
20632 None,
20633 None,
20634 Some(ClientOrderId::from("O-RECON-TRIGGERED")),
20635 );
20636 let cloid = order.client_order_id();
20637 let venue_order_id = VenueOrderId::from("281476929510124");
20638 cache_accepted_order(&cache, order, venue_order_id, None);
20639 let triggered = OrderEventAny::Triggered(OrderTriggered::new(
20640 trader_id(),
20641 strategy_id(),
20642 instrument_id,
20643 cloid,
20644 UUID4::new(),
20645 UnixNanos::from(1),
20646 UnixNanos::from(2),
20647 false,
20648 Some(venue_order_id),
20649 Some(account_id()),
20650 ));
20651 cache.borrow_mut().update_order(&triggered).unwrap();
20652 let (_, client_order_index) = forced_probed_index(cloid);
20653 let raw =
20654 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
20655
20656 restore_reconciled_order(
20657 &client.core,
20658 &client.dispatch,
20659 VenueOrderId::new(raw.order_id.as_str()),
20660 raw.client_order_index,
20661 false,
20662 );
20663
20664 assert!(client.dispatch.order_identities.contains_key(&cloid));
20665 assert!(client.dispatch.accepted_was_emitted(&cloid));
20666 assert!(client.dispatch.triggered_was_emitted(&cloid));
20667 }
20668
20669 #[rstest]
20670 fn reconciliation_routes_reused_index_by_exact_venue_order_id() {
20671 let (client, cache, _rx) = create_execution_client();
20672 let instrument_id = register_test_instrument(&client, &cache);
20673 let mut factory = test_order_factory();
20674 let active_order = test_limit_order(&mut factory, instrument_id, "O-RECON-LIVE-ACTIVE");
20675 let retired_order = test_limit_order(&mut factory, instrument_id, "O-RECON-LIVE-RETIRED");
20676 let active_cloid = active_order.client_order_id();
20677 let retired_cloid = retired_order.client_order_id();
20678 let active_venue_order_id = VenueOrderId::from("281476929510125");
20679 let retired_venue_order_id = VenueOrderId::from("281476929510126");
20680 cache_accepted_order(
20681 &cache,
20682 active_order,
20683 active_venue_order_id,
20684 Some(client_id()),
20685 );
20686 cache_canceled_order(&cache, retired_order, retired_venue_order_id);
20687 let (_, client_order_index) = forced_probed_index(active_cloid);
20688 let active_raw = reconciliation_raw_order(
20689 client_order_index,
20690 active_venue_order_id,
20691 LighterOrderStatus::Open,
20692 );
20693 let retired_raw = reconciliation_raw_order(
20694 client_order_index,
20695 retired_venue_order_id,
20696 LighterOrderStatus::Canceled,
20697 );
20698 let raw_client_id = client_order_index.to_string();
20699
20700 restore_reconciled_order(
20701 &client.core,
20702 &client.dispatch,
20703 VenueOrderId::new(active_raw.order_id.as_str()),
20704 active_raw.client_order_index,
20705 false,
20706 );
20707 restore_reconciled_order(
20708 &client.core,
20709 &client.dispatch,
20710 VenueOrderId::new(retired_raw.order_id.as_str()),
20711 retired_raw.client_order_index,
20712 true,
20713 );
20714
20715 assert_eq!(
20716 client
20717 .dispatch
20718 .resolve_live_order_cloid(&raw_client_id, active_venue_order_id,),
20719 Some(active_cloid),
20720 );
20721 assert_eq!(
20722 client
20723 .dispatch
20724 .resolve_live_order_cloid(&raw_client_id, retired_venue_order_id,),
20725 Some(retired_cloid),
20726 );
20727 assert_eq!(
20728 client
20729 .dispatch
20730 .resolve_live_trade_cloid(&raw_client_id, active_venue_order_id),
20731 Some(active_cloid),
20732 );
20733 assert_eq!(
20734 client
20735 .dispatch
20736 .resolve_live_trade_cloid(&raw_client_id, retired_venue_order_id),
20737 Some(retired_cloid),
20738 );
20739 }
20740
20741 #[rstest]
20742 fn reconciliation_does_not_restore_from_numeric_client_index_alone() {
20743 let (client, cache, _rx) = create_execution_client();
20744 let instrument_id = register_test_instrument(&client, &cache);
20745 let mut factory = test_order_factory();
20746 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-NUMERIC-ONLY");
20747 let cloid = order.client_order_id();
20748 let client_order_index = client.dispatch.derive_client_order_index(&cloid);
20749 let venue_order_id = VenueOrderId::from("281476929510130");
20750 cache_order(&cache, order);
20751 let raw =
20752 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
20753
20754 restore_reconciled_order(
20755 &client.core,
20756 &client.dispatch,
20757 VenueOrderId::new(raw.order_id.as_str()),
20758 raw.client_order_index,
20759 false,
20760 );
20761 let report = parse_http_order_to_report(
20762 &raw,
20763 &client.registry,
20764 account_id(),
20765 UnixNanos::from(1),
20766 &client.dispatch,
20767 )
20768 .unwrap();
20769 let translated = client.dispatch.translate_order_cloid(report);
20770
20771 assert_eq!(client.dispatch.client_order_index(&cloid), None);
20772 assert_eq!(
20773 client
20774 .dispatch
20775 .resolve_client_order_index(client_order_index),
20776 None,
20777 );
20778 assert!(!client.dispatch.order_identities.contains_key(&cloid));
20779 assert_eq!(
20780 translated.client_order_id,
20781 Some(ClientOrderId::new(venue_order_id.as_str())),
20782 );
20783 assert_eq!(translated.venue_order_id, venue_order_id);
20784 }
20785
20786 #[rstest]
20787 #[case::known_partner(true)]
20788 #[case::external_partner(false)]
20789 fn lookup_after_restart_preserves_collision_partner(#[case] partner_cached: bool) {
20790 let (client, cache, _rx) = create_execution_client();
20791 let instrument_id = register_test_instrument(&client, &cache);
20792 let mut factory = test_order_factory();
20793 let order = test_limit_order(&mut factory, instrument_id, "O-UNACKNOWLEDGED-PROBED");
20794 let cloid = order.client_order_id();
20795 let partner = test_limit_order(&mut factory, instrument_id, "O-COLLISION-PARTNER");
20796 let partner_cloid = partner.client_order_id();
20797 let venue_order_id = VenueOrderId::from("281476929510140");
20798 let source = WsDispatchState::new();
20799 let base_index = source.derive_client_order_index(&cloid);
20800 source.register_cloid(base_index, partner_cloid).unwrap();
20801 let selected_index = source.register_cloid(base_index, cloid).unwrap();
20802 cache_order(&cache, order);
20803
20804 let submitted = OrderSubmitted::new(
20805 trader_id(),
20806 strategy_id(),
20807 instrument_id,
20808 cloid,
20809 account_id(),
20810 UUID4::new(),
20811 UnixNanos::from(1),
20812 UnixNanos::from(2),
20813 );
20814 cache
20815 .borrow_mut()
20816 .update_order(&OrderEventAny::Submitted(submitted))
20817 .unwrap();
20818
20819 if partner_cached {
20820 cache_accepted_order(&cache, partner, venue_order_id, Some(client_id()));
20821 restore_reconciled_order(
20822 &client.core,
20823 &client.dispatch,
20824 venue_order_id,
20825 base_index,
20826 false,
20827 );
20828 }
20829
20830 let raw = reconciliation_raw_order(base_index, venue_order_id, LighterOrderStatus::Open);
20831 let lookup = OrderReportLookup::new(
20832 &client.registry,
20833 &client.dispatch,
20834 Some(instrument_id),
20835 Some(&cloid),
20836 None,
20837 )
20838 .unwrap();
20839 let parsed = lookup
20840 .parse(
20841 &raw,
20842 &client.registry,
20843 account_id(),
20844 UnixNanos::from(3),
20845 &client.dispatch,
20846 )
20847 .unwrap();
20848
20849 let expected_cloid = if partner_cached {
20850 partner_cloid
20851 } else {
20852 ClientOrderId::new(venue_order_id.as_str())
20853 };
20854
20855 let expected = parsed.0.clone().with_client_order_id(expected_cloid);
20856 let report = lookup.finish(&client.dispatch, parsed);
20857 let cached = cache.borrow().order_owned(&cloid).unwrap();
20858
20859 assert_eq!(selected_index, base_index + 1);
20860 assert_eq!(
20861 serde_json::to_value(&report).unwrap(),
20862 serde_json::to_value(&expected).unwrap()
20863 );
20864 assert_eq!(cached.status(), OrderStatus::Submitted);
20865 assert_eq!(cached.venue_order_id(), None);
20866 assert_eq!(client.dispatch.client_order_index(&cloid), None);
20867 }
20868
20869 fn forced_probed_index(cloid: ClientOrderId) -> (i64, i64) {
20870 let source = WsDispatchState::new();
20871 let base_index = source.derive_client_order_index(&cloid);
20872 source
20873 .cloid_map
20874 .insert(base_index, ClientOrderId::from("O-FORCED-COLLISION"));
20875 let client_order_index = source.register_cloid(base_index, cloid).unwrap();
20876 (base_index, client_order_index)
20877 }
20878
20879 fn reconciliation_raw_order(
20880 client_order_index: i64,
20881 venue_order_id: VenueOrderId,
20882 status: LighterOrderStatus,
20883 ) -> LighterOrder {
20884 let rig = dispatcher_rig("RECONCILIATION-RAW");
20885 let mut raw = dispatcher_test_order(&rig, status);
20886 raw.order_index = venue_order_id.as_str().parse().unwrap();
20887 raw.order_id = venue_order_id.to_string();
20888 raw.client_order_index = client_order_index;
20889 raw.client_order_id = client_order_index.to_string();
20890 raw
20891 }
20892
20893 fn cache_canceled_order(
20894 cache: &Rc<RefCell<Cache>>,
20895 order: OrderAny,
20896 venue_order_id: VenueOrderId,
20897 ) {
20898 let (instrument_id, client_order_id) =
20899 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
20900 let canceled = OrderEventAny::Canceled(OrderCanceled::new(
20901 trader_id(),
20902 strategy_id(),
20903 instrument_id,
20904 client_order_id,
20905 UUID4::new(),
20906 UnixNanos::from(1),
20907 UnixNanos::from(2),
20908 false,
20909 Some(venue_order_id),
20910 Some(account_id()),
20911 None,
20912 ));
20913 cache.borrow_mut().update_order(&canceled).unwrap();
20914 }
20915
20916 fn reconciliation_fill_report(
20917 instrument_id: InstrumentId,
20918 client_order_index: i64,
20919 venue_order_id: VenueOrderId,
20920 trade_id: &str,
20921 ) -> FillReport {
20922 FillReport::new(
20923 account_id(),
20924 instrument_id,
20925 venue_order_id,
20926 TradeId::new(trade_id),
20927 OrderSide::Buy,
20928 Quantity::from("0.1336"),
20929 Price::from("2352.73"),
20930 Money::from("0.000196 USDC"),
20931 LiquiditySide::Taker,
20932 Some(ClientOrderId::new(client_order_index.to_string())),
20933 None,
20934 UnixNanos::from(1),
20935 UnixNanos::from(2),
20936 Some(UUID4::new()),
20937 )
20938 }
20939
20940 #[rstest]
20941 fn dispatch_lighter_order_seeds_snapshot_after_synthesized_accept() {
20942 let mut rig = dispatcher_rig("9");
20947 register_identity(&rig);
20948
20949 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
20950 let mut trade = dispatcher_test_trade(&rig, true);
20951 trade.bid_id = order.order_index;
20952 trade.bid_id_str = Some(order.order_id.clone());
20953 dispatch_lighter_trade(
20954 &trade,
20955 &rig.dispatch,
20956 &rig.emitter,
20957 &rig.registry,
20958 account_id(),
20959 trader_id(),
20960 Some(TEST_ACCOUNT_INDEX_I64),
20961 UnixNanos::from(1),
20962 );
20963 assert_eq!(drain_events(&mut rig.rx).len(), 2);
20964 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
20965 assert!(
20966 rig.dispatch.snapshot_for(&rig.cloid).is_none(),
20967 "synthesized Accept has no snapshot until the Open frame seeds one",
20968 );
20969
20970 dispatch_lighter_order(
20974 &order,
20975 &rig.dispatch,
20976 &rig.emitter,
20977 &rig.registry,
20978 account_id(),
20979 trader_id(),
20980 UnixNanos::from(2),
20981 );
20982 assert!(
20983 rig.dispatch.snapshot_for(&rig.cloid).is_some(),
20984 "Open frame after synthesized accept must seed the snapshot",
20985 );
20986
20987 let mut modified = order;
20989 modified.price = Decimal::from_str("2400.00").unwrap();
20990 dispatch_lighter_order(
20991 &modified,
20992 &rig.dispatch,
20993 &rig.emitter,
20994 &rig.registry,
20995 account_id(),
20996 trader_id(),
20997 UnixNanos::from(3),
20998 );
20999 let events = drain_events(&mut rig.rx);
21000 let updated = events
21001 .iter()
21002 .find(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Updated(_))));
21003 assert!(
21004 updated.is_some(),
21005 "real modify must produce Updated, events={events:?}"
21006 );
21007 }
21008
21009 fn enqueue_create(client: &LighterExecutionClient, order: &OrderAny, nonce: i64) -> i64 {
21010 let client_order_index = client
21011 .dispatch
21012 .derive_client_order_index(&order.client_order_id());
21013 client
21014 .dispatch
21015 .register_cloid(client_order_index, order.client_order_id())
21016 .unwrap();
21017 client.dispatch.register_order_identity(
21018 order.client_order_id(),
21019 OrderIdentity::new(
21020 order.instrument_id(),
21021 order.strategy_id(),
21022 order.order_side(),
21023 order.order_type(),
21024 client_order_index,
21025 ),
21026 );
21027 let now = UnixNanos::from(1_000_000_000);
21028 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
21029 batch_id: None,
21030 connection_epoch: 0,
21031 kind: PendingSendTxKind::Create {
21032 order: Box::new(order.clone()),
21033 client_order_index,
21034 },
21035 submitted_at: now,
21036 nonce,
21037 api_key_index: TEST_API_KEY_INDEX,
21038 tx_hash: format!("hash{nonce:02x}"),
21039 });
21040 client.dispatch.nonce_manager.refresh(
21041 TEST_ACCOUNT_INDEX_I64,
21042 TEST_API_KEY_INDEX,
21043 nonce + 1,
21044 );
21045 let _ = client
21046 .dispatch
21047 .nonce_manager
21048 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX);
21049 client_order_index
21050 }
21051
21052 fn enqueue_other(client: &LighterExecutionClient, nonce: i64) {
21053 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
21054 batch_id: None,
21055 connection_epoch: 0,
21056 kind: PendingSendTxKind::Other,
21057 submitted_at: UnixNanos::from(1_000_000_000),
21058 nonce,
21059 api_key_index: TEST_API_KEY_INDEX,
21060 tx_hash: format!("hash{nonce:02x}"),
21061 });
21062 }
21063
21064 #[rstest]
21065 #[case(serde_json::json!({"self_trade_behavior_mode": "reduce"}), LighterSelfTradeBehaviorMode::Reduce, LighterSelfTradeEqualityMode::AccountIndex)]
21066 #[case(serde_json::json!({"self_trade_equality_mode": "master_account_index"}), LighterSelfTradeBehaviorMode::ExpireMaker, LighterSelfTradeEqualityMode::MasterAccountIndex)]
21067 #[case(serde_json::json!({"market_order_slippage_bps": 37}), LighterSelfTradeBehaviorMode::ExpireMaker, LighterSelfTradeEqualityMode::AccountIndex)]
21068 fn resolve_self_trade_partial_params(
21069 #[case] value: serde_json::Value,
21070 #[case] behavior: LighterSelfTradeBehaviorMode,
21071 #[case] equality: LighterSelfTradeEqualityMode,
21072 ) {
21073 let (client, _cache, _rx) = create_execution_client();
21074 let params: Params = serde_json::from_value(value).unwrap();
21075
21076 let attrs = client.resolve_create_attributes(Some(¶ms)).unwrap();
21077
21078 assert_eq!(
21079 attrs,
21080 L2TxAttributes {
21081 self_trade_behavior_mode: Some(behavior),
21082 self_trade_equality_mode: Some(equality),
21083 ..Default::default()
21084 }
21085 );
21086 }
21087
21088 fn enqueue_cancel(
21089 client: &LighterExecutionClient,
21090 instrument_id: InstrumentId,
21091 client_order_id: ClientOrderId,
21092 venue_order_id: VenueOrderId,
21093 nonce: i64,
21094 ) -> UUID4 {
21095 let command_id = UUID4::new();
21096 let pending = enqueue_cancel_pending(
21097 client,
21098 instrument_id,
21099 client_order_id,
21100 venue_order_id,
21101 nonce,
21102 command_id,
21103 );
21104 promote_cancel_submission(&client.dispatch, &pending);
21105 command_id
21106 }
21107
21108 fn enqueue_cancel_pending(
21109 client: &LighterExecutionClient,
21110 instrument_id: InstrumentId,
21111 client_order_id: ClientOrderId,
21112 venue_order_id: VenueOrderId,
21113 nonce: i64,
21114 command_id: UUID4,
21115 ) -> PendingSendTx {
21116 client
21117 .dispatch
21118 .register_cancel_command(client_order_id, command_id, true);
21119
21120 let pending = PendingSendTx {
21121 batch_id: None,
21122 connection_epoch: 0,
21123 kind: PendingSendTxKind::Cancel {
21124 command_id: Some(command_id),
21125 emit_cancel_rejected: true,
21126 strategy_id: strategy_id(),
21127 instrument_id,
21128 client_order_id,
21129 venue_order_id: Some(venue_order_id),
21130 },
21131 submitted_at: UnixNanos::from(1_000_000_000),
21132 nonce,
21133 api_key_index: TEST_API_KEY_INDEX,
21134 tx_hash: format!("hash{nonce:02x}"),
21135 };
21136
21137 client.dispatch.enqueue_pending_sendtx(pending.clone());
21138
21139 pending
21140 }
21141
21142 fn enqueue_modify(
21143 client: &LighterExecutionClient,
21144 instrument_id: InstrumentId,
21145 client_order_id: ClientOrderId,
21146 venue_order_id: VenueOrderId,
21147 nonce: i64,
21148 ) -> UUID4 {
21149 let command = ModifyOrder::new(
21150 trader_id(),
21151 Some(client_id()),
21152 strategy_id(),
21153 instrument_id,
21154 client_order_id,
21155 Some(venue_order_id),
21156 Some(Quantity::from("0.2000")),
21157 Some(Price::from("2362.00")),
21158 None,
21159 UUID4::new(),
21160 UnixNanos::default(),
21161 None,
21162 None,
21163 );
21164 client.dispatch.set_pending_modify_action(&command);
21165 client.dispatch.set_pending_modify_shape(
21166 &command,
21167 OrderShapeSnapshot {
21168 quantity: command.quantity.unwrap(),
21169 price: command.price,
21170 trigger_price: command.trigger_price,
21171 },
21172 );
21173
21174 client
21175 .dispatch
21176 .set_pending_modify_nonce(&client_order_id, command.command_id, 0, nonce);
21177 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
21178 batch_id: None,
21179 connection_epoch: 0,
21180 kind: PendingSendTxKind::Modify {
21181 command_id: command.command_id,
21182 strategy_id: strategy_id(),
21183 instrument_id,
21184 client_order_id,
21185 venue_order_id: Some(venue_order_id),
21186 },
21187 submitted_at: UnixNanos::from(1_000_000_000),
21188 nonce,
21189 api_key_index: TEST_API_KEY_INDEX,
21190 tx_hash: format!("hash{nonce:02x}"),
21191 });
21192
21193 command.command_id
21194 }
21195
21196 fn acked_probe_context(client: &LighterExecutionClient) -> AckedOrderProbeContext {
21197 AckedOrderProbeContext {
21198 http_client: client.http_client.clone(),
21199 registry: Arc::clone(&client.registry),
21200 credential: test_credential(),
21201 dispatch: client.dispatch.clone(),
21202 account_id: client.core.account_id,
21203 clock: client.clock,
21204 emitter: client.emitter.clone(),
21205 connection_epoch: client.ws_client.connection_epoch_atomic(),
21206 cancellation_token: client.cancellation_token.clone(),
21207 pending_tasks: client.pending_tasks.spawner().unwrap(),
21208 }
21209 }
21210
21211 fn failed_modify_tx(pending: &PendingSendTx) -> LighterTx {
21212 LighterTx {
21213 code: 200,
21214 message: None,
21215 hash: pending.tx_hash.clone(),
21216 tx_type: LighterTxType::ModifyOrder as u8,
21217 info: String::new(),
21218 event_info: serde_json::json!({"ae":"invalid order price"}).to_string(),
21219 status: LighterTxStatus::Failed,
21220 account_index: TEST_ACCOUNT_INDEX_I64,
21221 nonce: pending.nonce,
21222 api_key_index: pending.api_key_index,
21223 }
21224 }
21225
21226 #[rstest]
21227 #[case::hash("hash")]
21228 #[case::type_mismatch("type")]
21229 #[case::account("account")]
21230 #[case::key("key")]
21231 #[case::nonce("nonce")]
21232 #[tokio::test]
21233 async fn acknowledged_modify_requires_exact_transaction_identity(#[case] mismatch: &str) {
21234 let (client, cache, mut rx) = create_execution_client();
21235 let instrument = register_test_instrument(&client, &cache);
21236 let id = ClientOrderId::from("MODIFY-IDENTITY");
21237 enqueue_modify(&client, instrument, id, VenueOrderId::from("789"), 23);
21238 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21239 let mut tx = failed_modify_tx(&pending);
21240 match mismatch {
21241 "hash" => tx.hash = "different".to_string(),
21242 "type" => tx.tx_type = LighterTxType::CancelOrder as u8,
21243 "account" => tx.account_index += 1,
21244 "key" => tx.api_key_index += 1,
21245 "nonce" => tx.nonce += 1,
21246 _ => unreachable!(),
21247 }
21248
21249 let result = resolve_acked_action(&pending, &acked_probe_context(&client), tx).await;
21250 assert_eq!(
21251 result.unwrap_err().to_string(),
21252 "Lighter transaction lookup did not match acknowledged action identity"
21253 );
21254 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 23)));
21255 assert_eq!(
21256 client.dispatch.pending_order_action(&id),
21257 Some(PendingOrderAction::Modify)
21258 );
21259 assert!(rx.try_recv().is_err());
21260 }
21261
21262 #[rstest]
21263 #[case::failed_status(LighterTxStatus::Failed, "", true)]
21264 #[case::malformed_failed_status(LighterTxStatus::Failed, "invalid json", true)]
21265 #[case::application_error(LighterTxStatus::Executed, "application", true)]
21266 #[case::pending(LighterTxStatus::Pending, "", false)]
21267 #[case::pending_final(LighterTxStatus::PendingFinal, "", false)]
21268 #[tokio::test]
21269 async fn acknowledged_modify_outcomes_consume_only_matching_action(
21270 #[case] status: LighterTxStatus,
21271 #[case] event_info: &str,
21272 #[case] failed: bool,
21273 ) {
21274 let (client, cache, mut rx) = create_execution_client();
21275 let instrument = register_test_instrument(&client, &cache);
21276 let id = ClientOrderId::from("MODIFY-OUTCOME");
21277 let venue = VenueOrderId::from("789");
21278 let nonce = client
21279 .dispatch
21280 .nonce_manager
21281 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21282 .unwrap();
21283 enqueue_modify(&client, instrument, id, venue, nonce);
21284 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21285 let context = acked_probe_context(&client);
21286 let mut tx = failed_modify_tx(&pending);
21287 tx.status = status;
21288
21289 tx.event_info = if event_info == "application" {
21290 serde_json::json!({"ae":"invalid order price"}).to_string()
21291 } else {
21292 event_info.to_string()
21293 };
21294
21295 let resolved = resolve_acked_action(&pending, &context, tx.clone())
21296 .await
21297 .unwrap();
21298 assert_eq!(resolved, failed);
21299
21300 if failed {
21301 let OrderEventAny::ModifyRejected(event) = recv_order_event(&mut rx).await else {
21302 panic!("expected ModifyRejected")
21303 };
21304
21305 assert_eq!(event.client_order_id, id);
21306 assert_eq!(event.venue_order_id, Some(venue));
21307 assert_eq!(event.instrument_id, instrument);
21308 assert_eq!(event.strategy_id, strategy_id());
21309 assert_eq!(
21310 event.reason.as_str(),
21311 if event_info == "application" {
21312 "Lighter modify transaction failed: invalid order price"
21313 } else {
21314 "Lighter modify transaction failed: transaction failed without an application error"
21315 }
21316 );
21317 assert_eq!(client.dispatch.pending_modify_nonce(&id), None);
21318 assert_eq!(client.dispatch.pending_order_action(&id), None);
21319 assert!(resolve_acked_action(&pending, &context, tx).await.unwrap());
21320 } else {
21321 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, nonce)));
21322 }
21323
21324 assert!(rx.try_recv().is_err());
21325 assert_eq!(
21326 client
21327 .dispatch
21328 .nonce_manager
21329 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21330 .unwrap(),
21331 nonce + 1
21332 );
21333 }
21334
21335 #[rstest]
21336 #[case::empty_error("")]
21337 #[case::malformed_error("invalid json")]
21338 #[tokio::test]
21339 async fn acknowledged_cancel_failure_consumes_action_once(
21340 #[case] event_info: &str,
21341 #[values(false, true)] emit_rejected: bool,
21342 ) {
21343 let (client, cache, mut rx) = create_execution_client();
21344 let instrument = register_test_instrument(&client, &cache);
21345 let id = ClientOrderId::from("CANCEL-FAILED");
21346 let venue = VenueOrderId::from("789");
21347 let nonce = client
21348 .dispatch
21349 .nonce_manager
21350 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21351 .unwrap();
21352 enqueue_cancel(&client, instrument, id, venue, nonce);
21353 let mut pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21354
21355 if let PendingSendTxKind::Cancel {
21356 emit_cancel_rejected,
21357 ..
21358 } = &mut pending.kind
21359 {
21360 *emit_cancel_rejected = emit_rejected;
21361 }
21362
21363 let PendingSendTxKind::Cancel {
21364 command_id: Some(command_id),
21365 ..
21366 } = pending.kind
21367 else {
21368 unreachable!()
21369 };
21370
21371 client.dispatch.clear_pending_order_action(&id);
21372 client
21373 .dispatch
21374 .register_cancel_command(id, command_id, emit_rejected);
21375 client
21376 .dispatch
21377 .promote_pending_cancel(&id, command_id, 0, nonce, emit_rejected);
21378 let mut tx = failed_modify_tx(&pending);
21379 tx.tx_type = LighterTxType::CancelOrder as u8;
21380 tx.event_info = event_info.to_string();
21381 let context = acked_probe_context(&client);
21382 let resolved = resolve_acked_action(&pending, &context, tx.clone())
21383 .await
21384 .unwrap();
21385
21386 {
21387 let OrderEventAny::CancelRejected(event) = recv_order_event(&mut rx).await else {
21388 panic!("expected CancelRejected")
21389 };
21390
21391 let PendingSendTxKind::Cancel { command_id, .. } = pending.kind else {
21392 unreachable!()
21393 };
21394
21395 assert_eq!(event.causation_id, command_id);
21396 assert_eq!(event.client_order_id, id);
21397 assert_eq!(event.venue_order_id, Some(venue));
21398 assert_eq!(event.instrument_id, instrument);
21399 assert_eq!(event.strategy_id, strategy_id());
21400 assert_eq!(
21401 event.reason.as_str(),
21402 "Lighter cancel transaction failed: transaction failed without an application error"
21403 );
21404 }
21405
21406 let duplicate_resolved = resolve_acked_action(&pending, &context, tx).await.unwrap();
21407 assert!(resolved);
21408 assert!(duplicate_resolved);
21409 assert_eq!(client.dispatch.pending_cancel_nonce(&id), None);
21410 assert_eq!(client.dispatch.pending_order_action(&id), None);
21411 assert!(rx.try_recv().is_err());
21412 assert_eq!(
21413 client
21414 .dispatch
21415 .nonce_manager
21416 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX,)
21417 .unwrap(),
21418 nonce + 1
21419 );
21420 }
21421
21422 #[rstest]
21423 #[case::unknown_hash("unmatched", 0, LighterTxStatus::Failed, "")]
21424 #[case::old_connection("hash17", 1, LighterTxStatus::Failed, "")]
21425 #[case::pending("hash17", 0, LighterTxStatus::Pending, "")]
21426 #[case::pending_final("hash17", 0, LighterTxStatus::PendingFinal, "")]
21427 #[case::malformed_success("hash17", 0, LighterTxStatus::Executed, "invalid json")]
21428 fn account_tx_preserves_unconfirmed_modification(
21429 #[case] hash: &str,
21430 #[case] connection_epoch: u64,
21431 #[case] status: LighterTxStatus,
21432 #[case] event_info: &str,
21433 ) {
21434 let (client, cache, mut rx) = create_execution_client();
21435 let instrument = register_test_instrument(&client, &cache);
21436 let id = ClientOrderId::from("STREAM-UNCONFIRMED");
21437 enqueue_modify(&client, instrument, id, VenueOrderId::from("789"), 23);
21438 handle_account_tx(
21439 connection_epoch,
21440 &[LighterWsTx {
21441 hash: hash.to_owned(),
21442 status,
21443 event_info: event_info.to_owned(),
21444 }],
21445 &acked_probe_context(&client),
21446 );
21447
21448 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 23)));
21449 assert!(client.dispatch.pending_modify_executed(&id).is_none());
21450 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
21451 assert!(rx.try_recv().is_err());
21452 }
21453
21454 #[rstest]
21455 #[case::executed(LighterTxStatus::Executed)]
21456 #[case::failed(LighterTxStatus::Failed)]
21457 fn account_tx_preserves_ack_attribution(#[case] status: LighterTxStatus) {
21458 let (client, cache, mut rx) = create_execution_client();
21459 let instrument = register_test_instrument(&client, &cache);
21460 let id_a = ClientOrderId::from("STREAM-ACK-A");
21461 let id_b = ClientOrderId::from("STREAM-ACK-B");
21462 let nonce_a = client
21463 .dispatch
21464 .nonce_manager
21465 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21466 .unwrap();
21467 enqueue_modify(
21468 &client,
21469 instrument,
21470 id_a,
21471 VenueOrderId::from("789"),
21472 nonce_a,
21473 );
21474 let nonce_b = client
21475 .dispatch
21476 .nonce_manager
21477 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21478 .unwrap();
21479 let command_b = enqueue_modify(
21480 &client,
21481 instrument,
21482 id_b,
21483 VenueOrderId::from("790"),
21484 nonce_b,
21485 );
21486 handle_account_tx(
21487 0,
21488 &[LighterWsTx {
21489 hash: format!("hash{nonce_a:02x}"),
21490 status,
21491 event_info: String::new(),
21492 }],
21493 &acked_probe_context(&client),
21494 );
21495
21496 let stream_events = drain_events(&mut rx);
21497 assert_eq!(
21498 stream_events.len(),
21499 usize::from(status == LighterTxStatus::Failed)
21500 );
21501 assert!(
21502 handle_send_tx_ack_for_connection(
21503 &client.dispatch,
21504 Some(TEST_ACCOUNT_INDEX_I64),
21505 0,
21506 200,
21507 None,
21508 )
21509 .is_none()
21510 );
21511 handle_send_tx_rejection(
21512 &client.dispatch,
21513 &client.emitter,
21514 Some(TEST_ACCOUNT_INDEX_I64),
21515 UnixNanos::from(5),
21516 SendTxRejectionSource::Ack,
21517 Some(21702),
21518 "B rejected",
21519 Some(&format!("hash{nonce_b:02x}")),
21520 );
21521
21522 let ExecutionEvent::Order(OrderEventAny::ModifyRejected(rejected)) = rx.try_recv().unwrap()
21523 else {
21524 panic!("expected rejection of the second command");
21525 };
21526
21527 assert_eq!(rejected.client_order_id, id_b);
21528 assert_eq!(rejected.trader_id, trader_id());
21529 assert_eq!(rejected.strategy_id, strategy_id());
21530 assert_eq!(rejected.instrument_id, instrument);
21531 assert_eq!(rejected.account_id, Some(account_id()));
21532 assert_eq!(rejected.venue_order_id, Some(VenueOrderId::from("790")));
21533 assert_eq!(rejected.causation_id, Some(command_b));
21534 assert_eq!(rejected.reason.as_str(), "LIGHTER_21702: B rejected");
21535 assert_eq!(client.dispatch.pending_modify_nonce(&id_b), None);
21536 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
21537 assert_eq!(
21538 client
21539 .dispatch
21540 .nonce_manager
21541 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21542 .unwrap(),
21543 nonce_b
21544 );
21545 assert!(rx.try_recv().is_err());
21546 }
21547
21548 #[rstest]
21549 fn account_tx_rejection_cannot_publish_after_modification_resolves() {
21550 let (client, cache, mut rx) = create_execution_client();
21551 let instrument = register_test_instrument(&client, &cache);
21552 let id = ClientOrderId::from("STREAM-STALE-REJECTION");
21553 let venue = VenueOrderId::from("789");
21554 enqueue_modify(&client, instrument, id, venue, 23);
21555 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21556 assert!(client.dispatch.clear_pending_modify_if_nonce(&id, 0, 23));
21557 enqueue_modify(&client, instrument, id, venue, 24);
21558 reject_modify_order(
21559 &client.dispatch,
21560 &client.emitter,
21561 &pending,
21562 &CommandFailure::venue_rejected("stale execution failure"),
21563 UnixNanos::from(5),
21564 true,
21565 );
21566 client.dispatch.set_pending_modify_executed(&id, &pending);
21567 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 24)));
21568 assert!(client.dispatch.pending_modify_executed(&id).is_none());
21569 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
21570 assert!(rx.try_recv().is_err());
21571 }
21572
21573 #[rstest]
21574 #[case::newer_nonce(false)]
21575 #[case::reconnected(true)]
21576 #[tokio::test]
21577 async fn acknowledged_modify_does_not_consume_newer_submission(#[case] reconnected: bool) {
21578 let (client, cache, mut rx) = create_execution_client();
21579 let instrument = register_test_instrument(&client, &cache);
21580 let id = ClientOrderId::from("MODIFY-NEWER");
21581 let venue = VenueOrderId::from("789");
21582 enqueue_modify(&client, instrument, id, venue, 23);
21583 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21584 let context = acked_probe_context(&client);
21585 client.dispatch.clear_pending_modify_if_nonce(&id, 0, 23);
21586
21587 if reconnected {
21588 context.connection_epoch.store(1, Ordering::Release);
21589 enqueue_modify(&client, instrument, id, venue, 24);
21590 } else {
21591 enqueue_modify(&client, instrument, id, venue, 24);
21592 }
21593
21594 assert!(
21595 resolve_acked_action(&pending, &context, failed_modify_tx(&pending))
21596 .await
21597 .unwrap()
21598 );
21599 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 24)));
21600 assert!(rx.try_recv().is_err());
21601 }
21602
21603 #[rstest]
21604 fn queued_modify_cannot_bind_a_newer_command() {
21605 let (client, cache, _rx) = create_execution_client();
21606 let instrument = register_test_instrument(&client, &cache);
21607 let id = ClientOrderId::from("MODIFY-QUEUED");
21608
21609 let mut cmd = ModifyOrder::new(
21610 trader_id(),
21611 Some(client_id()),
21612 strategy_id(),
21613 instrument,
21614 id,
21615 Some(VenueOrderId::from("789")),
21616 Some(Quantity::from("0.2000")),
21617 Some(Price::from("2362.00")),
21618 None,
21619 UUID4::new(),
21620 UnixNanos::default(),
21621 None,
21622 None,
21623 );
21624 let old_command = cmd.command_id;
21625 let old_cmd = cmd.clone();
21626 client.dispatch.set_pending_modify_action(&cmd);
21627 assert!(
21628 client
21629 .dispatch
21630 .clear_pending_modify_if_command(&id, old_command)
21631 );
21632 cmd.command_id = UUID4::new();
21633 cmd.price = Some(Price::from("2363.00"));
21634 client.dispatch.set_pending_modify_action(&cmd);
21635 client.dispatch.set_pending_modify_shape(
21636 &cmd,
21637 OrderShapeSnapshot {
21638 quantity: cmd.quantity.unwrap(),
21639 price: cmd.price,
21640 trigger_price: cmd.trigger_price,
21641 },
21642 );
21643
21644 client.dispatch.set_pending_modify_shape(
21645 &old_cmd,
21646 OrderShapeSnapshot {
21647 quantity: old_cmd.quantity.unwrap(),
21648 price: old_cmd.price,
21649 trigger_price: old_cmd.trigger_price,
21650 },
21651 );
21652
21653 client
21654 .dispatch
21655 .set_pending_modify_nonce(&id, old_command, 0, 23);
21656 assert_eq!(client.dispatch.pending_modify_nonce(&id), None);
21657 assert!(
21658 !client
21659 .dispatch
21660 .clear_pending_modify_if_command(&id, old_command)
21661 );
21662 assert_eq!(
21663 client.dispatch.pending_order_action(&id),
21664 Some(PendingOrderAction::Modify)
21665 );
21666 client
21667 .dispatch
21668 .set_pending_modify_nonce(&id, cmd.command_id, 0, 24);
21669 client.dispatch.clear_pending_modify_if_update(
21670 &id,
21671 Quantity::from("0.2000"),
21672 Some(Price::from("2362.00")),
21673 None,
21674 );
21675 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 24)));
21676 client.dispatch.clear_pending_modify_if_update(
21677 &id,
21678 Quantity::from("0.2000"),
21679 Some(Price::from("2363.00")),
21680 None,
21681 );
21682 assert_eq!(client.dispatch.pending_order_action(&id), None);
21683 }
21684
21685 #[tokio::test]
21686 async fn handle_send_tx_ack_removes_hash_matched_entry() {
21687 let (client, cache, mut rx) = create_execution_client();
21688 let instrument_id = register_test_instrument(&client, &cache);
21689 let mut factory = test_order_factory();
21690 let order_a = test_limit_order(&mut factory, instrument_id, "ACK-A");
21691 let order_b = test_limit_order(&mut factory, instrument_id, "ACK-B");
21692 enqueue_create(&client, &order_a, 10);
21693 enqueue_create(&client, &order_b, 11);
21694
21695 let acked = handle_send_tx_ack(
21697 &client.dispatch,
21698 Some(TEST_ACCOUNT_INDEX_I64),
21699 200,
21700 Some("hash0b"),
21701 );
21702
21703 let acked = acked.expect("create ack");
21704 assert!(matches!(acked.kind, PendingSendTxKind::Create { .. }));
21705 assert!(matches!(
21706 AckedOrderProbe::from_pending(&acked),
21707 Some(AckedOrderProbe {
21708 nonce: 11,
21709 connection_epoch: 0,
21710 ..
21711 }),
21712 ));
21713 assert_eq!(client.dispatch.pending_sendtx_len(), 1, "only B pops");
21714 let head = client.dispatch.pop_pending_sendtx_head().unwrap();
21715 match head.kind {
21716 PendingSendTxKind::Create { order, .. } => {
21717 assert_eq!(order.client_order_id(), order_a.client_order_id());
21718 }
21719 _ => panic!("expected Create kind"),
21720 }
21721
21722 assert!(
21723 tokio::time::timeout(Duration::from_millis(50), rx.recv())
21724 .await
21725 .is_err(),
21726 "ack must not emit an event",
21727 );
21728 }
21729
21730 #[rstest]
21731 #[case::executed(LighterTxStatus::Executed, false)]
21732 #[case::pending_final(LighterTxStatus::PendingFinal, true)]
21733 #[tokio::test]
21734 async fn acknowledged_create_probe_retains_unqueryable_order(
21735 #[case] status: LighterTxStatus,
21736 #[case] pending: bool,
21737 ) {
21738 let (client, cache, mut rx) = create_execution_client();
21739 let instrument_id = register_test_instrument(&client, &cache);
21740 let mut factory = test_order_factory();
21741 let order = test_limit_order(&mut factory, instrument_id, "PROBE-UNQUERYABLE");
21742 let index = enqueue_create(&client, &order, TEST_SUBMISSION_NONCE);
21743 let submission = client.dispatch.pop_pending_sendtx_head().unwrap();
21744
21745 let tx = LighterTx {
21746 code: 200,
21747 message: None,
21748 hash: submission.tx_hash.clone(),
21749 tx_type: LighterTxType::CreateOrder as u8,
21750 info: serde_json::json!({"ClientOrderIndex": index}).to_string(),
21751 event_info: serde_json::json!({"ae": ""}).to_string(),
21752 status,
21753 account_index: TEST_ACCOUNT_INDEX_I64,
21754 nonce: TEST_SUBMISSION_NONCE,
21755 api_key_index: TEST_API_KEY_INDEX,
21756 };
21757
21758 let (http_client, server) = spawn_create_probe_server(&tx).await;
21759 let mut context = acked_probe_context(&client);
21760 context.http_client = http_client;
21761 let cloid = order.client_order_id();
21762 assert!(
21763 client
21764 .dispatch
21765 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE)
21766 );
21767
21768 probe_acked_create(
21769 &AckedOrderProbe::from_pending(&submission).unwrap(),
21770 &context,
21771 )
21772 .await
21773 .unwrap();
21774 server.abort();
21775 let _ = server.await;
21776
21777 let identity = client.dispatch.order_identity(&cloid).unwrap();
21778 assert_eq!(
21779 client
21780 .dispatch
21781 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE),
21782 pending
21783 );
21784 assert_eq!(identity.instrument_id, instrument_id);
21785 assert_eq!(identity.strategy_id, strategy_id());
21786 assert_eq!(identity.order_side, order.order_side());
21787 assert_eq!(identity.order_type, order.order_type());
21788 assert_eq!(identity.client_order_index, index);
21789 assert_eq!(identity.venue_order_id(), None);
21790 assert!(!client.dispatch.accepted_was_emitted(&cloid));
21791 assert!(!client.dispatch.venue_id_map.contains_key(&cloid));
21792 assert_eq!(
21793 client.dispatch.cloid_map.get(&index).map(|id| *id),
21794 Some(cloid)
21795 );
21796 assert!(drain_events(&mut rx).is_empty());
21797 }
21798
21799 #[rstest]
21800 #[case::hash("hash")]
21801 #[case::type_mismatch("type")]
21802 #[case::account("account")]
21803 #[case::key("key")]
21804 #[case::nonce("nonce")]
21805 #[case::index("index")]
21806 #[tokio::test]
21807 async fn acknowledged_create_probe_requires_exact_http_transaction_identity(
21808 #[case] mismatch: &str,
21809 ) {
21810 let (client, cache, mut rx) = create_execution_client();
21811 let instrument_id = register_test_instrument(&client, &cache);
21812 let mut factory = test_order_factory();
21813 let order = test_limit_order(&mut factory, instrument_id, "PROBE-IDENTITY");
21814 let index = enqueue_create(&client, &order, TEST_SUBMISSION_NONCE);
21815 let submission = client.dispatch.pop_pending_sendtx_head().unwrap();
21816
21817 let mut tx = LighterTx {
21818 code: 200,
21819 message: None,
21820 hash: submission.tx_hash.clone(),
21821 tx_type: LighterTxType::CreateOrder as u8,
21822 info: serde_json::json!({"ClientOrderIndex": index}).to_string(),
21823 event_info: serde_json::json!({"ae": "foreign transaction failed"}).to_string(),
21824 status: LighterTxStatus::Failed,
21825 account_index: TEST_ACCOUNT_INDEX_I64,
21826 nonce: TEST_SUBMISSION_NONCE,
21827 api_key_index: TEST_API_KEY_INDEX,
21828 };
21829
21830 match mismatch {
21831 "hash" => tx.hash = "foreign".to_string(),
21832 "type" => tx.tx_type = LighterTxType::CancelOrder as u8,
21833 "account" => tx.account_index += 1,
21834 "key" => tx.api_key_index += 1,
21835 "nonce" => tx.nonce += 1,
21836 "index" => tx.info = serde_json::json!({"ClientOrderIndex": index + 1}).to_string(),
21837 _ => unreachable!(),
21838 }
21839
21840 let (http_client, server) = spawn_create_probe_server(&tx).await;
21841 let mut context = acked_probe_context(&client);
21842 context.http_client = http_client;
21843 let error = probe_acked_create(
21844 &AckedOrderProbe::from_pending(&submission).unwrap(),
21845 &context,
21846 )
21847 .await
21848 .unwrap_err();
21849 server.abort();
21850 let _ = server.await;
21851
21852 let expected = if mismatch == "index" {
21853 format!(
21854 "Lighter transaction lookup returned client_order_index {} for acknowledged create {index}",
21855 index + 1
21856 )
21857 } else {
21858 "Lighter transaction lookup did not match acknowledged create identity".to_string()
21859 };
21860
21861 let cloid = order.client_order_id();
21862 assert_eq!(error.to_string(), expected);
21863 assert!(
21864 client
21865 .dispatch
21866 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE)
21867 );
21868 assert_eq!(
21869 client
21870 .dispatch
21871 .order_identity(&cloid)
21872 .unwrap()
21873 .client_order_index,
21874 index
21875 );
21876 assert_eq!(
21877 client.dispatch.cloid_map.get(&index).map(|id| *id),
21878 Some(cloid)
21879 );
21880 assert!(!client.dispatch.accepted_was_emitted(&cloid));
21881 assert!(!client.dispatch.venue_id_map.contains_key(&cloid));
21882 assert!(drain_events(&mut rx).is_empty());
21883 }
21884
21885 async fn spawn_create_probe_server(
21886 tx: &LighterTx,
21887 ) -> (LighterHttpClient, tokio::task::JoinHandle<()>) {
21888 let body = serde_json::to_string(tx).unwrap();
21889
21890 let app = Router::new()
21891 .route(
21892 "/api/v1/accountActiveOrders",
21893 get(|| async { r#"{"code":200,"orders":[]}"# }),
21894 )
21895 .route(
21896 "/api/v1/tx",
21897 get(move || {
21898 let body = body.clone();
21899 async move { body }
21900 }),
21901 );
21902
21903 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
21904 let addr = listener.local_addr().unwrap();
21905 let server = tokio::spawn(async move {
21906 axum::serve(listener, app).await.unwrap();
21907 });
21908
21909 let client = LighterHttpClient::new(
21910 LighterEnvironment::Testnet,
21911 Some(format!("http://{addr}")),
21912 1,
21913 None,
21914 )
21915 .unwrap();
21916 (client, server)
21917 }
21918
21919 #[rstest]
21920 fn acknowledged_create_tx_validation_requires_exact_identity() {
21921 let tx = LighterTx {
21922 code: 200,
21923 message: None,
21924 hash: "ABCDEF".to_string(),
21925 tx_type: LighterTxType::CreateOrder as u8,
21926 info: serde_json::json!({"ClientOrderIndex": 42}).to_string(),
21927 event_info: serde_json::json!({"ae": ""}).to_string(),
21928 status: LighterTxStatus::Failed,
21929 account_index: TEST_ACCOUNT_INDEX_I64,
21930 nonce: 10,
21931 api_key_index: TEST_API_KEY_INDEX,
21932 };
21933 assert!(
21934 validate_acked_create_tx(
21935 &tx,
21936 TEST_ACCOUNT_INDEX_I64,
21937 TEST_API_KEY_INDEX,
21938 42,
21939 10,
21940 "0xabcdef",
21941 )
21942 .is_ok(),
21943 );
21944
21945 let mismatches = [
21946 LighterTx {
21947 hash: "different".to_string(),
21948 ..tx.clone()
21949 },
21950 LighterTx {
21951 tx_type: LighterTxType::CancelOrder as u8,
21952 ..tx.clone()
21953 },
21954 LighterTx {
21955 account_index: TEST_ACCOUNT_INDEX_I64 + 1,
21956 ..tx.clone()
21957 },
21958 LighterTx {
21959 nonce: 11,
21960 ..tx.clone()
21961 },
21962 LighterTx {
21963 api_key_index: TEST_API_KEY_INDEX + 1,
21964 ..tx.clone()
21965 },
21966 ];
21967
21968 for mismatch in mismatches {
21969 let error = validate_acked_create_tx(
21970 &mismatch,
21971 TEST_ACCOUNT_INDEX_I64,
21972 TEST_API_KEY_INDEX,
21973 42,
21974 10,
21975 "abcdef",
21976 )
21977 .expect_err("mismatched transaction identity must fail");
21978 assert_eq!(
21979 error.to_string(),
21980 "Lighter transaction lookup did not match acknowledged create identity",
21981 );
21982 }
21983
21984 let wrong_index = LighterTx {
21985 info: serde_json::json!({"ClientOrderIndex": 43}).to_string(),
21986 ..tx
21987 };
21988
21989 let error = validate_acked_create_tx(
21990 &wrong_index,
21991 TEST_ACCOUNT_INDEX_I64,
21992 TEST_API_KEY_INDEX,
21993 42,
21994 10,
21995 "abcdef",
21996 )
21997 .expect_err("mismatched client order index must fail");
21998 assert_eq!(
21999 error.to_string(),
22000 "Lighter transaction lookup returned client_order_index 43 for acknowledged create 42",
22001 );
22002 }
22003
22004 #[rstest]
22005 #[case(41, true)]
22006 #[case(42, true)]
22007 #[case(43, true)]
22008 #[case(44, false)]
22009 fn acknowledged_group_tx_validation_requires_membership(
22010 #[case] client_order_index: i64,
22011 #[case] expected: bool,
22012 ) {
22013 let tx = LighterTx {
22014 code: 200,
22015 message: None,
22016 hash: "ABCDEF".to_string(),
22017 tx_type: LighterTxType::CreateGroupedOrders as u8,
22018 info: serde_json::json!({"Orders": [
22019 {"ClientOrderIndex": 41}, {"ClientOrderIndex": 42}, {"ClientOrderIndex": 43}
22020 ]})
22021 .to_string(),
22022 event_info: serde_json::json!({"ae": ""}).to_string(),
22023 status: LighterTxStatus::Executed,
22024 account_index: TEST_ACCOUNT_INDEX_I64,
22025 nonce: 10,
22026 api_key_index: TEST_API_KEY_INDEX,
22027 };
22028
22029 let result = validate_acked_create_tx(
22030 &tx,
22031 TEST_ACCOUNT_INDEX_I64,
22032 TEST_API_KEY_INDEX,
22033 client_order_index,
22034 10,
22035 "0xabcdef",
22036 );
22037 assert_eq!(result.is_ok(), expected);
22038
22039 if !expected {
22040 assert_eq!(
22041 result.unwrap_err().to_string(),
22042 "Lighter transaction lookup did not include acknowledged client_order_index 44"
22043 );
22044 }
22045 }
22046
22047 #[tokio::test]
22048 async fn send_tx_responses_only_attribute_within_their_connection_epoch() {
22049 let (client, _cache, mut rx) = create_execution_client();
22050 for (connection_epoch, nonce) in [(4, 20), (5, 21)] {
22051 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
22052 batch_id: None,
22053 connection_epoch,
22054 kind: PendingSendTxKind::Other,
22055 submitted_at: UnixNanos::from(1_000_000_000 + nonce as u64),
22056 nonce,
22057 api_key_index: TEST_API_KEY_INDEX,
22058 tx_hash: "same-hash".to_string(),
22059 });
22060 }
22061
22062 let acked =
22063 handle_send_tx_ack_for_connection(&client.dispatch, None, 5, 200, Some("same-hash"))
22064 .expect("replacement epoch response should match");
22065 assert_eq!(acked.connection_epoch, 5);
22066 assert_eq!(acked.nonce, 21);
22067 {
22068 let queue = client.dispatch.pending_sendtx.lock();
22069 assert_eq!(queue.len(), 1);
22070 assert_eq!(queue[0].connection_epoch, 4);
22071 assert_eq!(queue[0].nonce, 20);
22072 }
22073
22074 assert!(!handle_send_tx_rejection_for_connection(
22075 &client.dispatch,
22076 &client.emitter,
22077 None,
22078 4,
22079 UnixNanos::from(1_000_000_100),
22080 SendTxRejectionSource::Ack,
22081 Some(21702),
22082 "invalid price",
22083 Some("same-hash"),
22084 ));
22085 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
22086 assert!(
22087 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22088 .await
22089 .is_err(),
22090 "non-order responses must not emit order events",
22091 );
22092 }
22093
22094 #[tokio::test]
22095 async fn handle_send_tx_ack_unmatched_hash_pops_nothing() {
22096 let (client, cache, mut rx) = create_execution_client();
22097 let instrument_id = register_test_instrument(&client, &cache);
22098 let mut factory = test_order_factory();
22099 let order = test_limit_order(&mut factory, instrument_id, "ACK-UNMATCHED");
22100 enqueue_create(&client, &order, 10);
22101
22102 let acked = handle_send_tx_ack(
22103 &client.dispatch,
22104 Some(TEST_ACCOUNT_INDEX_I64),
22105 200,
22106 Some("0xabc"),
22107 );
22108
22109 assert!(acked.is_none());
22110 assert_eq!(
22111 client.dispatch.pending_sendtx_len(),
22112 1,
22113 "an echoed hash with no matching entry must not pop the head",
22114 );
22115 assert!(
22116 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22117 .await
22118 .is_err(),
22119 "unmatched ack must not emit an event",
22120 );
22121 }
22122
22123 #[tokio::test]
22124 async fn handle_send_tx_ack_hashless_with_multiple_pending_pops_nothing() {
22125 let (client, _cache, _rx) = create_execution_client();
22126 enqueue_other(&client, 10);
22127 enqueue_other(&client, 11);
22128
22129 let acked = handle_send_tx_ack(&client.dispatch, Some(TEST_ACCOUNT_INDEX_I64), 200, None);
22130
22131 assert!(acked.is_none());
22132 assert_eq!(
22133 client.dispatch.pending_sendtx_len(),
22134 2,
22135 "a hashless ack is ambiguous when multiple transactions are pending",
22136 );
22137 }
22138
22139 #[tokio::test]
22140 async fn prepared_create_tx_hash_round_trips_through_ack() {
22141 let (client, cache, _rx) = create_execution_client();
22144 let instrument_id = register_test_instrument(&client, &cache);
22145 let mut factory = test_order_factory();
22146 let order = test_limit_order(&mut factory, instrument_id, "ACK-REAL-HASH");
22147
22148 let credential = test_credential();
22149 let plan = client
22150 .prepare_create_order_plan(&order, None)
22151 .expect("prepare must validate");
22152 let prepared = client
22153 .fanout_dispatch_context(&credential)
22154 .expect("task admission")
22155 .sign_create_order(&plan)
22156 .expect("prepare must sign");
22157
22158 assert_eq!(prepared.tx_hash.len(), TX_HASH_BYTES * 2);
22159 assert!(
22160 prepared
22161 .tx_hash
22162 .chars()
22163 .all(|c| matches!(c, '0'..='9' | 'a'..='f')),
22164 "tx_hash must be lowercase hex, was `{}`",
22165 prepared.tx_hash,
22166 );
22167
22168 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
22169 batch_id: None,
22170 connection_epoch: 0,
22171 kind: PendingSendTxKind::Create {
22172 order: Box::new(order),
22173 client_order_index: prepared.client_order_index,
22174 },
22175 submitted_at: UnixNanos::from(1_000_000_000),
22176 nonce: prepared.nonce,
22177 api_key_index: prepared.api_key_index,
22178 tx_hash: prepared.tx_hash.clone(),
22179 });
22180
22181 let acked = handle_send_tx_ack(
22182 &client.dispatch,
22183 Some(TEST_ACCOUNT_INDEX_I64),
22184 200,
22185 Some(&prepared.tx_hash),
22186 );
22187
22188 assert!(acked.is_some());
22189 assert_eq!(
22190 client.dispatch.pending_sendtx_len(),
22191 0,
22192 "venue echo of the signed hash must match the enqueued entry",
22193 );
22194 }
22195
22196 #[tokio::test]
22197 async fn handle_send_tx_ack_retains_cancel_for_transaction_probe() {
22198 let (client, cache, mut rx) = create_execution_client();
22199 let instrument_id = register_test_instrument(&client, &cache);
22200 let client_order_id = ClientOrderId::from("ACK-CANCEL-TX");
22201 let venue_order_id = VenueOrderId::from("123");
22202 let expected_command_id =
22203 enqueue_cancel(&client, instrument_id, client_order_id, venue_order_id, 12);
22204
22205 let acked = handle_send_tx_ack(
22206 &client.dispatch,
22207 Some(TEST_ACCOUNT_INDEX_I64),
22208 200,
22209 Some("hash0c"),
22210 )
22211 .expect("acked cancel pending entry");
22212 assert!(AckedOrderProbe::from_pending(&acked).is_none());
22213
22214 let PendingSendTxKind::Cancel {
22215 instrument_id: actual_instrument_id,
22216 client_order_id: actual_client_order_id,
22217 venue_order_id: actual_venue_order_id,
22218 emit_cancel_rejected,
22219 strategy_id: actual_strategy_id,
22220 command_id,
22221 } = acked.kind
22222 else {
22223 panic!("expected acknowledged cancel")
22224 };
22225
22226 assert_eq!(command_id, Some(expected_command_id));
22227 assert_eq!(actual_instrument_id, instrument_id);
22228 assert_eq!(actual_client_order_id, client_order_id);
22229 assert_eq!(actual_venue_order_id, Some(venue_order_id));
22230 assert_eq!(actual_strategy_id, strategy_id());
22231 assert!(emit_cancel_rejected);
22232 assert_eq!(acked.connection_epoch, 0);
22233 assert_eq!(acked.nonce, 12);
22234 assert_eq!(acked.api_key_index, TEST_API_KEY_INDEX);
22235 assert_eq!(acked.tx_hash, "hash0c");
22236 assert_eq!(
22237 client.dispatch.pending_cancel_nonce(&client_order_id),
22238 Some((0, 12))
22239 );
22240 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
22241 assert!(
22242 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22243 .await
22244 .is_err(),
22245 "ack itself must not emit before the transaction probe runs",
22246 );
22247 }
22248
22249 #[rstest]
22250 #[tokio::test]
22251 async fn acknowledged_order_probe_is_retained_until_session_shutdown() {
22252 let (mut client, cache, _rx) = create_execution_client();
22253 let instrument_id = register_test_instrument(&client, &cache);
22254 let pending = PendingSendTx {
22255 batch_id: None,
22256 connection_epoch: 0,
22257 kind: PendingSendTxKind::Cancel {
22258 command_id: Some(UUID4::new()),
22259 emit_cancel_rejected: true,
22260 strategy_id: strategy_id(),
22261 instrument_id,
22262 client_order_id: ClientOrderId::from("ACK-RETAINED"),
22263 venue_order_id: Some(VenueOrderId::from("123")),
22264 },
22265 submitted_at: UnixNanos::from(1_000_000_000),
22266 nonce: 12,
22267 api_key_index: TEST_API_KEY_INDEX,
22268 tx_hash: "hash0c".to_string(),
22269 };
22270 let context = AckedOrderProbeContext {
22271 http_client: client.http_client.clone(),
22272 registry: Arc::clone(&client.registry),
22273 credential: test_credential(),
22274 dispatch: client.dispatch.clone(),
22275 account_id: client.core.account_id,
22276 clock: client.clock,
22277 emitter: client.emitter.clone(),
22278 connection_epoch: client.ws_client.connection_epoch_atomic(),
22279 cancellation_token: client.cancellation_token.clone(),
22280 pending_tasks: client.pending_tasks.spawner().expect("task spawner"),
22281 };
22282
22283 spawn_acked_order_probe(&pending, context);
22284
22285 assert_eq!(client.pending_tasks.len(), 1);
22286
22287 client.begin_session_shutdown();
22288 client
22289 .finish_session_shutdown()
22290 .await
22291 .expect("session shutdown");
22292
22293 assert!(client.pending_tasks.is_empty());
22294 }
22295
22296 #[tokio::test]
22297 async fn handle_send_tx_ack_retains_modify_for_transaction_probe() {
22298 let (client, cache, mut rx) = create_execution_client();
22299 let instrument_id = register_test_instrument(&client, &cache);
22300 let client_order_id = ClientOrderId::from("ACK-MODIFY-TX");
22301 let venue_order_id = VenueOrderId::from("456");
22302 let expected_command_id =
22303 enqueue_modify(&client, instrument_id, client_order_id, venue_order_id, 13);
22304
22305 let acked = handle_send_tx_ack(
22306 &client.dispatch,
22307 Some(TEST_ACCOUNT_INDEX_I64),
22308 200,
22309 Some("hash0d"),
22310 )
22311 .expect("acked modify pending entry");
22312 assert!(AckedOrderProbe::from_pending(&acked).is_none());
22313
22314 match acked.kind {
22315 PendingSendTxKind::Modify {
22316 instrument_id: actual_instrument_id,
22317 client_order_id: actual_client_order_id,
22318 venue_order_id: actual_venue_order_id,
22319 command_id,
22320 ..
22321 } => {
22322 assert_eq!(command_id, expected_command_id);
22323 assert_eq!(actual_instrument_id, instrument_id);
22324 assert_eq!(actual_client_order_id, client_order_id);
22325 assert_eq!(actual_venue_order_id, Some(venue_order_id));
22326 }
22327 other => panic!("expected acknowledged modify, was {other:?}"),
22328 }
22329
22330 assert_eq!(acked.connection_epoch, 0);
22331 assert_eq!(acked.nonce, 13);
22332 assert_eq!(acked.api_key_index, TEST_API_KEY_INDEX);
22333 assert_eq!(acked.tx_hash, "hash0d");
22334 assert_eq!(
22335 client.dispatch.pending_modify_nonce(&client_order_id),
22336 Some((0, 13))
22337 );
22338 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
22339 assert!(
22340 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22341 .await
22342 .is_err(),
22343 "ack itself must not emit before the transaction probe runs",
22344 );
22345 }
22346
22347 #[rstest]
22348 #[case::missing(false)]
22349 #[case::working(true)]
22350 #[tokio::test]
22351 async fn acknowledged_cancel_unresolved_status_retains_pending(#[case] found: bool) {
22352 let (client, cache, mut rx) = create_execution_client();
22353 let instrument = register_test_instrument(&client, &cache);
22354 let id = ClientOrderId::from("ACK-CANCEL-UNRESOLVED");
22355 let venue = VenueOrderId::from("123");
22356 let mut factory = test_order_factory();
22357 let order = test_limit_order(&mut factory, instrument, id.as_str());
22358 let index = client.dispatch.register_create_identity(&order).unwrap();
22359 enqueue_cancel(&client, instrument, id, venue, 12);
22360 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
22361 let order = reconciliation_raw_order(index, venue, LighterOrderStatus::Open);
22362
22363 let orders = if found { vec![order] } else { vec![] };
22364 let app = Router::new()
22365 .route(
22366 "/api/v1/accountActiveOrders",
22367 get(move || {
22368 let orders = orders.clone();
22369 async move { axum::Json(serde_json::json!({"code":200,"orders":orders})) }
22370 }),
22371 )
22372 .route(
22373 "/api/v1/accountInactiveOrders",
22374 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
22375 );
22376
22377 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22378 let addr = listener.local_addr().unwrap();
22379 let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
22380 let mut context = acked_probe_context(&client);
22381 context.http_client = LighterHttpClient::new(
22382 LighterEnvironment::Testnet,
22383 Some(format!("http://{addr}")),
22384 1,
22385 None,
22386 )
22387 .unwrap();
22388 let resolved = recover_acked_action_report(&pending, &context)
22389 .await
22390 .unwrap();
22391 server.abort();
22392 let _ = server.await;
22393 assert!(!resolved);
22394 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((0, 12)));
22395 assert_eq!(
22396 client.dispatch.pending_order_action(&id),
22397 Some(PendingOrderAction::Cancel)
22398 );
22399 let events = drain_events(&mut rx);
22400 if found {
22401 let [ExecutionEvent::Report(ExecutionReport::Order(report))] = events.as_slice() else {
22402 panic!("expected pending cancellation report, was {events:?}")
22403 };
22404
22405 assert_eq!(report.client_order_id, Some(id));
22406 assert_eq!(report.venue_order_id, venue);
22407 assert_eq!(report.order_status, OrderStatus::PendingCancel);
22408 } else {
22409 assert!(events.is_empty());
22410 }
22411 }
22412
22413 #[tokio::test]
22414 async fn send_tx_acks_recover_skip_window_past_window_size() {
22415 let (client, _cache, _rx) = create_execution_client();
22416 let window = i64::from(client.dispatch.nonce_manager.skip_window());
22417 for _ in 0..window {
22418 let nonce = client
22419 .dispatch
22420 .nonce_manager
22421 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22422 .unwrap();
22423 enqueue_other(&client, nonce);
22424 }
22425
22426 assert!(
22427 client
22428 .dispatch
22429 .nonce_manager
22430 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22431 .is_err(),
22432 "window must trip after {window} unacked txs",
22433 );
22434
22435 for i in 0..window {
22438 let acked = handle_send_tx_ack(
22439 &client.dispatch,
22440 Some(TEST_ACCOUNT_INDEX_I64),
22441 200,
22442 Some(&format!("hash{:02x}", TEST_NEXT_NONCE + i)),
22443 );
22444 assert!(matches!(
22445 acked.map(|pending| pending.kind),
22446 Some(PendingSendTxKind::Other),
22447 ));
22448 let nonce = client
22449 .dispatch
22450 .nonce_manager
22451 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22452 .unwrap();
22453 assert_eq!(nonce, TEST_NEXT_NONCE + window + i);
22454 enqueue_other(&client, nonce);
22455 }
22456 }
22457
22458 #[tokio::test]
22459 async fn handle_send_tx_rejection_rolls_back_latest_nonce() {
22460 let (client, cache, mut rx) = create_execution_client();
22461 let instrument_id = register_test_instrument(&client, &cache);
22462 let mut factory = test_order_factory();
22463 let order = test_limit_order(&mut factory, instrument_id, "REJECT-LATEST");
22464
22465 let nonce = client
22466 .dispatch
22467 .nonce_manager
22468 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22469 .unwrap();
22470 let client_order_index = client.dispatch.register_create_identity(&order).unwrap();
22471 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
22472 batch_id: None,
22473 connection_epoch: 0,
22474 kind: PendingSendTxKind::Create {
22475 order: Box::new(order.clone()),
22476 client_order_index,
22477 },
22478 submitted_at: UnixNanos::from(1_000_000_000),
22479 nonce,
22480 api_key_index: TEST_API_KEY_INDEX,
22481 tx_hash: format!("hash{nonce:02x}"),
22482 });
22483
22484 handle_send_tx_rejection(
22485 &client.dispatch,
22486 &client.emitter,
22487 Some(TEST_ACCOUNT_INDEX_I64),
22488 UnixNanos::from(1_000_000_000),
22489 SendTxRejectionSource::Ack,
22490 Some(21702),
22491 "invalid price",
22492 None,
22493 );
22494
22495 let event = recv_order_event(&mut rx).await;
22496 assert!(matches!(event, OrderEventAny::Rejected(_)));
22497 assert_nonce_reusable(&client.dispatch);
22498 }
22499
22500 #[tokio::test]
22501 async fn handle_send_tx_rejection_hashless_with_multiple_pending_is_unattributed() {
22502 let (client, cache, mut rx) = create_execution_client();
22503 let instrument_id = register_test_instrument(&client, &cache);
22504 let mut factory = test_order_factory();
22505 let order_a = test_limit_order(&mut factory, instrument_id, "REJECT-AMBIGUOUS-A");
22506 let order_b = test_limit_order(&mut factory, instrument_id, "REJECT-AMBIGUOUS-B");
22507 enqueue_create(&client, &order_a, 10);
22508 enqueue_create(&client, &order_b, 11);
22509
22510 handle_send_tx_rejection(
22511 &client.dispatch,
22512 &client.emitter,
22513 Some(TEST_ACCOUNT_INDEX_I64),
22514 UnixNanos::from(1_000_000_000),
22515 SendTxRejectionSource::Ack,
22516 Some(21702),
22517 "invalid price",
22518 None,
22519 );
22520
22521 assert_eq!(client.dispatch.pending_sendtx_len(), 2);
22522 assert!(
22523 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22524 .await
22525 .is_err(),
22526 "an ambiguous hashless rejection must not reject either order",
22527 );
22528 }
22529
22530 #[tokio::test]
22531 async fn handle_send_tx_rejection_with_newer_issuance_skips_rollback() {
22532 let (client, cache, mut rx) = create_execution_client();
22533 let instrument_id = register_test_instrument(&client, &cache);
22534 let mut factory = test_order_factory();
22535 let order = test_limit_order(&mut factory, instrument_id, "REJECT-STALE");
22536
22537 let rejected_nonce = client
22538 .dispatch
22539 .nonce_manager
22540 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22541 .unwrap();
22542 let newer_nonce = client
22543 .dispatch
22544 .nonce_manager
22545 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22546 .unwrap();
22547 let client_order_index = client.dispatch.register_create_identity(&order).unwrap();
22548 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
22549 batch_id: None,
22550 connection_epoch: 0,
22551 kind: PendingSendTxKind::Create {
22552 order: Box::new(order.clone()),
22553 client_order_index,
22554 },
22555 submitted_at: UnixNanos::from(1_000_000_000),
22556 nonce: rejected_nonce,
22557 api_key_index: TEST_API_KEY_INDEX,
22558 tx_hash: format!("hash{rejected_nonce:02x}"),
22559 });
22560
22561 handle_send_tx_rejection(
22562 &client.dispatch,
22563 &client.emitter,
22564 Some(TEST_ACCOUNT_INDEX_I64),
22565 UnixNanos::from(1_000_000_000),
22566 SendTxRejectionSource::Ack,
22567 Some(21702),
22568 "invalid price",
22569 None,
22570 );
22571
22572 let event = recv_order_event(&mut rx).await;
22573 assert!(matches!(event, OrderEventAny::Rejected(_)));
22574 assert_eq!(
22577 client
22578 .dispatch
22579 .nonce_manager
22580 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22581 Some(newer_nonce),
22582 );
22583 assert_eq!(
22584 client
22585 .dispatch
22586 .nonce_manager
22587 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22588 .unwrap(),
22589 newer_nonce + 1,
22590 );
22591 }
22592
22593 async fn spawn_next_nonce_server(nonce: i64) -> String {
22594 let body = serde_json::to_string(&LighterNextNonce {
22595 code: 200,
22596 message: None,
22597 nonce,
22598 })
22599 .unwrap();
22600
22601 let app = Router::new().route(
22602 "/api/v1/nextNonce",
22603 get(move || {
22604 let body = body.clone();
22605 async move { body }
22606 }),
22607 );
22608
22609 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22610 let addr = listener.local_addr().unwrap();
22611 tokio::spawn(async move {
22612 axum::serve(listener, app).await.unwrap();
22613 });
22614
22615 format!("http://{addr}")
22616 }
22617
22618 async fn spawn_counting_next_nonce_server(nonce: i64) -> (String, Arc<AtomicUsize>) {
22619 let requests = Arc::new(AtomicUsize::new(0));
22620 let requests_for_route = Arc::clone(&requests);
22621
22622 let body = serde_json::to_string(&LighterNextNonce {
22623 code: 200,
22624 message: None,
22625 nonce,
22626 })
22627 .unwrap();
22628
22629 let app = Router::new().route(
22630 "/api/v1/nextNonce",
22631 get(move || {
22632 requests_for_route.fetch_add(1, Ordering::SeqCst);
22633 let body = body.clone();
22634 async move { body }
22635 }),
22636 );
22637
22638 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22639 let addr = listener.local_addr().unwrap();
22640 tokio::spawn(async move {
22641 axum::serve(listener, app).await.unwrap();
22642 });
22643
22644 (format!("http://{addr}"), requests)
22645 }
22646
22647 #[tokio::test]
22648 async fn hard_nonce_refresh_waits_for_submission_and_replaces_its_baseline() {
22649 let (base_url_http, requests) = spawn_counting_next_nonce_server(100).await;
22650 let mut config = test_config();
22651 config.base_url_http = Some(base_url_http);
22652 let (client, _cache, _rx) = create_execution_client_with_config(config);
22653 let credential = test_credential();
22654 client
22655 .dispatch
22656 .nonce_manager
22657 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, 40);
22658
22659 let reserved = client.build_tx_context(&credential).unwrap();
22660 assert_eq!(reserved.context.nonce, 40);
22661 let mut refresh = Box::pin(client.refresh_nonce());
22662 assert!(
22663 tokio::time::timeout(Duration::from_millis(50), refresh.as_mut())
22664 .await
22665 .is_err(),
22666 "hard refresh must wait while a submission retains its nonce",
22667 );
22668 assert_eq!(requests.load(Ordering::SeqCst), 0);
22669 assert_eq!(
22670 client.nonce_ready_connection_epoch.load(Ordering::Acquire),
22671 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
22672 );
22673 assert_eq!(
22674 client
22675 .dispatch
22676 .nonce_manager
22677 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22678 Some(40),
22679 );
22680
22681 drop(reserved);
22682 refresh.await.unwrap();
22683
22684 assert_eq!(requests.load(Ordering::SeqCst), 1);
22685 assert_eq!(
22686 client.nonce_ready_connection_epoch.load(Ordering::Acquire),
22687 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
22688 );
22689 client
22690 .nonce_ready_connection_epoch
22691 .store(client.ws_client.connection_epoch(), Ordering::Release);
22692 let next = client.build_tx_context(&credential).unwrap();
22693 assert_eq!(next.context.nonce, 100);
22694 assert_eq!(
22695 client
22696 .dispatch
22697 .nonce_manager
22698 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22699 Some(100),
22700 );
22701 }
22702
22703 #[tokio::test]
22704 async fn nonce_refresh_retry_reopens_submission_with_replacement_baseline() {
22705 let (base_url_http, requests) = spawn_counting_next_nonce_server(100).await;
22706 let mut config = test_config();
22707 config.base_url_http = Some(base_url_http);
22708 let (client, _cache, _rx) = create_execution_client_with_config(config);
22709 let credential = test_credential();
22710 let connection_epoch = client.ws_client.connection_epoch();
22711 client
22712 .dispatch
22713 .nonce_manager
22714 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, 40);
22715 client
22716 .nonce_ready_connection_epoch
22717 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
22718
22719 NonceRefreshRetry {
22720 http_client: client.http_client.clone(),
22721 dispatch: client.dispatch.clone(),
22722 credential: Some(credential.clone()),
22723 submission_gate: Arc::clone(&client.nonce_submission_gate),
22724 ready_connection_epoch: Arc::clone(&client.nonce_ready_connection_epoch),
22725 ws_client: client.ws_client.clone(),
22726 cancellation_token: client.cancellation_token.clone(),
22727 pending_tasks: client.pending_tasks.spawner().expect("task spawner"),
22728 }
22729 .spawn(connection_epoch);
22730
22731 assert_eq!(client.pending_tasks.len(), 1);
22732
22733 let err = client
22734 .build_tx_context(&credential)
22735 .expect_err("submission must stay closed until retry succeeds");
22736 assert_eq!(
22737 err.to_string(),
22738 format!("Lighter nonce refresh is pending for connection epoch {connection_epoch}"),
22739 );
22740 wait_until_async(
22741 || async {
22742 client.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
22743 },
22744 Duration::from_secs(3),
22745 )
22746 .await;
22747
22748 let reserved = client.build_tx_context(&credential).unwrap();
22749 assert_eq!(requests.load(Ordering::SeqCst), 1);
22750 assert_eq!(reserved.context.nonce, 100);
22751 assert_eq!(
22752 client
22753 .dispatch
22754 .nonce_manager
22755 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22756 Some(99),
22757 );
22758 assert_eq!(
22759 client
22760 .dispatch
22761 .nonce_manager
22762 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22763 Some(100),
22764 );
22765 assert!(client.pending_tasks.all_finished());
22766 }
22767
22768 async fn spawn_integrator_approval_rejection_server() -> String {
22769 let app = Router::new()
22770 .route(
22771 "/api/v1/getMakerOnlyApiKeys",
22772 get(|| async { r#"{"code":200,"api_key_indexes":[]}"# }),
22773 )
22774 .route(
22775 "/api/v1/sendTx",
22776 post(|| async { r#"{"code":21702,"message":"invalid approval"}"# }),
22777 );
22778 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22779 let addr = listener.local_addr().unwrap();
22780 tokio::spawn(async move {
22781 axum::serve(listener, app).await.unwrap();
22782 });
22783
22784 format!("http://{addr}")
22785 }
22786
22787 #[tokio::test]
22788 async fn integrator_approval_api_rejection_releases_reservation_and_nonce() {
22789 let mut config = test_config();
22790 config.environment = LighterEnvironment::Mainnet;
22791 config.base_url_http = Some(spawn_integrator_approval_rejection_server().await);
22792 let (client, _cache, _rx) = create_execution_client_with_config(config);
22793
22794 let err = client.submit_integrator_auto_approval().await.unwrap_err();
22795
22796 assert!(format!("{err:#}").contains("invalid approval"));
22797 assert_nonce_reusable(&client.dispatch);
22798 }
22799
22800 #[tokio::test]
22801 async fn skip_window_exhaustion_resyncs_baseline_from_venue() {
22802 let venue_next_nonce = 100;
22803 let mut config = test_config();
22804 config.base_url_http = Some(spawn_next_nonce_server(venue_next_nonce).await);
22805 let (client, _cache, _rx) = create_execution_client_with_config(config);
22806 let credential = test_credential();
22807
22808 let window = i64::from(client.dispatch.nonce_manager.skip_window());
22809 for _ in 0..window {
22810 client
22811 .dispatch
22812 .nonce_manager
22813 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22814 .unwrap();
22815 }
22816
22817 let err = client
22818 .build_tx_context(&credential)
22819 .expect_err("window must trip");
22820 assert!(
22821 err.to_string().contains("skip-window exhausted"),
22822 "unexpected error, was {err}",
22823 );
22824
22825 wait_for_spawned_tasks(&client).await;
22826
22827 assert_eq!(
22828 client
22829 .dispatch
22830 .nonce_manager
22831 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22832 Some(venue_next_nonce - 1),
22833 "venue resync must advance the baseline",
22834 );
22835 assert!(
22836 !client.nonce_recovery_inflight.load(Ordering::Acquire),
22837 "recovery latch must release for the next exhaustion",
22838 );
22839 let context = client.build_tx_context(&credential).unwrap().context;
22840 assert_eq!(
22841 context.nonce, venue_next_nonce,
22842 "allocation must resume at the venue nonce",
22843 );
22844 }
22845
22846 #[tokio::test]
22847 async fn skip_window_recovery_fetch_failure_releases_latch() {
22848 let app = Router::new();
22850 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22851 let addr = listener.local_addr().unwrap();
22852 tokio::spawn(async move {
22853 axum::serve(listener, app).await.unwrap();
22854 });
22855
22856 let mut config = test_config();
22857 config.base_url_http = Some(format!("http://{addr}"));
22858 let (client, _cache, _rx) = create_execution_client_with_config(config);
22859 let credential = test_credential();
22860
22861 let window = i64::from(client.dispatch.nonce_manager.skip_window());
22862 for _ in 0..window {
22863 client
22864 .dispatch
22865 .nonce_manager
22866 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22867 .unwrap();
22868 }
22869
22870 client
22871 .build_tx_context(&credential)
22872 .expect_err("window must trip");
22873
22874 wait_for_spawned_tasks(&client).await;
22875
22876 assert_eq!(
22877 client
22878 .dispatch
22879 .nonce_manager
22880 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22881 Some(TEST_NEXT_NONCE - 1),
22882 "failed venue fetch must leave the baseline unchanged",
22883 );
22884 assert!(
22885 !client.nonce_recovery_inflight.load(Ordering::Acquire),
22886 "failed venue fetch must release the recovery latch",
22887 );
22888 }
22889
22890 #[tokio::test]
22891 async fn skip_window_recovery_dedupes_concurrent_fetches() {
22892 let hits = Arc::new(AtomicUsize::new(0));
22893 let server_hits = Arc::clone(&hits);
22894
22895 let body = serde_json::to_string(&LighterNextNonce {
22896 code: 200,
22897 message: None,
22898 nonce: 100,
22899 })
22900 .unwrap();
22901
22902 let app = Router::new().route(
22903 "/api/v1/nextNonce",
22904 get(move || {
22905 server_hits.fetch_add(1, Ordering::AcqRel);
22906 let body = body.clone();
22907 async move { body }
22908 }),
22909 );
22910 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22911 let addr = listener.local_addr().unwrap();
22912 tokio::spawn(async move {
22913 axum::serve(listener, app).await.unwrap();
22914 });
22915
22916 let mut config = test_config();
22917 config.base_url_http = Some(format!("http://{addr}"));
22918 let (client, _cache, _rx) = create_execution_client_with_config(config);
22919 let credential = test_credential();
22920
22921 let window = i64::from(client.dispatch.nonce_manager.skip_window());
22922 for _ in 0..window {
22923 client
22924 .dispatch
22925 .nonce_manager
22926 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22927 .unwrap();
22928 }
22929
22930 client
22933 .build_tx_context(&credential)
22934 .expect_err("window must trip");
22935 client
22936 .build_tx_context(&credential)
22937 .expect_err("window must still be exhausted");
22938
22939 wait_for_spawned_tasks(&client).await;
22940
22941 assert_eq!(
22942 hits.load(Ordering::Acquire),
22943 1,
22944 "burst exhaustion must trigger a single venue fetch",
22945 );
22946 assert!(!client.nonce_recovery_inflight.load(Ordering::Acquire));
22947 }
22948
22949 #[tokio::test]
22950 async fn refresh_nonce_releases_recovery_latch() {
22951 let venue_next_nonce = 77;
22952 let mut config = test_config();
22953 config.base_url_http = Some(spawn_next_nonce_server(venue_next_nonce).await);
22954 let (client, _cache, _rx) = create_execution_client_with_config(config);
22955
22956 client
22958 .nonce_recovery_inflight
22959 .store(true, Ordering::Release);
22960
22961 client.refresh_nonce().await.unwrap();
22962
22963 assert!(
22964 !client.nonce_recovery_inflight.load(Ordering::Acquire),
22965 "connect-time refresh must release a stuck recovery latch",
22966 );
22967 assert_eq!(
22968 client
22969 .dispatch
22970 .nonce_manager
22971 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22972 .unwrap(),
22973 venue_next_nonce,
22974 "refresh must hard-reset allocation to the venue nonce",
22975 );
22976 }
22977
22978 #[tokio::test]
22979 async fn handle_send_tx_rejection_ack_create_emits_order_rejected() {
22980 let (client, cache, mut rx) = create_execution_client();
22981 let instrument_id = register_test_instrument(&client, &cache);
22982 let mut factory = test_order_factory();
22983 let order = test_limit_order(&mut factory, instrument_id, "REJECT-CREATE");
22984 let client_order_index = enqueue_create(&client, &order, 42);
22985
22986 handle_send_tx_rejection(
22987 &client.dispatch,
22988 &client.emitter,
22989 Some(TEST_ACCOUNT_INDEX_I64),
22990 UnixNanos::from(1_000_000_000),
22991 SendTxRejectionSource::Ack,
22992 Some(21702),
22993 "invalid price",
22994 None,
22995 );
22996
22997 let event = recv_order_event(&mut rx).await;
22998 match event {
22999 OrderEventAny::Rejected(e) => {
23000 assert_eq!(e.client_order_id, order.client_order_id());
23001 assert_eq!(e.reason, "LIGHTER_21702: invalid price");
23002 assert!(!e.due_post_only);
23003 }
23004 other => panic!("expected Rejected, was {other:?}"),
23005 }
23006
23007 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23008 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
23009 }
23010
23011 #[tokio::test]
23012 async fn handle_send_tx_rejection_ack_create_sets_due_post_only() {
23013 let (client, cache, mut rx) = create_execution_client();
23014 let instrument_id = register_test_instrument(&client, &cache);
23015 let mut factory = test_order_factory();
23016 let order = test_limit_order(&mut factory, instrument_id, "REJECT-POST-ONLY");
23017 enqueue_create(&client, &order, 43);
23018
23019 handle_send_tx_rejection(
23020 &client.dispatch,
23021 &client.emitter,
23022 Some(TEST_ACCOUNT_INDEX_I64),
23023 UnixNanos::from(1_000_000_000),
23024 SendTxRejectionSource::Ack,
23025 Some(21700),
23026 "post-only order would execute",
23027 None,
23028 );
23029
23030 let event = recv_order_event(&mut rx).await;
23031 match event {
23032 OrderEventAny::Rejected(e) => {
23033 assert_eq!(e.client_order_id, order.client_order_id());
23034 assert!(e.due_post_only);
23035 }
23036 other => panic!("expected Rejected, was {other:?}"),
23037 }
23038
23039 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23040 }
23041
23042 #[tokio::test]
23043 async fn cancel_ack_promotes_before_send_continuation() {
23044 let (client, cache, mut rx) = create_execution_client();
23045 let instrument_id = register_test_instrument(&client, &cache);
23046 let id = ClientOrderId::from("CANCEL-ACK-RACE");
23047 let venue = VenueOrderId::from("123");
23048 let first = enqueue_cancel(&client, instrument_id, id, venue, 41);
23049 let old = client.dispatch.pending_cancel_token(&id, 0, 41).unwrap();
23050 let second = UUID4::new();
23051 let pending = enqueue_cancel_pending(&client, instrument_id, id, venue, 42, second);
23052
23053 assert_eq!(
23054 client.dispatch.pending_cancel_identity(&id),
23055 Some((first, (0, 41)))
23056 );
23057 assert!(!old.is_cancelled());
23058
23059 let acked = handle_send_tx_ack_for_connection(
23060 &client.dispatch,
23061 None,
23062 0,
23063 200,
23064 Some(&pending.tx_hash),
23065 )
23066 .unwrap();
23067 let current = client.dispatch.pending_cancel_token(&id, 0, 42).unwrap();
23068 promote_cancel_submission(&client.dispatch, &pending);
23069 handle_send_tx_ack_for_connection(&client.dispatch, None, 0, 200, Some("hash29")).unwrap();
23070
23071 assert_eq!(acked.nonce, 42);
23072 assert_eq!(
23073 client.dispatch.pending_cancel_identity(&id),
23074 Some((second, (0, 42)))
23075 );
23076 assert!(old.is_cancelled());
23077 assert!(!current.is_cancelled());
23078 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23079 assert!(rx.try_recv().is_err());
23080 }
23081
23082 #[tokio::test]
23083 async fn cancel_rejection_before_send_continuation_preserves_previous_probe() {
23084 let (client, cache, mut rx) = create_execution_client();
23085 let instrument_id = register_test_instrument(&client, &cache);
23086 let id = ClientOrderId::from("CANCEL-REJECTION-RACE");
23087 let venue = VenueOrderId::from("123");
23088 let first = enqueue_cancel(&client, instrument_id, id, venue, 41);
23089 let token = client.dispatch.pending_cancel_token(&id, 0, 41).unwrap();
23090 let second = UUID4::new();
23091 let pending = enqueue_cancel_pending(&client, instrument_id, id, venue, 42, second);
23092 handle_send_tx_rejection_for_connection(
23093 &client.dispatch,
23094 &client.emitter,
23095 None,
23096 0,
23097 UnixNanos::default(),
23098 SendTxRejectionSource::Ack,
23099 Some(21727),
23100 "later cancel failed",
23101 Some(&pending.tx_hash),
23102 );
23103 promote_cancel_submission(&client.dispatch, &pending);
23104
23105 let OrderEventAny::CancelRejected(event) = recv_order_event(&mut rx).await else {
23106 panic!("Expected venue rejection");
23107 };
23108
23109 assert_eq!(event.causation_id, Some(second));
23110 assert_eq!(event.client_order_id, id);
23111 assert_eq!(event.venue_order_id, Some(venue));
23112 assert_eq!(event.reason, "LIGHTER_21727: later cancel failed");
23113 assert_eq!(
23114 client.dispatch.pending_cancel_identity(&id),
23115 Some((first, (0, 41)))
23116 );
23117 assert!(!token.is_cancelled());
23118 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
23119 assert!(rx.try_recv().is_err());
23120 }
23121
23122 #[rstest]
23123 #[case::explicit_enqueued(true)]
23124 #[case::explicit_waiting(false)]
23125 #[tokio::test]
23126 async fn implicit_cancel_enqueue_preserves_explicit_rejection(
23127 #[case] explicit_enqueued: bool,
23128 #[values(false, true)] implicit_marked: bool,
23129 ) {
23130 let (client, cache, mut rx) = create_execution_client();
23131 let instrument_id = register_test_instrument(&client, &cache);
23132 let mut factory = test_order_factory();
23133 let order = test_limit_order(&mut factory, instrument_id, "CANCEL-ELIGIBILITY");
23134 let id = order.client_order_id();
23135 let venue_id = VenueOrderId::from("123");
23136 let cause = UUID4::new();
23137 cache_pending_cancel_order(&cache, order, venue_id, Some(cause));
23138 let context = client
23139 .fanout_dispatch_context(client.credential.as_ref().unwrap())
23140 .unwrap();
23141
23142 let plan = CancelOrderPlan {
23143 command_id: Some(cause),
23144 client_order_id: id,
23145 strategy_id: strategy_id(),
23146 instrument_id,
23147 venue_order_id: Some(venue_id),
23148 market_index: TEST_MARKET_INDEX,
23149 venue_index: 123,
23150 };
23151
23152 let first = context.sign_cancel_order(&plan).unwrap();
23153 let second = context.sign_cancel_order(&plan).unwrap();
23154 let (explicit, batch) = if explicit_enqueued {
23155 (first, second)
23156 } else {
23157 (second, first)
23158 };
23159
23160 client.dispatch.register_cancel_command(id, cause, true);
23161
23162 if explicit_enqueued {
23163 context.enqueue_cancel_order(&explicit, true);
23164 client
23165 .dispatch
23166 .promote_pending_cancel(&id, cause, 0, explicit.nonce, true);
23167 }
23168
23169 let mut batch = PreparedBatchTx::from((batch, implicit_marked));
23170 if let PendingSendTxKind::Cancel { command_id, .. } = &mut batch.pending.kind {
23171 *command_id = None;
23172 }
23173
23174 batch.pending.batch_id = Some("batch-unmarked".to_string());
23175 context.enqueue_batch_tx(&batch);
23176 assert_eq!(
23177 client.dispatch.pending_cancel_nonce(&id),
23178 explicit_enqueued.then_some((0, explicit.nonce))
23179 );
23180
23181 if !explicit_enqueued {
23182 context.enqueue_cancel_order(&explicit, true);
23183 client
23184 .dispatch
23185 .promote_pending_cancel(&id, cause, 0, explicit.nonce, true);
23186 }
23187
23188 handle_send_tx_rejection_for_connection(
23189 &client.dispatch,
23190 &client.emitter,
23191 Some(TEST_ACCOUNT_INDEX_I64),
23192 0,
23193 UnixNanos::default(),
23194 SendTxRejectionSource::Ack,
23195 Some(21727),
23196 "unmarked batch failed",
23197 Some(&batch.pending.tx_hash),
23198 );
23199 assert_eq!(
23200 client.dispatch.pending_cancel_nonce(&id),
23201 Some((0, explicit.nonce))
23202 );
23203 assert!(rx.try_recv().is_err());
23204 handle_send_tx_rejection_for_connection(
23205 &client.dispatch,
23206 &client.emitter,
23207 Some(TEST_ACCOUNT_INDEX_I64),
23208 0,
23209 UnixNanos::default(),
23210 SendTxRejectionSource::Ack,
23211 Some(21727),
23212 "explicit cancel failed",
23213 Some(&explicit.tx_hash),
23214 );
23215 let event = recv_order_event(&mut rx).await;
23216 cache.borrow_mut().update_order(&event).unwrap();
23217 match event {
23218 OrderEventAny::CancelRejected(event) => {
23219 assert_eq!(event.causation_id, Some(cause));
23220 assert_eq!(event.client_order_id, id);
23221 assert_eq!(event.venue_order_id, Some(venue_id));
23222 assert_eq!(event.instrument_id, instrument_id);
23223 assert_eq!(event.strategy_id, strategy_id());
23224 assert_eq!(event.reason, "LIGHTER_21727: explicit cancel failed");
23225 }
23226 other => panic!("expected explicit cancel rejection, was {other:?}"),
23227 }
23228
23229 assert_eq!(
23230 cache.borrow().order(&id).unwrap().status(),
23231 OrderStatus::Accepted
23232 );
23233 assert_eq!(client.dispatch.pending_cancel_nonce(&id), None);
23234 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23235 }
23236
23237 #[rstest]
23238 #[case::pending(PendingOrderAction::Cancel, true)]
23239 #[case::unmarked(PendingOrderAction::CancelUnmarked, false)]
23240 #[tokio::test]
23241 async fn cancel_rejection_preserves_newer_nonce(
23242 #[case] action: PendingOrderAction,
23243 #[case] emit_cancel_rejected: bool,
23244 ) {
23245 let (client, cache, mut rx) = create_execution_client();
23246 let instrument_id = register_test_instrument(&client, &cache);
23247 let id = ClientOrderId::from("CANCEL-NEWER");
23248
23249 let cause = emit_cancel_rejected.then(UUID4::new);
23250
23251 let pending = PendingSendTx {
23252 batch_id: Some("batch-old".to_string()),
23253 connection_epoch: 3,
23254 kind: PendingSendTxKind::Cancel {
23255 command_id: cause,
23256 emit_cancel_rejected,
23257 strategy_id: strategy_id(),
23258 instrument_id,
23259 client_order_id: id,
23260 venue_order_id: Some(VenueOrderId::from("123")),
23261 },
23262 submitted_at: UnixNanos::default(),
23263 nonce: 41,
23264 api_key_index: TEST_API_KEY_INDEX,
23265 tx_hash: "old-hash".to_string(),
23266 };
23267
23268 client.dispatch.set_pending_order_action(id, action);
23269 client
23270 .dispatch
23271 .set_pending_cancel_nonce(&id, 3, 42, emit_cancel_rejected);
23272 reject_cancel_order(
23273 &client.dispatch,
23274 &client.emitter,
23275 &pending,
23276 &CommandFailure::venue_rejected("old failure"),
23277 UnixNanos::default(),
23278 );
23279 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((3, 42)));
23280 assert_eq!(client.dispatch.pending_order_action(&id), Some(action));
23281
23282 if let Some(cause) = cause {
23283 let event = recv_order_event(&mut rx).await;
23284
23285 let OrderEventAny::CancelRejected(event) = event else {
23286 panic!("expected cancel rejection");
23287 };
23288
23289 assert_eq!(event.causation_id, Some(cause));
23290 assert_eq!(event.client_order_id, id);
23291 assert_eq!(event.reason, "old failure");
23292 }
23293
23294 assert!(rx.try_recv().is_err());
23295 }
23296
23297 #[tokio::test]
23298 async fn handle_send_tx_rejection_cancel_emits_cancel_rejected() {
23299 let (client, cache, mut rx) = create_execution_client();
23300 let instrument_id = register_test_instrument(&client, &cache);
23301 let client_order_id = ClientOrderId::from("REJECT-CANCEL");
23302 let venue_order_id = VenueOrderId::from("123");
23303 let nonce = client
23304 .dispatch
23305 .nonce_manager
23306 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23307 .unwrap();
23308
23309 let cause = enqueue_cancel(
23310 &client,
23311 instrument_id,
23312 client_order_id,
23313 venue_order_id,
23314 nonce,
23315 );
23316
23317 handle_send_tx_rejection(
23318 &client.dispatch,
23319 &client.emitter,
23320 Some(TEST_ACCOUNT_INDEX_I64),
23321 UnixNanos::from(1_000_000_000),
23322 SendTxRejectionSource::Ack,
23323 Some(21727),
23324 "order is not cancelable",
23325 None,
23326 );
23327
23328 let event = recv_order_event(&mut rx).await;
23329 match event {
23330 OrderEventAny::CancelRejected(e) => {
23331 assert_eq!(e.causation_id, Some(cause));
23332 assert_eq!(e.client_order_id, client_order_id);
23333 assert_eq!(e.instrument_id, instrument_id);
23334 assert_eq!(e.venue_order_id, Some(venue_order_id));
23335 assert_eq!(e.reason, "LIGHTER_21727: order is not cancelable",);
23336 }
23337 other => panic!("expected CancelRejected, was {other:?}"),
23338 }
23339
23340 assert_eq!(
23341 client.dispatch.pending_cancel_identity(&client_order_id),
23342 None
23343 );
23344 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23345 assert_nonce_reusable(&client.dispatch);
23346 }
23347
23348 #[tokio::test]
23349 async fn handle_send_tx_rejection_modify_emits_modify_rejected() {
23350 let (client, cache, mut rx) = create_execution_client();
23351 let instrument_id = register_test_instrument(&client, &cache);
23352 let client_order_id = ClientOrderId::from("REJECT-MODIFY");
23353 let venue_order_id = VenueOrderId::from("456");
23354 let nonce = client
23355 .dispatch
23356 .nonce_manager
23357 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23358 .unwrap();
23359
23360 enqueue_modify(
23361 &client,
23362 instrument_id,
23363 client_order_id,
23364 venue_order_id,
23365 nonce,
23366 );
23367
23368 handle_send_tx_rejection(
23369 &client.dispatch,
23370 &client.emitter,
23371 Some(TEST_ACCOUNT_INDEX_I64),
23372 UnixNanos::from(1_000_000_000),
23373 SendTxRejectionSource::Ack,
23374 Some(21702),
23375 "modify rejected by venue",
23376 None,
23377 );
23378
23379 let event = recv_order_event(&mut rx).await;
23380 match event {
23381 OrderEventAny::ModifyRejected(e) => {
23382 assert_eq!(e.client_order_id, client_order_id);
23383 assert_eq!(e.instrument_id, instrument_id);
23384 assert_eq!(e.venue_order_id, Some(venue_order_id));
23385 assert_eq!(e.reason, "LIGHTER_21702: modify rejected by venue",);
23386 }
23387 other => panic!("expected ModifyRejected, was {other:?}"),
23388 }
23389
23390 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23391 assert_nonce_reusable(&client.dispatch);
23392 }
23393
23394 #[rstest]
23395 #[case::newer_modify(PendingOrderAction::Modify)]
23396 #[case::newer_cancel(PendingOrderAction::Cancel)]
23397 fn handle_send_tx_rejection_preserves_superseding_modify_action(
23398 #[case] action: PendingOrderAction,
23399 ) {
23400 let (client, cache, mut rx) = create_execution_client();
23401 let instrument = register_test_instrument(&client, &cache);
23402 let id = ClientOrderId::from("REJECT-MODIFY-SUPERSEDED");
23403 let venue = VenueOrderId::from("456");
23404 let nonce = client
23405 .dispatch
23406 .nonce_manager
23407 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23408 .unwrap();
23409 enqueue_modify(&client, instrument, id, venue, nonce);
23410 let current_nonce = client
23411 .dispatch
23412 .nonce_manager
23413 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23414 .unwrap();
23415
23416 let pending = client
23417 .dispatch
23418 .pending_sendtx
23419 .lock()
23420 .front()
23421 .unwrap()
23422 .clone();
23423
23424 let PendingSendTxKind::Modify {
23425 command_id: cause, ..
23426 } = pending.kind
23427 else {
23428 unreachable!()
23429 };
23430
23431 assert!(client.dispatch.clear_pending_modify_if_nonce(&id, 0, nonce));
23432
23433 if action == PendingOrderAction::Modify {
23434 enqueue_modify(&client, instrument, id, venue, current_nonce);
23435 } else {
23436 client.dispatch.set_pending_order_action(id, action);
23437 client
23438 .dispatch
23439 .set_pending_cancel_nonce(&id, 0, current_nonce, true);
23440 }
23441
23442 handle_send_tx_rejection(
23443 &client.dispatch,
23444 &client.emitter,
23445 Some(TEST_ACCOUNT_INDEX_I64),
23446 UnixNanos::from(1_000_000_000),
23447 SendTxRejectionSource::Ack,
23448 Some(21702),
23449 "older modify rejected by venue",
23450 Some(&format!("hash{nonce:02x}")),
23451 );
23452
23453 let ExecutionEvent::Order(OrderEventAny::ModifyRejected(event)) = rx.try_recv().unwrap()
23454 else {
23455 panic!("expected older command rejection");
23456 };
23457
23458 assert_eq!(event.client_order_id, id);
23459 assert_eq!(
23460 event.reason,
23461 "LIGHTER_21702: older modify rejected by venue"
23462 );
23463 assert_eq!(event.causation_id, Some(cause));
23464 assert!(rx.try_recv().is_err());
23465 assert_eq!(client.dispatch.pending_order_action(&id), Some(action));
23466 assert_eq!(
23467 client.dispatch.pending_modify_nonce(&id),
23468 (action == PendingOrderAction::Modify).then_some((0, current_nonce))
23469 );
23470 assert_eq!(
23471 client.dispatch.pending_cancel_nonce(&id),
23472 (action == PendingOrderAction::Cancel).then_some((0, current_nonce))
23473 );
23474 assert_eq!(
23475 client.dispatch.pending_sendtx_len(),
23476 usize::from(action == PendingOrderAction::Modify)
23477 );
23478 assert_eq!(
23479 client
23480 .dispatch
23481 .nonce_manager
23482 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23483 .unwrap(),
23484 current_nonce + 1
23485 );
23486 }
23487
23488 #[tokio::test]
23489 async fn handle_send_tx_rejection_hash_match_attributes_past_the_head() {
23490 let (client, cache, mut rx) = create_execution_client();
23491 let instrument_id = register_test_instrument(&client, &cache);
23492 let mut factory = test_order_factory();
23493 let order_a = test_limit_order(&mut factory, instrument_id, "REJECT-HASH-A");
23494 let order_b = test_limit_order(&mut factory, instrument_id, "REJECT-HASH-B");
23495 enqueue_create(&client, &order_a, 10);
23496 enqueue_create(&client, &order_b, 11);
23497
23498 handle_send_tx_rejection(
23500 &client.dispatch,
23501 &client.emitter,
23502 Some(TEST_ACCOUNT_INDEX_I64),
23503 UnixNanos::from(1_000_000_000),
23504 SendTxRejectionSource::Ack,
23505 Some(21702),
23506 "invalid price",
23507 Some("hash0b"),
23508 );
23509
23510 let event = recv_order_event(&mut rx).await;
23511 match event {
23512 OrderEventAny::Rejected(e) => {
23513 assert_eq!(e.client_order_id, order_b.client_order_id());
23514 }
23515 other => panic!("expected Rejected, was {other:?}"),
23516 }
23517
23518 assert_eq!(client.dispatch.pending_sendtx_len(), 1, "A must survive");
23519 let head = client.dispatch.pop_pending_sendtx_head().unwrap();
23520 match head.kind {
23521 PendingSendTxKind::Create { order, .. } => {
23522 assert_eq!(order.client_order_id(), order_a.client_order_id());
23523 }
23524 _ => panic!("expected Create kind"),
23525 }
23526 }
23527
23528 #[tokio::test]
23529 async fn handle_send_tx_rejection_unmatched_hash_pops_nothing() {
23530 let (client, cache, mut rx) = create_execution_client();
23531 let instrument_id = register_test_instrument(&client, &cache);
23532 let mut factory = test_order_factory();
23533 let order = test_limit_order(&mut factory, instrument_id, "REJECT-UNMATCHED");
23534 enqueue_create(&client, &order, 10);
23535
23536 handle_send_tx_rejection(
23537 &client.dispatch,
23538 &client.emitter,
23539 Some(TEST_ACCOUNT_INDEX_I64),
23540 UnixNanos::from(1_000_000_000),
23541 SendTxRejectionSource::Ack,
23542 Some(21702),
23543 "invalid price",
23544 Some("0xbeef"),
23545 );
23546
23547 assert_eq!(
23548 client.dispatch.pending_sendtx_len(),
23549 1,
23550 "an echoed hash with no matching entry must not pop the head",
23551 );
23552 assert!(
23553 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23554 .await
23555 .is_err(),
23556 "unmatched rejection must not emit an event",
23557 );
23558 }
23559
23560 #[tokio::test]
23561 async fn handle_send_tx_rejection_bare_error_within_window_attributes() {
23562 let (client, cache, mut rx) = create_execution_client();
23563 let instrument_id = register_test_instrument(&client, &cache);
23564 let mut factory = test_order_factory();
23565 let order = test_limit_order(&mut factory, instrument_id, "BARE-IN");
23566 enqueue_create(&client, &order, 50);
23567
23568 let within_window = UnixNanos::from(1_000_000_000 + 500 * 1_000_000);
23569 handle_send_tx_rejection(
23570 &client.dispatch,
23571 &client.emitter,
23572 Some(TEST_ACCOUNT_INDEX_I64),
23573 within_window,
23574 SendTxRejectionSource::BareError,
23575 Some(21149),
23576 "integrator is not approved",
23577 None,
23578 );
23579
23580 let event = recv_order_event(&mut rx).await;
23581 assert!(matches!(event, OrderEventAny::Rejected(_)));
23582 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23583 }
23584
23585 #[tokio::test]
23586 async fn handle_send_tx_rejection_bare_error_outside_window_skips() {
23587 let (client, cache, mut rx) = create_execution_client();
23588 let instrument_id = register_test_instrument(&client, &cache);
23589 let mut factory = test_order_factory();
23590 let order = test_limit_order(&mut factory, instrument_id, "BARE-OUT");
23591 enqueue_create(&client, &order, 60);
23592
23593 let outside_window = UnixNanos::from(1_000_000_000 + 2_000 * 1_000_000);
23594 handle_send_tx_rejection(
23595 &client.dispatch,
23596 &client.emitter,
23597 Some(TEST_ACCOUNT_INDEX_I64),
23598 outside_window,
23599 SendTxRejectionSource::BareError,
23600 Some(99),
23601 "late error",
23602 None,
23603 );
23604
23605 assert_eq!(
23606 client.dispatch.pending_sendtx_len(),
23607 1,
23608 "head must remain queued past the 1s attribution window",
23609 );
23610 assert!(
23611 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23612 .await
23613 .is_err(),
23614 "no event must be emitted outside the window",
23615 );
23616 }
23617
23618 #[tokio::test]
23619 async fn handle_send_tx_rejection_bare_error_with_multiple_pending_skips() {
23620 let (client, cache, mut rx) = create_execution_client();
23621 let instrument_id = register_test_instrument(&client, &cache);
23622 let mut factory = test_order_factory();
23623 let order_a = test_limit_order(&mut factory, instrument_id, "BARE-MULTI-A");
23624 let order_b = test_limit_order(&mut factory, instrument_id, "BARE-MULTI-B");
23625 enqueue_create(&client, &order_a, 70);
23626 enqueue_create(&client, &order_b, 71);
23627
23628 handle_send_tx_rejection(
23629 &client.dispatch,
23630 &client.emitter,
23631 Some(TEST_ACCOUNT_INDEX_I64),
23632 UnixNanos::from(1_000_000_000 + 500 * 1_000_000),
23633 SendTxRejectionSource::BareError,
23634 Some(21149),
23635 "integrator is not approved",
23636 None,
23637 );
23638
23639 assert_eq!(client.dispatch.pending_sendtx_len(), 2);
23640 assert!(
23641 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23642 .await
23643 .is_err(),
23644 "a bare error cannot be attributed with multiple transactions pending",
23645 );
23646 }
23647
23648 #[tokio::test]
23649 async fn handle_send_tx_rejection_other_kind_rolls_back_latest_nonce() {
23650 let (client, _cache, _rx) = create_execution_client();
23651 let nonce = client
23652 .dispatch
23653 .nonce_manager
23654 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23655 .unwrap();
23656 enqueue_other(&client, nonce);
23657
23658 let needs_resync = handle_send_tx_rejection(
23659 &client.dispatch,
23660 &client.emitter,
23661 Some(TEST_ACCOUNT_INDEX_I64),
23662 UnixNanos::from(1_000_000_000),
23663 SendTxRejectionSource::Ack,
23664 Some(23000),
23665 "Too Many Requests",
23666 None,
23667 );
23668
23669 assert!(!needs_resync, "rate-limit rejection must not force resync");
23670 assert_nonce_reusable(&client.dispatch);
23671 }
23672
23673 #[tokio::test]
23674 async fn handle_send_tx_rejection_other_kind_skips_rollback_with_newer_issuance() {
23675 let (client, _cache, _rx) = create_execution_client();
23676 let rejected_nonce = client
23677 .dispatch
23678 .nonce_manager
23679 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23680 .unwrap();
23681 let newer_nonce = client
23682 .dispatch
23683 .nonce_manager
23684 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23685 .unwrap();
23686 enqueue_other(&client, rejected_nonce);
23687
23688 handle_send_tx_rejection(
23689 &client.dispatch,
23690 &client.emitter,
23691 Some(TEST_ACCOUNT_INDEX_I64),
23692 UnixNanos::from(1_000_000_000),
23693 SendTxRejectionSource::Ack,
23694 Some(23000),
23695 "Too Many Requests",
23696 None,
23697 );
23698
23699 assert_eq!(
23700 client
23701 .dispatch
23702 .nonce_manager
23703 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
23704 Some(newer_nonce),
23705 "non-latest rejection must leave last_issued alone",
23706 );
23707 }
23708
23709 #[tokio::test]
23710 async fn handle_send_tx_rejection_invalid_nonce_signals_resync() {
23711 let (client, _cache, _rx) = create_execution_client();
23712
23713 enqueue_other(&client, 70);
23714 let attributed = handle_send_tx_rejection(
23715 &client.dispatch,
23716 &client.emitter,
23717 Some(TEST_ACCOUNT_INDEX_I64),
23718 UnixNanos::from(1_000_000_000),
23719 SendTxRejectionSource::Ack,
23720 Some(LIGHTER_ERROR_CODE_INVALID_NONCE),
23721 "invalid nonce",
23722 None,
23723 );
23724 assert!(attributed, "attributed invalid nonce must signal resync");
23725
23726 let unattributed = handle_send_tx_rejection(
23727 &client.dispatch,
23728 &client.emitter,
23729 Some(TEST_ACCOUNT_INDEX_I64),
23730 UnixNanos::from(1_000_000_000),
23731 SendTxRejectionSource::Ack,
23732 Some(LIGHTER_ERROR_CODE_INVALID_NONCE),
23733 "invalid nonce",
23734 None,
23735 );
23736 assert!(
23737 unattributed,
23738 "unattributed invalid nonce must still signal resync",
23739 );
23740
23741 enqueue_other(&client, 71);
23742 let other_code = handle_send_tx_rejection(
23743 &client.dispatch,
23744 &client.emitter,
23745 Some(TEST_ACCOUNT_INDEX_I64),
23746 UnixNanos::from(1_000_000_000),
23747 SendTxRejectionSource::Ack,
23748 Some(21702),
23749 "invalid price",
23750 None,
23751 );
23752 assert!(!other_code, "other rejection codes must not force resync");
23753 }
23754
23755 #[tokio::test]
23756 async fn handle_send_tx_rejection_other_kind_logs_and_skips_emit() {
23757 let (client, _cache, mut rx) = create_execution_client();
23758 enqueue_other(&client, 70);
23759
23760 handle_send_tx_rejection(
23761 &client.dispatch,
23762 &client.emitter,
23763 Some(TEST_ACCOUNT_INDEX_I64),
23764 UnixNanos::from(1_000_000_000),
23765 SendTxRejectionSource::Ack,
23766 Some(21727),
23767 "invalid client order index",
23768 None,
23769 );
23770
23771 assert_eq!(client.dispatch.pending_sendtx_len(), 0, "Other head pops");
23772 assert!(
23773 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23774 .await
23775 .is_err(),
23776 "Other-kind rejection must not emit OrderRejected",
23777 );
23778 }
23779
23780 #[tokio::test]
23781 async fn handle_send_tx_rejection_empty_queue_logs_warn() {
23782 let (client, _cache, mut rx) = create_execution_client();
23783
23784 handle_send_tx_rejection(
23785 &client.dispatch,
23786 &client.emitter,
23787 Some(TEST_ACCOUNT_INDEX_I64),
23788 UnixNanos::from(1_000_000_000),
23789 SendTxRejectionSource::Ack,
23790 Some(1),
23791 "no pending",
23792 None,
23793 );
23794
23795 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23796 assert!(
23797 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23798 .await
23799 .is_err(),
23800 );
23801 }
23802
23803 #[rstest]
23804 #[case::standard_no_override(LighterAccountTier::Standard, None, 60, None)]
23805 #[case::standard_zero_is_default(LighterAccountTier::Standard, Some(0), 60, None)]
23806 #[case::standard_override_above_tier(
23807 LighterAccountTier::Standard,
23808 Some(24_000),
23809 24_000,
23810 Some(TierCrossCheck::AboveTier { documented: 60 })
23811 )]
23812 #[case::premium_raise_hint(
23813 LighterAccountTier::Premium,
23814 None,
23815 60,
23816 Some(TierCrossCheck::RaiseHint { documented: 24_000 })
23817 )]
23818 #[case::premium_configured_no_advisory(LighterAccountTier::Premium, Some(24_000), 24_000, None)]
23819 #[case::unknown_no_advisory(LighterAccountTier::Unknown(7), None, 60, None)]
23820 fn test_tier_quota_report(
23821 #[case] tier: LighterAccountTier,
23822 #[case] configured: Option<u32>,
23823 #[case] expected_active: u32,
23824 #[case] expected_cross_check: Option<TierCrossCheck>,
23825 ) {
23826 assert_eq!(
23827 tier_quota_report(tier, configured, 60),
23828 (expected_active, expected_cross_check),
23829 );
23830 }
23831
23832 #[rstest]
23833 fn cold_first_frame_updates_cached_shape() {
23834 let mut rig = dispatcher_rig("COLD-SHAPE");
23835 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
23836 rig.dispatch.forget_cloid(frame.client_order_index);
23837 let order = OrderTestBuilder::new(OrderType::Limit)
23838 .trader_id(trader_id())
23839 .strategy_id(strategy_id())
23840 .instrument_id(rig.instrument_id)
23841 .client_order_id(rig.cloid)
23842 .side(OrderSide::Buy)
23843 .quantity(Quantity::from("0.1000"))
23844 .price(Price::from("2352.74"))
23845 .build();
23846 let accepted =
23847 TestOrderEventStubs::accepted(&order, account_id(), VenueOrderId::new(&frame.order_id));
23848 let mut order = order;
23849 order.apply(accepted).unwrap();
23850 rig.dispatch.seed_order_restorations(vec![order]);
23851 frame.initial_base_amount = Decimal::from_str("0.0400").unwrap();
23852 frame.remaining_base_amount = frame.initial_base_amount;
23853 dispatch_lighter_order(
23854 &frame,
23855 &rig.dispatch,
23856 &rig.emitter,
23857 &rig.registry,
23858 account_id(),
23859 trader_id(),
23860 UnixNanos::from(2),
23861 );
23862 let events = drain_events(&mut rig.rx);
23863 assert_eq!(events.len(), 1);
23864
23865 let ExecutionEvent::Order(OrderEventAny::Updated(update)) = &events[0] else {
23866 panic!("expected restored shape update");
23867 };
23868
23869 assert_eq!(update.quantity, Quantity::from("0.0400"));
23870 assert_eq!(update.price, Some(Price::from("2352.74")));
23871 assert_eq!(update.client_order_id, rig.cloid);
23872 assert_eq!(
23873 update.venue_order_id,
23874 Some(VenueOrderId::new(&frame.order_id))
23875 );
23876 }
23877
23878 #[rstest]
23879 #[case(false)]
23880 #[case(true)]
23881 fn native_group_child_sizing_preserves_real_fill(#[case] trade_first: bool) {
23882 let mut rig = dispatcher_rig("CHILD-SIZING");
23883 let order = OrderTestBuilder::new(OrderType::StopLimit)
23884 .trader_id(trader_id())
23885 .strategy_id(strategy_id())
23886 .instrument_id(rig.instrument_id)
23887 .client_order_id(rig.cloid)
23888 .side(OrderSide::Buy)
23889 .quantity(Quantity::from("0.1000"))
23890 .price(Price::from("2352.74"))
23891 .trigger_price(Price::from("2300.00"))
23892 .parent_order_id(ClientOrderId::new("PARENT-CHILD-SIZING"))
23893 .reduce_only(true)
23894 .build();
23895 rig.dispatch.register_create_identity(&order).unwrap();
23896 rig.dispatch
23897 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
23898 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
23899 frame.initial_base_amount = Decimal::ZERO;
23900 frame.remaining_base_amount = Decimal::ZERO;
23901 frame.order_type = LighterOrderKind::StopLossLimit;
23902 frame.trigger_price = Decimal::from_str("2300.00").unwrap();
23903 frame.reduce_only = true;
23904 let cache = Rc::new(RefCell::new(Cache::default()));
23905 cache
23906 .borrow_mut()
23907 .add_instrument(
23908 LIGHTER_INSTRUMENT_CACHE
23909 .get(&rig.instrument_id)
23910 .unwrap()
23911 .value()
23912 .clone(),
23913 )
23914 .unwrap();
23915 cache
23916 .borrow_mut()
23917 .add_order(order, None, Some(client_id()), false)
23918 .unwrap();
23919 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
23920 let mut engine = ExecutionEngine::new(clock, cache.clone(), None);
23921 dispatch_lighter_order(
23922 &frame,
23923 &rig.dispatch,
23924 &rig.emitter,
23925 &rig.registry,
23926 account_id(),
23927 trader_id(),
23928 UnixNanos::from(1),
23929 );
23930
23931 for event in drain_events(&mut rig.rx) {
23932 let ExecutionEvent::Order(event) = event else {
23933 panic!("expected acceptance");
23934 };
23935
23936 engine.process(&event);
23937 }
23938
23939 let mut trade = dispatcher_test_trade(&rig, true);
23940 trade.bid_id = frame.order_index;
23941 trade.bid_id_str = Some(frame.order_id.clone());
23942 trade.size = Decimal::from_str("0.0400").unwrap();
23943 trade.usd_amount = trade.size * trade.price;
23944 frame.status = LighterOrderStatus::Filled;
23945 frame.initial_base_amount = trade.size;
23946 frame.filled_base_amount = trade.size;
23947 frame.filled_quote_amount = trade.usd_amount;
23948 frame.order_type = LighterOrderKind::Limit;
23949 frame.trigger_price = Decimal::ZERO;
23950
23951 if trade_first {
23952 dispatch_lighter_trade(
23953 &trade,
23954 &rig.dispatch,
23955 &rig.emitter,
23956 &rig.registry,
23957 account_id(),
23958 trader_id(),
23959 Some(TEST_ACCOUNT_INDEX_I64),
23960 UnixNanos::from(2),
23961 );
23962 }
23963
23964 dispatch_lighter_order(
23965 &frame,
23966 &rig.dispatch,
23967 &rig.emitter,
23968 &rig.registry,
23969 account_id(),
23970 trader_id(),
23971 UnixNanos::from(3),
23972 );
23973
23974 if !trade_first {
23975 dispatch_lighter_trade(
23976 &trade,
23977 &rig.dispatch,
23978 &rig.emitter,
23979 &rig.registry,
23980 account_id(),
23981 trader_id(),
23982 Some(TEST_ACCOUNT_INDEX_I64),
23983 UnixNanos::from(4),
23984 );
23985 }
23986
23987 let events = drain_events(&mut rig.rx);
23988 for event in events {
23989 match event {
23990 ExecutionEvent::Order(event) => engine.process(&event),
23991 ExecutionEvent::Report(report) => engine.reconcile_execution_report(&report),
23992 _ => panic!("unexpected child event"),
23993 }
23994 }
23995
23996 let cache = cache.borrow();
23997 let order = cache.order(&rig.cloid).unwrap();
23998
23999 let fills = order
24000 .events()
24001 .into_iter()
24002 .filter_map(|event| match event {
24003 OrderEventAny::Filled(fill) => Some(fill),
24004 _ => None,
24005 })
24006 .collect::<Vec<_>>();
24007
24008 assert_eq!(order.status(), OrderStatus::Filled);
24009 assert_eq!(order.quantity(), Quantity::from("0.0400"));
24010 assert_eq!(order.filled_qty(), Quantity::from("0.0400"));
24011 assert_eq!(order.leaves_qty(), Quantity::from("0.0000"));
24012 assert_eq!(order.is_triggered(), Some(true));
24013 assert_eq!(fills.len(), 1);
24014 assert_eq!(fills[0].trade_id, parse_lighter_trade_id(&trade).unwrap());
24015 assert_eq!(fills[0].last_qty, Quantity::from("0.0400"));
24016 assert_eq!(fills[0].last_px, Price::from("2352.73"));
24017 assert_eq!(fills[0].commission, Some(Money::from("0.00263506 USDC")));
24018 }
24019
24020 #[rstest]
24021 #[case::distinct_timestamps(false)]
24022 #[case::same_timestamp(true)]
24023 fn native_group_preserves_position_fill_order(#[case] same_timestamp: bool) {
24024 let mut rig = dispatcher_rig("POSITION-PARENT");
24025 let parent_id = rig.cloid;
24026 let child_id = ClientOrderId::new("POSITION-CHILD");
24027 let extra_id = ClientOrderId::new("POSITION-EXTRA");
24028 let parent = OrderTestBuilder::new(OrderType::Limit)
24029 .trader_id(trader_id())
24030 .strategy_id(strategy_id())
24031 .instrument_id(rig.instrument_id)
24032 .client_order_id(parent_id)
24033 .side(OrderSide::Buy)
24034 .quantity(Quantity::from("0.1000"))
24035 .price(Price::from("100.00"))
24036 .build();
24037 rig.dispatch.register_create_identity(&parent).unwrap();
24038 rig.dispatch
24039 .mark_order_submission(&parent_id, TEST_SUBMISSION_NONCE);
24040 let mut parent_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24041 parent_frame.order_index = 562_947_905_631_053;
24042 parent_frame.order_id = parent_frame.order_index.to_string();
24043 parent_frame.initial_base_amount = Decimal::from_str("0.1000").unwrap();
24044 parent_frame.remaining_base_amount = parent_frame.initial_base_amount;
24045 parent_frame.price = Decimal::from_str("100.00").unwrap();
24046 dispatch_lighter_order(
24047 &parent_frame,
24048 &rig.dispatch,
24049 &rig.emitter,
24050 &rig.registry,
24051 account_id(),
24052 trader_id(),
24053 UnixNanos::from(1),
24054 );
24055 let mut opening = dispatcher_test_trade(&rig, true);
24056 opening.bid_id = parent_frame.order_index;
24057 opening.bid_id_str = Some(parent_frame.order_id);
24058 opening.trade_id_str = Some("900001".to_string());
24059 opening.size = Decimal::from_str("0.1000").unwrap();
24060 opening.price = Decimal::from_str("100.00").unwrap();
24061 opening.usd_amount = opening.size * opening.price;
24062 opening.maker_fee = Some(0);
24063 opening.taker_fee = Some(0);
24064 dispatch_lighter_trade(
24065 &opening,
24066 &rig.dispatch,
24067 &rig.emitter,
24068 &rig.registry,
24069 account_id(),
24070 trader_id(),
24071 Some(TEST_ACCOUNT_INDEX_I64),
24072 UnixNanos::from(2),
24073 );
24074 rig.cloid = child_id;
24075 let child = OrderTestBuilder::new(OrderType::StopLimit)
24076 .trader_id(trader_id())
24077 .strategy_id(strategy_id())
24078 .instrument_id(rig.instrument_id)
24079 .client_order_id(child_id)
24080 .side(OrderSide::Sell)
24081 .quantity(Quantity::from("0.1000"))
24082 .price(Price::from("200.00"))
24083 .trigger_price(Price::from("150.00"))
24084 .parent_order_id(parent_id)
24085 .reduce_only(true)
24086 .build();
24087 rig.dispatch.register_create_identity(&child).unwrap();
24088 rig.dispatch
24089 .mark_order_submission(&child_id, TEST_SUBMISSION_NONCE);
24090 let mut child_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24091 child_frame.initial_base_amount = Decimal::from_str("0.0400").unwrap();
24092 child_frame.remaining_base_amount = child_frame.initial_base_amount;
24093 child_frame.price = Decimal::from_str("200.00").unwrap();
24094 child_frame.side = Some(LighterOrderSide::Sell);
24095 child_frame.is_ask = true;
24096 child_frame.reduce_only = true;
24097 dispatch_lighter_order(
24098 &child_frame,
24099 &rig.dispatch,
24100 &rig.emitter,
24101 &rig.registry,
24102 account_id(),
24103 trader_id(),
24104 UnixNanos::from(3),
24105 );
24106 let mut first = dispatcher_test_trade(&rig, false);
24107 first.ask_id = child_frame.order_index;
24108 first.ask_id_str = Some(child_frame.order_id.clone());
24109 first.trade_id_str = Some("900002".to_string());
24110 first.timestamp = opening.timestamp + i64::from(!same_timestamp);
24111 first.size = Decimal::from_str("0.0200").unwrap();
24112 first.price = Decimal::from_str("200.00").unwrap();
24113 first.usd_amount = first.size * first.price;
24114 first.maker_fee = Some(0);
24115 first.taker_fee = Some(0);
24116 dispatch_lighter_trade(
24117 &first,
24118 &rig.dispatch,
24119 &rig.emitter,
24120 &rig.registry,
24121 account_id(),
24122 trader_id(),
24123 Some(TEST_ACCOUNT_INDEX_I64),
24124 UnixNanos::from(4),
24125 );
24126 rig.cloid = extra_id;
24127 let extra = OrderTestBuilder::new(OrderType::Limit)
24128 .trader_id(trader_id())
24129 .strategy_id(strategy_id())
24130 .instrument_id(rig.instrument_id)
24131 .client_order_id(extra_id)
24132 .side(OrderSide::Buy)
24133 .quantity(Quantity::from("0.1000"))
24134 .price(Price::from("100.00"))
24135 .time_in_force(TimeInForce::Ioc)
24136 .build();
24137 rig.dispatch.register_create_identity(&extra).unwrap();
24138 rig.dispatch
24139 .mark_order_submission(&extra_id, TEST_SUBMISSION_NONCE + 1);
24140 let mut extra_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24141 extra_frame.order_index += 5;
24142 extra_frame.order_id = extra_frame.order_index.to_string();
24143 extra_frame.initial_base_amount = Decimal::from_str("0.1000").unwrap();
24144 extra_frame.remaining_base_amount = extra_frame.initial_base_amount;
24145 extra_frame.price = Decimal::from_str("100.00").unwrap();
24146 dispatch_lighter_order(
24147 &extra_frame,
24148 &rig.dispatch,
24149 &rig.emitter,
24150 &rig.registry,
24151 account_id(),
24152 trader_id(),
24153 UnixNanos::from(5),
24154 );
24155 let mut addition = dispatcher_test_trade(&rig, true);
24156 addition.bid_id = extra_frame.order_index;
24157 addition.bid_id_str = Some(extra_frame.order_id);
24158 addition.trade_id_str = Some("900003".to_string());
24159 addition.timestamp = opening.timestamp + 2 * i64::from(!same_timestamp);
24160 addition.size = Decimal::from_str("0.0600").unwrap();
24161 addition.price = Decimal::from_str("50.00").unwrap();
24162 addition.usd_amount = addition.size * addition.price;
24163 addition.maker_fee = Some(0);
24164 addition.taker_fee = Some(0);
24165 dispatch_lighter_trade(
24166 &addition,
24167 &rig.dispatch,
24168 &rig.emitter,
24169 &rig.registry,
24170 account_id(),
24171 trader_id(),
24172 Some(TEST_ACCOUNT_INDEX_I64),
24173 UnixNanos::from(6),
24174 );
24175 rig.cloid = child_id;
24176 let mut last = first;
24177 last.trade_id_str = Some("900004".to_string());
24178 last.timestamp = opening.timestamp + 3 * i64::from(!same_timestamp);
24179 child_frame.status = LighterOrderStatus::Filled;
24180 child_frame.remaining_base_amount = Decimal::ZERO;
24181 child_frame.filled_base_amount = Decimal::from_str("0.0400").unwrap();
24182 child_frame.filled_quote_amount = Decimal::from(8);
24183 dispatch_lighter_order(
24184 &child_frame,
24185 &rig.dispatch,
24186 &rig.emitter,
24187 &rig.registry,
24188 account_id(),
24189 trader_id(),
24190 UnixNanos::from(7),
24191 );
24192 dispatch_lighter_trade(
24193 &last,
24194 &rig.dispatch,
24195 &rig.emitter,
24196 &rig.registry,
24197 account_id(),
24198 trader_id(),
24199 Some(TEST_ACCOUNT_INDEX_I64),
24200 UnixNanos::from(8),
24201 );
24202 let mut fills = Vec::new();
24203
24204 for event in drain_events(&mut rig.rx) {
24205 match event {
24206 ExecutionEvent::Order(OrderEventAny::Filled(fill)) => fills.push(fill),
24207 ExecutionEvent::Report(EngineExecutionReport::OrderWithFills(_, reports)) => {
24208 for report in reports {
24209 fills.push(OrderFilled::new(
24210 trader_id(),
24211 strategy_id(),
24212 rig.instrument_id,
24213 report.client_order_id.unwrap(),
24214 report.venue_order_id,
24215 account_id(),
24216 report.trade_id,
24217 report.order_side,
24218 OrderType::StopLimit,
24219 report.last_qty,
24220 report.last_px,
24221 Currency::from("USDC"),
24222 report.liquidity_side,
24223 UUID4::new(),
24224 report.ts_event,
24225 report.ts_init,
24226 true,
24227 None,
24228 Some(report.commission),
24229 None,
24230 ));
24231 }
24232 }
24233 _ => {}
24234 }
24235 }
24236
24237 let ids = fills.iter().map(|fill| fill.trade_id).collect::<Vec<_>>();
24238 let position_id = PositionId::new("POSITION-ORDER");
24239 for fill in &mut fills {
24240 fill.position_id = Some(position_id);
24241 }
24242
24243 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&rig.instrument_id).unwrap();
24244 let mut position = Position::new(&instrument, fills[0].clone());
24245 for fill in &fills[1..] {
24246 position.apply(fill);
24247 }
24248
24249 let expected_pnl = Money::from_decimal_native(
24251 Decimal::from(31) / Decimal::from(7),
24252 Currency::from("USDC"),
24253 )
24254 .unwrap();
24255
24256 assert_eq!(
24257 ids,
24258 ["900001", "900002", "900003", "900004"].map(TradeId::new)
24259 );
24260 assert_eq!(position.quantity, Quantity::from("0.1200"));
24261 assert_eq!(position.realized_pnl, Some(expected_pnl));
24262 }
24263
24264 #[rstest]
24265 #[case::filled(LighterOrderStatus::Filled)]
24266 #[case::partial_then_canceled(LighterOrderStatus::Canceled)]
24267 fn terminal_after_fill_retires_identity_with_publication(#[case] status: LighterOrderStatus) {
24268 let mut rig = dispatcher_rig("TERMINAL-PUBLICATION");
24269 register_identity(&rig);
24270 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24271 dispatch_lighter_order(
24272 &frame,
24273 &rig.dispatch,
24274 &rig.emitter,
24275 &rig.registry,
24276 account_id(),
24277 trader_id(),
24278 UnixNanos::from(1),
24279 );
24280 let mut trade = dispatcher_test_trade(&rig, true);
24281 trade.bid_id = frame.order_index;
24282 trade.bid_id_str = Some(frame.order_id.clone());
24283 trade.size = if status == LighterOrderStatus::Filled {
24284 frame.initial_base_amount
24285 } else {
24286 frame.initial_base_amount / Decimal::from(2)
24287 };
24288
24289 trade.usd_amount = trade.size * trade.price;
24290 dispatch_lighter_trade(
24291 &trade,
24292 &rig.dispatch,
24293 &rig.emitter,
24294 &rig.registry,
24295 account_id(),
24296 trader_id(),
24297 Some(TEST_ACCOUNT_INDEX_I64),
24298 UnixNanos::from(2),
24299 );
24300 frame.status = status;
24301 frame.filled_base_amount = trade.size;
24302 frame.remaining_base_amount = Decimal::ZERO;
24303 dispatch_lighter_order(
24304 &frame,
24305 &rig.dispatch,
24306 &rig.emitter,
24307 &rig.registry,
24308 account_id(),
24309 trader_id(),
24310 UnixNanos::from(3),
24311 );
24312 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
24313 let events = drain_events(&mut rig.rx);
24314 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
24315 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
24316 assert!(
24317 !rig.dispatch
24318 .cloid_map
24319 .contains_key(&frame.client_order_index)
24320 );
24321 assert!(!rig.dispatch.order_snapshots.contains_key(&rig.cloid));
24322 assert!(identity.has_publication());
24323 assert!(
24324 rig.dispatch
24325 .any_order_identity(|id, other, _| *id == rig.cloid && other.has_publication())
24326 );
24327 assert_eq!(
24328 events
24329 .iter()
24330 .filter(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Filled(_))))
24331 .count(),
24332 1
24333 );
24334 }
24335
24336 #[rstest]
24337 #[case::modify("modify", false)]
24338 #[case::terminal("terminal", false)]
24339 #[case::fill("fill", false)]
24340 #[case::update("update", false)]
24341 #[case::retired_fill("fill", true)]
24342 #[case::retired_update("update", true)]
24343 fn position_reports_wait_for_execution(#[case] hold: &str, #[case] retired: bool) {
24344 let (client, cache, _) = create_execution_client();
24345 let instrument_id = register_test_instrument(&client, &cache);
24346 let mut factory = test_order_factory();
24347 let order = test_limit_order(&mut factory, instrument_id, "POSITION-HOLD");
24348 let cloid = order.client_order_id();
24349 client.dispatch.register_create_identity(&order).unwrap();
24350 let identity = client.dispatch.order_identity(&cloid).unwrap();
24351
24352 match hold {
24353 "modify" => {
24354 enqueue_modify(&client, instrument_id, cloid, VenueOrderId::new("700"), 23);
24355 }
24356 "terminal" => {
24357 let rig = dispatcher_rig("POSITION-TERMINAL");
24358 client
24359 .dispatch
24360 .order_events
24361 .lock()
24362 .entry(cloid)
24363 .or_default()
24364 .order = Some(dispatcher_test_order(&rig, LighterOrderStatus::Canceled));
24365 }
24366 "update" => identity.publish_update(UUID4::new()),
24367 "fill" => {
24368 let instrument = cache.borrow().instrument(&instrument_id).unwrap().clone();
24369
24370 let OrderEventAny::Filled(fill) = TestOrderEventStubs::filled(
24371 &order,
24372 &instrument,
24373 Some(TradeId::new("HELD-FILL")),
24374 None,
24375 Some(Price::from("2360.00")),
24376 Some(Quantity::from("0.0050")),
24377 None,
24378 Some(Money::from("0.01000000 USDC")),
24379 None,
24380 Some(account_id()),
24381 ) else {
24382 unreachable!()
24383 };
24384
24385 identity.publish_fill(&fill);
24386 }
24387 _ => unreachable!(),
24388 }
24389
24390 if retired {
24391 client.dispatch.retire_order_identity(&cloid);
24392 }
24393
24394 let report = PositionStatusReport::new(
24395 account_id(),
24396 instrument_id,
24397 PositionSide::Long,
24398 Quantity::from("0.0050"),
24399 UnixNanos::from(4),
24400 UnixNanos::from(5),
24401 None,
24402 None,
24403 Some(Decimal::from_str("2360.00").unwrap()),
24404 );
24405 client
24406 .dispatch
24407 .replace_position_snapshot(std::slice::from_ref(&report), &[], &[]);
24408
24409 let cmd = GeneratePositionStatusReports::new(
24410 UUID4::new(),
24411 UnixNanos::from(6),
24412 Some(instrument_id),
24413 None,
24414 None,
24415 None,
24416 None,
24417 );
24418 let (reports, complete, coverage) = client.cached_position_reports(&cmd).unwrap();
24419 assert!(reports.is_empty());
24420 assert!(!complete);
24421 assert_eq!(coverage, Some(AHashSet::from_iter([TEST_MARKET_INDEX])));
24422 let other_id = client.registry.insert(1, "BTC", LighterProductType::Perp);
24423 let mut other_cmd = cmd;
24424 other_cmd.instrument_id = Some(other_id);
24425 let (reports, complete, _) = client.cached_position_reports(&other_cmd).unwrap();
24426 assert!(reports.is_empty());
24427 assert!(complete);
24428 }
24429
24430 #[rstest]
24431 #[case::explicit_flat(true)]
24432 #[case::empty_snapshot(false)]
24433 #[tokio::test]
24434 async fn restart_held_child_fill_does_not_synthesize_position(#[case] explicit_flat: bool) {
24435 let (mut client, cache, mut rx) = create_execution_client();
24436 let core = client.core.clone();
24437 let instrument_id = register_test_instrument(&client, &cache);
24438 let mut rig = dispatcher_rig("RESTART-CHILD");
24439 rig.dispatch = client.dispatch.clone();
24440 rig.registry = Arc::clone(&client.registry);
24441
24442 let account = MarginAccount::new(
24443 AccountState::new(
24444 account_id(),
24445 AccountType::Margin,
24446 vec![AccountBalance::new(
24447 Money::from("10000.00000000 USDC"),
24448 Money::from("0.00000000 USDC"),
24449 Money::from("10000.00000000 USDC"),
24450 )],
24451 Vec::new(),
24452 true,
24453 UUID4::new(),
24454 UnixNanos::from(1),
24455 UnixNanos::from(1),
24456 Some(Currency::from("USDC")),
24457 ),
24458 true,
24459 );
24460 cache
24461 .borrow_mut()
24462 .add_account(AccountAny::Margin(account))
24463 .unwrap();
24464 let parent_id = ClientOrderId::new("RESTART-PARENT");
24465 let position_id = PositionId::new("RESTART-POSITION");
24466 let parent = OrderTestBuilder::new(OrderType::Limit)
24467 .trader_id(trader_id())
24468 .strategy_id(strategy_id())
24469 .instrument_id(instrument_id)
24470 .client_order_id(parent_id)
24471 .side(OrderSide::Buy)
24472 .quantity(Quantity::from("0.1000"))
24473 .price(Price::from("2352.73"))
24474 .build();
24475 let instrument = LIGHTER_INSTRUMENT_CACHE
24476 .get(&instrument_id)
24477 .unwrap()
24478 .value()
24479 .clone();
24480 let opening = TestOrderEventStubs::filled(
24481 &parent,
24482 &instrument,
24483 Some(TradeId::new("RESTART-OPENING")),
24484 Some(position_id),
24485 Some(Price::from("2352.73")),
24486 Some(Quantity::from("0.1000")),
24487 Some(LiquiditySide::Taker),
24488 Some(Money::from("0.00000000 USDC")),
24489 Some(UnixNanos::from(1)),
24490 Some(account_id()),
24491 );
24492 let mut parent = parent;
24493 parent
24494 .apply(TestOrderEventStubs::accepted(
24495 &parent,
24496 account_id(),
24497 VenueOrderId::new("600"),
24498 ))
24499 .unwrap();
24500 parent.apply(opening.clone()).unwrap();
24501 cache
24502 .borrow_mut()
24503 .add_order(parent, Some(position_id), Some(client_id()), false)
24504 .unwrap();
24505
24506 let OrderEventAny::Filled(opening) = opening else {
24507 unreachable!()
24508 };
24509
24510 let position = Position::new(&instrument, opening);
24511 cache
24512 .borrow_mut()
24513 .add_position(&position, OmsType::Netting)
24514 .unwrap();
24515 let child = OrderTestBuilder::new(OrderType::StopLimit)
24516 .trader_id(trader_id())
24517 .strategy_id(strategy_id())
24518 .instrument_id(instrument_id)
24519 .client_order_id(rig.cloid)
24520 .side(OrderSide::Sell)
24521 .quantity(Quantity::from("0.1000"))
24522 .price(Price::from("2352.74"))
24523 .trigger_price(Price::from("2300.00"))
24524 .parent_order_id(parent_id)
24525 .reduce_only(true)
24526 .build();
24527 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24528 frame.side = Some(LighterOrderSide::Sell);
24529 frame.is_ask = true;
24530 frame.order_type = LighterOrderKind::Limit;
24531 frame.reduce_only = true;
24532 let venue_id = VenueOrderId::new(&frame.order_id);
24533 cache_accepted_order(&cache, child, venue_id, Some(client_id()));
24534 cache
24535 .borrow_mut()
24536 .add_position_id(
24537 &position_id,
24538 &instrument_id.venue,
24539 &rig.cloid,
24540 &strategy_id(),
24541 )
24542 .unwrap();
24543 let child = cache.borrow().order_owned(&rig.cloid).unwrap();
24544 client
24545 .dispatch
24546 .restore_reconciled_order(&child, frame.client_order_index, venue_id, false)
24547 .unwrap();
24548 let mut trade = dispatcher_test_trade(&rig, false);
24549 trade.ask_id = frame.order_index;
24550 trade.ask_id_str = Some(frame.order_id.clone());
24551 trade.size = Decimal::from_str("0.1000").unwrap();
24552 trade.usd_amount = trade.size * trade.price;
24553 trade.maker_fee = Some(0);
24554 trade.taker_fee = Some(0);
24555 dispatch_lighter_trade(
24556 &trade,
24557 &client.dispatch,
24558 &client.emitter,
24559 &client.registry,
24560 account_id(),
24561 trader_id(),
24562 Some(TEST_ACCOUNT_INDEX_I64),
24563 UnixNanos::from(2),
24564 );
24565
24566 let flat = PositionStatusReport::new(
24567 account_id(),
24568 instrument_id,
24569 PositionSide::Flat,
24570 Quantity::from("0.0000"),
24571 UnixNanos::from(3),
24572 UnixNanos::from(4),
24573 None,
24574 None,
24575 None,
24576 );
24577
24578 let positions = if explicit_flat {
24579 vec![flat]
24580 } else {
24581 Vec::new()
24582 };
24583
24584 client
24585 .dispatch
24586 .replace_position_snapshot(&positions, &[], &[]);
24587 let mut external = frame.clone();
24588 external.order_index += 1;
24589 external.order_id = external.order_index.to_string();
24590 external.client_order_index = 0;
24591 external.client_order_id = "0".to_string();
24592 external.reduce_only = false;
24593 let active = serde_json::json!({"code":200,"orders":[external]});
24594 let app = Router::new()
24595 .route(
24596 "/api/v1/accountActiveOrders",
24597 get(move || {
24598 let active = active.clone();
24599 async move { axum::Json(active) }
24600 }),
24601 )
24602 .route(
24603 "/api/v1/accountInactiveOrders",
24604 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
24605 )
24606 .route(
24607 "/api/v1/trades",
24608 get(|| async { axum::Json(serde_json::json!({"code":200,"trades":[]})) }),
24609 );
24610
24611 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
24612 let addr = listener.local_addr().unwrap();
24613 let server = tokio::spawn(async move {
24614 axum::serve(listener, app).await.unwrap();
24615 });
24616
24617 client.http_client = LighterHttpClient::new(
24618 LighterEnvironment::Testnet,
24619 Some(format!("http://{addr}")),
24620 1,
24621 None,
24622 )
24623 .unwrap();
24624 let mass = client
24625 .generate_mass_status(Some(60))
24626 .await
24627 .unwrap()
24628 .unwrap();
24629 server.abort();
24630 let _ = server.await;
24631
24632 let command = GeneratePositionStatusReports::new(
24633 UUID4::new(),
24634 UnixNanos::from(5),
24635 None,
24636 None,
24637 None,
24638 None,
24639 None,
24640 );
24641 let direct = client.generate_position_status_reports(&command).await;
24642 let worker = client
24643 .generate_position_status_reports_task(&command)
24644 .unwrap();
24645 worker.collection.await;
24646 let worker = (worker.finish)();
24647 assert!(mass.position_reports().is_empty());
24648 assert!(!mass.reports_complete());
24649 assert!(direct.is_err());
24650 assert!(worker.is_err());
24651 assert!(drain_events(&mut rx).is_empty());
24652 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
24653
24654 let engine = Rc::new(RefCell::new(ExecutionEngine::new(
24655 clock.clone(),
24656 cache.clone(),
24657 None,
24658 )));
24659 let mut manager =
24660 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default()).unwrap();
24661 let result = manager.reconcile_execution_mass_status(&mass, &engine);
24662 assert!(
24663 result
24664 .events
24665 .iter()
24666 .all(|event| !matches!(event, OrderEventAny::Filled(_)))
24667 );
24668 assert_eq!(
24669 cache.borrow().position(&position_id).unwrap().quantity,
24670 Quantity::from("0.1000")
24671 );
24672 frame.status = LighterOrderStatus::Filled;
24673 frame.initial_base_amount = trade.size;
24674 frame.filled_base_amount = trade.size;
24675 frame.remaining_base_amount = Decimal::ZERO;
24676 frame.filled_quote_amount = trade.usd_amount;
24677 dispatch_lighter_order(
24678 &frame,
24679 &client.dispatch,
24680 &client.emitter,
24681 &client.registry,
24682 account_id(),
24683 trader_id(),
24684 UnixNanos::from(6),
24685 );
24686 engine
24687 .borrow_mut()
24688 .register_client(Box::new(client))
24689 .unwrap();
24690 let released = drain_events(&mut rx);
24691 assert!(
24692 released
24693 .iter()
24694 .any(|event| matches!(event, ExecutionEvent::Order(OrderEventAny::Filled(_)))),
24695 "released child events: {released:?}"
24696 );
24697
24698 for event in released {
24699 match event {
24700 ExecutionEvent::Order(event) => engine.borrow_mut().process(&event),
24701 ExecutionEvent::Report(report) => {
24702 engine.borrow_mut().reconcile_execution_report(&report);
24703 }
24704 _ => panic!("unexpected restart event"),
24705 }
24706 }
24707
24708 let cache_ref = cache.borrow();
24709 let position = cache_ref.position(&position_id).unwrap();
24710 let child = cache_ref.order(&rig.cloid).unwrap();
24711 assert_eq!(position.quantity, Quantity::from("0.0000"));
24712 assert_eq!(position.realized_pnl, Some(Money::from("0.00000000 USDC")));
24713 assert_eq!(
24714 child.trade_ids(),
24715 vec![&parse_lighter_trade_id(&trade).unwrap()]
24716 );
24717 assert_eq!(child.filled_qty(), Quantity::from("0.1000"));
24718 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
24719 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
24720
24721 let command = GeneratePositionStatusReports::new(
24722 UUID4::new(),
24723 UnixNanos::from(7),
24724 None,
24725 None,
24726 None,
24727 None,
24728 None,
24729 );
24730 let (reports, complete, coverage) =
24731 cached_position_reports(&core, &rig.dispatch, &rig.registry, &command).unwrap();
24732 assert!(complete);
24733 assert_eq!(coverage, Some(AHashSet::new()));
24734 assert_eq!(reports.len(), usize::from(explicit_flat));
24735 }
24736
24737 #[rstest]
24738 fn resting_child_does_not_withhold_position_reports() {
24739 let (client, cache, _) = create_execution_client();
24740 let instrument_id = register_test_instrument(&client, &cache);
24741 let rig = dispatcher_rig("RESTING-POSITION");
24742 let order = OrderTestBuilder::new(OrderType::StopLimit)
24743 .trader_id(trader_id())
24744 .strategy_id(strategy_id())
24745 .instrument_id(instrument_id)
24746 .client_order_id(rig.cloid)
24747 .side(OrderSide::Sell)
24748 .quantity(Quantity::from("0.1000"))
24749 .price(Price::from("2300.00"))
24750 .trigger_price(Price::from("2301.00"))
24751 .parent_order_id(ClientOrderId::new("RESTING-PARENT"))
24752 .reduce_only(true)
24753 .build();
24754 client.dispatch.register_create_identity(&order).unwrap();
24755 client
24756 .dispatch
24757 .order_events
24758 .lock()
24759 .entry(rig.cloid)
24760 .or_default()
24761 .order = Some(dispatcher_test_order(&rig, LighterOrderStatus::Pending));
24762
24763 let report = PositionStatusReport::new(
24764 account_id(),
24765 instrument_id,
24766 PositionSide::Long,
24767 Quantity::from("0.1000"),
24768 UnixNanos::from(4),
24769 UnixNanos::from(5),
24770 None,
24771 None,
24772 Some(Decimal::from_str("2360.00").unwrap()),
24773 );
24774 client
24775 .dispatch
24776 .replace_position_snapshot(std::slice::from_ref(&report), &[], &[]);
24777
24778 let cmd = GeneratePositionStatusReports::new(
24779 UUID4::new(),
24780 UnixNanos::from(6),
24781 None,
24782 None,
24783 None,
24784 None,
24785 None,
24786 );
24787 let (reports, complete, coverage) = client.cached_position_reports(&cmd).unwrap();
24788 assert_eq!(reports, vec![report]);
24789 assert!(complete);
24790 assert_eq!(coverage, Some(AHashSet::new()));
24791 }
24792
24793 #[rstest]
24794 #[case::cached_fill("2352.73", false)]
24795 #[case::price_restatement("2352.74", true)]
24796 fn cached_parent_fill_is_not_deferred_during_child_recovery(
24797 #[case] price: &str,
24798 #[case] deferred: bool,
24799 ) {
24800 let (client, cache, mut rx) = create_execution_client();
24801 let instrument_id = register_test_instrument(&client, &cache);
24802 let instrument = cache.borrow().instrument(&instrument_id).unwrap().clone();
24803 LIGHTER_INSTRUMENT_CACHE.insert(instrument_id, instrument.clone());
24804 let parent_id = ClientOrderId::new("HTTP-CACHED-PARENT");
24805 let parent_venue = VenueOrderId::new("710001");
24806 let child_id = ClientOrderId::new("HTTP-RECOVERING-CHILD");
24807 let child_venue = VenueOrderId::new("710002");
24808 let parent = OrderTestBuilder::new(OrderType::Limit)
24809 .trader_id(trader_id())
24810 .strategy_id(strategy_id())
24811 .instrument_id(instrument_id)
24812 .client_order_id(parent_id)
24813 .side(OrderSide::Buy)
24814 .quantity(Quantity::from("0.1336"))
24815 .price(Price::from("2352.73"))
24816 .build();
24817 cache_accepted_order(&cache, parent, parent_venue, Some(client_id()));
24818 let parent = cache.borrow().order_owned(&parent_id).unwrap();
24819 let mut report =
24820 reconciliation_fill_report(instrument_id, 101, parent_venue, "CACHED-PARENT-FILL");
24821 let opening = TestOrderEventStubs::filled(
24822 &parent,
24823 &instrument,
24824 Some(report.trade_id),
24825 None,
24826 Some(report.last_px),
24827 Some(report.last_qty),
24828 Some(report.liquidity_side),
24829 Some(report.commission),
24830 Some(report.ts_event),
24831 Some(account_id()),
24832 );
24833 cache.borrow_mut().update_order(&opening).unwrap();
24834 let child = OrderTestBuilder::new(OrderType::StopLimit)
24835 .trader_id(trader_id())
24836 .strategy_id(strategy_id())
24837 .instrument_id(instrument_id)
24838 .client_order_id(child_id)
24839 .side(OrderSide::Sell)
24840 .quantity(Quantity::from("0.1336"))
24841 .price(Price::from("2301.00"))
24842 .trigger_price(Price::from("2302.00"))
24843 .parent_order_id(parent_id)
24844 .reduce_only(true)
24845 .build();
24846 cache_accepted_order(&cache, child, child_venue, Some(client_id()));
24847 let child = cache.borrow().order_owned(&child_id).unwrap();
24848 client
24849 .dispatch
24850 .restore_reconciled_order(&child, 102, child_venue, false)
24851 .unwrap();
24852 let identity = client.dispatch.order_identity(&child_id).unwrap();
24853 let mut child_report =
24854 reconciliation_fill_report(instrument_id, 102, child_venue, "NEW-CHILD-FILL");
24855 child_report.order_side = OrderSide::Sell;
24856 child_report.last_qty = Quantity::from("0.0444");
24857 child_report.last_px = Price::from("2301.23");
24858 let child_fill = child_fill_from_report(&child_report, &identity, trader_id()).unwrap();
24859 client.dispatch.defer_fill(
24860 client
24861 .dispatch
24862 .order_events
24863 .lock()
24864 .entry(child_id)
24865 .or_default(),
24866 child_fill.clone(),
24867 );
24868 report.last_px = Price::from(price);
24869
24870 let sweep = finish_fill_reports(
24871 &client.core,
24872 &client.dispatch,
24873 BTreeSet::new(),
24874 Ok(FillSweep {
24875 reports: vec![report.clone()],
24876 covers_window: true,
24877 withheld_instruments: AHashSet::new(),
24878 }),
24879 )
24880 .unwrap();
24881
24882 let events = client.dispatch.order_events.lock();
24883 assert!(sweep.reports.is_empty());
24884 assert!(!sweep.covers_window);
24885 assert_eq!(
24886 sweep.withheld_instruments,
24887 AHashSet::from_iter([instrument_id])
24888 );
24889 assert_eq!(events.get(&parent_id).is_some(), deferred);
24890 assert_eq!(events[&child_id].fills, vec![child_fill]);
24891
24892 if deferred {
24893 let fills = &events[&parent_id].fills;
24894 assert_eq!(fills.len(), 1);
24895 assert_eq!(fills[0].trade_id, report.trade_id);
24896 assert_eq!(fills[0].client_order_id, parent_id);
24897 assert_eq!(fills[0].venue_order_id, parent_venue);
24898 assert_eq!(fills[0].last_qty, report.last_qty);
24899 assert_eq!(fills[0].last_px, report.last_px);
24900 assert_eq!(fills[0].commission, Some(report.commission));
24901 }
24902
24903 assert_eq!(
24904 cache.borrow().order(&parent_id).unwrap().trade_ids(),
24905 vec![&report.trade_id]
24906 );
24907 assert!(drain_events(&mut rx).is_empty());
24908 }
24909
24910 #[rstest]
24911 #[case::immediate(false, false, 1)]
24912 #[case::deferred(false, true, 1)]
24913 #[case::repeated(false, false, 16)]
24914 #[case::deferred_repeated(false, true, 16)]
24915 #[case::first_frame(true, false, 1)]
24916 fn self_trade_reduce_updates_cache_and_reports(
24917 #[case] first_frame: bool,
24918 #[case] deferred: bool,
24919 #[case] repeats: usize,
24920 ) {
24921 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
24922 let mut events = Vec::new();
24923
24924 if !first_frame {
24925 dispatch_reduce_frame(&rig, &frame, false);
24926 if !deferred {
24927 events.extend(consume_reduce_events(&mut rig, &mut engine));
24928 }
24929 }
24930
24931 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
24932 frame.updated_at += 1_000;
24933 for _ in 0..repeats {
24934 dispatch_reduce_frame(&rig, &frame, false);
24935 if !deferred {
24936 events.extend(consume_reduce_events(&mut rig, &mut engine));
24937 }
24938 }
24939
24940 events.extend(consume_reduce_events(&mut rig, &mut engine));
24941 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
24942 let report = rig.dispatch.translate_order_cloid(
24943 parse_http_order_to_report(
24944 &frame,
24945 &rig.registry,
24946 account_id(),
24947 UnixNanos::from(5),
24948 &rig.dispatch,
24949 )
24950 .unwrap(),
24951 );
24952 engine.reconcile_order_status_report(&report);
24953 engine.reconcile_order_status_report(&report);
24954 let reconciled = cache.borrow().order(&rig.cloid).unwrap().clone();
24955 rig.dispatch = WsDispatchState::new();
24956 rig.dispatch
24957 .seed_order_restorations(vec![reconciled.clone()]);
24958 dispatch_reduce_frame(&rig, &frame, false);
24959 let replayed = consume_reduce_events(&mut rig, &mut engine);
24960
24961 assert_eq!(events.len(), 2);
24962 assert!(matches!(events[0], OrderEventAny::Accepted(_)));
24963
24964 let OrderEventAny::Updated(update) = &events[1] else {
24965 panic!("expected quantity update");
24966 };
24967
24968 assert_eq!(update.quantity, Quantity::from("0.0001"));
24969 assert!(update.reconciliation);
24970 assert_eq!(order.status(), OrderStatus::Accepted);
24971 assert_eq!(order.quantity(), Quantity::from("0.0001"));
24972 assert_eq!(order.leaves_qty(), Quantity::from("0.0001"));
24973 assert_eq!(order.filled_qty(), Quantity::from("0.0000"));
24974 assert_eq!(report.quantity, Quantity::from("0.0001"));
24975 assert_eq!(report.filled_qty, Quantity::from("0.0000"));
24976 assert_eq!(reconciled, order);
24977 assert!(replayed.is_empty());
24978 }
24979
24980 #[rstest]
24981 #[case::frame_first(false, false, false)]
24982 #[case::trade_first(true, false, false)]
24983 #[case::reduced_then_frame(false, true, false)]
24984 #[case::reduced_then_trade(true, true, false)]
24985 #[case::deferred_frame(false, false, true)]
24986 #[case::deferred_trade(true, false, true)]
24987 fn self_trade_reduce_final_fill_ordering(
24988 #[case] trade_first: bool,
24989 #[case] reduced_open: bool,
24990 #[case] deferred: bool,
24991 #[values(false, true)] first_frame: bool,
24992 ) {
24993 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
24994 if !first_frame {
24995 dispatch_reduce_frame(&rig, &frame, false);
24996 consume_reduce_events(&mut rig, &mut engine);
24997 }
24998
24999 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
25000 frame.updated_at += 1_000;
25001 if reduced_open {
25002 dispatch_reduce_frame(&rig, &frame, false);
25003 consume_reduce_events(&mut rig, &mut engine);
25004 }
25005
25006 let mut trade = dispatcher_test_trade(&rig, true);
25007 trade.bid_id = frame.order_index;
25008 trade.bid_id_str = Some(frame.order_id.clone());
25009 trade.size = Decimal::from_str("0.0001").unwrap();
25010 trade.usd_amount = trade.size * trade.price;
25011 frame.status = LighterOrderStatus::Filled;
25012 frame.remaining_base_amount = Decimal::ZERO;
25013 frame.filled_base_amount = trade.size;
25014 frame.filled_quote_amount = trade.usd_amount;
25015 frame.updated_at += 1_000;
25016
25017 if trade_first {
25018 dispatch_lighter_trade(
25019 &trade,
25020 &rig.dispatch,
25021 &rig.emitter,
25022 &rig.registry,
25023 account_id(),
25024 trader_id(),
25025 Some(TEST_ACCOUNT_INDEX_I64),
25026 UnixNanos::from(3),
25027 );
25028 consume_reduce_events(&mut rig, &mut engine);
25029 }
25030
25031 dispatch_reduce_frame(&rig, &frame, deferred);
25032 consume_reduce_events(&mut rig, &mut engine);
25033
25034 if !trade_first {
25035 dispatch_lighter_trade(
25036 &trade,
25037 &rig.dispatch,
25038 &rig.emitter,
25039 &rig.registry,
25040 account_id(),
25041 trader_id(),
25042 Some(TEST_ACCOUNT_INDEX_I64),
25043 UnixNanos::from(4),
25044 );
25045 consume_reduce_events(&mut rig, &mut engine);
25046 }
25047
25048 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
25049
25050 assert_eq!(order.quantity(), Quantity::from("0.0001"));
25051 assert_eq!(order.filled_qty(), Quantity::from("0.0001"));
25052 assert_eq!(order.leaves_qty(), Quantity::from("0.0000"));
25053 assert_eq!(order.status(), OrderStatus::Filled);
25054 assert_eq!(
25055 order.trade_ids(),
25056 vec![&parse_lighter_trade_id(&trade).unwrap()]
25057 );
25058 }
25059
25060 #[rstest]
25061 #[case::stream(false)]
25062 #[case::report(true)]
25063 fn self_trade_reduce_cancellation_remains_canceled(#[case] report: bool) {
25064 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
25065 dispatch_reduce_frame(&rig, &frame, false);
25066 consume_reduce_events(&mut rig, &mut engine);
25067 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
25068 frame.updated_at += 1_000;
25069 dispatch_reduce_frame(&rig, &frame, false);
25070 consume_reduce_events(&mut rig, &mut engine);
25071 frame.status = LighterOrderStatus::Canceled;
25072 frame.remaining_base_amount = Decimal::ZERO;
25073 frame.updated_at += 1_000;
25074
25075 if report {
25076 let report = rig.dispatch.translate_order_cloid(
25077 parse_http_order_to_report(
25078 &frame,
25079 &rig.registry,
25080 account_id(),
25081 UnixNanos::from(5),
25082 &rig.dispatch,
25083 )
25084 .unwrap(),
25085 );
25086 engine.reconcile_order_status_report(&report);
25087 } else {
25088 dispatch_reduce_frame(&rig, &frame, false);
25089 consume_reduce_events(&mut rig, &mut engine);
25090 }
25091
25092 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
25093
25094 assert_eq!(order.status(), OrderStatus::Canceled);
25095 assert_eq!(order.filled_qty(), Quantity::from("0.0000"));
25096 assert!(order.trade_ids().is_empty());
25097 assert_eq!(
25098 order.quantity(),
25099 Quantity::from(if report { "0.0027" } else { "0.0001" })
25100 );
25101 }
25102
25103 fn reduce_order_replay() -> (
25104 DispatcherRig,
25105 Rc<RefCell<Cache>>,
25106 ExecutionEngine,
25107 LighterOrder,
25108 ) {
25109 let rig = dispatcher_rig("REDUCE-REPLAY");
25110 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
25111 frame.initial_base_amount = Decimal::from_str("0.0027").unwrap();
25112 frame.remaining_base_amount = frame.initial_base_amount;
25113 let order = OrderTestBuilder::new(OrderType::Limit)
25114 .trader_id(trader_id())
25115 .strategy_id(strategy_id())
25116 .instrument_id(rig.instrument_id)
25117 .client_order_id(rig.cloid)
25118 .side(OrderSide::Buy)
25119 .quantity(Quantity::from("0.0027"))
25120 .price(Price::from("2352.74"))
25121 .build();
25122 rig.dispatch.register_create_identity(&order).unwrap();
25123 rig.dispatch
25124 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
25125 let cache = Rc::new(RefCell::new(Cache::default()));
25126 cache
25127 .borrow_mut()
25128 .add_instrument(
25129 LIGHTER_INSTRUMENT_CACHE
25130 .get(&rig.instrument_id)
25131 .unwrap()
25132 .value()
25133 .clone(),
25134 )
25135 .unwrap();
25136 cache
25137 .borrow_mut()
25138 .add_order(order, None, Some(client_id()), false)
25139 .unwrap();
25140 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
25141 let mut engine = ExecutionEngine::new(clock, cache.clone(), None);
25142 engine.process(&OrderEventAny::Submitted(OrderSubmitted::new(
25143 trader_id(),
25144 strategy_id(),
25145 rig.instrument_id,
25146 rig.cloid,
25147 account_id(),
25148 UUID4::new(),
25149 UnixNanos::from(1),
25150 UnixNanos::from(1),
25151 )));
25152 (rig, cache, engine, frame)
25153 }
25154
25155 fn dispatch_reduce_frame(rig: &DispatcherRig, frame: &LighterOrder, deferred: bool) {
25156 if deferred {
25157 dispatch_deferred_order(rig.cloid, frame, &rig.dispatch, &rig.emitter);
25158 } else {
25159 dispatch_lighter_order(
25160 frame,
25161 &rig.dispatch,
25162 &rig.emitter,
25163 &rig.registry,
25164 account_id(),
25165 trader_id(),
25166 UnixNanos::from(3),
25167 );
25168 }
25169 }
25170
25171 fn consume_reduce_events(
25172 rig: &mut DispatcherRig,
25173 engine: &mut ExecutionEngine,
25174 ) -> Vec<OrderEventAny> {
25175 let mut orders = Vec::new();
25176
25177 for event in drain_events(&mut rig.rx) {
25178 match event {
25179 ExecutionEvent::Order(event) => {
25180 engine.process(&event);
25181 orders.push(event);
25182 }
25183 ExecutionEvent::Report(report) => engine.reconcile_execution_report(&report),
25184 other => panic!("unexpected event: {other:?}"),
25185 }
25186 }
25187
25188 orders
25189 }
25190
25191 #[rstest]
25192 #[case::ioc(LighterOrderStatus::CanceledNotEnoughLiquidity, false)]
25193 #[case::expiry(LighterOrderStatus::CanceledExpired, false)]
25194 #[case::post_only(LighterOrderStatus::CanceledPostOnly, false)]
25195 #[case::resting(LighterOrderStatus::Canceled, true)]
25196 fn terminal_waits_for_all_distinct_fills(
25197 #[case] status: LighterOrderStatus,
25198 #[case] trade_first: bool,
25199 ) {
25200 let mut rig = dispatcher_rig("TERMINAL-FILLS");
25201 register_identity(&rig);
25202 rig.dispatch.mark_accepted_emitted(rig.cloid);
25203 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
25204 let mut order = dispatcher_test_order(&rig, status);
25205 order.filled_base_amount = Decimal::from_str("0.0030").unwrap();
25206 let mut first = dispatcher_test_trade(&rig, true);
25207 first.bid_id = order.order_index;
25208 first.bid_id_str = Some(order.order_id.clone());
25209 first.size = Decimal::from_str("0.0010").unwrap();
25210 let mut second = first.clone();
25211 second.trade_id += 1;
25212 second.trade_id_str = Some(second.trade_id.to_string());
25213 second.size = Decimal::from_str("0.0020").unwrap();
25214
25215 let send_trade = |trade: &LighterTrade| {
25216 dispatch_lighter_trade(
25217 trade,
25218 &rig.dispatch,
25219 &rig.emitter,
25220 &rig.registry,
25221 account_id(),
25222 trader_id(),
25223 Some(TEST_ACCOUNT_INDEX_I64),
25224 UnixNanos::from(2),
25225 );
25226 };
25227
25228 if trade_first {
25229 send_trade(&first);
25230 }
25231
25232 dispatch_lighter_order(
25233 &order,
25234 &rig.dispatch,
25235 &rig.emitter,
25236 &rig.registry,
25237 account_id(),
25238 trader_id(),
25239 UnixNanos::from(1),
25240 );
25241 let deadline = rig.dispatch.terminals_pending.lock()[&rig.cloid].retry_at;
25242 dispatch_lighter_order(
25243 &order,
25244 &rig.dispatch,
25245 &rig.emitter,
25246 &rig.registry,
25247 account_id(),
25248 trader_id(),
25249 UnixNanos::from(3),
25250 );
25251 assert_eq!(
25252 rig.dispatch.terminals_pending.lock()[&rig.cloid].retry_at,
25253 deadline
25254 );
25255 send_trade(&first);
25256 send_trade(&first);
25257 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
25258 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
25259 let mut events = drain_events(&mut rig.rx);
25260 assert_eq!(events.len(), 1);
25261 assert!(
25262 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
25263 if fill.trade_id.as_str() == first.trade_id_str.as_deref().unwrap()
25264 && fill.last_qty == Quantity::from("0.0010"))
25265 );
25266 dispatch_lighter_trade(
25267 &second,
25268 &rig.dispatch,
25269 &rig.emitter,
25270 &rig.registry,
25271 account_id(),
25272 trader_id(),
25273 Some(TEST_ACCOUNT_INDEX_I64),
25274 UnixNanos::from(4),
25275 );
25276 events.extend(drain_events(&mut rig.rx));
25277 assert_eq!(events.len(), 3);
25278 assert!(
25279 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Filled(fill))
25280 if fill.last_qty == Quantity::from("0.0020"))
25281 );
25282
25283 let terminal_ts = match &events[2] {
25284 ExecutionEvent::Order(OrderEventAny::Canceled(event)) => {
25285 assert_ne!(status, LighterOrderStatus::CanceledExpired);
25286 event.ts_event
25287 }
25288 ExecutionEvent::Order(OrderEventAny::Expired(event)) => {
25289 assert_eq!(status, LighterOrderStatus::CanceledExpired);
25290 event.ts_event
25291 }
25292 event => panic!("expected terminal after fills, was {event:?}"),
25293 };
25294
25295 assert_eq!(terminal_ts, UnixNanos::from(1_777_941_383_900_000_000));
25296 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0030").unwrap());
25297 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25298 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
25299 }
25300
25301 #[rstest]
25302 fn terminal_after_fills_has_no_delay() {
25303 let mut rig = dispatcher_rig("TERMINAL-COVERED");
25304 register_identity(&rig);
25305 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
25306 let mut trade = dispatcher_test_trade(&rig, true);
25307 trade.bid_id = order.order_index;
25308 trade.bid_id_str = Some(order.order_id.clone());
25309 order.filled_base_amount = trade.size;
25310 dispatch_lighter_trade(
25311 &trade,
25312 &rig.dispatch,
25313 &rig.emitter,
25314 &rig.registry,
25315 account_id(),
25316 trader_id(),
25317 Some(TEST_ACCOUNT_INDEX_I64),
25318 UnixNanos::from(1),
25319 );
25320 dispatch_lighter_order(
25321 &order,
25322 &rig.dispatch,
25323 &rig.emitter,
25324 &rig.registry,
25325 account_id(),
25326 trader_id(),
25327 UnixNanos::from(2),
25328 );
25329 let events = drain_events(&mut rig.rx);
25330 assert_eq!(events.len(), 3);
25331 assert!(matches!(
25332 &events[0],
25333 ExecutionEvent::Order(OrderEventAny::Accepted(_))
25334 ));
25335 assert!(matches!(
25336 &events[1],
25337 ExecutionEvent::Order(OrderEventAny::Filled(_))
25338 ));
25339 assert!(matches!(
25340 &events[2],
25341 ExecutionEvent::Order(OrderEventAny::Canceled(_))
25342 ));
25343 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25344 }
25345
25346 #[rstest]
25347 #[case::modify_hold(true)]
25348 #[case::fill_hold(false)]
25349 #[tokio::test]
25350 async fn terminal_preserves_later_holds_and_emits_once(#[case] modify_pending: bool) {
25351 let mut rig = dispatcher_rig("TERMINAL-MODIFY");
25352 register_identity(&rig);
25353 rig.dispatch.mark_accepted_emitted(rig.cloid);
25354 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
25355 let mut trade = dispatcher_test_trade(&rig, true);
25356 trade.bid_id = order.order_index;
25357 trade.bid_id_str = Some(order.order_id.clone());
25358 order.filled_base_amount = trade.size;
25359 dispatch_lighter_order(
25360 &order,
25361 &rig.dispatch,
25362 &rig.emitter,
25363 &rig.registry,
25364 account_id(),
25365 trader_id(),
25366 UnixNanos::from(1),
25367 );
25368
25369 let command = ModifyOrder::new(
25370 trader_id(),
25371 Some(client_id()),
25372 strategy_id(),
25373 rig.instrument_id,
25374 rig.cloid,
25375 Some(VenueOrderId::new(&order.order_id)),
25376 None,
25377 Some(Price::from("2350.00")),
25378 None,
25379 UUID4::new(),
25380 UnixNanos::from(2),
25381 None,
25382 None,
25383 );
25384 assert!(rig.dispatch.set_pending_modify_action(&command));
25385 dispatch_lighter_order(
25386 &order,
25387 &rig.dispatch,
25388 &rig.emitter,
25389 &rig.registry,
25390 account_id(),
25391 trader_id(),
25392 UnixNanos::from(3),
25393 );
25394 dispatch_lighter_trade(
25395 &trade,
25396 &rig.dispatch,
25397 &rig.emitter,
25398 &rig.registry,
25399 account_id(),
25400 trader_id(),
25401 Some(TEST_ACCOUNT_INDEX_I64),
25402 UnixNanos::from(4),
25403 );
25404
25405 if !modify_pending {
25406 assert!(
25407 rig.dispatch
25408 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
25409 );
25410 }
25411
25412 let (http_client, hits, server) = terminal_trades_server(&order, Vec::new()).await;
25413 let (client, _, _) = create_execution_client();
25414 let mut context = acked_probe_context(&client);
25415 context.http_client = http_client;
25416 context.dispatch = rig.dispatch.clone();
25417 context.registry = rig.registry.clone();
25418 context.emitter = rig.emitter.clone();
25419 for _ in 0..=ACKED_ORDER_PROBE_ATTEMPTS {
25420 recover_terminals(&context, true).await;
25421 }
25422
25423 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
25424 {
25425 let mut events = rig.dispatch.order_events.lock();
25426 release_terminal(rig.cloid, true, &rig.dispatch, &rig.emitter, &events);
25427 assert!(drain_events(&mut rig.rx).is_empty());
25428 assert!(
25429 rig.dispatch
25430 .terminals_pending
25431 .lock()
25432 .contains_key(&rig.cloid)
25433 );
25434 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
25435
25436 if modify_pending {
25437 assert!(
25438 rig.dispatch
25439 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
25440 );
25441 }
25442
25443 let deferred = events.remove(&rig.cloid).unwrap();
25444 flush_modify_events(
25445 rig.cloid,
25446 deferred,
25447 &rig.dispatch,
25448 &rig.emitter,
25449 &mut events,
25450 );
25451 }
25452
25453 let events = drain_events(&mut rig.rx);
25454 assert_eq!(events.len(), 2);
25455 assert!(
25456 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.trade_id == parse_lighter_trade_id(&trade).unwrap())
25457 );
25458 assert!(matches!(
25459 &events[1],
25460 ExecutionEvent::Order(OrderEventAny::Canceled(_))
25461 ));
25462 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25463 assert!(rig.dispatch.order_events.lock().is_empty());
25464 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
25465 server.abort();
25466 let _ = server.await;
25467 }
25468
25469 async fn terminal_trades_server(
25470 order: &LighterOrder,
25471 trades: Vec<LighterTrade>,
25472 ) -> (
25473 LighterHttpClient,
25474 Arc<AtomicUsize>,
25475 tokio::task::JoinHandle<()>,
25476 ) {
25477 let hits = Arc::new(AtomicUsize::new(0));
25478 let count = hits.clone();
25479 let order_index = order.order_index.to_string();
25480 let market_index = order.market_index.to_string();
25481
25482 let raw = order.clone();
25483
25484 let app = Router::new()
25485 .route(
25486 "/api/v1/accountActiveOrders",
25487 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
25488 )
25489 .route(
25490 "/api/v1/accountInactiveOrders",
25491 get(move || {
25492 let raw = raw.clone();
25493 async move { axum::Json(serde_json::json!({"code":200,"orders":[raw]})) }
25494 }),
25495 )
25496 .route(
25497 "/api/v1/trades",
25498 get(
25499 move |axum::extract::Query(query): axum::extract::Query<
25500 BTreeMap<String, String>,
25501 >| {
25502 let trades = trades.clone();
25503 let count = count.clone();
25504 let order_index = order_index.clone();
25505 let market_index = market_index.clone();
25506 async move {
25507 count.fetch_add(1, Ordering::Relaxed);
25508 assert_eq!(query.get("order_index"), Some(&order_index));
25509 assert_eq!(query.get("market_id"), Some(&market_index));
25510 assert_eq!(
25511 query.get("account_index"),
25512 Some(&TEST_ACCOUNT_INDEX_I64.to_string())
25513 );
25514 assert_eq!(query.get("sort_by").map(String::as_str), Some("timestamp"));
25515 assert_eq!(query.get("sort_dir").map(String::as_str), Some("desc"));
25516 assert_eq!(query.get("limit").map(String::as_str), Some("100"));
25517 assert!(!query.contains_key("from"));
25518 assert!(query.get("auth").is_some_and(|auth| !auth.is_empty()));
25519 axum::Json(serde_json::json!({"code":200,"trades":trades}))
25520 }
25521 },
25522 ),
25523 );
25524
25525 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
25526 let addr = listener.local_addr().unwrap();
25527 let server = tokio::spawn(async move {
25528 axum::serve(listener, app).await.unwrap();
25529 });
25530
25531 let http_client = LighterHttpClient::new(
25532 LighterEnvironment::Testnet,
25533 Some(format!("http://{addr}")),
25534 1,
25535 None,
25536 )
25537 .unwrap();
25538 (http_client, hits, server)
25539 }
25540
25541 #[rstest]
25542 #[case::recovered(true, false)]
25543 #[case::exhausted(false, false)]
25544 #[case::foreign_order(false, true)]
25545 #[tokio::test]
25546 async fn terminal_recovery_is_order_scoped_and_bounded(
25547 #[case] recovered: bool,
25548 #[case] foreign_order: bool,
25549 ) {
25550 let mut rig = dispatcher_rig("TERMINAL-RECOVERY");
25551 register_identity(&rig);
25552 rig.dispatch.mark_accepted_emitted(rig.cloid);
25553 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledNotEnoughLiquidity);
25554 let mut trade = dispatcher_test_trade(&rig, true);
25555 trade.bid_id = order.order_index;
25556 trade.bid_id_str = Some(order.order_id.clone());
25557 trade.size = Decimal::from_str("0.0025").unwrap();
25558 order.filled_base_amount = trade.size;
25559 dispatch_lighter_order(
25560 &order,
25561 &rig.dispatch,
25562 &rig.emitter,
25563 &rig.registry,
25564 account_id(),
25565 trader_id(),
25566 UnixNanos::from(1),
25567 );
25568
25569 let trades = if foreign_order {
25570 trade.bid_id += 1;
25571 trade.bid_id_str = Some(trade.bid_id.to_string());
25572 vec![trade]
25573 } else if recovered {
25574 vec![trade.clone(), trade]
25575 } else {
25576 Vec::new()
25577 };
25578
25579 let (http_client, hits, server) = terminal_trades_server(&order, trades).await;
25580 let (client, _, _) = create_execution_client();
25581 let mut context = acked_probe_context(&client);
25582 context.http_client = http_client;
25583 context.dispatch = rig.dispatch.clone();
25584 context.registry = rig.registry.clone();
25585 context.emitter = rig.emitter.clone();
25586 recover_terminals(&context, false).await;
25587 assert_eq!(hits.load(Ordering::Relaxed), 0, "deadline must be honored");
25588
25589 if foreign_order {
25590 let error = recover_terminal_trades(order.market_index, order.order_index, &context)
25591 .await
25592 .unwrap_err();
25593 assert_eq!(
25594 error.to_string(),
25595 "Lighter terminal recovery returned a trade outside the requested order"
25596 );
25597 hits.store(0, Ordering::Relaxed);
25598 }
25599
25600 let attempts = if recovered {
25601 1
25602 } else {
25603 ACKED_ORDER_PROBE_ATTEMPTS
25604 };
25605
25606 for attempt in 1..=attempts {
25607 recover_terminals(&context, true).await;
25608 assert_eq!(hits.load(Ordering::Relaxed), attempt);
25609
25610 if attempt < attempts {
25611 assert!(drain_events(&mut rig.rx).is_empty());
25612 assert_eq!(
25613 rig.dispatch.terminals_pending.lock()[&rig.cloid].attempts,
25614 attempt
25615 );
25616 }
25617 }
25618
25619 let events = drain_events(&mut rig.rx);
25620 assert_eq!(events.len(), if recovered { 2 } else { 1 });
25621
25622 if recovered {
25623 assert!(
25624 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
25625 if fill.last_qty == Quantity::from("0.0025"))
25626 );
25627 }
25628
25629 assert!(matches!(
25630 events.last(),
25631 Some(ExecutionEvent::Order(OrderEventAny::Canceled(_)))
25632 ));
25633 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25634 recover_terminals(&context, true).await;
25635 assert!(drain_events(&mut rig.rx).is_empty());
25636 assert_eq!(hits.load(Ordering::Relaxed), attempts);
25637 server.abort();
25638 let _ = server.await;
25639 }
25640
25641 #[rstest]
25642 #[case::reconnect(false)]
25643 #[case::stop(true)]
25644 #[tokio::test]
25645 async fn terminal_worker_recovers_immediately_on_lifecycle(#[case] stop: bool) {
25646 let mut rig = dispatcher_rig("TERMINAL-LIFECYCLE");
25647 register_identity(&rig);
25648 rig.dispatch.mark_accepted_emitted(rig.cloid);
25649 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
25650 let mut trade = dispatcher_test_trade(&rig, true);
25651 trade.bid_id = order.order_index;
25652 trade.bid_id_str = Some(order.order_id.clone());
25653 trade.size = Decimal::from_str("0.0025").unwrap();
25654 order.filled_base_amount = trade.size;
25655 dispatch_lighter_order(
25656 &order,
25657 &rig.dispatch,
25658 &rig.emitter,
25659 &rig.registry,
25660 account_id(),
25661 trader_id(),
25662 UnixNanos::from(1),
25663 );
25664 rig.dispatch
25665 .terminals_pending
25666 .lock()
25667 .get_mut(&rig.cloid)
25668 .unwrap()
25669 .retry_at = tokio::time::Instant::now() + Duration::from_secs(3600);
25670 let (http_client, hits, server) = terminal_trades_server(&order, vec![trade]).await;
25671 let (mut client, _, _) = create_execution_client();
25672 client.http_client = http_client;
25673 client.dispatch = rig.dispatch.clone();
25674 client.registry = rig.registry.clone();
25675 client.emitter = rig.emitter.clone();
25676 let context = acked_probe_context(&client);
25677 client
25678 .terminal_recovery_handle
25679 .spawn(run_terminal_recovery(
25680 context,
25681 client.terminal_producers_stopped.clone(),
25682 ))
25683 .unwrap();
25684
25685 if stop {
25686 client.stop().unwrap();
25687 } else {
25688 client.dispatch.terminals_recover.notify_one();
25689 }
25690
25691 if !stop {
25692 wait_until_async(
25693 || async { rig.dispatch.terminals_pending.lock().is_empty() },
25694 Duration::from_secs(2),
25695 )
25696 .await;
25697 client.stop().unwrap();
25698 }
25699
25700 client.finish_session_shutdown().await.unwrap();
25701 let events = drain_events(&mut rig.rx);
25702 assert_eq!(hits.load(Ordering::Relaxed), 1);
25703 assert_eq!(events.len(), 2);
25704 assert!(
25705 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
25706 if fill.last_qty == Quantity::from("0.0025"))
25707 );
25708 assert!(
25709 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Expired(event))
25710 if event.ts_event == UnixNanos::from(1_777_941_383_900_000_000))
25711 );
25712 assert!(client.terminal_recovery_handle.is_none());
25713 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25714 server.abort();
25715 let _ = server.await;
25716 }
25717
25718 #[rstest]
25719 #[case::restored(true, false)]
25720 #[case::reconciled(false, false)]
25721 #[case::declined(false, true)]
25722 fn terminal_preserves_applied_fills_and_post_only_state(
25723 #[case] restored: bool,
25724 #[case] declined: bool,
25725 ) {
25726 let mut rig = dispatcher_rig("TERMINAL-BASELINE");
25727 register_identity(&rig);
25728 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
25729 let instrument = LIGHTER_INSTRUMENT_CACHE
25730 .get(&rig.instrument_id)
25731 .unwrap()
25732 .value()
25733 .clone();
25734 let mut order = OrderTestBuilder::new(OrderType::Limit)
25735 .trader_id(trader_id())
25736 .strategy_id(strategy_id())
25737 .instrument_id(rig.instrument_id)
25738 .client_order_id(rig.cloid)
25739 .side(OrderSide::Buy)
25740 .quantity(Quantity::from("0.0050"))
25741 .price(Price::from("2352.74"))
25742 .post_only(true)
25743 .build();
25744 dispatch_lighter_order(
25745 &frame,
25746 &rig.dispatch,
25747 &rig.emitter,
25748 &rig.registry,
25749 account_id(),
25750 trader_id(),
25751 UnixNanos::from(1),
25752 );
25753 frame.price = Decimal::from_str("2353.00").unwrap();
25754 dispatch_lighter_order(
25755 &frame,
25756 &rig.dispatch,
25757 &rig.emitter,
25758 &rig.registry,
25759 account_id(),
25760 trader_id(),
25761 UnixNanos::from(2),
25762 );
25763
25764 for event in drain_events(&mut rig.rx) {
25765 let ExecutionEvent::Order(event) = event else {
25766 panic!("expected order event");
25767 };
25768
25769 order.apply(event).unwrap();
25770 }
25771
25772 let mut first = dispatcher_test_trade(&rig, true);
25773 first.bid_id = frame.order_index;
25774 first.bid_id_str = Some(frame.order_id.clone());
25775 first.size = Decimal::from_str("0.0010").unwrap();
25776 let fill = parse_lighter_order_filled(
25777 &first,
25778 &instrument,
25779 &rig.dispatch.order_identity(&rig.cloid).unwrap(),
25780 rig.cloid,
25781 account_id(),
25782 trader_id(),
25783 TEST_ACCOUNT_INDEX_I64,
25784 UnixNanos::from(3),
25785 )
25786 .unwrap()
25787 .unwrap();
25788 let (mut client, cache, _) = create_execution_client();
25789 client.dispatch = rig.dispatch.clone();
25790 client.emitter = rig.emitter.clone();
25791 client.subscribe_applied_fills();
25792 cache
25793 .borrow_mut()
25794 .add_order(order.clone(), None, Some(client_id()), false)
25795 .unwrap();
25796
25797 if !restored {
25798 let mut report = parse_ws_fill_report(
25799 &first,
25800 TEST_ACCOUNT_INDEX_I64,
25801 &instrument,
25802 account_id(),
25803 UnixNanos::from(3),
25804 )
25805 .unwrap()
25806 .unwrap();
25807 report.client_order_id = Some(rig.cloid);
25808
25809 let sweep = finish_fill_reports(
25810 &client.core,
25811 &rig.dispatch,
25812 BTreeSet::new(),
25813 Ok(FillSweep {
25814 reports: vec![report.clone()],
25815 covers_window: true,
25816 withheld_instruments: AHashSet::new(),
25817 }),
25818 )
25819 .unwrap();
25820
25821 assert_eq!(
25822 rig.dispatch
25823 .order_identity(&rig.cloid)
25824 .unwrap()
25825 .filled_qty(),
25826 Decimal::ZERO
25827 );
25828 assert_eq!(sweep.reports, vec![report.clone()]);
25829 assert!(sweep.covers_window);
25830 assert!(sweep.withheld_instruments.is_empty());
25831 assert!(!order_shape_pending(
25832 &client.core,
25833 &rig.dispatch,
25834 &rig.cloid
25835 ));
25836 assert_eq!(rig.dispatch.trade_source(fill.trade_id), None);
25837
25838 let repeated = finish_fill_reports(
25839 &client.core,
25840 &rig.dispatch,
25841 BTreeSet::new(),
25842 Ok(FillSweep {
25843 reports: vec![report.clone()],
25844 covers_window: true,
25845 withheld_instruments: AHashSet::new(),
25846 }),
25847 )
25848 .unwrap();
25849
25850 assert_eq!(repeated.reports, vec![report]);
25851 }
25852
25853 msgbus::publish_order_event(
25854 "events.order.S-001".into(),
25855 &OrderEventAny::Filled(fill.clone()),
25856 );
25857 assert_eq!(
25858 rig.dispatch
25859 .order_identity(&rig.cloid)
25860 .unwrap()
25861 .filled_qty(),
25862 Decimal::ZERO
25863 );
25864
25865 if !declined {
25866 cache
25867 .borrow_mut()
25868 .update_order(&OrderEventAny::Filled(fill.clone()))
25869 .unwrap();
25870 msgbus::publish_order_event(
25871 "events.order.S-001".into(),
25872 &OrderEventAny::Filled(fill.clone()),
25873 );
25874 order.apply(OrderEventAny::Filled(fill.clone())).unwrap();
25875 }
25876
25877 if restored {
25878 rig.dispatch = WsDispatchState::new();
25879 rig.dispatch.seed_order_restorations(vec![order.clone()]);
25880 }
25881
25882 if !restored {
25883 assert_eq!(
25884 rig.dispatch
25885 .order_identity(&rig.cloid)
25886 .unwrap()
25887 .filled_qty(),
25888 if declined { Decimal::ZERO } else { first.size }
25889 );
25890 }
25891
25892 frame.status = LighterOrderStatus::CanceledPostOnly;
25893 frame.filled_base_amount = Decimal::from_str("0.0030").unwrap();
25894 dispatch_lighter_order(
25895 &frame,
25896 &rig.dispatch,
25897 &rig.emitter,
25898 &rig.registry,
25899 account_id(),
25900 trader_id(),
25901 UnixNanos::from(4),
25902 );
25903 let report =
25904 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(4))
25905 .unwrap()
25906 .with_client_order_id(rig.cloid);
25907 assert!(!order_report_ready(
25908 &client.core,
25909 &rig.dispatch,
25910 &report,
25911 &[]
25912 ));
25913 assert_eq!(
25914 execution_instruments_pending(&client.core, &rig.dispatch),
25915 AHashSet::from_iter([rig.instrument_id])
25916 );
25917 dispatch_lighter_trade(
25918 &first,
25919 &rig.dispatch,
25920 &rig.emitter,
25921 &rig.registry,
25922 account_id(),
25923 trader_id(),
25924 Some(TEST_ACCOUNT_INDEX_I64),
25925 UnixNanos::from(5),
25926 );
25927 let recovered = drain_events(&mut rig.rx);
25928
25929 if declined {
25930 assert_eq!(recovered.len(), 1);
25931
25932 let ExecutionEvent::Order(OrderEventAny::Filled(recovered_fill)) = &recovered[0] else {
25933 panic!("discarded HTTP report must remain recoverable as a real fill");
25934 };
25935
25936 assert_eq!(recovered_fill.trade_id, fill.trade_id);
25937 assert_eq!(recovered_fill.last_qty, fill.last_qty);
25938 cache
25939 .borrow_mut()
25940 .update_order(&OrderEventAny::Filled(recovered_fill.clone()))
25941 .unwrap();
25942 msgbus::publish_order_event(
25943 "events.order.S-001".into(),
25944 &OrderEventAny::Filled(recovered_fill.clone()),
25945 );
25946 order
25947 .apply(OrderEventAny::Filled(recovered_fill.clone()))
25948 .unwrap();
25949 } else {
25950 assert!(
25951 recovered.is_empty(),
25952 "historical fill must not repeat or release terminal"
25953 );
25954 }
25955
25956 let mut second = first;
25957 second.trade_id += 1;
25958 second.trade_id_str = Some(second.trade_id.to_string());
25959 second.size = Decimal::from_str("0.0020").unwrap();
25960 dispatch_lighter_trade(
25961 &second,
25962 &rig.dispatch,
25963 &rig.emitter,
25964 &rig.registry,
25965 account_id(),
25966 trader_id(),
25967 Some(TEST_ACCOUNT_INDEX_I64),
25968 UnixNanos::from(6),
25969 );
25970 let events = drain_events(&mut rig.rx);
25971 assert_eq!(events.len(), 2);
25972 assert!(matches!(
25973 &events[0],
25974 ExecutionEvent::Order(OrderEventAny::Filled(_))
25975 ));
25976 assert!(matches!(
25977 &events[1],
25978 ExecutionEvent::Order(OrderEventAny::Canceled(_))
25979 ));
25980
25981 for event in events {
25982 let ExecutionEvent::Order(event) = event else {
25983 panic!("expected order event");
25984 };
25985
25986 order.apply(event).unwrap();
25987 }
25988
25989 assert_eq!(order.status(), OrderStatus::Canceled);
25990 assert_eq!(order.filled_qty(), Quantity::from("0.0030"));
25991 assert_eq!(order.price(), Some(Price::from("2353.00")));
25992 assert_eq!(order.trade_ids().len(), 2);
25993 assert_eq!(
25994 rig.dispatch
25995 .order_identity(&rig.cloid)
25996 .unwrap()
25997 .filled_qty(),
25998 Decimal::from_str("0.0030").unwrap()
25999 );
26000 }
26001
26002 #[rstest]
26003 #[case::paged(false)]
26004 #[case::repeated_cursor(true)]
26005 #[tokio::test]
26006 async fn terminal_recovery_paginates_without_duplicate_fills(#[case] repeat_cursor: bool) {
26007 let mut rig = dispatcher_rig("TERMINAL-PAGES");
26008 register_identity(&rig);
26009 rig.dispatch.mark_accepted_emitted(rig.cloid);
26010 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
26011 order.filled_base_amount = Decimal::from_str("0.0030").unwrap();
26012 let mut first = dispatcher_test_trade(&rig, true);
26013 first.bid_id = order.order_index;
26014 first.bid_id_str = Some(order.order_id.clone());
26015 first.size = Decimal::from_str("0.0010").unwrap();
26016 let mut second = first.clone();
26017 second.trade_id += 1;
26018 second.trade_id_str = Some(second.trade_id.to_string());
26019 second.size = Decimal::from_str("0.0020").unwrap();
26020 let hits = Arc::new(AtomicUsize::new(0));
26021 let count = hits.clone();
26022
26023 let app = Router::new().route("/api/v1/trades", get(move |axum::extract::Query(query): axum::extract::Query<BTreeMap<String, String>>| {
26024 let first = first.clone();
26025 let second = second.clone();
26026 let count = count.clone();
26027 async move {
26028 count.fetch_add(1, Ordering::Relaxed);
26029
26030 if let Some(cursor) = query.get("cursor") {
26031 assert_eq!(cursor, "older");
26032 axum::Json(serde_json::json!({"code":200,"trades":[first],"next_cursor":if repeat_cursor { Some("older") } else { None }}))
26033 } else {
26034 axum::Json(serde_json::json!({"code":200,"trades":[second],"next_cursor":"older"}))
26035 }
26036 }
26037 }));
26038
26039 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
26040 let addr = listener.local_addr().unwrap();
26041 let server = tokio::spawn(async move {
26042 axum::serve(listener, app).await.unwrap();
26043 });
26044
26045 let (client, _, _) = create_execution_client();
26046 let mut context = acked_probe_context(&client);
26047 context.http_client = LighterHttpClient::new(
26048 LighterEnvironment::Testnet,
26049 Some(format!("http://{addr}")),
26050 1,
26051 None,
26052 )
26053 .unwrap();
26054 context.dispatch = rig.dispatch.clone();
26055 context.registry = rig.registry.clone();
26056 context.emitter = rig.emitter.clone();
26057 dispatch_lighter_order(
26058 &order,
26059 &rig.dispatch,
26060 &rig.emitter,
26061 &rig.registry,
26062 account_id(),
26063 trader_id(),
26064 UnixNanos::from(1),
26065 );
26066 let result = recover_terminal_trades(order.market_index, order.order_index, &context).await;
26067 assert_eq!(hits.load(Ordering::Relaxed), 2);
26068
26069 if repeat_cursor {
26070 assert_eq!(
26071 result.unwrap_err().to_string(),
26072 "Lighter terminal recovery repeated cursor"
26073 );
26074 assert!(drain_events(&mut rig.rx).is_empty());
26075 assert!(
26076 rig.dispatch
26077 .terminals_pending
26078 .lock()
26079 .contains_key(&rig.cloid)
26080 );
26081 } else {
26082 result.unwrap();
26083 let events = drain_events(&mut rig.rx);
26084 assert_eq!(events.len(), 3);
26085 assert!(
26086 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.last_qty == Quantity::from("0.0010"))
26087 );
26088 assert!(
26089 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.last_qty == Quantity::from("0.0020"))
26090 );
26091 assert!(matches!(
26092 &events[2],
26093 ExecutionEvent::Order(OrderEventAny::Canceled(_))
26094 ));
26095 assert!(rig.dispatch.terminals_pending.lock().is_empty());
26096 }
26097
26098 server.abort();
26099 let _ = server.await;
26100 }
26101
26102 #[rstest]
26103 #[case::normal(false, 0)]
26104 #[case::interrupted_shutdown(true, 0)]
26105 #[case::interrupted_lookup(false, 1)]
26106 #[case::repeated_reconnect(false, ACKED_ORDER_PROBE_ATTEMPTS)]
26107 #[tokio::test]
26108 async fn terminal_shutdown_bounds_unresponsive_http(
26109 #[case] interrupted: bool,
26110 #[case] reconnects: usize,
26111 ) {
26112 let mut rig = dispatcher_rig("TERMINAL-TIMEOUT");
26113 register_identity(&rig);
26114 rig.dispatch.mark_accepted_emitted(rig.cloid);
26115 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
26116 order.filled_base_amount = Decimal::from_str("0.0025").unwrap();
26117 dispatch_lighter_order(
26118 &order,
26119 &rig.dispatch,
26120 &rig.emitter,
26121 &rig.registry,
26122 account_id(),
26123 trader_id(),
26124 UnixNanos::from(1),
26125 );
26126 let hits = Arc::new(AtomicUsize::new(0));
26127 let count = hits.clone();
26128 let app = Router::new().route(
26129 "/api/v1/trades",
26130 get(move || {
26131 count.fetch_add(1, Ordering::Relaxed);
26132 std::future::pending::<String>()
26133 }),
26134 );
26135
26136 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
26137 let addr = listener.local_addr().unwrap();
26138 let server = tokio::spawn(async move {
26139 axum::serve(listener, app).await.unwrap();
26140 });
26141
26142 let (mut client, _, _) = create_execution_client();
26143 client.http_client = LighterHttpClient::new(
26144 LighterEnvironment::Testnet,
26145 Some(format!("http://{addr}")),
26146 60,
26147 None,
26148 )
26149 .unwrap();
26150 client.dispatch = rig.dispatch.clone();
26151 client.registry = rig.registry.clone();
26152 client.emitter = rig.emitter.clone();
26153 let context = acked_probe_context(&client);
26154 client
26155 .terminal_recovery_handle
26156 .spawn(run_terminal_recovery(
26157 context,
26158 client.terminal_producers_stopped.clone(),
26159 ))
26160 .unwrap();
26161
26162 for expected in 1..=reconnects {
26163 rig.dispatch.terminals_recover.notify_one();
26164 wait_until_async(
26165 || async { hits.load(Ordering::Relaxed) == expected },
26166 Duration::from_secs(2),
26167 )
26168 .await;
26169 }
26170
26171 client.stop().unwrap();
26172
26173 if interrupted {
26174 {
26175 let shutdown = client.finish_session_shutdown();
26176 tokio::pin!(shutdown);
26177 tokio::select! {
26178 result = &mut shutdown => panic!("shutdown completed before blocked recovery: {result:?}"),
26179 () = wait_until_async(|| async { hits.load(Ordering::Relaxed) == 1 }, Duration::from_secs(2)) => {},
26180 }
26181 }
26182
26183 assert!(client.terminal_recovery_handle.is_some());
26184 assert!(client.terminal_producers_stopped.is_cancelled());
26185 }
26186
26187 tokio::time::timeout(
26188 TERMINAL_SHUTDOWN_TIMEOUT + DISCONNECT_TIMEOUT,
26189 client.finish_session_shutdown(),
26190 )
26191 .await
26192 .unwrap()
26193 .unwrap();
26194 let events = drain_events(&mut rig.rx);
26195 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
26196 assert_eq!(events.len(), 1);
26197 assert!(
26198 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Expired(event))
26199 if event.client_order_id == rig.cloid)
26200 );
26201 assert!(rig.dispatch.terminals_pending.lock().is_empty());
26202 assert!(client.terminal_recovery_handle.is_none());
26203 server.abort();
26204 let _ = server.await;
26205 }
26206
26207 #[rstest]
26208 #[case::query_terminal(0, 0)]
26209 #[case::query_deferred(0, 1)]
26210 #[case::cancel_terminal(1, 0)]
26211 #[case::cancel_deferred(1, 1)]
26212 #[case::query_partial(0, 2)]
26213 #[case::cancel_partial(1, 2)]
26214 #[case::create_partial(2, 2)]
26215 #[case::query_queued_fill(0, 3)]
26216 #[case::cancel_queued_fill(1, 3)]
26217 #[case::create_queued_fill(2, 3)]
26218 #[tokio::test]
26219 async fn terminal_blocks_http_order_publication(#[case] route: u8, #[case] phase: u8) {
26220 let (mut client, cache, mut rx) = create_execution_client();
26221 let instrument = register_test_instrument(&client, &cache);
26222 let id = ClientOrderId::from("TERMINAL-HTTP-BARRIER");
26223 let venue = VenueOrderId::from("123");
26224 let mut factory = test_order_factory();
26225 let cached = test_limit_order(&mut factory, instrument, id.as_str());
26226 let index = client.dispatch.register_create_identity(&cached).unwrap();
26227 client
26228 .dispatch
26229 .mark_order_submission(&id, TEST_SUBMISSION_NONCE);
26230 cache_accepted_order(&cache, cached, venue, Some(client_id()));
26231 client.dispatch.mark_accepted_emitted(id);
26232 let mut order = reconciliation_raw_order(index, venue, LighterOrderStatus::Canceled);
26233 order.filled_base_amount = Decimal::from_str("0.0025").unwrap();
26234 if phase >= 2 {
26235 order.status = LighterOrderStatus::Open;
26236 order.remaining_base_amount = order.initial_base_amount - order.filled_base_amount;
26237
26238 if phase == 3 {
26239 let mut report =
26240 reconciliation_fill_report(instrument, index, venue, "QUEUED-REAL-FILL");
26241 report.client_order_id = Some(id);
26242 report.last_qty = Quantity::from("0.0025");
26243 let identity = client.dispatch.order_identity(&id).unwrap();
26244 let cached = cache.borrow().order(&id).unwrap().clone();
26245 let inst = cache.borrow().instrument(&instrument).unwrap().clone();
26246
26247 let OrderEventAny::Filled(fill) = TestOrderEventStubs::filled(
26248 &cached,
26249 &inst,
26250 Some(report.trade_id),
26251 None,
26252 Some(report.last_px),
26253 Some(report.last_qty),
26254 None,
26255 Some(report.commission),
26256 None,
26257 Some(account_id()),
26258 ) else {
26259 panic!("expected fill");
26260 };
26261
26262 assert!(!identity.has_publication());
26263 dispatch_lighter_fill(
26264 fill,
26265 &client.emitter,
26266 &client.dispatch,
26267 &client.dispatch.order_events.lock(),
26268 );
26269 }
26270 } else {
26271 dispatch_lighter_order(
26272 &order,
26273 &client.dispatch,
26274 &client.emitter,
26275 &client.registry,
26276 account_id(),
26277 trader_id(),
26278 UnixNanos::from(1),
26279 );
26280 assert!(client.dispatch.terminals_pending.lock().contains_key(&id));
26281
26282 if phase == 1 {
26283 client.dispatch.terminals_pending.lock().remove(&id);
26284 client
26285 .dispatch
26286 .order_events
26287 .lock()
26288 .entry(id)
26289 .or_default()
26290 .store_order(&order);
26291 }
26292 }
26293
26294 let app = Router::new()
26295 .route(
26296 "/api/v1/accountActiveOrders",
26297 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
26298 )
26299 .route(
26300 "/api/v1/accountInactiveOrders",
26301 get(move || {
26302 let order = order.clone();
26303 async move { axum::Json(serde_json::json!({"code":200,"orders":[order]})) }
26304 }),
26305 );
26306
26307 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
26308 let addr = listener.local_addr().unwrap();
26309 let server = tokio::spawn(async move {
26310 axum::serve(listener, app).await.unwrap();
26311 });
26312
26313 client.http_client = LighterHttpClient::new(
26314 LighterEnvironment::Testnet,
26315 Some(format!("http://{addr}")),
26316 1,
26317 None,
26318 )
26319 .unwrap();
26320
26321 if route == 1 {
26322 enqueue_cancel(&client, instrument, id, venue, 12);
26323 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
26324 assert!(
26325 !recover_acked_action_report(&pending, &acked_probe_context(&client))
26326 .await
26327 .unwrap()
26328 );
26329 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((0, 12)));
26330 } else if route == 2 {
26331 let cached = cache.borrow().order(&id).unwrap().clone();
26332
26333 let probe = AckedOrderProbe {
26334 order: Box::new(cached),
26335 client_order_index: index,
26336 connection_epoch: 0,
26337 nonce: TEST_SUBMISSION_NONCE,
26338 api_key_index: TEST_API_KEY_INDEX,
26339 tx_hash: "create-probe".to_string(),
26340 };
26341
26342 assert!(!probe_acked_create_report(&probe, &acked_probe_context(&client), 1).await);
26343 } else {
26344 client
26345 .query_order(QueryOrder::new(
26346 trader_id(),
26347 Some(client_id()),
26348 strategy_id(),
26349 instrument,
26350 id,
26351 Some(venue),
26352 UUID4::new(),
26353 UnixNanos::from(2),
26354 None,
26355 None,
26356 ))
26357 .unwrap();
26358 client.pending_tasks.begin_shutdown();
26359 client
26360 .pending_tasks
26361 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
26362 .await
26363 .unwrap();
26364 }
26365
26366 let events = drain_events(&mut rx);
26367 assert_eq!(events.len(), usize::from(phase == 3));
26368 assert_eq!(
26369 cache.borrow().order(&id).unwrap().filled_qty(),
26370 Quantity::from("0.0000")
26371 );
26372
26373 if phase == 3 {
26374 assert!(matches!(
26375 &events[0],
26376 ExecutionEvent::Order(OrderEventAny::Filled(_))
26377 ));
26378 }
26379
26380 assert!(client.dispatch.order_identities.contains_key(&id));
26381 server.abort();
26382 let _ = server.await;
26383 }
26384
26385 #[rstest]
26386 #[case::normal(false, false)]
26387 #[case::interrupted(true, false)]
26388 #[case::incomplete(false, true)]
26389 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
26390 async fn terminal_shutdown_drains_after_producer_drop(
26391 #[case] interrupted: bool,
26392 #[case] incomplete: bool,
26393 ) {
26394 let mut rig = dispatcher_rig("TERMINAL-LATE-PRODUCER");
26395 register_identity(&rig);
26396 rig.dispatch.mark_accepted_emitted(rig.cloid);
26397 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
26398 let mut trade = dispatcher_test_trade(&rig, true);
26399 trade.bid_id = order.order_index;
26400 trade.bid_id_str = Some(order.order_id.clone());
26401 order.filled_base_amount = trade.size;
26402 let (http_client, hits, server) = terminal_trades_server(&order, vec![trade]).await;
26403 let (mut client, _, _) = create_execution_client();
26404 client.http_client = http_client;
26405 client.dispatch = rig.dispatch.clone();
26406 client.registry = rig.registry.clone();
26407 client.emitter = rig.emitter.clone();
26408 client.subscribe_applied_fills();
26409 let context = acked_probe_context(&client);
26410 client
26411 .terminal_recovery_handle
26412 .spawn(run_terminal_recovery(
26413 context.clone(),
26414 client.terminal_producers_stopped.clone(),
26415 ))
26416 .unwrap();
26417 let (ready, started) = tokio::sync::oneshot::channel();
26418 let (release, released) = std::sync::mpsc::sync_channel::<()>(1);
26419
26420 client
26421 .pending_tasks
26422 .spawn(async move {
26423 let _guard = TaskGroupGuard::new(&[], move || {
26424 dispatch_lighter_order(
26425 &order,
26426 &context.dispatch,
26427 &context.emitter,
26428 &context.registry,
26429 account_id(),
26430 trader_id(),
26431 UnixNanos::from(2),
26432 );
26433 });
26434
26435 ready.send(()).unwrap();
26436
26437 if incomplete {
26438 released.recv().unwrap();
26439 } else {
26440 std::future::pending::<()>().await;
26441 }
26442 })
26443 .unwrap();
26444
26445 started.await.unwrap();
26446 client.stop().unwrap();
26447
26448 if interrupted {
26449 assert!(
26450 tokio::time::timeout(Duration::from_millis(20), client.finish_session_shutdown())
26451 .await
26452 .is_err()
26453 );
26454 assert!(!client.terminal_producers_stopped.is_cancelled());
26455 assert!(client.terminal_recovery_handle.is_some());
26456 assert!(!client.pending_tasks.is_empty());
26457 }
26458
26459 if incomplete {
26460 assert!(client.finish_session_shutdown().await.is_err());
26461 assert!(!client.terminal_producers_stopped.is_cancelled());
26462 assert!(client.terminal_recovery_handle.is_some());
26463 assert!(!client.pending_tasks.is_empty());
26464 assert!(client.fills_applied.is_some());
26465 release.send(()).unwrap();
26466 }
26467
26468 client.finish_session_shutdown().await.unwrap();
26469 let events = drain_events(&mut rig.rx);
26470 assert_eq!(hits.load(Ordering::Relaxed), 1);
26471 assert_eq!(events.len(), 2);
26472 assert!(matches!(
26473 &events[0],
26474 ExecutionEvent::Order(OrderEventAny::Filled(_))
26475 ));
26476 assert!(matches!(
26477 &events[1],
26478 ExecutionEvent::Order(OrderEventAny::Expired(_))
26479 ));
26480 assert!(rig.dispatch.terminals_pending.lock().is_empty());
26481 assert!(client.session_tasks_finished());
26482 assert!(client.fills_applied.is_none());
26483 server.abort();
26484 let _ = server.await;
26485 }
26486
26487 #[rstest]
26488 #[case::canceled(LighterOrderStatus::Canceled, true, false, false)]
26489 #[case::expired(LighterOrderStatus::CanceledExpired, true, false, false)]
26490 #[case::missing_trade(LighterOrderStatus::Canceled, false, false, false)]
26491 #[case::canceled_bundle_first(LighterOrderStatus::Canceled, true, true, true)]
26492 #[case::canceled_mass_first(LighterOrderStatus::Canceled, true, true, false)]
26493 #[case::expired_bundle_first(LighterOrderStatus::CanceledExpired, true, true, true)]
26494 #[case::expired_mass_first(LighterOrderStatus::CanceledExpired, true, true, false)]
26495 #[case::early_missing_trade(LighterOrderStatus::Canceled, false, true, false)]
26496 #[tokio::test]
26497 async fn terminal_mass_status_pairs_missing_fills(
26498 #[case] status: LighterOrderStatus,
26499 #[case] complete: bool,
26500 #[case] early_timestamp: bool,
26501 #[case] bundle_first: bool,
26502 ) {
26503 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
26504 cache
26505 .borrow_mut()
26506 .add_account(AccountAny::Margin(test_margin_account()))
26507 .unwrap();
26508 dispatch_reduce_frame(&rig, &frame, false);
26509 consume_reduce_events(&mut rig, &mut engine);
26510 let instrument = LIGHTER_INSTRUMENT_CACHE
26511 .get(&rig.instrument_id)
26512 .unwrap()
26513 .value()
26514 .clone();
26515 let mut first = dispatcher_test_trade(&rig, true);
26516 first.bid_id = frame.order_index;
26517 first.bid_id_str = Some(frame.order_id.clone());
26518 first.size = Decimal::from_str("0.0005").unwrap();
26519 first.usd_amount = first.size * first.price;
26520 let mut initial_fill = parse_ws_fill_report(
26521 &first,
26522 TEST_ACCOUNT_INDEX_I64,
26523 &instrument,
26524 account_id(),
26525 UnixNanos::from(2),
26526 )
26527 .unwrap()
26528 .unwrap();
26529 initial_fill.client_order_id = Some(rig.cloid);
26530 engine.reconcile_fill_report(&initial_fill);
26531 let mut second = first.clone();
26532 second.trade_id += 1;
26533 second.trade_id_str = Some(second.trade_id.to_string());
26534 second.timestamp += 1;
26535 second.size = Decimal::from_str("0.0010").unwrap();
26536 second.usd_amount = second.size * second.price;
26537 let second_id = parse_lighter_trade_id(&second).unwrap();
26538 frame.updated_at = if early_timestamp {
26539 second.timestamp - 2
26540 } else {
26541 second.timestamp + 1
26542 };
26543
26544 let terminal_timestamp = UnixNanos::from(frame.updated_at as u64 * 1_000_000);
26545 frame.status = status;
26546 frame.filled_base_amount =
26547 Decimal::from_str(if complete { "0.0015" } else { "0.0020" }).unwrap();
26548 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
26549 frame.remaining_base_amount = Decimal::ZERO;
26550
26551 let mut client = create_execution_client_with_cache(cache.clone());
26552 client.registry = rig.registry.clone();
26553 client.emitter = rig.emitter.clone();
26554 client
26555 .dispatch
26556 .seed_order_restorations(vec![cache.borrow().order(&rig.cloid).unwrap().clone()]);
26557 client.dispatch.note_active_market(frame.market_index);
26558
26559 let position = PositionStatusReport::new(
26560 account_id(),
26561 rig.instrument_id,
26562 PositionSide::Long,
26563 Quantity::from("0.0015"),
26564 UnixNanos::from(3),
26565 UnixNanos::from(3),
26566 None,
26567 None,
26568 None,
26569 );
26570 client
26571 .dispatch
26572 .replace_position_snapshot(&[position], &[], &[]);
26573 let trades = vec![first, second.clone(), second];
26574
26575 let app = Router::new()
26576 .route(
26577 "/api/v1/accountActiveOrders",
26578 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
26579 )
26580 .route(
26581 "/api/v1/accountInactiveOrders",
26582 get(move || {
26583 let frame = frame.clone();
26584 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
26585 }),
26586 )
26587 .route(
26588 "/api/v1/trades",
26589 get(move || {
26590 let trades = trades.clone();
26591 async move { axum::Json(serde_json::json!({"code":200,"trades":trades})) }
26592 }),
26593 );
26594
26595 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
26596 let addr = listener.local_addr().unwrap();
26597 let server = tokio::spawn(async move {
26598 axum::serve(listener, app).await.unwrap();
26599 });
26600
26601 client.http_client = LighterHttpClient::new(
26602 LighterEnvironment::Testnet,
26603 Some(format!("http://{addr}")),
26604 1,
26605 None,
26606 )
26607 .unwrap();
26608 let mass = client.generate_mass_status(None).await.unwrap().unwrap();
26609 assert_eq!(
26610 mass.order_reports().len(),
26611 usize::from(complete && !early_timestamp)
26612 );
26613 let bundles = drain_events(&mut rig.rx);
26614 assert_eq!(bundles.len(), usize::from(complete && early_timestamp));
26615 assert_eq!(mass.fill_reports().values().map(Vec::len).sum::<usize>(), 2);
26616 assert_eq!(mass.position_reports().len(), usize::from(complete));
26617 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
26618 let engine = Rc::new(RefCell::new(engine));
26619 engine
26620 .borrow_mut()
26621 .register_client(Box::new(client))
26622 .unwrap();
26623 let mut manager =
26624 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default()).unwrap();
26625 let applied = Rc::new(RefCell::new(Vec::new()));
26626 let captured = applied.clone();
26627 let handler = TypedHandler::from(move |event: &OrderEventAny| {
26628 captured.borrow_mut().push(event.clone());
26629 });
26630
26631 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
26632 let subscription = AppliedFillSubscription(handler);
26633
26634 if bundle_first {
26635 for event in &bundles {
26636 let ExecutionEvent::Report(report) = event else {
26637 panic!("expected bundled report")
26638 };
26639
26640 engine.borrow_mut().reconcile_execution_report(report);
26641 }
26642 }
26643
26644 manager.reconcile_execution_mass_status(&mass, &engine);
26645
26646 if !bundle_first {
26647 for event in &bundles {
26648 let ExecutionEvent::Report(report) = event else {
26649 panic!("expected bundled report")
26650 };
26651
26652 engine.borrow_mut().reconcile_execution_report(report);
26653 }
26654 }
26655
26656 drop(subscription);
26657 let applied = applied.take();
26658 assert_eq!(applied.len(), if complete { 2 } else { 1 });
26659
26660 let OrderEventAny::Filled(fill) = &applied[0] else {
26661 panic!("expected real fill first")
26662 };
26663
26664 assert_eq!(fill.trade_id, second_id);
26665 assert_eq!(fill.last_qty, Quantity::from("0.0010"));
26666
26667 if complete {
26668 assert!(matches!(
26669 applied.last(),
26670 Some(OrderEventAny::Canceled(_) | OrderEventAny::Expired(_))
26671 ));
26672 assert_eq!(applied.last().unwrap().ts_event(), terminal_timestamp);
26673 }
26674
26675 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
26676 assert_eq!(cached.filled_qty(), Quantity::from("0.0015"));
26677 assert_eq!(cached.trade_ids(), vec![&initial_fill.trade_id, &second_id]);
26678 assert_eq!(
26679 cached.status(),
26680 if !complete {
26681 OrderStatus::PartiallyFilled
26682 } else if status == LighterOrderStatus::CanceledExpired {
26683 OrderStatus::Expired
26684 } else {
26685 OrderStatus::Canceled
26686 }
26687 );
26688 assert!(
26689 nautilus_execution::reconciliation::inferred_reconciliation_trade_ids(&cached)
26690 .unwrap()
26691 .is_empty()
26692 );
26693 server.abort();
26694 let _ = server.await;
26695 }
26696
26697 #[rstest]
26698 #[case::observed(0)]
26699 #[case::synced(1)]
26700 #[case::restored(2)]
26701 #[case::restored_child(3)]
26702 #[tokio::test]
26703 async fn terminal_inferred_fills_exhaust_without_replay(#[case] mode: u8) {
26704 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
26705
26706 cache
26707 .borrow_mut()
26708 .add_account(AccountAny::Margin(test_margin_account()))
26709 .unwrap();
26710
26711 let mut client = create_execution_client_with_cache(cache.clone());
26712 client.dispatch = rig.dispatch.clone();
26713 client.emitter = rig.emitter.clone();
26714 client.registry = rig.registry.clone();
26715 if mode == 0 {
26716 client.subscribe_applied_fills();
26717 }
26718
26719 dispatch_reduce_frame(&rig, &frame, false);
26720 consume_reduce_events(&mut rig, &mut engine);
26721 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
26722 frame.remaining_base_amount -= frame.filled_base_amount;
26723 let instrument = LIGHTER_INSTRUMENT_CACHE
26724 .get(&rig.instrument_id)
26725 .unwrap()
26726 .value()
26727 .clone();
26728 let mut report =
26729 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(2))
26730 .unwrap()
26731 .with_client_order_id(rig.cloid);
26732 report.order_status = OrderStatus::PartiallyFilled;
26733 engine.reconcile_order_status_report(&report);
26734 let mut cached = cache.borrow().order(&rig.cloid).unwrap().clone();
26735 let inferred =
26736 nautilus_execution::reconciliation::inferred_reconciliation_trade_ids(&cached).unwrap();
26737 assert_eq!(inferred.len(), 1);
26738 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
26739
26740 if mode == 1 {
26741 assert!(order_shape_pending(&client.core, &rig.dispatch, &rig.cloid));
26742 } else if mode >= 2 {
26743 if mode == 3 {
26744 let mut history = cached.events().into_iter().cloned().collect::<Vec<_>>();
26745
26746 let OrderEventAny::Initialized(init) = &mut history[0] else {
26747 panic!("expected initialization");
26748 };
26749
26750 init.parent_order_id = Some(ClientOrderId::new("INFERRED-PARENT"));
26751 cached = OrderAny::from_events(history).unwrap();
26752 cache
26753 .borrow_mut()
26754 .add_order(cached.clone(), None, Some(client_id()), true)
26755 .unwrap();
26756 }
26757
26758 rig.dispatch = WsDispatchState::new();
26759 rig.dispatch.seed_order_restorations(vec![cached]);
26760 client.dispatch = rig.dispatch.clone();
26761 dispatch_reduce_frame(&rig, &frame, false);
26762 assert!(drain_events(&mut rig.rx).is_empty());
26763 }
26764
26765 let positions_before = cache
26766 .borrow()
26767 .positions_open(None, None, None, None, None)
26768 .iter()
26769 .map(|position| position.quantity)
26770 .collect::<Vec<_>>();
26771 assert_eq!(positions_before, vec![Quantity::from("0.0010")]);
26772 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
26773 assert!(identity.has_inferred_fills());
26774 assert_eq!(identity.filled_qty(), Decimal::ZERO);
26775 assert_eq!(rig.dispatch.trade_source(inferred[0]), None);
26776 assert_eq!(
26777 execution_instruments_pending(&client.core, &client.dispatch),
26778 AHashSet::from_iter([rig.instrument_id])
26779 );
26780 assert!(!order_report_ready(
26781 &client.core,
26782 &rig.dispatch,
26783 &report,
26784 &[]
26785 ));
26786 let mut trade = dispatcher_test_trade(&rig, true);
26787 trade.bid_id = frame.order_index;
26788 trade.bid_id_str = Some(frame.order_id.clone());
26789 trade.size = frame.filled_base_amount;
26790 let mut fill = parse_ws_fill_report(
26791 &trade,
26792 TEST_ACCOUNT_INDEX_I64,
26793 &instrument,
26794 account_id(),
26795 UnixNanos::from(3),
26796 )
26797 .unwrap()
26798 .unwrap();
26799 fill.client_order_id = Some(rig.cloid);
26800 frame.status = LighterOrderStatus::Canceled;
26801 frame.remaining_base_amount = Decimal::ZERO;
26802 dispatch_reduce_frame(&rig, &frame, false);
26803 assert!(drain_events(&mut rig.rx).is_empty());
26804 let (http, hits, server) = terminal_trades_server(&frame, vec![trade]).await;
26805 client.http_client = http;
26806 let context = acked_probe_context(&client);
26807
26808 if mode == 3 {
26809 let command = GenerateOrderStatusReport::new(
26810 UUID4::new(),
26811 UnixNanos::from(4),
26812 Some(rig.instrument_id),
26813 Some(rig.cloid),
26814 Some(VenueOrderId::new(&frame.order_id)),
26815 None,
26816 None,
26817 );
26818 assert!(client.generate_order_status_report(&command).await.is_err());
26819 wait_until_async(
26820 || async {
26821 client
26822 .dispatch
26823 .order_events
26824 .lock()
26825 .get(&rig.cloid)
26826 .is_none_or(|entry| entry.recovery_epoch.is_none())
26827 },
26828 Duration::from_secs(6),
26829 )
26830 .await;
26831 }
26832
26833 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
26834 recover_terminals(&context, true).await;
26835 let events = consume_reduce_events(&mut rig, &mut engine);
26836
26837 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
26838 assert!(events.is_empty());
26839 } else {
26840 assert_eq!(events.len(), 1);
26841 assert!(
26842 matches!(&events[0], OrderEventAny::Canceled(event) if event.ts_event == report.ts_last)
26843 );
26844 }
26845 }
26846
26847 let sweep = finish_fill_reports(
26848 &client.core,
26849 &rig.dispatch,
26850 BTreeSet::new(),
26851 Ok(FillSweep {
26852 reports: vec![fill],
26853 covers_window: true,
26854 withheld_instruments: AHashSet::new(),
26855 }),
26856 )
26857 .unwrap();
26858
26859 assert!(sweep.reports.is_empty());
26860 assert!(!sweep.covers_window);
26861 assert_eq!(
26862 sweep.withheld_instruments,
26863 AHashSet::from_iter([rig.instrument_id])
26864 );
26865 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
26866 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
26867 assert_eq!(cached.trade_ids(), inferred.iter().collect::<Vec<_>>());
26868 assert_eq!(cached.status(), OrderStatus::Canceled);
26869 assert_eq!(
26870 cache
26871 .borrow()
26872 .positions_open(None, None, None, None, None)
26873 .iter()
26874 .map(|position| position.quantity)
26875 .collect::<Vec<_>>(),
26876 positions_before
26877 );
26878 assert_eq!(identity.filled_qty(), Decimal::ZERO);
26879 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
26880 assert!(rig.dispatch.terminals_pending.lock().is_empty());
26881 inferred_reconciliation_trade_ids(&cached).unwrap();
26882 assert_eq!(
26883 client
26884 .dispatch
26885 .order_events
26886 .lock()
26887 .get(&rig.cloid)
26888 .map_or(0, |entry| entry.fills.len()),
26889 0
26890 );
26891 client.dispatch.replace_position_snapshot(&[], &[], &[]);
26892 assert!(execution_instruments_pending(&client.core, &client.dispatch).is_empty());
26893
26894 let command = GeneratePositionStatusReports::new(
26895 UUID4::new(),
26896 UnixNanos::from(5),
26897 Some(rig.instrument_id),
26898 None,
26899 None,
26900 None,
26901 None,
26902 );
26903 assert!(
26904 client
26905 .generate_position_status_reports(&command)
26906 .await
26907 .unwrap()
26908 .is_empty()
26909 );
26910 let task = client
26911 .generate_position_status_reports_task(&command)
26912 .unwrap();
26913 task.collection.await;
26914 assert!((task.finish)().unwrap().is_empty());
26915 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
26916
26917 let config = ExecutionManagerConfig {
26918 position_check_threshold_ns: DurationNanos::from_secs(0),
26919 ..ExecutionManagerConfig::default()
26920 };
26921
26922 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
26923 for event in manager.check_positions_consistency(&[&client]).await {
26924 engine.process(&event);
26925 }
26926
26927 assert_eq!(
26928 cache
26929 .borrow()
26930 .positions_open(None, None, None, None, None)
26931 .iter()
26932 .map(|position| position.signed_decimal_qty())
26933 .sum::<Decimal>(),
26934 Decimal::ZERO
26935 );
26936 assert_eq!(
26937 cache.borrow().order(&rig.cloid).unwrap().trade_ids(),
26938 inferred.iter().collect::<Vec<_>>()
26939 );
26940 assert_eq!(identity.filled_qty(), Decimal::ZERO);
26941 cache
26942 .borrow_mut()
26943 .purge_closed_orders(UnixNanos::from(u64::MAX), 0, false)
26944 .unwrap();
26945 assert!(cache.borrow().order(&rig.cloid).is_none());
26946 assert!(execution_instruments_pending(&client.core, &client.dispatch).is_empty());
26947 server.abort();
26948 let _ = server.await;
26949 }
26950
26951 #[rstest]
26952 #[case::applied(true, false)]
26953 #[case::discarded(false, false)]
26954 #[case::venue_uuid(true, true)]
26955 fn terminal_tracks_engine_applied_reconciliation(
26956 #[case] applied: bool,
26957 #[case] venue_uuid: bool,
26958 ) {
26959 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
26960
26961 let mut client = create_execution_client_with_cache(cache.clone());
26962 client.dispatch = rig.dispatch.clone();
26963 client.emitter = rig.emitter.clone();
26964 client.subscribe_applied_fills();
26965 client.subscribe_applied_fills();
26966 dispatch_reduce_frame(&rig, &frame, false);
26967 consume_reduce_events(&mut rig, &mut engine);
26968 let mut trade = dispatcher_test_trade(&rig, true);
26969 trade.bid_id = frame.order_index;
26970 trade.bid_id_str = Some(frame.order_id.clone());
26971 trade.size = Decimal::from_str("0.0010").unwrap();
26972 if venue_uuid {
26973 trade.trade_id_str = Some("4b1da1f0-0d43-5a14-99b0-7245e76569af".to_string());
26974 }
26975
26976 let instrument = LIGHTER_INSTRUMENT_CACHE
26977 .get(&rig.instrument_id)
26978 .unwrap()
26979 .value()
26980 .clone();
26981 let mut report = parse_ws_fill_report(
26982 &trade,
26983 TEST_ACCOUNT_INDEX_I64,
26984 &instrument,
26985 account_id(),
26986 UnixNanos::from(2),
26987 )
26988 .unwrap()
26989 .unwrap();
26990 report.client_order_id = Some(rig.cloid);
26991
26992 for _ in 0..2 {
26993 let reports = finish_fill_reports(
26994 &client.core,
26995 &rig.dispatch,
26996 BTreeSet::new(),
26997 Ok(FillSweep {
26998 reports: vec![report.clone()],
26999 covers_window: true,
27000 withheld_instruments: AHashSet::new(),
27001 }),
27002 )
27003 .unwrap();
27004
27005 assert_eq!(reports.reports, vec![report.clone()]);
27006 assert!(!order_shape_pending(
27007 &client.core,
27008 &rig.dispatch,
27009 &rig.cloid
27010 ));
27011 }
27012
27013 if applied {
27014 engine.reconcile_fill_report(&report);
27015 }
27016
27017 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
27018 assert_eq!(
27019 identity.filled_qty(),
27020 if applied { trade.size } else { Decimal::ZERO }
27021 );
27022 assert_eq!(
27023 rig.dispatch.trade_source(report.trade_id),
27024 if applied {
27025 Some(TradeDedupSource::Reconciliation)
27026 } else {
27027 None
27028 }
27029 );
27030 assert!(!identity.has_publication());
27031 frame.status = LighterOrderStatus::Canceled;
27032 frame.filled_base_amount = trade.size;
27033 frame.remaining_base_amount = Decimal::ZERO;
27034 dispatch_reduce_frame(&rig, &frame, false);
27035
27036 if !applied {
27037 assert!(drain_events(&mut rig.rx).is_empty());
27038 dispatch_lighter_trade(
27039 &trade,
27040 &rig.dispatch,
27041 &rig.emitter,
27042 &rig.registry,
27043 account_id(),
27044 trader_id(),
27045 Some(TEST_ACCOUNT_INDEX_I64),
27046 UnixNanos::from(3),
27047 );
27048 }
27049
27050 let events = consume_reduce_events(&mut rig, &mut engine);
27051 assert_eq!(events.len(), if applied { 1 } else { 2 });
27052 assert!(matches!(events.last(), Some(OrderEventAny::Canceled(_))));
27053
27054 if !applied {
27055 assert!(
27056 matches!(&events[0], OrderEventAny::Filled(fill) if fill.trade_id == report.trade_id)
27057 );
27058 }
27059
27060 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
27061 assert_eq!(order.status(), OrderStatus::Canceled);
27062 assert_eq!(order.filled_qty(), Quantity::from("0.0010"));
27063 assert_eq!(order.trade_ids(), vec![&report.trade_id]);
27064 assert!(rig.dispatch.terminals_pending.lock().is_empty());
27065 frame.filled_base_amount += Decimal::from_str("0.0001").unwrap();
27066 dispatch_reduce_frame(&rig, &frame, false);
27067 assert!(drain_events(&mut rig.rx).is_empty());
27068 }
27069
27070 #[rstest]
27071 #[case::drop(true)]
27072 #[case::shutdown(false)]
27073 #[tokio::test]
27074 async fn terminal_fill_subscription_detaches(#[case] drop_client: bool) {
27075 let (mut rig, cache, mut engine, frame) = reduce_order_replay();
27076
27077 let mut client = create_execution_client_with_cache(cache.clone());
27078 client.dispatch = rig.dispatch.clone();
27079 client.subscribe_applied_fills();
27080 client.subscribe_applied_fills();
27081 dispatch_reduce_frame(&rig, &frame, false);
27082 consume_reduce_events(&mut rig, &mut engine);
27083 let mut report = reconciliation_fill_report(
27084 rig.instrument_id,
27085 frame.client_order_index,
27086 VenueOrderId::new(&frame.order_id),
27087 "APPLIED-FIRST",
27088 );
27089 report.client_order_id = Some(rig.cloid);
27090 report.last_qty = Quantity::from("0.0010");
27091 engine.reconcile_fill_report(&report);
27092 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
27093 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
27094
27095 if !drop_client {
27096 client.stop().unwrap();
27097 client.finish_session_shutdown().await.unwrap();
27098 assert!(client.fills_applied.is_none());
27099 }
27100
27101 drop(client);
27102 report.trade_id = TradeId::new("APPLIED-SECOND");
27103 engine.reconcile_fill_report(&report);
27104 assert_eq!(
27105 cache.borrow().order(&rig.cloid).unwrap().filled_qty(),
27106 Quantity::from("0.0020")
27107 );
27108 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
27109 assert_eq!(rig.dispatch.trade_source(report.trade_id), None);
27110 }
27111
27112 fn test_margin_account() -> MarginAccount {
27113 MarginAccount::new(
27114 AccountState::new(
27115 account_id(),
27116 AccountType::Margin,
27117 vec![AccountBalance::new(
27118 Money::from("10000.00000000 USDC"),
27119 Money::from("0.00000000 USDC"),
27120 Money::from("10000.00000000 USDC"),
27121 )],
27122 Vec::new(),
27123 true,
27124 UUID4::new(),
27125 UnixNanos::from(1),
27126 UnixNanos::from(1),
27127 Some(Currency::from("USDC")),
27128 ),
27129 true,
27130 )
27131 }
27132
27133 #[rstest]
27134 #[case::inline_cancel(0, LighterOrderStatus::Canceled)]
27135 #[case::worker_cancel(1, LighterOrderStatus::Canceled)]
27136 #[case::query_cancel(2, LighterOrderStatus::Canceled)]
27137 #[case::inline_expire(0, LighterOrderStatus::CanceledExpired)]
27138 #[case::worker_expire(1, LighterOrderStatus::CanceledExpired)]
27139 #[case::query_expire(2, LighterOrderStatus::CanceledExpired)]
27140 #[case::query_filled(2, LighterOrderStatus::Filled)]
27141 #[tokio::test]
27142 async fn terminal_query_recovers_missed_frames(
27143 #[case] route: u8,
27144 #[case] status: LighterOrderStatus,
27145 ) {
27146 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
27147 cache
27148 .borrow_mut()
27149 .add_account(AccountAny::Margin(test_margin_account()))
27150 .unwrap();
27151 dispatch_reduce_frame(&rig, &frame, false);
27152 consume_reduce_events(&mut rig, &mut engine);
27153
27154 let mut client = create_execution_client_with_cache(cache.clone());
27155 client.dispatch = rig.dispatch.clone();
27156 client.registry = rig.registry.clone();
27157 client.emitter = rig.emitter.clone();
27158 frame.status = status;
27159 let filled = status == LighterOrderStatus::Filled;
27160 let qty = Quantity::from(if filled { "0.0027" } else { "0.0010" });
27161 frame.filled_base_amount = qty.as_decimal();
27162 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27163 frame.remaining_base_amount = Decimal::ZERO;
27164 let mut trade = dispatcher_test_trade(&rig, true);
27165 trade.bid_id = frame.order_index;
27166 trade.bid_id_str = Some(frame.order_id.clone());
27167 trade.size = frame.filled_base_amount;
27168 trade.usd_amount = trade.size * trade.price;
27169 let trade_id = parse_lighter_trade_id(&trade).unwrap();
27170 let terminal_timestamp = UnixNanos::from(frame.updated_at as u64 * 1_000_000);
27171 let venue = VenueOrderId::from(frame.order_id.as_str());
27172 let raw = frame.clone();
27173 let hits = Arc::new(AtomicUsize::new(0));
27174 let counted = hits.clone();
27175
27176 let app = Router::new()
27177 .route(
27178 "/api/v1/accountActiveOrders",
27179 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
27180 )
27181 .route(
27182 "/api/v1/accountInactiveOrders",
27183 get(move || {
27184 let raw = raw.clone();
27185 async move { axum::Json(serde_json::json!({"code":200,"orders":[raw]})) }
27186 }),
27187 )
27188 .route(
27189 "/api/v1/trades",
27190 get(move || {
27191 let trade = trade.clone();
27192 let counted = counted.clone();
27193 async move {
27194 counted.fetch_add(1, Ordering::Relaxed);
27195 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
27196 }
27197 }),
27198 );
27199
27200 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
27201 let addr = listener.local_addr().unwrap();
27202 let server = tokio::spawn(async move {
27203 axum::serve(listener, app).await.unwrap();
27204 });
27205
27206 client.http_client = LighterHttpClient::new(
27207 LighterEnvironment::Testnet,
27208 Some(format!("http://{addr}")),
27209 1,
27210 None,
27211 )
27212 .unwrap();
27213
27214 let command = GenerateOrderStatusReport::new(
27215 UUID4::new(),
27216 UnixNanos::from(4),
27217 Some(rig.instrument_id),
27218 Some(rig.cloid),
27219 Some(venue),
27220 None,
27221 None,
27222 );
27223
27224 if filled && route == 2 {
27225 dispatch_reduce_frame(&rig, &frame, false);
27226 assert!(!client.dispatch.order_identities.contains_key(&rig.cloid));
27227 }
27228
27229 let context = acked_probe_context(&client);
27230 let report = match route {
27231 0 => client.generate_order_status_report(&command).await.unwrap(),
27232 1 => {
27233 let task = client.generate_order_status_report_task(&command).unwrap();
27234 task.collection.await;
27235 (task.finish)().unwrap()
27236 }
27237 _ => {
27238 client
27239 .query_order(QueryOrder::new(
27240 trader_id(),
27241 Some(client_id()),
27242 strategy_id(),
27243 rig.instrument_id,
27244 rig.cloid,
27245 Some(venue),
27246 UUID4::new(),
27247 UnixNanos::from(4),
27248 None,
27249 None,
27250 ))
27251 .unwrap();
27252 client.pending_tasks.begin_shutdown();
27253 client
27254 .pending_tasks
27255 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
27256 .await
27257 .unwrap();
27258 None
27259 }
27260 };
27261
27262 if !filled || route != 2 {
27263 assert!(drain_events(&mut rig.rx).is_empty());
27264 }
27265
27266 assert_eq!(
27267 client
27268 .dispatch
27269 .terminals_pending
27270 .lock()
27271 .contains_key(&rig.cloid),
27272 route == 2 && !filled
27273 );
27274 let applied = Rc::new(RefCell::new(Vec::new()));
27275 let captured = applied.clone();
27276 let handler = TypedHandler::from(move |event: &OrderEventAny| {
27277 captured.borrow_mut().push(event.clone());
27278 });
27279
27280 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
27281 let subscription = AppliedFillSubscription(handler);
27282
27283 if let Some(report) = report {
27284 assert_eq!(report.filled_qty, qty);
27285
27286 let command = GenerateFillReports::new(
27287 UUID4::new(),
27288 UnixNanos::from(5),
27289 Some(rig.instrument_id),
27290 Some(venue),
27291 None,
27292 None,
27293 None,
27294 None,
27295 );
27296 let fills = client.generate_fill_reports(command).await.unwrap();
27297 let bundle = EngineExecutionReport::OrderWithFills(Box::new(report), fills);
27298 engine.reconcile_execution_report(&bundle);
27299 } else {
27300 if !filled {
27301 recover_terminals(&context, true).await;
27302 }
27303
27304 consume_reduce_events(&mut rig, &mut engine);
27305 }
27306
27307 drop(subscription);
27308 let events = applied.take();
27309 assert_eq!(events.len(), if filled { 1 } else { 2 });
27310
27311 let OrderEventAny::Filled(fill) = &events[0] else {
27312 panic!("expected real fill first");
27313 };
27314
27315 assert_eq!(fill.trade_id, trade_id);
27316 assert_eq!(fill.last_qty, qty);
27317
27318 if !filled {
27319 assert!(matches!(
27320 events[1],
27321 OrderEventAny::Canceled(_) | OrderEventAny::Expired(_)
27322 ));
27323 assert_eq!(events[1].ts_event(), terminal_timestamp);
27324 }
27325
27326 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
27327 assert_eq!(cached.trade_ids(), vec![&trade_id]);
27328 assert_eq!(cached.filled_qty(), qty);
27329 assert_eq!(hits.load(Ordering::Relaxed), 1);
27330 server.abort();
27331 let _ = server.await;
27332 }
27333
27334 #[rstest]
27335 #[case::bulk_complete(true, 2)]
27336 #[case::targeted_complete(false, 2)]
27337 #[case::bulk_partial(true, 1)]
27338 #[case::targeted_partial(false, 1)]
27339 #[case::bulk_empty(true, 0)]
27340 #[case::targeted_empty(false, 0)]
27341 #[tokio::test]
27342 async fn terminal_targeted_reconciliation_recovers_offsetting_fills(
27343 #[case] bulk_report: bool,
27344 #[case] coverage: u8,
27345 ) {
27346 let (mut rig, cache, mut engine, mut buy) = reduce_order_replay();
27347 cache
27348 .borrow_mut()
27349 .add_account(AccountAny::Margin(test_margin_account()))
27350 .unwrap();
27351 dispatch_reduce_frame(&rig, &buy, false);
27352 consume_reduce_events(&mut rig, &mut engine);
27353 let sell_id = ClientOrderId::from("MISSED-SELL");
27354 let sell_order = OrderTestBuilder::new(OrderType::Limit)
27355 .trader_id(trader_id())
27356 .strategy_id(strategy_id())
27357 .instrument_id(rig.instrument_id)
27358 .client_order_id(sell_id)
27359 .side(OrderSide::Sell)
27360 .quantity(Quantity::from("0.0027"))
27361 .price(Price::from("2352.74"))
27362 .build();
27363 let sell_index = rig.dispatch.register_create_identity(&sell_order).unwrap();
27364 rig.dispatch
27365 .mark_order_submission(&sell_id, TEST_SUBMISSION_NONCE + 1);
27366 cache
27367 .borrow_mut()
27368 .add_order(sell_order, None, Some(client_id()), false)
27369 .unwrap();
27370 engine.process(&OrderEventAny::Submitted(OrderSubmitted::new(
27371 trader_id(),
27372 strategy_id(),
27373 rig.instrument_id,
27374 sell_id,
27375 account_id(),
27376 UUID4::new(),
27377 UnixNanos::from(1),
27378 UnixNanos::from(1),
27379 )));
27380 let mut sell = buy.clone();
27381 sell.order_index += 1;
27382 sell.order_id = sell.order_index.to_string();
27383 sell.client_order_index = sell_index;
27384 sell.client_order_id = sell_index.to_string();
27385 sell.is_ask = true;
27386 sell.side = Some(LighterOrderSide::Sell);
27387 dispatch_reduce_frame(&rig, &sell, false);
27388 consume_reduce_events(&mut rig, &mut engine);
27389 buy.status = LighterOrderStatus::Canceled;
27390 sell.status = LighterOrderStatus::CanceledExpired;
27391 for frame in [&mut buy, &mut sell] {
27392 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
27393 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27394 frame.remaining_base_amount = Decimal::ZERO;
27395 }
27396
27397 let mut buy_trade = dispatcher_test_trade(&rig, true);
27398 buy_trade.bid_id = buy.order_index;
27399 buy_trade.bid_id_str = Some(buy.order_id.clone());
27400
27401 let qty = Quantity::from(match coverage {
27402 0 => "0.0000",
27403 1 => "0.0005",
27404 _ => "0.0010",
27405 });
27406
27407 buy_trade.size = qty.as_decimal();
27408 buy_trade.usd_amount = buy_trade.size * buy_trade.price;
27409 let mut sell_trade = dispatcher_test_trade(&rig, false);
27410 sell_trade.trade_id += 1;
27411 sell_trade.trade_id_str = Some(sell_trade.trade_id.to_string());
27412 sell_trade.ask_id = sell.order_index;
27413 sell_trade.ask_id_str = Some(sell.order_id.clone());
27414 sell_trade.ask_client_id = sell_index;
27415 sell_trade.ask_client_id_str = Some(sell_index.to_string());
27416 sell_trade.size = buy_trade.size;
27417 sell_trade.usd_amount = sell_trade.size * sell_trade.price;
27418 buy.updated_at = buy_trade.timestamp + 1;
27419 sell.updated_at = sell_trade.timestamp + 1;
27420 let expected = [
27421 (
27422 rig.cloid,
27423 parse_lighter_trade_id(&buy_trade).unwrap(),
27424 OrderStatus::Canceled,
27425 ),
27426 (
27427 sell_id,
27428 parse_lighter_trade_id(&sell_trade).unwrap(),
27429 OrderStatus::Expired,
27430 ),
27431 ];
27432 let orders = vec![buy, sell];
27433
27434 let trades = if coverage == 0 {
27435 Vec::new()
27436 } else {
27437 vec![buy_trade, sell_trade]
27438 };
27439
27440 let history_hits = Arc::new(AtomicUsize::new(0));
27441 let hits = Arc::new(AtomicUsize::new(0));
27442 let counted = hits.clone();
27443
27444 let app = Router::new()
27445 .route(
27446 "/api/v1/accountActiveOrders",
27447 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
27448 )
27449 .route(
27450 "/api/v1/accountInactiveOrders",
27451 get(move || {
27452 let orders = orders.clone();
27453 let history_hits = history_hits.clone();
27454 async move {
27455 let first = history_hits.fetch_add(1, Ordering::Relaxed) == 0;
27456
27457 let orders = if !bulk_report && first {
27458 Vec::new()
27459 } else {
27460 orders
27461 };
27462
27463 axum::Json(serde_json::json!({"code":200,"orders":orders}))
27464 }
27465 }),
27466 )
27467 .route(
27468 "/api/v1/trades",
27469 get(move || {
27470 let trades = trades.clone();
27471 let counted = counted.clone();
27472 async move {
27473 counted.fetch_add(1, Ordering::Relaxed);
27474 axum::Json(serde_json::json!({"code":200,"trades":trades}))
27475 }
27476 }),
27477 );
27478
27479 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
27480 let addr = listener.local_addr().unwrap();
27481 let server = tokio::spawn(async move {
27482 axum::serve(listener, app).await.unwrap();
27483 });
27484
27485 let mut client = create_execution_client_with_cache(cache.clone());
27486 client.dispatch = rig.dispatch.clone();
27487 client.registry = rig.registry.clone();
27488 client.emitter = rig.emitter.clone();
27489 client.dispatch.note_active_market(TEST_MARKET_INDEX);
27490 client.dispatch.replace_position_snapshot(&[], &[], &[]);
27491 client.http_client = LighterHttpClient::new(
27492 LighterEnvironment::Testnet,
27493 Some(format!("http://{addr}")),
27494 1,
27495 None,
27496 )
27497 .unwrap();
27498 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
27499
27500 let config = ExecutionManagerConfig {
27501 open_check_lookback_mins: None,
27502 open_check_missing_retries: 0,
27503 open_check_open_only: false,
27504 open_check_threshold_ns: DurationNanos::from_secs(0),
27505 single_order_query_delay_ms: 0,
27506 ..ExecutionManagerConfig::default()
27507 };
27508
27509 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
27510 assert!(
27511 cache
27512 .borrow()
27513 .positions_open(None, None, None, None, None)
27514 .is_empty()
27515 );
27516 let events = manager.check_open_orders(&[&client]).await;
27517 for event in &events {
27518 engine.process(event);
27519 }
27520
27521 assert_eq!(hits.load(Ordering::Relaxed), 2);
27522 assert_eq!(events.len(), usize::from(coverage) * 2);
27523
27524 for (id, trade_id, status) in expected {
27525 let order_events = events
27526 .iter()
27527 .filter(|event| event.client_order_id() == id)
27528 .collect::<Vec<_>>();
27529
27530 if coverage > 0 {
27531 assert!(
27532 matches!(order_events[0], OrderEventAny::Filled(fill) if fill.trade_id == trade_id && fill.last_qty == qty)
27533 );
27534 }
27535
27536 if coverage == 2 {
27537 assert!(matches!(
27538 order_events[1],
27539 OrderEventAny::Canceled(_) | OrderEventAny::Expired(_)
27540 ));
27541 }
27542
27543 let order = cache.borrow().order(&id).unwrap().clone();
27544 assert_eq!(
27545 order.status(),
27546 match coverage {
27547 0 => OrderStatus::Accepted,
27548 1 => OrderStatus::PartiallyFilled,
27549 _ => status,
27550 }
27551 );
27552 assert_eq!(order.filled_qty(), qty);
27553 let trade_ids: Vec<_> = std::iter::once(&trade_id)
27554 .take(usize::from(coverage > 0))
27555 .collect();
27556 assert_eq!(order.trade_ids(), trade_ids);
27557 assert!(
27558 inferred_reconciliation_trade_ids(&order)
27559 .unwrap()
27560 .is_empty()
27561 );
27562 }
27563
27564 assert_eq!(
27565 cache
27566 .borrow()
27567 .positions_open(None, None, None, None, None)
27568 .iter()
27569 .map(|position| position.signed_decimal_qty())
27570 .sum::<Decimal>(),
27571 Decimal::ZERO
27572 );
27573 assert!(drain_events(&mut rig.rx).is_empty());
27574 server.abort();
27575 let _ = server.await;
27576 }
27577
27578 #[rstest]
27579 #[case::bulk_partial(true, false)]
27580 #[case::targeted_partial(false, false)]
27581 #[case::bulk_filled(true, true)]
27582 #[case::targeted_filled(false, true)]
27583 #[tokio::test]
27584 async fn terminal_reconciliation_preserves_real_fill_gate(
27585 #[case] bulk: bool,
27586 #[case] filled: bool,
27587 ) {
27588 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
27589 cache
27590 .borrow_mut()
27591 .add_account(AccountAny::Margin(test_margin_account()))
27592 .unwrap();
27593 dispatch_reduce_frame(&rig, &frame, false);
27594 consume_reduce_events(&mut rig, &mut engine);
27595
27596 frame.status = if filled {
27597 LighterOrderStatus::Filled
27598 } else {
27599 LighterOrderStatus::Open
27600 };
27601
27602 let qty = Quantity::from(if filled { "0.0027" } else { "0.0010" });
27603 frame.filled_base_amount = qty.as_decimal();
27604 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27605 frame.remaining_base_amount = frame.initial_base_amount - frame.filled_base_amount;
27606 let mut trade = dispatcher_test_trade(&rig, true);
27607 trade.bid_id = frame.order_index;
27608 trade.bid_id_str = Some(frame.order_id.clone());
27609 trade.size = frame.filled_base_amount;
27610 trade.usd_amount = trade.size * trade.price;
27611 let trade_id = parse_lighter_trade_id(&trade).unwrap();
27612 let response_trade = trade.clone();
27613 let trade_hits = Arc::new(AtomicUsize::new(0));
27614 let hits = trade_hits.clone();
27615 let active = frame.clone();
27616 let inactive = frame.clone();
27617 let history_hits = Arc::new(AtomicUsize::new(0));
27618 let active_hits = history_hits.clone();
27619
27620 let app = Router::new()
27621 .route(
27622 "/api/v1/trades",
27623 get(move || {
27624 let trade = response_trade.clone();
27625 let hits = hits.clone();
27626 async move {
27627 hits.fetch_add(1, Ordering::Relaxed);
27628 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
27629 }
27630 }),
27631 )
27632 .route(
27633 "/api/v1/accountActiveOrders",
27634 get(move || {
27635 let active = active.clone();
27636 let hits = active_hits.clone();
27637 async move {
27638 let orders =
27639 if !filled && (bulk || hits.fetch_add(1, Ordering::Relaxed) > 0) {
27640 vec![active]
27641 } else {
27642 Vec::new()
27643 };
27644
27645 axum::Json(serde_json::json!({"code":200,"orders":orders}))
27646 }
27647 }),
27648 )
27649 .route(
27650 "/api/v1/accountInactiveOrders",
27651 get(move || {
27652 let inactive = inactive.clone();
27653 let hits = history_hits.clone();
27654 async move {
27655 let orders = if filled && (bulk || hits.fetch_add(1, Ordering::Relaxed) > 0)
27656 {
27657 vec![inactive]
27658 } else {
27659 Vec::new()
27660 };
27661
27662 axum::Json(serde_json::json!({"code":200,"orders":orders}))
27663 }
27664 }),
27665 );
27666
27667 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
27668 let addr = listener.local_addr().unwrap();
27669 let server = tokio::spawn(async move {
27670 axum::serve(listener, app).await.unwrap();
27671 });
27672
27673 let mut client = create_execution_client_with_cache(cache.clone());
27674 client.dispatch = rig.dispatch.clone();
27675 client.registry = rig.registry.clone();
27676 client.emitter = rig.emitter.clone();
27677 client.dispatch.note_active_market(TEST_MARKET_INDEX);
27678 client.http_client = LighterHttpClient::new(
27679 LighterEnvironment::Testnet,
27680 Some(format!("http://{addr}")),
27681 1,
27682 None,
27683 )
27684 .unwrap();
27685 client.subscribe_applied_fills();
27686 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
27687
27688 let config = ExecutionManagerConfig {
27689 open_check_lookback_mins: None,
27690 open_check_missing_retries: 0,
27691 open_check_open_only: false,
27692 open_check_threshold_ns: DurationNanos::from_secs(0),
27693 single_order_query_delay_ms: 0,
27694 ..ExecutionManagerConfig::default()
27695 };
27696
27697 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
27698 let events = manager.check_open_orders(&[&client]).await;
27699 assert!(
27700 events.is_empty(),
27701 "uncovered reports must not create inferred fills"
27702 );
27703
27704 let command = GenerateOrderStatusReport::new(
27705 UUID4::new(),
27706 UnixNanos::from(4),
27707 Some(rig.instrument_id),
27708 Some(rig.cloid),
27709 Some(VenueOrderId::new(&frame.order_id)),
27710 None,
27711 None,
27712 );
27713 assert!(client.generate_order_status_report(&command).await.is_err());
27714 let task = client.generate_order_status_report_task(&command).unwrap();
27715 task.collection.await;
27716 assert!((task.finish)().is_err());
27717 assert_eq!(rig.rx.len(), usize::from(filled));
27718 assert_eq!(trade_hits.load(Ordering::Relaxed), usize::from(filled));
27719 dispatch_lighter_trade(
27720 &trade,
27721 &client.dispatch,
27722 &client.emitter,
27723 &client.registry,
27724 account_id(),
27725 trader_id(),
27726 Some(TEST_ACCOUNT_INDEX_I64),
27727 UnixNanos::from(5),
27728 );
27729 consume_reduce_events(&mut rig, &mut engine);
27730 let report = client
27731 .generate_order_status_report(&command)
27732 .await
27733 .unwrap()
27734 .unwrap();
27735 assert_eq!(report.filled_qty, qty);
27736 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
27737 assert_eq!(cached.trade_ids(), vec![&trade_id]);
27738 assert_eq!(cached.filled_qty(), qty);
27739 assert_eq!(
27740 cached.status(),
27741 if filled {
27742 OrderStatus::Filled
27743 } else {
27744 OrderStatus::PartiallyFilled
27745 }
27746 );
27747 assert!(
27748 inferred_reconciliation_trade_ids(&cached)
27749 .unwrap()
27750 .is_empty()
27751 );
27752 server.abort();
27753 let _ = server.await;
27754 }
27755
27756 #[rstest]
27757 #[case::canceled(LighterOrderStatus::Canceled)]
27758 #[case::filled(LighterOrderStatus::Filled)]
27759 #[tokio::test]
27760 async fn terminal_mass_status_recovers_child_without_sweep_fills(
27761 #[case] status: LighterOrderStatus,
27762 ) {
27763 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
27764 cache
27765 .borrow_mut()
27766 .add_account(AccountAny::Margin(test_margin_account()))
27767 .unwrap();
27768 dispatch_reduce_frame(&rig, &frame, false);
27769 consume_reduce_events(&mut rig, &mut engine);
27770 let mut history = cache
27771 .borrow()
27772 .order(&rig.cloid)
27773 .unwrap()
27774 .events()
27775 .into_iter()
27776 .cloned()
27777 .collect::<Vec<_>>();
27778
27779 let OrderEventAny::Initialized(init) = &mut history[0] else {
27780 panic!("expected initialization");
27781 };
27782
27783 init.parent_order_id = Some(ClientOrderId::new("OFFLINE-PARENT"));
27784 let cached = OrderAny::from_events(history).unwrap();
27785 cache
27786 .borrow_mut()
27787 .add_order(cached.clone(), None, Some(client_id()), true)
27788 .unwrap();
27789
27790 let mut client = create_execution_client_with_cache(cache.clone());
27791 client.registry = rig.registry.clone();
27792 client.emitter = rig.emitter.clone();
27793 client.dispatch.seed_order_restorations(vec![cached]);
27794 client.dispatch.note_active_market(TEST_MARKET_INDEX);
27795 client.dispatch.replace_position_snapshot(&[], &[], &[]);
27796 frame.status = status;
27797
27798 let qty = Quantity::from(if status == LighterOrderStatus::Filled {
27799 "0.0027"
27800 } else {
27801 "0.0010"
27802 });
27803
27804 frame.filled_base_amount = qty.as_decimal();
27805 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27806 frame.remaining_base_amount = Decimal::ZERO;
27807 let mut trade = dispatcher_test_trade(&rig, true);
27808 trade.bid_id = frame.order_index;
27809 trade.bid_id_str = Some(frame.order_id.clone());
27810 trade.size = qty.as_decimal();
27811 trade.usd_amount = trade.size * trade.price;
27812 let trade_id = parse_lighter_trade_id(&trade).unwrap();
27813 let hits = Arc::new(AtomicUsize::new(0));
27814 let counted = hits.clone();
27815
27816 let venue = VenueOrderId::new(&frame.order_id);
27817
27818 let app = Router::new()
27819 .route(
27820 "/api/v1/accountActiveOrders",
27821 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
27822 )
27823 .route(
27824 "/api/v1/accountInactiveOrders",
27825 get(move || {
27826 let frame = frame.clone();
27827 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
27828 }),
27829 )
27830 .route(
27831 "/api/v1/trades",
27832 get(move |Query(query): Query<AHashMap<String, String>>| {
27833 let trade = trade.clone();
27834 let hits = counted.clone();
27835 async move {
27836 let trades = if query.contains_key("order_index") {
27837 hits.fetch_add(1, Ordering::Relaxed);
27838 vec![trade]
27839 } else {
27840 Vec::new()
27841 };
27842
27843 axum::Json(serde_json::json!({"code":200,"trades":trades}))
27844 }
27845 }),
27846 );
27847
27848 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
27849 let addr = listener.local_addr().unwrap();
27850 let server = tokio::spawn(async move {
27851 axum::serve(listener, app).await.unwrap();
27852 });
27853
27854 client.http_client = LighterHttpClient::new(
27855 LighterEnvironment::Testnet,
27856 Some(format!("http://{addr}")),
27857 1,
27858 None,
27859 )
27860 .unwrap();
27861 assert_eq!(
27862 cache.borrow().client_order_id(&venue).copied(),
27863 Some(rig.cloid)
27864 );
27865 let mass = client.generate_mass_status(None).await.unwrap().unwrap();
27866 assert!(
27867 client
27868 .dispatch
27869 .order_identity(&rig.cloid)
27870 .unwrap()
27871 .child_fill_baseline()
27872 .is_some()
27873 );
27874 wait_until_async(
27875 || async {
27876 client
27877 .dispatch
27878 .order_identity(&rig.cloid)
27879 .is_some_and(|identity| identity.has_child_publication())
27880 },
27881 Duration::from_secs(3),
27882 )
27883 .await;
27884
27885 assert!(mass.fill_reports().is_empty());
27886 client.pending_tasks.begin_shutdown();
27887 client
27888 .pending_tasks
27889 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
27890 .await
27891 .unwrap();
27892
27893 for event in drain_events(&mut rig.rx) {
27894 match event {
27895 ExecutionEvent::Order(event) => engine.process(&event),
27896 ExecutionEvent::Report(report) => {
27897 engine.reconcile_execution_report(&report);
27898 }
27899 _ => panic!("unexpected child recovery event"),
27900 }
27901 }
27902
27903 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
27904 assert_eq!(hits.load(Ordering::Relaxed), 1);
27905 assert_eq!(
27906 cached.status(),
27907 if status == LighterOrderStatus::Filled {
27908 OrderStatus::Filled
27909 } else {
27910 OrderStatus::Canceled
27911 }
27912 );
27913 assert_eq!(cached.filled_qty(), qty);
27914 assert_eq!(cached.trade_ids(), vec![&trade_id]);
27915 assert!(
27916 inferred_reconciliation_trade_ids(&cached)
27917 .unwrap()
27918 .is_empty()
27919 );
27920 server.abort();
27921 let _ = server.await;
27922 }
27923
27924 #[tokio::test]
27925 async fn terminal_child_recovery_rechecks_inferred_overlap_after_http() {
27926 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
27927 cache
27928 .borrow_mut()
27929 .add_account(AccountAny::Margin(test_margin_account()))
27930 .unwrap();
27931 dispatch_reduce_frame(&rig, &frame, false);
27932 consume_reduce_events(&mut rig, &mut engine);
27933 let mut history = cache
27934 .borrow()
27935 .order(&rig.cloid)
27936 .unwrap()
27937 .events()
27938 .into_iter()
27939 .cloned()
27940 .collect::<Vec<_>>();
27941
27942 let OrderEventAny::Initialized(init) = &mut history[0] else {
27943 panic!("expected initialization");
27944 };
27945
27946 init.parent_order_id = Some(ClientOrderId::new("RACING-PARENT"));
27947 let cached = OrderAny::from_events(history).unwrap();
27948 cache
27949 .borrow_mut()
27950 .add_order(cached.clone(), None, Some(client_id()), true)
27951 .unwrap();
27952 rig.dispatch = WsDispatchState::new();
27953 rig.dispatch
27954 .restore_reconciled_order(
27955 &cached,
27956 frame.client_order_index,
27957 VenueOrderId::new(&frame.order_id),
27958 false,
27959 )
27960 .unwrap();
27961 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
27962
27963 let mut client = create_execution_client_with_cache(cache.clone());
27964 client.dispatch = rig.dispatch.clone();
27965 client.registry = rig.registry.clone();
27966 client.emitter = rig.emitter.clone();
27967 client.subscribe_applied_fills();
27968 frame.status = LighterOrderStatus::Canceled;
27969 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
27970 frame.remaining_base_amount = Decimal::ZERO;
27971 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27972 let instrument = LIGHTER_INSTRUMENT_CACHE
27973 .get(&rig.instrument_id)
27974 .unwrap()
27975 .value()
27976 .clone();
27977 let mut partial =
27978 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(3))
27979 .unwrap()
27980 .with_client_order_id(rig.cloid);
27981 partial.order_status = OrderStatus::PartiallyFilled;
27982 let mut trade = dispatcher_test_trade(&rig, true);
27983 trade.bid_id = frame.order_index;
27984 trade.bid_id_str = Some(frame.order_id.clone());
27985 trade.size = frame.filled_base_amount;
27986 trade.usd_amount = trade.size * trade.price;
27987 let requested = Arc::new(tokio::sync::Notify::new());
27988 let resume = Arc::new(tokio::sync::Notify::new());
27989 let signal = requested.clone();
27990 let gate = resume.clone();
27991
27992 let app = Router::new()
27993 .route(
27994 "/api/v1/accountActiveOrders",
27995 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
27996 )
27997 .route(
27998 "/api/v1/accountInactiveOrders",
27999 get(move || {
28000 let frame = frame.clone();
28001 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
28002 }),
28003 )
28004 .route(
28005 "/api/v1/trades",
28006 get(move || {
28007 let trade = trade.clone();
28008 let signal = signal.clone();
28009 let gate = gate.clone();
28010 async move {
28011 signal.notify_one();
28012 gate.notified().await;
28013 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
28014 }
28015 }),
28016 );
28017
28018 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
28019 let addr = listener.local_addr().unwrap();
28020 let server = tokio::spawn(async move {
28021 axum::serve(listener, app).await.unwrap();
28022 });
28023
28024 client.http_client = LighterHttpClient::new(
28025 LighterEnvironment::Testnet,
28026 Some(format!("http://{addr}")),
28027 1,
28028 None,
28029 )
28030 .unwrap();
28031 let context = acked_probe_context(&client);
28032 let (result, ()) = tokio::join!(recover_child_order(rig.cloid, 0, &context), async {
28033 tokio::time::timeout(Duration::from_secs(2), requested.notified())
28034 .await
28035 .unwrap();
28036 engine.reconcile_order_status_report(&partial);
28037 assert!(identity.has_inferred_fills());
28038 resume.notify_one();
28039 });
28040 assert!(result.unwrap());
28041 assert_eq!(
28042 client
28043 .dispatch
28044 .order_events
28045 .lock()
28046 .get(&rig.cloid)
28047 .map_or(0, |entry| entry.fills.len()),
28048 0
28049 );
28050 assert!(drain_events(&mut rig.rx).is_empty());
28051 assert_eq!(identity.filled_qty(), Decimal::ZERO);
28052 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
28053 let inferred = inferred_reconciliation_trade_ids(&cached).unwrap();
28054 assert_eq!(inferred.len(), 1);
28055 assert_eq!(cached.trade_ids(), inferred.iter().collect::<Vec<_>>());
28056 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
28057 server.abort();
28058 let _ = server.await;
28059 }
28060
28061 fn create_execution_client_with_cache(cache: Rc<RefCell<Cache>>) -> LighterExecutionClient {
28062 let core = ExecutionClientCore::new(
28063 trader_id(),
28064 client_id(),
28065 test_config().resolved_venue(),
28066 OmsType::Netting,
28067 account_id(),
28068 AccountType::Margin,
28069 None,
28070 cache,
28071 );
28072 LighterExecutionClient::new(core, test_config()).unwrap()
28073 }
28074}