1use std::{
29 collections::{BTreeMap, BTreeSet},
30 future::Future,
31 ops::ControlFlow,
32 sync::{
33 Arc,
34 atomic::{AtomicBool, AtomicU64, Ordering},
35 },
36 time::Duration,
37};
38
39use ahash::{AHashMap, AHashSet};
40use anyhow::Context;
41use async_trait::async_trait;
42use futures_util::{StreamExt, stream};
43#[cfg(test)]
44use nautilus_common::live::get_runtime;
45use nautilus_common::{
46 clients::{ExecutionClient, ExecutionReportTask},
47 enums::{LogColor, LogLevel},
48 live::runner::{get_exec_event_sender, try_get_exec_event_sender},
49 log_debug,
50 messages::{
51 ExecutionReport,
52 execution::{
53 BatchCancelOrders, BatchModifyOrders, CancelAllOrders, CancelOrder,
54 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
55 GeneratePositionStatusReports, ModifyOrder, QueryAccount, QueryOrder, SubmitOrder,
56 SubmitOrderList,
57 },
58 },
59 msgbus::{self, TypedHandler},
60};
61use nautilus_core::{
62 DurationNanos, UUID4, UnixNanos,
63 datetime::unix_nanos_to_iso8601,
64 params::Params,
65 string::secret::SecretString,
66 time::{AtomicTime, get_atomic_clock_realtime},
67};
68use nautilus_execution::reconciliation::{
69 inferred_reconciliation_trade_ids, is_inferred_reconciliation_trade_id_format,
70};
71use nautilus_live::{
72 ExecutionClientCore, ExecutionEventEmitter, SocketControlFactory,
73 book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS,
74 execution::failure::CommandFailure,
75 task::{TaskGroup, TaskGroupGuard, TaskJoinOutcome, TaskSlot, TaskSpawner, finish_task},
76};
77use nautilus_model::{
78 accounts::AccountAny,
79 enums::{
80 AccountType, ContingencyType, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
81 TimeInForce,
82 },
83 events::{OrderAccepted, OrderDeniedReason, OrderEventAny, OrderFilled, OrderTriggered},
84 identifiers::{
85 AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, TradeId, TraderId, Venue,
86 VenueOrderId,
87 },
88 instruments::{Instrument, InstrumentAny},
89 orders::{LIMIT_ORDER_TYPES, Order, OrderAny, STOP_ORDER_TYPES, TRIGGERABLE_ORDER_TYPES},
90 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
91 types::{AccountBalance, MarginBalance, Quantity},
92};
93use nautilus_network::error::SendError;
94use parking_lot::Mutex;
95use rust_decimal::Decimal;
96use serde::Deserialize;
97use tokio_util::sync::CancellationToken;
98use zeroize::Zeroizing;
99
100use crate::{
101 common::{
102 consts::{
103 DISCONNECT_TIMEOUT, LIGHTER_ERROR_CODE_INVALID_NONCE, LIGHTER_MAX_BATCH_TX,
104 SCHEDULED_CANCEL_MAX_DELAY_SECS, SCHEDULED_CANCEL_MIN_DELAY_SECS,
105 SCHEDULED_CANCEL_REQUEST_TIMEOUT,
106 },
107 credential::{Credential, scrub_auth},
108 deployment,
109 enums::{
110 LighterAccountTier, LighterCancelAllTimeInForce, LighterGroupingType,
111 LighterOrderStatus, LighterPositionMarginMode, LighterProductType,
112 LighterSelfTradeBehaviorMode, LighterSelfTradeEqualityMode, LighterTxStatus,
113 LighterTxType,
114 },
115 parse::{parse_price_from_ticks, parse_quantity_from_ticks, quantity_from_decimal},
116 rate_limit::{LighterTxRateLimiter, await_tx_quota, build_tx_rate_limiter, resolve_quota},
117 symbol::{MarketRegistry, product_type_from_instrument_id},
118 },
119 config::{
120 LighterExecutionClientConfig, validate_execution_venue, validate_execution_ws_url,
121 validate_market_order_slippage_bps, validate_proxy_url, validate_scheduled_cancel_timeout,
122 },
123 http::{
124 client::{LIGHTER_REST_PAGE_SIZE, LighterHttpClient, LighterRawHttpClient},
125 error::LighterHttpError,
126 models::{
127 LighterAccountDetail, LighterOrder, LighterSendTxRequest, LighterTrade, LighterTx,
128 },
129 query::{
130 LighterAccountActiveOrdersQuery, LighterAccountInactiveOrdersQuery,
131 LighterSortDirection, LighterTradeSortBy, LighterTradesQuery,
132 },
133 },
134 signing::{
135 auth_token::{build_auth_token_for, fresh_k},
136 curve::Scalar,
137 nonce::NonceError,
138 tx::{
139 ApproveIntegratorTxInfo, CancelAllOrdersTxInfo, CancelOrderTxInfo,
140 CreateGroupedOrdersTxInfo, CreateOrderTxInfo, L2TxAttributes, ModifyOrderTxInfo,
141 OrderInfo, TxContext, TxInfoJson, UpdateLeverageTxInfo, sign_tx,
142 },
143 },
144 websocket::{
145 LighterWsError, USER_STREAMS_ENDPOINT,
146 client::{LighterWebSocketClient, RetainedTaskSlot, TaskRetentionGuard},
147 dispatch::{
148 DeferredOrderEvents, LIGHTER_INSTRUMENT_CACHE, MAX_RECONCILIATION_PAGES, OrderIdentity,
149 OrderReportLookup, OrderShapeSnapshot, PendingOrderAction, PendingSendTx,
150 PendingSendTxKind, PendingTerminal, TradeDedupSource, WsDispatchState,
151 cache_instruments_for_reports, collect_order, collect_order_while_pending,
152 derive_market_order_price_ticks, evict_terminal_mappings, fills_for_order,
153 lookup_create_order_status_report, nautilus_to_lighter_order_type,
154 nautilus_to_lighter_tif, order_expiry_for, parse_http_order_to_report, price_to_ticks,
155 quantity_to_ticks, tx_hash_matches,
156 },
157 messages::{
158 AccountStream, ExecutionReport as LighterExecutionReport, LighterWsChannel,
159 LighterWsTx, NautilusWsMessage, SendTxRejectionSource, TX_BATCH_ID_PREFIX,
160 },
161 parse::{
162 LighterCommissionError, OpenFrameContext, ParsedOrderEvent, is_order_triggered,
163 lighter_order_shape, parse_lighter_order_event, parse_lighter_order_filled,
164 parse_lighter_order_updated, parse_lighter_trade_id, parse_ws_fill_report,
165 parse_ws_order_status_report,
166 },
167 },
168};
169
170const DEFAULT_TX_EXPIRY_MS: i64 = 5 * 60 * 1_000;
173
174const ACKED_ORDER_LOOKUP_DELAY: Duration = Duration::from_secs(2);
176const ACKED_ORDER_PROBE_ATTEMPTS: usize = 3;
177const TERMINAL_FILL_DELAY: Duration = Duration::from_secs(2);
178const TERMINAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(2);
179const TERMINAL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(7);
180const ORDER_EVENTS_PATTERN: &str = "events.order.*";
181const BATCH_ACK_TIMEOUT: Duration = Duration::from_secs(10);
182const ACCOUNT_CANCEL_CONFIRM_TIMEOUT: Duration = Duration::from_secs(10);
183
184const STRATEGY_REASON_MAX_CHARS: usize = 512;
185
186const AUTH_TOKEN_REFRESH_INTERVAL: std::time::Duration =
191 std::time::Duration::from_secs(6 * 60 * 60);
192
193const _: () = assert!(
195 AUTH_TOKEN_REFRESH_INTERVAL.as_secs()
196 < crate::signing::auth_token::DEFAULT_AUTH_TOKEN_TTL_SECS as u64,
197 "AUTH_TOKEN_REFRESH_INTERVAL must stay below DEFAULT_AUTH_TOKEN_TTL_SECS",
198);
199
200const AUTH_TOKEN_REFRESH_RETRY_WINDOW: Duration = Duration::from_secs(60 * 60);
203const AUTH_TOKEN_REFRESH_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(30);
204
205const AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY: Duration = Duration::from_secs(5 * 60);
207const AUTH_TOKEN_REFRESH_BACKOFF: AuthTokenRefreshBackoff = AuthTokenRefreshBackoff {
208 initial_delay: AUTH_TOKEN_REFRESH_RETRY_INITIAL_DELAY,
209 max_delay: AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY,
210 window: AUTH_TOKEN_REFRESH_RETRY_WINDOW,
211};
212const NONCE_REFRESH_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(1);
213const NONCE_REFRESH_RETRY_MAX_DELAY: Duration = Duration::from_secs(30);
214const ACCOUNT_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(10);
217const REFERRAL_ATTRIBUTION_TIMEOUT: Duration = Duration::from_secs(10);
218
219const SENDTX_BARE_ERROR_WINDOW_MS: u64 = 1_000;
224const INTEGRATOR_AUTO_APPROVAL_MAX_TTL_MS: i64 = 5 * 365 * 24 * 60 * 60 * 1_000;
225const INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK: u32 = 0;
226const NONCE_CONNECTION_EPOCH_UNAVAILABLE: u64 = u64::MAX;
227
228#[derive(Debug)]
229pub struct LighterExecutionClient {
230 core: ExecutionClientCore,
231 clock: &'static AtomicTime,
232 config: LighterExecutionClientConfig,
233 emitter: ExecutionEventEmitter,
234 credential: Option<Credential>,
235 http_client: LighterHttpClient,
236 ws_client: LighterWebSocketClient,
237 tx_rate_limiter: Arc<LighterTxRateLimiter>,
238 tx_send_sequencer: TxSendSequencer,
239 nonce_submission_gate: Arc<tokio::sync::RwLock<()>>,
240 nonce_ready_connection_epoch: Arc<AtomicU64>,
241 registry: Arc<MarketRegistry>,
242 socket_factory: SocketControlFactory,
243 pending_tasks: TaskGroup,
244 ws_stream_handle: TaskSlot<()>,
245 terminal_recovery_handle: TaskSlot<()>,
246 terminal_producers_stopped: CancellationToken,
247 fills_applied: Option<AppliedFillSubscription>,
248 ws_disconnect_handle: TaskSlot<Result<(), LighterWsError>>,
249 ws_handler_retained: Arc<RetainedTaskSlot>,
250 auth_refresh_handle: TaskSlot<()>,
251 scheduled_cancel_handle: TaskSlot<()>,
252 scheduled_cancel_notify: Arc<tokio::sync::Notify>,
253 shutdown_errors: Vec<String>,
254 cancellation_token: CancellationToken,
255 dispatch: WsDispatchState,
256 nonce_recovery_inflight: Arc<AtomicBool>,
257 auth_refresh_notify: Arc<tokio::sync::Notify>,
258}
259
260impl LighterExecutionClient {
261 fn log_report_receipt(count: usize, report_type: &str, log_level: LogLevel) {
262 let plural = if count == 1 { "" } else { "s" };
263 let message = format!("Received {count} {report_type}{plural}");
264
265 match log_level {
266 LogLevel::Off => {}
267 LogLevel::Trace => log::trace!("{message}"),
268 LogLevel::Debug => log::debug!("{message}"),
269 LogLevel::Info => log::info!("{message}"),
270 LogLevel::Warning => log::warn!("{message}"),
271 LogLevel::Error => log::error!("{message}"),
272 }
273 }
274
275 pub fn new(
288 core: ExecutionClientCore,
289 config: LighterExecutionClientConfig,
290 ) -> anyhow::Result<Self> {
291 validate_market_order_slippage_bps(config.market_order_slippage_bps)?;
292 validate_scheduled_cancel_timeout(config.scheduled_cancel_timeout_secs)?;
293 validate_execution_venue(config.venue)?;
294 validate_execution_ws_url(config.base_url_ws.as_deref())?;
295 validate_proxy_url(config.proxy_url.as_ref())?;
296 anyhow::ensure!(
297 core.venue == config.resolved_venue(),
298 "Lighter execution core venue {} does not match configured venue {}",
299 core.venue,
300 config.resolved_venue(),
301 );
302
303 anyhow::ensure!(
304 config.account_id.get_issuer() == core.venue,
305 "Lighter account ID issuer {} does not match configured venue {}",
306 config.account_id.get_issuer(),
307 core.venue,
308 );
309
310 let credential = Credential::resolve_for_deployment(
311 config.private_key.clone().map(SecretString::into_inner),
312 config.account_index,
313 config.api_key_index,
314 config.deployment,
315 config.environment,
316 )
317 .context("failed to resolve Lighter credentials")?;
318
319 let registry = Arc::new(MarketRegistry::new_with_venue_and_settlement_currency(
320 core.venue,
321 config.settlement_currency(),
322 ));
323 let socket_factory = SocketControlFactory::new(core.client_id, Some(core.venue));
324
325 let tx_rate_limiter = build_tx_rate_limiter(config.sendtx_quota_per_min);
328
329 let raw_http = LighterRawHttpClient::new_with_quotas(
330 config.environment,
331 Some(config.http_url()),
332 config.http_timeout_secs,
333 config
334 .proxy_url
335 .as_ref()
336 .map(|value| value.expose_secret().to_owned()),
337 resolve_quota(config.rest_quota_per_min),
338 Some(Arc::clone(&tx_rate_limiter)),
339 )
340 .context("failed to construct Lighter raw HTTP client")?;
341
342 let http_client =
343 LighterHttpClient::from_raw_with_registry(raw_http, Arc::clone(®istry));
344
345 let ws_client = Self::create_ws_client(&config, Arc::clone(®istry), &socket_factory);
346
347 let clock = get_atomic_clock_realtime();
348
349 let mut emitter = ExecutionEventEmitter::new(
350 clock,
351 core.trader_id,
352 core.account_id,
353 AccountType::Margin,
354 None,
355 );
356
357 if let Some(sender) = try_get_exec_event_sender() {
358 emitter.set_sender(sender);
359 }
360
361 let pending_tasks = TaskGroup::new();
362
363 Ok(Self {
364 core,
365 clock,
366 config,
367 emitter,
368 credential,
369 http_client,
370 ws_client,
371 tx_rate_limiter,
372 tx_send_sequencer: TxSendSequencer::new(),
373 nonce_submission_gate: Arc::new(tokio::sync::RwLock::new(())),
374 nonce_ready_connection_epoch: Arc::new(AtomicU64::new(0)),
375 registry,
376 socket_factory,
377 cancellation_token: pending_tasks.cancellation_token(),
378 pending_tasks,
379 ws_stream_handle: TaskSlot::new(),
380 terminal_recovery_handle: TaskSlot::new(),
381 terminal_producers_stopped: CancellationToken::new(),
382 fills_applied: None,
383 ws_disconnect_handle: TaskSlot::new(),
384 ws_handler_retained: Arc::new(RetainedTaskSlot::new()),
385 auth_refresh_handle: TaskSlot::new(),
386 scheduled_cancel_handle: TaskSlot::new(),
387 scheduled_cancel_notify: Arc::new(tokio::sync::Notify::new()),
388 shutdown_errors: Vec::new(),
389 dispatch: WsDispatchState::new(),
390 nonce_recovery_inflight: Arc::new(AtomicBool::new(false)),
391 auth_refresh_notify: Arc::new(tokio::sync::Notify::new()),
392 })
393 }
394
395 #[must_use]
397 pub fn config(&self) -> &LighterExecutionClientConfig {
398 &self.config
399 }
400
401 #[must_use]
403 pub fn has_credentials(&self) -> bool {
404 self.credential.is_some()
405 }
406
407 #[must_use]
410 pub fn pending_tasks_all_finished(&self) -> bool {
411 self.pending_tasks.all_finished()
412 }
413
414 pub async fn cancel_all_orders_account(&self) -> anyhow::Result<String> {
431 self.send_cancel_all(LighterCancelAllTimeInForce::Immediate, 0)
432 .await
433 }
434
435 pub async fn schedule_cancel(&self, time: Option<u64>) -> anyhow::Result<String> {
455 let (time_in_force, scheduled_time_ms) = match time {
456 Some(time) => {
457 let now = self.clock.get_time_ms();
458 let delay = time
459 .checked_sub(now)
460 .context("scheduled cancellation is in the past")?;
461 anyhow::ensure!(
462 (SCHEDULED_CANCEL_MIN_DELAY_SECS * 1_000
463 ..=SCHEDULED_CANCEL_MAX_DELAY_SECS * 1_000)
464 .contains(&delay),
465 "scheduled cancellation must be between five minutes and fifteen days ahead",
466 );
467 let time = i64::try_from(time)
468 .context("scheduled cancellation timestamp overflows milliseconds")?;
469 (LighterCancelAllTimeInForce::Scheduled, time)
470 }
471 None => (LighterCancelAllTimeInForce::AbortScheduled, 0),
472 };
473
474 self.send_cancel_all(time_in_force, scheduled_time_ms).await
475 }
476
477 async fn send_cancel_all(
478 &self,
479 time_in_force: LighterCancelAllTimeInForce,
480 scheduled_time_ms: i64,
481 ) -> anyhow::Result<String> {
482 let credential = self
483 .credential
484 .as_ref()
485 .context("Lighter account cancellation requires credentials")?;
486 self.fanout_dispatch_context(credential)
487 .context("failed to spawn Lighter account cancellation")?
488 .send_cancel_all(
489 time_in_force,
490 scheduled_time_ms,
491 false,
492 self.cancellation_token.clone(),
493 )
494 .await
495 }
496
497 async fn start_scheduled_cancel(&mut self) -> anyhow::Result<()> {
498 let Some(timeout_secs) = self.config.scheduled_cancel_timeout_secs else {
499 return Ok(());
500 };
501
502 let credential = self
503 .credential
504 .as_ref()
505 .context("scheduled cancellation requires credentials")?;
506 let context = self.fanout_dispatch_context(credential)?;
507 context
508 .refresh_scheduled_cancel(timeout_secs)
509 .await
510 .context("failed to arm Lighter scheduled cancellation")?;
511 let cancellation = self.cancellation_token.clone();
512 let notify = Arc::clone(&self.scheduled_cancel_notify);
513 let refresh_interval =
514 Duration::from_secs(((timeout_secs - SCHEDULED_CANCEL_MIN_DELAY_SECS) / 3).min(60));
515
516 self.scheduled_cancel_handle.spawn(async move {
517 let mut interval = tokio::time::interval_at(tokio::time::Instant::now() + refresh_interval, refresh_interval);
518 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
519
520 loop {
521 tokio::select! {
522 biased;
523 () = cancellation.cancelled() => break,
524 () = notify.notified() => {},
525 _ = interval.tick() => {},
526 }
527
528 if !context.ws_client.is_active()
529 || context.nonce_ready_connection_epoch.load(Ordering::Acquire) != context.ws_client.connection_epoch() {
530 continue;
531 }
532
533 tokio::select! {
534 biased;
535 () = cancellation.cancelled() => break,
536 result = context.refresh_scheduled_cancel(timeout_secs) => {
537 if let Err(e) = result {
538 log::error!("Scheduled cancellation refresh failed; the last deadline may expire: {e:#}");
539 }
540 }
541 }
542 }
543 })?;
544
545 log::info!("Scheduled cancellation armed: timeout_secs={timeout_secs}");
546 Ok(())
547 }
548
549 fn create_ws_client(
550 config: &LighterExecutionClientConfig,
551 registry: Arc<MarketRegistry>,
552 socket_factory: &SocketControlFactory,
553 ) -> LighterWebSocketClient {
554 let ws_client = LighterWebSocketClient::new(
555 Some(config.ws_url()),
556 config.environment,
557 registry,
558 config.transport_backend,
559 config.ws_timeout_secs,
560 Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
562 config
563 .proxy_url
564 .as_ref()
565 .map(|value| value.expose_secret().to_owned()),
566 );
567
568 ws_client.with_socket_control(socket_factory.control(USER_STREAMS_ENDPOINT))
569 }
570
571 fn take_ws_client(&mut self) -> LighterWebSocketClient {
572 let cached_instruments = self.ws_client.instruments_cache();
573 let ws_cache = cached_instruments
574 .iter()
575 .map(|entry| (*entry.key(), entry.value().clone()))
576 .collect();
577 let replacement = Self::create_ws_client(
578 &self.config,
579 Arc::clone(&self.registry),
580 &self.socket_factory,
581 );
582 replacement.cache_instruments(ws_cache);
583
584 std::mem::replace(&mut self.ws_client, replacement)
585 }
586
587 fn spawn_task<F>(&self, description: &'static str, fut: F)
588 where
589 F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
590 {
591 let future = async move {
592 if let Err(e) = fut.await {
593 log::warn!("{description} failed: {e:?}");
594 }
595 };
596
597 if let Err(e) = self.pending_tasks.spawn(future) {
598 log::warn!("Skipping Lighter {description} after shutdown began: {e}");
599 }
600 }
601
602 fn subscribe_applied_fills(&mut self) {
603 if self.fills_applied.is_some() {
604 return;
605 }
606
607 let core = self.core.clone();
608 let dispatch = self.dispatch.clone();
609 let emitter = self.emitter.clone();
610
611 let handler = TypedHandler::from(move |event: &OrderEventAny| {
612 let OrderEventAny::Filled(fill) = event else {
613 return;
614 };
615
616 if fill.account_id != core.account_id || fill.trader_id != core.trader_id {
617 return;
618 }
619
620 let Some(identity) = dispatch.order_identity(&fill.client_order_id) else {
621 return;
622 };
623
624 if identity.strategy_id != fill.strategy_id
625 || identity.instrument_id != fill.instrument_id
626 {
627 return;
628 }
629
630 let cache = core.cache();
631
632 let Some(order) = cache.order(&fill.client_order_id) else {
633 return;
634 };
635
636 if !order.is_duplicate_fill(fill) {
637 return;
638 }
639
640 let events = dispatch.order_events.lock();
641
642 if is_inferred_reconciliation_trade_id_format(&fill.trade_id) {
643 identity.sync_fills(&order);
644 }
645
646 if identity.has_inferred_fills() {
647 return;
648 }
649
650 identity.publish_fill(fill);
651 identity.publication_pending(Some(&order));
652 dispatch.mark_trade_reconciled(fill.trade_id);
653 release_terminal(fill.client_order_id, false, &dispatch, &emitter, &events);
654 });
655
656 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), Some(10));
657 self.fills_applied = Some(AppliedFillSubscription(handler));
658 }
659
660 fn begin_session_shutdown(&mut self) {
661 self.pending_tasks.begin_shutdown();
662 self.cancellation_token.cancel();
663 self.ws_client.begin_shutdown();
664 self.core.set_disconnected();
665
666 if self.ws_disconnect_handle.is_none()
667 && (self.ws_client.is_active() || self.ws_stream_handle.is_some())
668 {
669 let ws_client = self.take_ws_client();
670 let retained = Arc::clone(&self.ws_handler_retained);
671
672 if let Err(e) = self
673 .ws_disconnect_handle
674 .spawn(ws_client.disconnect_with_task_retention(retained))
675 {
676 log::error!("Failed to start Lighter WebSocket disconnect task: {e}");
677 }
678 }
679 }
680
681 fn session_tasks_finished(&self) -> bool {
682 self.ws_stream_handle.is_none()
683 && self.terminal_recovery_handle.is_none()
684 && self.ws_disconnect_handle.is_none()
685 && self.ws_handler_retained.is_empty()
686 && self.auth_refresh_handle.is_none()
687 && self.scheduled_cancel_handle.is_none()
688 && self.pending_tasks.is_empty()
689 && self.shutdown_errors.is_empty()
690 }
691
692 async fn finish_session_shutdown(&mut self) -> anyhow::Result<()> {
693 self.pending_tasks.begin_shutdown();
694
695 Self::finish_disconnect_task(
696 &mut self.ws_disconnect_handle,
697 "WebSocket disconnect",
698 &mut self.shutdown_errors,
699 )
700 .await;
701
702 if let Err(e) = self.ws_handler_retained.finish().await {
703 self.shutdown_errors.push(e.to_string());
704 }
705 Self::finish_owned_task(
706 &mut self.ws_stream_handle,
707 "execution consumer",
708 &mut self.shutdown_errors,
709 )
710 .await;
711
712 Self::finish_owned_task(
713 &mut self.auth_refresh_handle,
714 "auth-token refresh",
715 &mut self.shutdown_errors,
716 )
717 .await;
718
719 if let Err(e) = self
720 .pending_tasks
721 .finish_shutdown(Duration::from_secs(1), DISCONNECT_TIMEOUT)
722 .await
723 {
724 self.shutdown_errors
725 .push(format!("client-owned tasks failed: {e}"));
726 }
727
728 Self::finish_owned_task(
729 &mut self.scheduled_cancel_handle,
730 "scheduled cancellation refresh",
731 &mut self.shutdown_errors,
732 )
733 .await;
734
735 if self.ws_disconnect_handle.is_none()
736 && self.ws_handler_retained.is_empty()
737 && self.ws_stream_handle.is_none()
738 && self.pending_tasks.is_empty()
739 {
740 self.terminal_producers_stopped.cancel();
741
742 if let Some(outcome) = finish_task(
743 &mut self.terminal_recovery_handle,
744 TERMINAL_SHUTDOWN_TIMEOUT + DISCONNECT_TIMEOUT,
745 DISCONNECT_TIMEOUT,
746 )
747 .await
748 && !matches!(outcome, TaskJoinOutcome::Completed(()))
749 {
750 self.shutdown_errors
751 .push(format!("terminal recovery task failed: {outcome:?}"));
752 }
753 }
754
755 if self.session_tasks_finished() {
756 self.fills_applied = None;
757 }
758
759 let stale = self.dispatch.take_pending_sendtx();
760 warn_pending_sendtx_unknown(&stale, "session shutdown");
761 self.nonce_recovery_inflight.store(false, Ordering::Release);
762
763 if !self.shutdown_errors.is_empty() {
764 let errors = std::mem::take(&mut self.shutdown_errors);
765 anyhow::bail!("Failed to terminate Lighter tasks: {}", errors.join("; "));
766 }
767 Ok(())
768 }
769
770 async fn finish_owned_task(
771 slot: &mut TaskSlot<()>,
772 description: &str,
773 errors: &mut Vec<String>,
774 ) {
775 let Some(outcome) = finish_task(slot, DISCONNECT_TIMEOUT, DISCONNECT_TIMEOUT).await else {
776 return;
777 };
778
779 match outcome {
780 TaskJoinOutcome::Completed(()) => {
781 log::debug!("Lighter {description} task completed");
782 }
783 TaskJoinOutcome::Aborted => {
784 log::debug!("Lighter {description} task cancelled");
785 }
786 TaskJoinOutcome::Failed(e) => {
787 errors.push(format!("{description} task failed: {e}"));
788 }
789 TaskJoinOutcome::Incomplete => {
790 errors.push(format!("{description} task did not stop after abort"));
791 }
792 }
793 }
794
795 async fn finish_disconnect_task(
796 slot: &mut TaskSlot<Result<(), LighterWsError>>,
797 description: &str,
798 errors: &mut Vec<String>,
799 ) {
800 let Some(outcome) = finish_task(slot, DISCONNECT_TIMEOUT, DISCONNECT_TIMEOUT).await else {
801 return;
802 };
803
804 match outcome {
805 TaskJoinOutcome::Completed(Ok(())) | TaskJoinOutcome::Aborted => {}
806 TaskJoinOutcome::Completed(Err(e)) => {
807 errors.push(format!("{description} failed: {e}"));
808 }
809 TaskJoinOutcome::Failed(e) => {
810 errors.push(format!("{description} task failed: {e}"));
811 }
812 TaskJoinOutcome::Incomplete => {
813 errors.push(format!("{description} task did not stop after abort"));
814 }
815 }
816 }
817
818 async fn ensure_instruments_initialized_async(&self) -> anyhow::Result<()> {
819 if self.core.instruments_initialized() {
820 return Ok(());
821 }
822
823 let instruments = self
824 .http_client
825 .request_instruments()
826 .await
827 .context("failed to request Lighter instruments")?;
828
829 let ws_cache: Vec<(i64, InstrumentAny)> = instruments
830 .iter()
831 .filter_map(|instrument| {
832 self.registry
833 .market_index(&instrument.id())
834 .map(|market_index| (market_index, instrument.clone()))
835 })
836 .collect();
837 self.ws_client.cache_instruments(ws_cache);
838 cache_instruments_for_reports(&instruments);
839
840 log::debug!(
841 "Bootstrapped {} Lighter instruments ({} registry entries)",
842 instruments.len(),
843 self.registry.len(),
844 );
845
846 self.core.set_instruments_initialized();
847 Ok(())
848 }
849
850 async fn await_account_streams_ready(&self, timeout_secs: f64) -> anyhow::Result<()> {
851 let timeout = Duration::from_secs_f64(timeout_secs);
852 self.dispatch.account_streams_ready.await_all(timeout).await
853 }
854
855 async fn refresh_nonce(&self) -> anyhow::Result<()> {
856 let Some(credential) = &self.credential else {
857 return Ok(());
858 };
859 self.nonce_ready_connection_epoch
860 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
861 let _refresh_guard = self.nonce_submission_gate.write().await;
862
863 let response = self
864 .http_client
865 .get_next_nonce(credential.account_index(), credential.api_key_index())
866 .await
867 .context("failed to fetch Lighter nextNonce")?;
868
869 self.dispatch.nonce_manager.refresh(
870 credential.account_index(),
871 credential.api_key_index(),
872 response.nonce,
873 );
874
875 self.nonce_recovery_inflight.store(false, Ordering::Release);
877
878 log::debug!(
879 "Refreshed Lighter nonce baseline: account_index={}, api_key_index={}, next_nonce={}",
880 credential.account_index(),
881 credential.api_key_index(),
882 response.nonce,
883 );
884 Ok(())
885 }
886
887 async fn sync_nonce_from_venue(&self) -> anyhow::Result<()> {
888 let Some(credential) = &self.credential else {
889 return Ok(());
890 };
891
892 let _refresh_guard = self.nonce_submission_gate.write().await;
893 let response = self
894 .http_client
895 .get_next_nonce(credential.account_index(), credential.api_key_index())
896 .await
897 .context("failed to fetch Lighter nextNonce")?;
898 self.dispatch.nonce_manager.sync_from_venue(
899 credential.account_index(),
900 credential.api_key_index(),
901 response.nonce,
902 )?;
903 Ok(())
904 }
905
906 async fn fetch_account_detail(&self) -> Option<LighterAccountDetail> {
907 let Some(credential) = &self.credential else {
908 return None;
909 };
910 let account_index = credential.account_index();
911
912 match tokio::time::timeout(
913 ACCOUNT_SNAPSHOT_TIMEOUT,
914 self.http_client.get_account_detail(account_index),
915 )
916 .await
917 {
918 Ok(Ok(detail)) => Some(detail),
919 Ok(Err(e)) => {
920 log::warn!(
921 "Failed to fetch Lighter account detail for account_index={account_index}; \
922 continuing startup without account metadata: {e}"
923 );
924 None
925 }
926 Err(_) => {
927 log::warn!(
928 "Lighter account detail timed out after {}s for account_index={account_index}; \
929 continuing startup without account metadata",
930 ACCOUNT_SNAPSHOT_TIMEOUT.as_secs()
931 );
932 None
933 }
934 }
935 }
936
937 fn log_account_tier(&self, detail: &LighterAccountDetail) {
942 let account_index = detail.account_index;
943 let code = detail.account_type;
944 let tier = LighterAccountTier::from_code(code);
945 let standard_rest = LighterAccountTier::Standard
946 .documented_rest_quota_per_min()
947 .unwrap_or(60);
948 let (active_rest, cross_check) =
949 tier_quota_report(tier, self.config.rest_quota_per_min, standard_rest);
950
951 log_debug!(
952 "Lighter execution account {account_index} reported tier {tier} \
953 (account_type={code}); active REST quota {active_rest} req/min",
954 color = LogColor::Blue
955 );
956
957 match cross_check {
958 Some(TierCrossCheck::AboveTier { documented }) => {
959 log::warn!(
960 "Configured Lighter rest_quota_per_min={active_rest} exceeds the {tier} tier \
961 limit of {documented} req/min; the venue may reject requests unless the \
962 caller IP is registered for the higher limit"
963 );
964 }
965 Some(TierCrossCheck::RaiseHint { documented }) => {
966 log_debug!(
967 "Lighter {tier} tier permits up to {documented} REST req/min; set \
968 rest_quota_per_min (and register the caller IP with Lighter) to use it",
969 color = LogColor::Blue
970 );
971 }
972 None => {}
973 }
974 }
975
976 fn integrator_account_index(&self) -> Option<u64> {
977 deployment::integrator_account_index(self.config.deployment, self.config.environment)
978 }
979
980 async fn apply_referral_attribution(
981 &self,
982 detail: &LighterAccountDetail,
983 ) -> anyhow::Result<()> {
984 let Some(referral_code) =
985 deployment::referral_code(self.config.deployment, self.config.environment)
986 else {
987 return Ok(());
988 };
989
990 let Some(credential) = &self.credential else {
991 return Ok(());
992 };
993
994 anyhow::ensure!(
995 !detail.l1_address.trim().is_empty(),
996 "Lighter account detail returned an empty L1 address"
997 );
998
999 let auth_token = build_auth_token_for(credential)
1000 .context("failed to mint Lighter auth token for referral attribution")?;
1001 let referral_code = Zeroizing::new(referral_code.to_string());
1002
1003 self.http_client
1004 .use_referral(&detail.l1_address, referral_code.as_str(), &auth_token)
1005 .await
1006 .context("failed to apply Lighter referral code")?;
1007
1008 Ok(())
1009 }
1010
1011 async fn is_maker_only_api_key(&self, credential: &Credential) -> anyhow::Result<bool> {
1015 let auth_token = build_auth_token_for(credential)
1016 .context("failed to mint Lighter auth token for maker-only check")?;
1017 let response = self
1018 .http_client
1019 .get_maker_only_api_keys(credential.account_index(), auth_token)
1020 .await
1021 .context("failed to query getMakerOnlyApiKeys")?;
1022 let api_key_index = i64::from(credential.api_key_index());
1023 Ok(response.api_key_indexes.contains(&api_key_index))
1024 }
1025
1026 async fn submit_integrator_auto_approval(&self) -> anyhow::Result<()> {
1027 let Some(integrator_account_index) = self.integrator_account_index() else {
1028 return Ok(());
1029 };
1030
1031 let Some(credential) = &self.credential else {
1032 return Ok(());
1033 };
1034
1035 let mut maker_only_check_failed = false;
1036
1037 match self.is_maker_only_api_key(credential).await {
1038 Ok(true) => {
1039 log::warn!(
1040 "Skipping Lighter integrator auto-approval: api_key_index={} is maker-only; \
1041 ensure the account has been approved by a non-maker-only key",
1042 credential.api_key_index(),
1043 );
1044 return Ok(());
1045 }
1046 Ok(false) => {}
1047 Err(e) => {
1048 maker_only_check_failed = true;
1049
1050 log::debug!(
1051 "Unable to determine whether the Lighter API key is maker-only; proceeding \
1052 with integrator auto-approval: {e:?}"
1053 );
1054 }
1055 }
1056
1057 let mut approval =
1058 self.prepare_integrator_auto_approval(credential, integrator_account_index)?;
1059
1060 let request = LighterSendTxRequest::new(
1061 LighterTxType::ApproveIntegrator as u8,
1062 approval.tx_info.clone(),
1063 );
1064
1065 approval.send_reservation.wait_for_turn().await;
1066
1067 let response = match self.http_client.send_tx(&request).await {
1068 Ok(response) => response,
1069 Err(e) => {
1070 if lighter_http_error_is_definite_api_rejection(&e) {
1071 let _ = self.dispatch.nonce_manager.ack_failure_if_latest(
1072 credential.account_index(),
1073 approval.api_key_index,
1074 approval.nonce,
1075 );
1076 }
1077 approval.send_reservation.release();
1078 let hint = if maker_only_check_failed {
1079 " (maker-only pre-flight check failed earlier; venue may reject with 62007 \
1080 if this key is maker-only)"
1081 } else {
1082 ""
1083 };
1084 return Err(anyhow::Error::new(e).context(format!(
1085 "failed to submit Lighter integrator approval nonce={} api_key_index={}{hint}",
1086 approval.nonce, approval.api_key_index,
1087 )));
1088 }
1089 };
1090
1091 let _ = self.dispatch.nonce_manager.ack_success(
1092 credential.account_index(),
1093 approval.api_key_index,
1094 approval.nonce,
1095 );
1096 approval.send_reservation.release();
1097
1098 log::debug!(
1099 "Submitted Lighter integrator approval: integrator={}, nonce={}, \
1100 api_key_index={}, approval_expiry={}, tx_hash={}",
1101 integrator_account_index,
1102 approval.nonce,
1103 approval.api_key_index,
1104 approval.approval_expiry,
1105 response.tx_hash,
1106 );
1107 Ok(())
1108 }
1109
1110 fn prepare_integrator_auto_approval(
1111 &self,
1112 credential: &Credential,
1113 integrator_account_index: u64,
1114 ) -> anyhow::Result<PreparedIntegratorApproval> {
1115 let k = fresh_k()?;
1116 let ReservedTxContext {
1117 context,
1118 send_reservation,
1119 } = self.build_tx_context(credential)?;
1120
1121 let now_ms = (self.clock.get_time_ns().as_u64() as i64) / 1_000_000;
1122 let approval_expiry = now_ms.saturating_add(INTEGRATOR_AUTO_APPROVAL_MAX_TTL_MS);
1123 let nonce = context.nonce;
1124 let api_key_index = context.api_key_index;
1125
1126 let tx = ApproveIntegratorTxInfo {
1127 context,
1128 integrator_account_index: integrator_account_index as i64,
1129 max_perps_taker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1130 max_perps_maker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1131 max_spot_taker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1132 max_spot_maker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1133 approval_expiry,
1134 skip_nonce: 0,
1135 };
1136
1137 let signed = sign_tx(&tx, self.config.chain_id(), &credential.private_key()?, k)?;
1138
1139 let tx_info = TxInfoJson::approve_integrator(&tx, &signed, "");
1140
1141 Ok(PreparedIntegratorApproval {
1142 tx_info,
1143 nonce,
1144 api_key_index,
1145 approval_expiry,
1146 send_reservation,
1147 })
1148 }
1149
1150 async fn spawn_ws_consumer(&mut self) -> anyhow::Result<()> {
1151 let mut ws_guard = TaskRetentionGuard::new(
1155 self.ws_client.clone(),
1156 Arc::clone(&self.ws_handler_retained),
1157 );
1158 ws_guard
1159 .client_mut()
1160 .connect_with_cancellation(self.cancellation_token.clone())
1161 .await
1162 .context("failed to connect to Lighter WebSocket")?;
1163
1164 let post_connect = async {
1168 ws_guard
1169 .client_mut()
1170 .wait_until_active()
1171 .await
1172 .context("Lighter WebSocket did not reach active state")?;
1173
1174 if let Some(credential) = &self.credential {
1175 let auth_token = build_auth_token_for(credential)
1176 .context("failed to mint Lighter auth token")?;
1177 let account_index = credential.account_index();
1178
1179 ws_guard
1180 .client_mut()
1181 .set_execution_context(self.core.account_id, account_index)
1182 .await
1183 .map_err(|e| anyhow::anyhow!("failed to set Lighter execution context: {e}"))?;
1184
1185 let channels = auth_token_rotation_channels(account_index);
1186
1187 for channel in channels {
1188 ws_guard
1189 .client_mut()
1190 .subscribe_account(channel.clone(), auth_token.clone())
1191 .await
1192 .map_err(|e| {
1193 anyhow::anyhow!(
1194 "failed to subscribe to Lighter account channel {channel:?}: {e}",
1195 )
1196 })?;
1197 }
1198
1199 log::debug!("Subscribed to Lighter account streams: account_index={account_index}",);
1200 } else {
1201 log::warn!(
1202 "Lighter execution client has no credentials: account streams not subscribed; \
1203 typed execution reports will not flow"
1204 );
1205 }
1206
1207 Ok::<(), anyhow::Error>(())
1208 };
1209
1210 if let Err(e) = post_connect.await {
1211 log::warn!("Lighter post-connect setup failed, tearing down WS: {e}");
1212 let ws_client = ws_guard.disarm();
1213 let mut rollback_errors = Vec::new();
1214
1215 if let Err(e) = ws_client
1216 .disconnect_with_task_retention(Arc::clone(&self.ws_handler_retained))
1217 .await
1218 {
1219 rollback_errors.push(e.to_string());
1220 }
1221
1222 if let Err(e) = self.ws_handler_retained.finish().await {
1223 rollback_errors.push(e.to_string());
1224 }
1225
1226 if rollback_errors.is_empty() {
1227 return Err(e);
1228 }
1229 return Err(e.context(format!(
1230 "Lighter post-connect rollback failed: {}",
1231 rollback_errors.join("; ")
1232 )));
1233 }
1234
1235 let mut ws_client = ws_guard.disarm();
1236 self.ws_client.set_task_slot(ws_client.take_task_slot());
1237
1238 let cancellation_token = self.cancellation_token.clone();
1239 let emitter = self.emitter.clone();
1240 let dispatch = self.dispatch.clone();
1241 let credential_for_loop = self.credential.clone();
1242 let http_client_for_loop = self.http_client.clone();
1243 let registry_for_loop = Arc::clone(&self.registry);
1244 let auth_refresh_notify = Arc::clone(&self.auth_refresh_notify);
1245 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
1246 let nonce_ready_connection_epoch = Arc::clone(&self.nonce_ready_connection_epoch);
1247 let scheduled_cancel_notify = Arc::clone(&self.scheduled_cancel_notify);
1248 let pending_tasks_for_loop = self
1249 .pending_tasks
1250 .spawner()
1251 .context("Lighter task admission is closed")?;
1252 let account_id_for_loop = self.core.account_id;
1253 let clock_for_loop = self.clock;
1254
1255 let orders_cached = {
1256 let cache = self.core.cache();
1257 cache
1258 .orders_open_refs(
1259 Some(&self.core.venue),
1260 None,
1261 None,
1262 Some(&self.core.account_id),
1263 None,
1264 )
1265 .into_iter()
1266 .map(|order| order.clone())
1267 .collect()
1268 };
1269
1270 dispatch.seed_order_restorations(orders_cached);
1271
1272 let nonce_refresh_retry = NonceRefreshRetry {
1273 http_client: self.http_client.clone(),
1274 dispatch: self.dispatch.clone(),
1275 credential: self.credential.clone(),
1276 submission_gate: Arc::clone(&self.nonce_submission_gate),
1277 ready_connection_epoch: Arc::clone(&self.nonce_ready_connection_epoch),
1278 ws_client: ws_client.clone(),
1279 cancellation_token: self.cancellation_token.clone(),
1280 pending_tasks: pending_tasks_for_loop.clone(),
1281 };
1282
1283 let child_context = credential_for_loop
1284 .as_ref()
1285 .map(|credential| AckedOrderProbeContext {
1286 http_client: http_client_for_loop.clone(),
1287 registry: Arc::clone(®istry_for_loop),
1288 credential: credential.clone(),
1289 dispatch: dispatch.clone(),
1290 account_id: account_id_for_loop,
1291 clock: clock_for_loop,
1292 emitter: emitter.clone(),
1293 connection_epoch: ws_client.connection_epoch_atomic(),
1294 cancellation_token: cancellation_token.clone(),
1295 pending_tasks: pending_tasks_for_loop.clone(),
1296 });
1297
1298 if let Some(context) = child_context.clone() {
1299 self.terminal_producers_stopped = CancellationToken::new();
1300
1301 self.terminal_recovery_handle.spawn(run_terminal_recovery(
1302 context,
1303 self.terminal_producers_stopped.clone(),
1304 ))?;
1305 }
1306
1307 self.ws_stream_handle.spawn(async move {
1308 log::debug!("Lighter execution WebSocket consumption loop started");
1309
1310 loop {
1311 tokio::select! {
1312 biased;
1313 () = cancellation_token.cancelled() => {
1314 log::debug!("Lighter execution consumption loop cancelled");
1315 break;
1316 }
1317 msg_opt = ws_client.next_event() => {
1318 match msg_opt {
1319 Some(NautilusWsMessage::ExecutionReports(reports)) => {
1320 let mut order_count = 0_usize;
1321 let mut fill_count = 0_usize;
1322 let trader_id = emitter.trader_id();
1323 let account_index = credential_for_loop
1324 .as_ref()
1325 .map(|c| c.account_index());
1326
1327 for report in reports {
1328 match report {
1329 LighterExecutionReport::Order(order) => {
1330 order_count += 1;
1331 dispatch_lighter_order(
1332 &order,
1333 &dispatch,
1334 &emitter,
1335 ®istry_for_loop,
1336 account_id_for_loop,
1337 trader_id,
1338 clock_for_loop.get_time_ns(),
1339 );
1340
1341 if let Some(context) = &child_context {
1342 confirm_executed_modify(&order, context);
1343 }
1344 }
1345 LighterExecutionReport::Fill(trade) => {
1346 fill_count += 1;
1347 dispatch_lighter_trade(
1348 &trade,
1349 &dispatch,
1350 &emitter,
1351 ®istry_for_loop,
1352 account_id_for_loop,
1353 trader_id,
1354 account_index,
1355 clock_for_loop.get_time_ns(),
1356 );
1357 }
1358 }
1359 }
1360
1361 if let Some(context) = &child_context {
1362 spawn_child_recoveries(context);
1363 }
1364 log::debug!(
1365 "Lighter execution batch: orders={order_count} fills={fill_count}",
1366 );
1367 }
1368 Some(NautilusWsMessage::AccountTx { connection_epoch, txs }) => {
1369 if let Some(context) = &child_context {
1370 handle_account_tx(connection_epoch, &txs, context);
1371 }
1372 }
1373 Some(NautilusWsMessage::PositionSnapshot {
1374 reports,
1375 skipped_market_ids,
1376 }) => {
1377 for r in &reports {
1380 if let Some(idx) =
1381 registry_for_loop.market_index(&r.instrument_id)
1382 {
1383 dispatch.note_active_market(idx);
1384 }
1385 }
1386 let position_count = reports.len();
1387 let retained_positions: Vec<InstrumentId> = skipped_market_ids
1388 .iter()
1389 .filter_map(|market_id| {
1390 registry_for_loop.instrument_id(*market_id)
1391 })
1392 .collect();
1393 let removed = dispatch.replace_position_snapshot(
1394 &reports,
1395 &retained_positions,
1396 &skipped_market_ids,
1397 );
1398 log::debug!(
1399 "Lighter position snapshot: positions={position_count}, skipped_markets={}, removed={}",
1400 skipped_market_ids.len(),
1401 removed.len(),
1402 );
1403 emit_lighter_position_reports(
1404 reports,
1405 removed,
1406 &emitter,
1407 account_id_for_loop,
1408 clock_for_loop.get_time_ns(),
1409 );
1410 }
1411 Some(NautilusWsMessage::PositionUpdate {
1412 reports,
1413 closed_market_ids,
1414 skipped_market_ids,
1415 }) => {
1416 let mut covered_market_ids = closed_market_ids.clone();
1417
1418 for report in &reports {
1419 if let Some(market_id) =
1420 registry_for_loop.market_index(&report.instrument_id)
1421 {
1422 dispatch.note_active_market(market_id);
1423 covered_market_ids.push(market_id);
1424 }
1425 }
1426 let position_count = reports.len();
1427 let closed_positions: Vec<InstrumentId> = closed_market_ids
1428 .iter()
1429 .filter_map(|market_id| {
1430 registry_for_loop.instrument_id(*market_id)
1431 })
1432 .collect();
1433 let removed = dispatch.apply_position_update(
1434 &reports,
1435 &closed_positions,
1436 &covered_market_ids,
1437 &skipped_market_ids,
1438 );
1439 log::debug!(
1440 "Lighter position update: positions={position_count}, closed_markets={}, skipped_markets={}, removed={}",
1441 closed_market_ids.len(),
1442 skipped_market_ids.len(),
1443 removed.len(),
1444 );
1445 emit_lighter_position_reports(
1446 reports,
1447 removed,
1448 &emitter,
1449 account_id_for_loop,
1450 clock_for_loop.get_time_ns(),
1451 );
1452 }
1453 Some(NautilusWsMessage::AccountState(state)) => {
1454 log::debug!(
1455 "Lighter AccountState: balances={} margins={}",
1456 state.balances.len(),
1457 state.margins.len(),
1458 );
1459 dispatch.cache_account_state((*state).clone());
1465 emitter.send_account_state(*state);
1466 }
1467 Some(NautilusWsMessage::Reconnected { connection_epoch }) => {
1468 dispatch.terminals_recover.notify_one();
1469 dispatch.invalidate_position_snapshot();
1470 dispatch.stop_cancel_probes_before(connection_epoch);
1471 nonce_ready_connection_epoch.store(
1472 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
1473 Ordering::Release,
1474 );
1475 log::debug!("Lighter WebSocket reconnected (execution stream)");
1483 auth_refresh_notify.notify_one();
1484
1485 let refresh_guard = nonce_submission_gate.write().await;
1499 let disconnected_epoch = connection_epoch.saturating_sub(1);
1500 let stale = dispatch.drain_pending_sendtx(disconnected_epoch);
1501 warn_pending_sendtx_unknown(&stale, "reconnect");
1502
1503 if let Some(credential) = &credential_for_loop {
1504 match http_client_for_loop
1505 .get_next_nonce(
1506 credential.account_index(),
1507 credential.api_key_index(),
1508 )
1509 .await
1510 {
1511 Ok(response) => {
1512 dispatch.nonce_manager.refresh(
1513 credential.account_index(),
1514 credential.api_key_index(),
1515 response.nonce,
1516 );
1517 nonce_ready_connection_epoch
1518 .store(connection_epoch, Ordering::Release);
1519 drop(refresh_guard);
1520 scheduled_cancel_notify.notify_one();
1521 log::debug!(
1522 "Refreshed Lighter nonce after reconnect: \
1523 account_index={}, next_nonce={}",
1524 credential.account_index(),
1525 response.nonce,
1526 );
1527 }
1528 Err(e) => {
1529 log::warn!(
1530 "Failed to refresh Lighter nonce after reconnect: {e}",
1531 );
1532 nonce_refresh_retry
1533 .clone()
1534 .spawn(connection_epoch);
1535 }
1536 }
1537 }
1538 }
1539 Some(message @ (NautilusWsMessage::SendTxAck { .. } | NautilusWsMessage::SendTxBatchResult { .. })) => {
1540 let account_index = credential_for_loop.as_ref().map(|c| c.account_index());
1541 let acked = match message {
1542 NautilusWsMessage::SendTxAck { connection_epoch, tx_hash, code } => {
1543 handle_send_tx_ack_for_connection(&dispatch, account_index, connection_epoch, code, tx_hash.as_deref()).into_iter().collect()
1544 }
1545 NautilusWsMessage::SendTxBatchResult { connection_epoch, id, code, message, tx_hashes } => {
1546 let (acked, resync) = handle_send_tx_batch_result(
1547 &dispatch, &emitter, account_index, connection_epoch,
1548 clock_for_loop.get_time_ns(), &id, code, &message, &tx_hashes,
1549 );
1550
1551 if resync && ws_client.connection_epoch() == connection_epoch {
1552 nonce_ready_connection_epoch.store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
1553
1554 nonce_refresh_retry.clone().spawn(connection_epoch);
1555 }
1556 acked
1557 }
1558 _ => unreachable!("matched transaction response"),
1559 };
1560
1561 for pending in acked {
1562 let Some(credential) = credential_for_loop.clone() else { continue; };
1563
1564 if matches!(pending.kind, PendingSendTxKind::Modify { .. }) { continue; }
1565 spawn_acked_order_probe(
1566 &pending,
1567 AckedOrderProbeContext {
1568 http_client: http_client_for_loop.clone(),
1569 registry: Arc::clone(®istry_for_loop),
1570 credential,
1571 dispatch: dispatch.clone(),
1572 account_id: account_id_for_loop,
1573 clock: clock_for_loop,
1574 emitter: emitter.clone(),
1575 connection_epoch: ws_client.connection_epoch_atomic(),
1576 cancellation_token: cancellation_token.clone(),
1577 pending_tasks: pending_tasks_for_loop.clone(),
1578 },
1579 );
1580 }
1581 }
1582 Some(NautilusWsMessage::SendTxRejected {
1583 connection_epoch,
1584 source,
1585 code,
1586 message,
1587 tx_hash,
1588 }) => {
1589 let account_index = credential_for_loop
1590 .as_ref()
1591 .map(|c| c.account_index());
1592 let needs_nonce_resync = handle_send_tx_rejection_for_connection(
1593 &dispatch,
1594 &emitter,
1595 account_index,
1596 connection_epoch,
1597 clock_for_loop.get_time_ns(),
1598 source,
1599 code,
1600 &message,
1601 tx_hash.as_deref(),
1602 );
1603
1604 if needs_nonce_resync
1608 && let Some(credential) = &credential_for_loop
1609 {
1610 if ws_client.connection_epoch() != connection_epoch {
1611 log::warn!(
1612 "Skipping stale Lighter nonce rejection from connection \
1613 epoch {connection_epoch}",
1614 );
1615 continue;
1616 }
1617 nonce_ready_connection_epoch.store(
1618 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
1619 Ordering::Release,
1620 );
1621 let _refresh_guard = nonce_submission_gate.write().await;
1622
1623 match http_client_for_loop
1624 .get_next_nonce(
1625 credential.account_index(),
1626 credential.api_key_index(),
1627 )
1628 .await
1629 {
1630 Ok(response) => {
1631 dispatch.nonce_manager.refresh(
1632 credential.account_index(),
1633 credential.api_key_index(),
1634 response.nonce,
1635 );
1636 nonce_ready_connection_epoch
1637 .store(connection_epoch, Ordering::Release);
1638 log::debug!(
1639 "Hard-refreshed Lighter nonce after invalid-nonce \
1640 rejection: account_index={}, next_nonce={}",
1641 credential.account_index(),
1642 response.nonce,
1643 );
1644 }
1645 Err(e) => {
1646 log::warn!(
1647 "Failed to refresh Lighter nonce after \
1648 invalid-nonce rejection: {e}",
1649 );
1650 nonce_refresh_retry
1651 .clone()
1652 .spawn(connection_epoch);
1653 }
1654 }
1655 }
1656 }
1657 Some(NautilusWsMessage::Raw(value)) => {
1658 log::debug!("Unhandled Lighter raw frame on execution stream: {value}");
1659 }
1660 Some(NautilusWsMessage::AccountStreamFirstFrame(stream)) => {
1661 match stream {
1667 AccountStream::Orders => {
1668 dispatch.account_streams_ready.mark_orders();
1669 }
1670 AccountStream::Trades => {
1671 dispatch.account_streams_ready.mark_trades();
1672 }
1673 AccountStream::Positions => {
1674 dispatch.account_streams_ready.mark_positions();
1675 }
1676 AccountStream::Assets => {
1677 dispatch.account_streams_ready.mark_assets();
1678 }
1679 AccountStream::UserStats => {
1680 dispatch.account_streams_ready.mark_user_stats();
1681 }
1682 }
1683 }
1684 Some(
1688 NautilusWsMessage::Trades(_)
1689 | NautilusWsMessage::Quote(_)
1690 | NautilusWsMessage::Deltas(_)
1691 | NautilusWsMessage::Depth(_)
1692 | NautilusWsMessage::Bar(_)
1693 | NautilusWsMessage::MarkPrice(_)
1694 | NautilusWsMessage::IndexPrice(_)
1695 | NautilusWsMessage::FundingRate(_),
1696 ) => {}
1697 None => {
1698 log::debug!("Lighter execution next_event returned None");
1699 tokio::select! {
1700 () = cancellation_token.cancelled() => {
1701 log::debug!(
1702 "Lighter execution consumption loop cancelled"
1703 );
1704 break;
1705 }
1706 () = tokio::time::sleep(Duration::from_secs(1)) => {}
1707 }
1708 }
1709 }
1710 }
1711 }
1712 }
1713
1714 log::debug!("Lighter execution WebSocket consumption loop finished");
1715 })?;
1716
1717 if let Some(credential) = &self.credential {
1718 self.spawn_auth_token_refresh(credential.clone())?;
1719 }
1720
1721 Ok(())
1722 }
1723
1724 fn spawn_auth_token_refresh(&mut self, credential: Credential) -> anyhow::Result<()> {
1725 let ws_client = self.ws_client.clone();
1726 let cancellation_token = self.cancellation_token.clone();
1727 let account_index = credential.account_index();
1728 let channels = auth_token_rotation_channels(account_index);
1729 let refresh_notify = Arc::clone(&self.auth_refresh_notify);
1730
1731 self.auth_refresh_handle.spawn(async move {
1732 log::debug!(
1733 "Lighter auth-token refresh task started: interval={}s, account_index={account_index}",
1734 AUTH_TOKEN_REFRESH_INTERVAL.as_secs(),
1735 );
1736
1737 let mut next_refresh_delay = AUTH_TOKEN_REFRESH_INTERVAL;
1738
1739 loop {
1740 tokio::select! {
1741 biased;
1742 () = cancellation_token.cancelled() => {
1743 log::debug!("Lighter auth-token refresh task cancelled");
1744 break;
1745 }
1746 () = refresh_notify.notified() => {
1747 log::debug!(
1748 "Refreshing Lighter auth token immediately after reconnect: account_index={account_index}",
1749 );
1750 }
1751 () = tokio::time::sleep(next_refresh_delay) => {}
1752 }
1753
1754 let outcome = refresh_auth_token_until_rotated(
1755 &credential,
1756 &channels,
1757 &cancellation_token,
1758 AUTH_TOKEN_REFRESH_BACKOFF,
1759 build_auth_token_for,
1760 |channel, token| {
1761 let ws_client = ws_client.clone();
1762 async move { ws_client.subscribe_account(channel, token).await }
1763 },
1764 )
1765 .await;
1766
1767 if let Some(delay) = auth_token_refresh_next_delay(outcome) {
1768 if outcome == AuthTokenRefreshOutcome::Exhausted {
1769 log::error!(
1770 "Lighter auth-token rotation exhausted retry window; retrying again in {}s: account_index={account_index}",
1771 delay.as_secs(),
1772 );
1773 }
1774 next_refresh_delay = delay;
1775 } else {
1776 log::debug!("Lighter auth-token refresh task cancelled");
1777 break;
1778 }
1779 }
1780 })?;
1781 Ok(())
1782 }
1783
1784 fn resolve_slippage_bps(&self, params: Option<&Params>) -> anyhow::Result<u32> {
1786 let Some(value) = params.and_then(|p| p.get("market_order_slippage_bps")) else {
1787 return Ok(self.config.market_order_slippage_bps);
1788 };
1789
1790 let value = value
1791 .as_u64()
1792 .context("Lighter market_order_slippage_bps must be an unsigned integer")?;
1793 let value =
1794 u32::try_from(value).context("Lighter market_order_slippage_bps exceeds u32 range")?;
1795 validate_market_order_slippage_bps(value)?;
1796 Ok(value)
1797 }
1798
1799 fn build_tx_context(&self, credential: &Credential) -> anyhow::Result<ReservedTxContext> {
1800 let connection_epoch = self.ws_client.connection_epoch();
1801 anyhow::ensure!(
1802 self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
1803 TxPreparationError::NonceRefreshPending(connection_epoch),
1804 );
1805 let nonce_guard = Arc::clone(&self.nonce_submission_gate)
1806 .try_read_owned()
1807 .context("Lighter nonce refresh is in progress")?;
1808 anyhow::ensure!(
1809 self.ws_client.connection_epoch() == connection_epoch
1810 && self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
1811 TxPreparationError::StateChanged,
1812 );
1813
1814 let send_reservation = match self.tx_send_sequencer.reserve(
1815 credential.account_index(),
1816 credential.api_key_index(),
1817 || {
1818 self.dispatch
1819 .nonce_manager
1820 .next_nonce(credential.account_index(), credential.api_key_index())
1821 },
1822 ) {
1823 Ok(reservation) => reservation,
1824 Err(e @ NonceError::SkipWindowExhausted { .. }) => {
1825 self.spawn_nonce_window_recovery(credential);
1826 anyhow::bail!("failed to allocate Lighter nonce: {e}");
1827 }
1828 Err(e) => anyhow::bail!("failed to allocate Lighter nonce: {e}"),
1829 };
1830
1831 let nonce = send_reservation.nonce;
1832
1833 let now_ns = self.clock.get_time_ns().as_u64() as i64;
1834 let expired_at = (now_ns / 1_000_000).saturating_add(DEFAULT_TX_EXPIRY_MS);
1835 let send_reservation = send_reservation.with_nonce_ownership(nonce_guard, connection_epoch);
1836
1837 let context = TxContext {
1838 account_index: credential.account_index(),
1839 api_key_index: credential.api_key_index(),
1840 nonce,
1841 expired_at,
1842 };
1843
1844 Ok(ReservedTxContext {
1845 context,
1846 send_reservation,
1847 })
1848 }
1849
1850 fn spawn_nonce_window_recovery(&self, credential: &Credential) {
1851 if self.nonce_recovery_inflight.swap(true, Ordering::AcqRel) {
1852 return;
1853 }
1854
1855 let inflight = Arc::clone(&self.nonce_recovery_inflight);
1856 let http_client = self.http_client.clone();
1857 let dispatch = self.dispatch.clone();
1858 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
1859 let account_index = credential.account_index();
1860 let api_key_index = credential.api_key_index();
1861
1862 self.spawn_task("nonce_window_recovery", async move {
1863 let _refresh_guard = nonce_submission_gate.write().await;
1864 let result = http_client
1865 .get_next_nonce(account_index, api_key_index)
1866 .await;
1867 inflight.store(false, Ordering::Release);
1868
1869 match result {
1870 Ok(response) => {
1871 let _ = dispatch.nonce_manager.sync_from_venue(
1874 account_index,
1875 api_key_index,
1876 response.nonce,
1877 );
1878 log::debug!(
1879 "Resynced Lighter nonce baseline after skip-window exhaustion: \
1880 account_index={account_index}, api_key_index={api_key_index}, \
1881 next_nonce={}",
1882 response.nonce,
1883 );
1884 }
1885 Err(e) => {
1886 log::error!(
1887 "Failed to resync Lighter nonce after skip-window exhaustion: {e}",
1888 );
1889 }
1890 }
1891 Ok(())
1892 });
1893 }
1894
1895 fn dispatch_create_order_plan(
1896 &self,
1897 plan: &CreateOrderPlan,
1898 credential: &Credential,
1899 ) -> anyhow::Result<()> {
1900 let context = self.fanout_dispatch_context(credential)?;
1901 let prepared = context.sign_create_order(plan)?;
1902 self.spawn_task("submit_order", async move {
1903 context.send_create_order(prepared).await;
1904 Ok(())
1905 });
1906
1907 Ok(())
1908 }
1909
1910 fn prepare_create_order_plan(
1911 &self,
1912 order: &OrderAny,
1913 params: Option<&Params>,
1914 ) -> anyhow::Result<CreateOrderPlan> {
1915 let slippage_bps = self.resolve_slippage_bps(params)?;
1916 let attributes = self.resolve_create_attributes(params)?;
1917 let instrument_id = order.instrument_id();
1918 let market_index = self.registry.market_index(&instrument_id).ok_or_else(|| {
1919 anyhow::anyhow!("no Lighter market_index registered for instrument {instrument_id}")
1920 })?;
1921
1922 let instrument = self.core.cache().try_instrument(&instrument_id)?.clone();
1923
1924 let order_kind = nautilus_to_lighter_order_type(order.order_type())?;
1925
1926 let tif = nautilus_to_lighter_tif(
1927 order.order_type(),
1928 order.time_in_force(),
1929 order.is_post_only(),
1930 )?;
1931 let now_ms = (self.clock.get_time_ns().as_u64() / 1_000_000) as i64;
1932 let order_expiry = order_expiry_for(
1933 order.order_type(),
1934 &order.time_in_force(),
1935 order.expire_time(),
1936 now_ms,
1937 self.config.use_gtd,
1938 )?;
1939
1940 let base_amount = quantity_to_ticks(&order.quantity(), instrument.size_precision())?;
1941
1942 anyhow::ensure!(
1944 base_amount > 0,
1945 "quantity `{}` rounds to 0 ticks at size_precision {}",
1946 order.quantity(),
1947 instrument.size_precision(),
1948 );
1949 let price_precision = instrument.price_precision();
1950 let is_buy = matches!(order.order_side(), OrderSide::Buy);
1951
1952 let price_ticks = match order.order_type() {
1955 OrderType::Market => {
1956 let quote = self
1957 .core
1958 .cache()
1959 .quote(&instrument_id)
1960 .copied()
1961 .ok_or_else(|| {
1962 anyhow::anyhow!(
1963 "no cached quote for {instrument_id}: subscribe to quotes before submitting MARKET orders",
1964 )
1965 })?;
1966 let base = if is_buy {
1967 quote.ask_price.as_decimal()
1968 } else {
1969 quote.bid_price.as_decimal()
1970 };
1971 derive_market_order_price_ticks(base, is_buy, price_precision, slippage_bps)?
1972 }
1973 OrderType::StopMarket | OrderType::MarketIfTouched => {
1974 let trigger = order.trigger_price().ok_or_else(|| {
1975 anyhow::anyhow!("{:?} orders require a trigger_price", order.order_type(),)
1976 })?;
1977 derive_market_order_price_ticks(
1978 trigger.as_decimal(),
1979 is_buy,
1980 price_precision,
1981 slippage_bps,
1982 )?
1983 }
1984 _ => order
1985 .price()
1986 .map(|p| price_to_ticks(&p, price_precision))
1987 .transpose()?
1988 .unwrap_or(0),
1989 };
1990
1991 let trigger_price_ticks = order
1992 .trigger_price()
1993 .map(|p| price_to_ticks(&p, price_precision))
1994 .transpose()?
1995 .unwrap_or(0);
1996
1997 if matches!(
2000 order.order_type(),
2001 OrderType::StopMarket
2002 | OrderType::StopLimit
2003 | OrderType::MarketIfTouched
2004 | OrderType::LimitIfTouched
2005 ) {
2006 anyhow::ensure!(
2007 trigger_price_ticks > 0,
2008 "trigger_price `{:?}` rounds to 0 ticks at precision {price_precision}",
2009 order.trigger_price(),
2010 );
2011 }
2012 validate_order_amount(&instrument, order.quantity(), price_ticks, price_precision)?;
2013
2014 Ok(CreateOrderPlan {
2015 order: order.clone(),
2016 market_index,
2017 base_amount,
2018 price: price_ticks,
2019 order_type: order_kind as u8,
2020 time_in_force: tif as u8,
2021 trigger_price: trigger_price_ticks,
2022 order_expiry,
2023 attributes,
2024 })
2025 }
2026
2027 fn resolve_create_attributes(&self, params: Option<&Params>) -> anyhow::Result<L2TxAttributes> {
2028 let mut attributes = integrator_attributes(self.integrator_account_index());
2029
2030 let behavior = match params.and_then(|p| p.get("self_trade_behavior_mode")) {
2031 Some(value) => value
2032 .as_str()
2033 .context("Lighter self_trade_behavior_mode must be a string")?
2034 .parse::<LighterSelfTradeBehaviorMode>()
2035 .context("invalid Lighter self_trade_behavior_mode")?,
2036 None => LighterSelfTradeBehaviorMode::default(),
2037 };
2038
2039 let equality = match params.and_then(|p| p.get("self_trade_equality_mode")) {
2040 Some(value) => value
2041 .as_str()
2042 .context("Lighter self_trade_equality_mode must be a string")?
2043 .parse::<LighterSelfTradeEqualityMode>()
2044 .context("invalid Lighter self_trade_equality_mode")?,
2045 None => LighterSelfTradeEqualityMode::default(),
2046 };
2047
2048 attributes.self_trade_behavior_mode = Some(behavior);
2049 attributes.self_trade_equality_mode = Some(equality);
2050 attributes.validate()?;
2051 Ok(attributes)
2052 }
2053
2054 fn prepare_create_group_plan(
2055 &self,
2056 orders: &[OrderAny],
2057 params: Option<&Params>,
2058 ) -> anyhow::Result<CreateGroupPlan> {
2059 let (grouping_type, indices) = grouped_order_indices(orders)?;
2060 let mut plans = Vec::with_capacity(indices.len());
2061 for index in indices {
2062 let order = &orders[index];
2063 let instrument = self
2064 .core
2065 .cache()
2066 .instrument(&order.instrument_id())
2067 .cloned();
2068
2069 if let Some(reason) = local_submit_denial_reason(order, instrument.as_ref()) {
2070 return Err(reason);
2071 }
2072
2073 anyhow::ensure!(
2074 !is_lighter_spot_order(order, instrument.as_ref()),
2075 "Lighter grouped orders require perpetual markets"
2076 );
2077 plans.push(self.prepare_create_order_plan(order, params)?);
2078 }
2079
2080 prepare_group_bodies(grouping_type, &mut plans, self.config.use_gtd)?;
2081 Ok(CreateGroupPlan {
2082 grouping_type,
2083 orders: plans,
2084 })
2085 }
2086
2087 fn fanout_dispatch_context(
2088 &self,
2089 credential: &Credential,
2090 ) -> anyhow::Result<FanoutDispatchContext> {
2091 let pending_tasks = self
2092 .pending_tasks
2093 .spawner()
2094 .context("Lighter task admission is closed")?;
2095 Ok(FanoutDispatchContext {
2096 clock: self.clock,
2097 chain_id: self.config.chain_id(),
2098 integrator_account_index: self.integrator_account_index(),
2099 emitter: self.emitter.clone(),
2100 credential: credential.clone(),
2101 http_client: self.http_client.clone(),
2102 registry: Arc::clone(&self.registry),
2103 ws_client: self.ws_client.clone(),
2104 tx_rate_limiter: Arc::clone(&self.tx_rate_limiter),
2105 tx_send_sequencer: self.tx_send_sequencer.clone(),
2106 nonce_submission_gate: Arc::clone(&self.nonce_submission_gate),
2107 nonce_ready_connection_epoch: Arc::clone(&self.nonce_ready_connection_epoch),
2108 dispatch: self.dispatch.clone(),
2109 nonce_recovery_inflight: Arc::clone(&self.nonce_recovery_inflight),
2110 pending_tasks,
2111 })
2112 }
2113
2114 fn dispatch_signed_cancel_order(&self, cmd: &CancelOrder, credential: &Credential) {
2115 let context = match self.fanout_dispatch_context(credential) {
2116 Ok(context) => context,
2117 Err(e) => {
2118 log::warn!("Skipping Lighter cancel_order after shutdown began: {e}");
2119 self.deny_cancel_commands(std::slice::from_ref(cmd), &e);
2120 return;
2121 }
2122 };
2123
2124 let emit_cancel_rejected = self.can_emit_order_cancel_rejected(&cmd.client_order_id);
2125
2126 let prepared = match self
2127 .prepare_cancel_order_plan(cmd)
2128 .and_then(|plan| context.sign_cancel_order(&plan))
2129 {
2130 Ok(prepared) => prepared,
2131 Err(e) => {
2132 let reason = format!("Lighter cancel_order failed: {e:#}");
2133
2134 log::warn!("{reason} for {}", cmd.client_order_id);
2135 self.emitter.emit_order_cancel_denied_event(
2136 cmd.strategy_id,
2137 cmd.instrument_id,
2138 cmd.client_order_id,
2139 cmd.venue_order_id,
2140 &reason,
2141 self.clock.get_time_ns(),
2142 Some(cmd.command_id),
2143 );
2144
2145 return;
2146 }
2147 };
2148
2149 self.dispatch.register_cancel_command(
2150 cmd.client_order_id,
2151 cmd.command_id,
2152 emit_cancel_rejected,
2153 );
2154
2155 self.spawn_task("cancel_order", async move {
2156 context
2157 .send_cancel_order(prepared, emit_cancel_rejected)
2158 .await;
2159 Ok(())
2160 });
2161 }
2162
2163 fn dispatch_cancel_orders(&self, cancels: &[CancelOrder]) -> anyhow::Result<()> {
2164 let credential = self
2165 .credential
2166 .as_ref()
2167 .ok_or_else(|| {
2168 anyhow::anyhow!("Lighter execution client cannot cancel without credentials")
2169 })
2170 .inspect_err(|e| self.deny_cancel_commands(cancels, e))?;
2171
2172 if cancels.is_empty() {
2173 log::debug!("batch_cancel_orders called with empty cancel list");
2174 return Ok(());
2175 }
2176
2177 let context = self
2178 .fanout_dispatch_context(credential)
2179 .inspect_err(|e| self.deny_cancel_commands(cancels, e))?;
2180
2181 let mut plans = Vec::with_capacity(cancels.len());
2182 let mut failures = Vec::new();
2183
2184 let command_ids: AHashMap<_, _> = cancels
2185 .iter()
2186 .map(|cancel| (cancel.client_order_id, cancel.command_id))
2187 .collect();
2188
2189 for cancel in cancels {
2190 if command_ids
2191 .get(&cancel.client_order_id)
2192 .is_some_and(|command_id| *command_id != cancel.command_id)
2193 {
2194 let (strategy_id, instrument_id, venue_order_id) =
2195 self.core.get_order(&cancel.client_order_id).map_or(
2196 (
2197 cancel.strategy_id,
2198 cancel.instrument_id,
2199 cancel.venue_order_id,
2200 ),
2201 |order| {
2202 (
2203 order.strategy_id(),
2204 order.instrument_id(),
2205 order.venue_order_id(),
2206 )
2207 },
2208 );
2209
2210 self.emitter.emit_order_cancel_denied_event(
2211 strategy_id,
2212 instrument_id,
2213 cancel.client_order_id,
2214 venue_order_id,
2215 "Duplicate cancel target in batch",
2216 self.clock.get_time_ns(),
2217 Some(cancel.command_id),
2218 );
2219 continue;
2220 }
2221
2222 match self.prepare_cancel_order_plan(cancel) {
2223 Ok(plan) => {
2224 let group = self
2225 .prepare_oco_cancel_plans(cancel)
2226 .unwrap_or_else(|| vec![plan]);
2227 plans.extend(group.into_iter().map(|plan| {
2228 let emit = self.can_emit_order_cancel_rejected(&plan.client_order_id);
2229 (plan, emit)
2230 }));
2231 }
2232 Err(e) => failures.push((cancel, format!("Lighter cancel_order failed: {e}"))),
2233 }
2234 }
2235
2236 for (plan, _) in &mut plans {
2237 plan.command_id = command_ids.get(&plan.client_order_id).copied();
2238 }
2239
2240 let mut seen = AHashSet::new();
2241 plans.retain(|(plan, _)| seen.insert(plan.client_order_id));
2242 for (cancel, reason) in failures {
2243 if !seen.contains(&cancel.client_order_id) {
2244 self.emitter.emit_order_cancel_denied_event(
2245 cancel.strategy_id,
2246 cancel.instrument_id,
2247 cancel.client_order_id,
2248 cancel.venue_order_id,
2249 &reason,
2250 self.clock.get_time_ns(),
2251 Some(cancel.command_id),
2252 );
2253 }
2254 }
2255
2256 for (plan, emit) in &plans {
2257 if let Some(command_id) = plan.command_id {
2258 self.dispatch
2259 .register_cancel_command(plan.client_order_id, command_id, *emit);
2260 }
2261 }
2262
2263 if plans.is_empty() {
2264 return Ok(());
2265 }
2266
2267 self.spawn_task("batch_cancel_orders", async move {
2268 context
2269 .dispatch_batch_plans(
2270 plans
2271 .into_iter()
2272 .map(|(plan, emit)| BatchOrderPlan::Cancel(plan, emit))
2273 .collect(),
2274 )
2275 .await;
2276 Ok(())
2277 });
2278
2279 Ok(())
2280 }
2281
2282 fn can_emit_order_cancel_rejected(&self, client_order_id: &ClientOrderId) -> bool {
2283 self.core
2284 .cache()
2285 .order(client_order_id)
2286 .is_none_or(|order| order.is_pending_cancel())
2287 }
2288
2289 fn prepare_cancel_order_plan(&self, cmd: &CancelOrder) -> anyhow::Result<CancelOrderPlan> {
2290 let market_index = self
2291 .registry
2292 .market_index(&cmd.instrument_id)
2293 .ok_or_else(|| {
2294 anyhow::anyhow!(
2295 "no Lighter market_index registered for instrument {}",
2296 cmd.instrument_id,
2297 )
2298 })?;
2299
2300 self.core.cache().try_order(&cmd.client_order_id)?;
2301
2302 let voi = cmd
2305 .venue_order_id
2306 .or_else(|| self.dispatch.lookup_venue_order_id(&cmd.client_order_id))
2307 .ok_or_else(|| {
2308 anyhow::anyhow!(
2309 "cannot cancel Lighter order {}: venue order_id not yet known \
2310 (await OrderAccepted before issuing cancel)",
2311 cmd.client_order_id,
2312 )
2313 })?;
2314
2315 let venue_index: i64 = voi
2316 .as_str()
2317 .parse()
2318 .with_context(|| format!("Lighter venue_order_id `{voi}` is not an integer index"))?;
2319
2320 Ok(CancelOrderPlan {
2321 command_id: Some(cmd.command_id),
2322 client_order_id: cmd.client_order_id,
2323 strategy_id: cmd.strategy_id,
2324 instrument_id: cmd.instrument_id,
2325 venue_order_id: Some(voi),
2326 market_index,
2327 venue_index,
2328 })
2329 }
2330
2331 fn deny_modify_commands(&self, commands: &[ModifyOrder], error: &anyhow::Error) {
2332 let reason = error.to_string();
2333
2334 for command in commands {
2335 self.emitter.emit_order_modify_denied_event(
2336 command.strategy_id,
2337 command.instrument_id,
2338 command.client_order_id,
2339 command.venue_order_id,
2340 &reason,
2341 self.clock.get_time_ns(),
2342 Some(command.command_id),
2343 );
2344 }
2345 }
2346
2347 fn deny_cancel_commands(&self, commands: &[CancelOrder], error: &anyhow::Error) {
2348 let reason = error.to_string();
2349
2350 for command in commands {
2351 self.emitter.emit_order_cancel_denied_event(
2352 command.strategy_id,
2353 command.instrument_id,
2354 command.client_order_id,
2355 command.venue_order_id,
2356 &reason,
2357 self.clock.get_time_ns(),
2358 Some(command.command_id),
2359 );
2360 }
2361 }
2362
2363 fn dispatch_signed_modify_order(&self, cmd: &ModifyOrder, credential: &Credential) {
2364 let probe_context = match self.fanout_dispatch_context(credential) {
2365 Ok(context) => context.probe_context(),
2366 Err(e) => {
2367 log::warn!("Lighter modify probe unavailable: {e}");
2368 self.deny_modify_commands(std::slice::from_ref(cmd), &e);
2369 return;
2370 }
2371 };
2372
2373 if !self.admit_modify_order(cmd) {
2374 return;
2375 }
2376
2377 let prepared = match self.prepare_signed_modify_order(cmd, credential) {
2378 Ok(prepared) => prepared,
2379 Err(e) => {
2380 let mut events = self.dispatch.order_events.lock();
2381 self.dispatch
2382 .clear_pending_modify_if_command(&cmd.client_order_id, cmd.command_id);
2383 let reason = format!("Lighter modify_order failed: {e:#}");
2384 log::warn!("{reason} for {}", cmd.client_order_id);
2385 self.emitter.emit_order_modify_denied_event(
2386 cmd.strategy_id,
2387 cmd.instrument_id,
2388 cmd.client_order_id,
2389 cmd.venue_order_id,
2390 &reason,
2391 self.clock.get_time_ns(),
2392 Some(cmd.command_id),
2393 );
2394
2395 if let Some(deferred) = events.remove(&cmd.client_order_id) {
2396 flush_modify_events(
2397 cmd.client_order_id,
2398 deferred,
2399 &self.dispatch,
2400 &self.emitter,
2401 &mut events,
2402 );
2403 }
2404
2405 return;
2406 }
2407 };
2408 let PreparedModifyOrder {
2409 command_id,
2410 client_order_id,
2411 strategy_id,
2412 instrument_id,
2413 venue_order_id,
2414 tx_info,
2415 nonce,
2416 api_key_index,
2417 tx_hash,
2418 mut send_reservation,
2419 } = prepared;
2420 let connection_epoch = send_reservation.connection_epoch;
2421
2422 let ws_client = self.ws_client.clone();
2423 let dispatch = self.dispatch.clone();
2424 let credential = credential.clone();
2425 let emitter = self.emitter.clone();
2426 let clock = self.clock;
2427
2428 let tx_rate_limiter = self.tx_rate_limiter.clone();
2429
2430 self.spawn_task("modify_order", async move {
2431 send_reservation.wait_for_turn().await;
2432 await_tx_quota(&tx_rate_limiter).await;
2433
2434 let pending = PendingSendTx {
2435 batch_id: None,
2436 connection_epoch,
2437 kind: PendingSendTxKind::Modify {
2438 command_id,
2439 strategy_id,
2440 instrument_id,
2441 client_order_id,
2442 venue_order_id,
2443 },
2444 submitted_at: clock.get_time_ns(),
2445 nonce,
2446 api_key_index,
2447 tx_hash,
2448 };
2449
2450 dispatch.enqueue_pending_sendtx(pending.clone());
2451 spawn_acked_order_probe(&pending, probe_context);
2452
2453 if let Err(e) = ws_client
2454 .send_tx_on_connection(LighterTxType::ModifyOrder as u8, tx_info, connection_epoch)
2455 .await
2456 {
2457 let failure = classify_lighter_ws_command_failure("modify_order", &e);
2458 let reason = command_failure_reason(&failure);
2459 if matches!(&failure, CommandFailure::Ambiguous(_)) {
2460 log::warn!(
2461 "Lighter modify_order dispatch outcome unknown for {client_order_id}: {reason}; \
2462 retaining pending state for venue reconciliation; diagnostic={e:?}",
2463 );
2464 } else {
2465 log::warn!("{reason} for {client_order_id}; diagnostic={e:?}");
2466 dispatch.remove_pending_sendtx_by_nonce(connection_epoch, nonce);
2467 rollback_tx_dispatch(&dispatch, &credential, None, nonce);
2468 reject_modify_order(
2469 &dispatch,
2470 &emitter,
2471 &pending,
2472 &failure,
2473 clock.get_time_ns(),
2474 false,
2475 );
2476 }
2477 }
2478 send_reservation.release();
2479 Ok(())
2480 });
2481 }
2482
2483 fn admit_modify_order(&self, cmd: &ModifyOrder) -> bool {
2484 let _events = self.dispatch.order_events.lock();
2485
2486 let publication_pending = self
2487 .dispatch
2488 .order_identity(&cmd.client_order_id)
2489 .is_some_and(|identity| {
2490 identity
2491 .publication_pending(self.core.cache().order(&cmd.client_order_id).as_deref())
2492 });
2493
2494 if !publication_pending && self.dispatch.set_pending_modify_action(cmd) {
2495 return true;
2496 }
2497
2498 self.emitter.emit_order_modify_denied_event(
2499 cmd.strategy_id,
2500 cmd.instrument_id,
2501 cmd.client_order_id,
2502 cmd.venue_order_id,
2503 "Lighter permits one unresolved modification per order; wait for its update or rejection",
2504 self.clock.get_time_ns(),
2505 Some(cmd.command_id),
2506 );
2507 false
2508 }
2509
2510 fn prepare_signed_modify_order(
2511 &self,
2512 cmd: &ModifyOrder,
2513 credential: &Credential,
2514 ) -> anyhow::Result<PreparedModifyOrder> {
2515 let plan = self.prepare_modify_order_plan(cmd)?;
2516 let context = self.fanout_dispatch_context(credential)?;
2517 let k = fresh_k()?;
2518 let reserved = self.build_tx_context(credential)?;
2519 context.sign_modify_order(&plan, reserved, k)
2520 }
2521
2522 fn prepare_modify_order_plan(&self, cmd: &ModifyOrder) -> anyhow::Result<ModifyOrderPlan> {
2523 let market_index = self
2524 .registry
2525 .market_index(&cmd.instrument_id)
2526 .ok_or_else(|| {
2527 anyhow::anyhow!(
2528 "no Lighter market_index registered for instrument {}",
2529 cmd.instrument_id,
2530 )
2531 })?;
2532
2533 let voi = cmd
2534 .venue_order_id
2535 .or_else(|| self.dispatch.lookup_venue_order_id(&cmd.client_order_id))
2536 .ok_or_else(|| {
2537 anyhow::anyhow!(
2538 "cannot modify Lighter order {}: venue order_id not yet known \
2539 (await OrderAccepted before issuing modify)",
2540 cmd.client_order_id,
2541 )
2542 })?;
2543
2544 let venue_index: i64 = voi
2545 .as_str()
2546 .parse()
2547 .with_context(|| format!("Lighter venue_order_id `{voi}` is not an integer index"))?;
2548
2549 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
2550 let instrument = self
2551 .core
2552 .cache()
2553 .try_instrument(&cmd.instrument_id)?
2554 .clone();
2555
2556 let new_qty = cmd.quantity.unwrap_or(order.quantity());
2557 anyhow::ensure!(
2558 !(order.parent_order_id().is_some()
2559 && self
2560 .dispatch
2561 .snapshot_for(&cmd.client_order_id)
2562 .is_some_and(|shape| shape.quantity.is_zero())),
2563 "Lighter cannot modify an OTO child while it waits for parent execution"
2564 );
2565 let price_precision = instrument.price_precision();
2566 let new_trigger = cmd.trigger_price.or(order.trigger_price());
2567 let triggered = order.is_triggered() == Some(true)
2568 || self.dispatch.triggered_was_emitted(&cmd.client_order_id);
2569 anyhow::ensure!(
2570 !triggered || cmd.trigger_price.is_none(),
2571 "Lighter cannot modify a trigger price after the order has triggered"
2572 );
2573
2574 let price_ticks = match order.order_type() {
2577 OrderType::StopMarket | OrderType::MarketIfTouched => {
2578 let trigger = new_trigger.ok_or_else(|| {
2579 anyhow::anyhow!("{:?} orders require a trigger_price", order.order_type())
2580 })?;
2581 let is_buy = matches!(order.order_side(), OrderSide::Buy);
2582 let slippage_bps = self.resolve_slippage_bps(cmd.params.as_ref())?;
2583
2584 derive_market_order_price_ticks(
2585 trigger.as_decimal(),
2586 is_buy,
2587 price_precision,
2588 slippage_bps,
2589 )?
2590 }
2591 _ => {
2592 let new_price = cmd.price.or(order.price()).ok_or_else(|| {
2593 anyhow::anyhow!("modify_order requires a price (none on order or command)")
2594 })?;
2595
2596 price_to_ticks(&new_price, price_precision)?
2597 }
2598 };
2599
2600 let base_amount = quantity_to_ticks(&new_qty, instrument.size_precision())?;
2601 anyhow::ensure!(
2602 base_amount > 0,
2603 "quantity `{new_qty}` rounds to 0 ticks at size_precision {}",
2604 instrument.size_precision(),
2605 );
2606
2607 let reduced_qty = self
2609 .dispatch
2610 .order_identity(&cmd.client_order_id)
2611 .map_or(Decimal::ZERO, |identity| identity.reduced_qty());
2612 let reduced_qty = quantity_from_decimal(reduced_qty, instrument.size_precision())?;
2613 let reduced_amount = quantity_to_ticks(&reduced_qty, instrument.size_precision())?;
2614 let base_amount = base_amount
2615 .checked_add(reduced_amount)
2616 .context("Lighter modified quantity overflow")?;
2617
2618 let trigger_price_ticks = match new_trigger.filter(|_| !triggered) {
2619 Some(trigger) if !trigger.is_zero() => price_to_ticks(&trigger, price_precision)?,
2620 _ => 0,
2621 };
2622
2623 if !triggered
2624 && matches!(
2625 order.order_type(),
2626 OrderType::StopMarket
2627 | OrderType::StopLimit
2628 | OrderType::MarketIfTouched
2629 | OrderType::LimitIfTouched
2630 )
2631 {
2632 anyhow::ensure!(
2633 trigger_price_ticks > 0,
2634 "trigger_price `{new_trigger:?}` rounds to 0 ticks at precision {price_precision}",
2635 );
2636 }
2637 validate_order_amount(&instrument, new_qty, price_ticks, price_precision)?;
2638 self.dispatch.set_pending_modify_shape(
2639 cmd,
2640 OrderShapeSnapshot {
2641 quantity: parse_quantity_from_ticks(base_amount, instrument.size_precision())?,
2642 price: LIMIT_ORDER_TYPES
2643 .contains(&order.order_type())
2644 .then(|| parse_price_from_ticks(price_ticks, price_precision))
2645 .transpose()?,
2646 trigger_price: STOP_ORDER_TYPES
2647 .contains(&order.order_type())
2648 .then(|| {
2649 if triggered {
2650 Ok(None)
2651 } else {
2652 parse_price_from_ticks(trigger_price_ticks, price_precision).map(Some)
2653 }
2654 })
2655 .transpose()?
2656 .flatten(),
2657 },
2658 );
2659
2660 Ok(ModifyOrderPlan {
2661 command_id: cmd.command_id,
2662 client_order_id: cmd.client_order_id,
2663 strategy_id: cmd.strategy_id,
2664 instrument_id: cmd.instrument_id,
2665 venue_order_id: Some(voi),
2666 market_index,
2667 venue_index,
2668 base_amount,
2669 price: price_ticks,
2670 trigger_price: trigger_price_ticks,
2671 })
2672 }
2673
2674 pub fn update_leverage(
2693 &self,
2694 instrument_id: InstrumentId,
2695 initial_margin_fraction: u16,
2696 margin_mode: LighterPositionMarginMode,
2697 ) -> anyhow::Result<()> {
2698 let credential = self.credential.as_ref().ok_or_else(|| {
2699 anyhow::anyhow!("Lighter execution client cannot update leverage without credentials")
2700 })?;
2701
2702 let market_index = self.registry.market_index(&instrument_id).ok_or_else(|| {
2703 anyhow::anyhow!("no Lighter market_index registered for instrument {instrument_id}")
2704 })?;
2705
2706 anyhow::ensure!(
2707 (1..=10_000).contains(&initial_margin_fraction),
2708 "initial_margin_fraction must be in 1..=10_000, was {initial_margin_fraction}",
2709 );
2710
2711 let k = fresh_k()?;
2712 let ReservedTxContext {
2713 context,
2714 mut send_reservation,
2715 } = self.build_tx_context(credential)?;
2716
2717 let connection_epoch = send_reservation.connection_epoch;
2718
2719 let captured_nonce = context.nonce;
2720 let captured_api_key_index = context.api_key_index;
2721 let mut rollback_guard =
2722 TxDispatchGuard::new(self.dispatch.clone(), credential, None, captured_nonce);
2723
2724 let tx = UpdateLeverageTxInfo {
2725 context,
2726 market_index,
2727 initial_margin_fraction,
2728 margin_mode: margin_mode as u8,
2729 skip_nonce: 0,
2730 };
2731
2732 let signed = sign_tx(&tx, self.config.chain_id(), &credential.private_key()?, k)?;
2733
2734 let tx_info_str = TxInfoJson::update_leverage(&tx, &signed);
2735 let tx_info = serde_json::value::RawValue::from_string(tx_info_str)
2736 .context("failed to wrap signed Lighter update_leverage tx_info JSON")?;
2737
2738 rollback_guard.disarm();
2739 let captured_tx_hash = signed.tx_hash_hex();
2740
2741 let ws_client = self.ws_client.clone();
2742 let dispatch = self.dispatch.clone();
2743 let credential = credential.clone();
2744 let clock = self.clock;
2745
2746 let tx_rate_limiter = self.tx_rate_limiter.clone();
2747
2748 self.spawn_task("update_leverage", async move {
2749 send_reservation.wait_for_turn().await;
2750 await_tx_quota(&tx_rate_limiter).await;
2751 dispatch.enqueue_pending_sendtx(PendingSendTx {
2752 batch_id: None,
2753 connection_epoch,
2754 kind: PendingSendTxKind::Other,
2755 submitted_at: clock.get_time_ns(),
2756 nonce: captured_nonce,
2757 api_key_index: captured_api_key_index,
2758 tx_hash: captured_tx_hash,
2759 });
2760
2761 if let Err(e) = ws_client
2762 .send_tx_on_connection(
2763 LighterTxType::UpdateLeverage as u8,
2764 tx_info,
2765 connection_epoch,
2766 )
2767 .await
2768 {
2769 let failure = classify_lighter_ws_command_failure("update_leverage", &e);
2770 let reason = command_failure_reason(&failure);
2771 if matches!(&failure, CommandFailure::Ambiguous(_)) {
2772 log::warn!(
2773 "Lighter update_leverage dispatch outcome unknown for {instrument_id}: \
2774 {reason}; retaining pending nonce for venue reconciliation; diagnostic={e:?}",
2775 );
2776 } else {
2777 log::error!("{reason} for {instrument_id}; diagnostic={e:?}");
2778 dispatch.remove_pending_sendtx_by_nonce(connection_epoch, captured_nonce);
2779 rollback_tx_dispatch(&dispatch, &credential, None, captured_nonce);
2780 }
2781 }
2782 send_reservation.release();
2783 Ok(())
2784 });
2785
2786 Ok(())
2787 }
2788}
2789
2790fn emit_lighter_position_reports(
2791 reports: Vec<PositionStatusReport>,
2792 removed: Vec<InstrumentId>,
2793 emitter: &ExecutionEventEmitter,
2794 account_id: AccountId,
2795 now: UnixNanos,
2796) {
2797 for report in reports {
2798 log::debug!(
2799 "Lighter PositionStatusReport: instrument={} side={:?} qty={}",
2800 report.instrument_id,
2801 report.position_side,
2802 report.quantity,
2803 );
2804 emitter.send_position_report(report);
2805 }
2806
2807 for instrument_id in removed {
2809 let flat = PositionStatusReport::new(
2810 account_id,
2811 instrument_id,
2812 PositionSide::Flat,
2813 Quantity::zero(0),
2814 now,
2815 now,
2816 Some(UUID4::new()),
2817 None,
2818 None,
2819 );
2820 emitter.send_position_report(flat);
2821 }
2822}
2823
2824#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2825enum AuthTokenRefreshOutcome {
2826 Rotated,
2827 Cancelled,
2828 Exhausted,
2829}
2830
2831#[derive(Debug, Clone, Copy)]
2832struct AuthTokenRefreshBackoff {
2833 initial_delay: Duration,
2834 max_delay: Duration,
2835 window: Duration,
2836}
2837
2838#[derive(Clone)]
2839struct NonceRefreshRetry {
2840 http_client: LighterHttpClient,
2841 dispatch: WsDispatchState,
2842 credential: Option<Credential>,
2843 submission_gate: Arc<tokio::sync::RwLock<()>>,
2844 ready_connection_epoch: Arc<AtomicU64>,
2845 ws_client: LighterWebSocketClient,
2846 cancellation_token: CancellationToken,
2847 pending_tasks: TaskSpawner,
2848}
2849
2850impl NonceRefreshRetry {
2851 fn spawn(self, connection_epoch: u64) {
2852 let pending_tasks = self.pending_tasks.clone();
2853
2854 let future = async move {
2855 let Some(credential) = self.credential else {
2856 return;
2857 };
2858 let mut retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
2859
2860 loop {
2861 tokio::select! {
2862 () = self.cancellation_token.cancelled() => return,
2863 () = tokio::time::sleep(retry_delay) => {}
2864 }
2865
2866 if self.ws_client.connection_epoch() != connection_epoch
2867 || self.ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
2868 {
2869 return;
2870 }
2871
2872 let _refresh_guard = tokio::select! {
2873 () = self.cancellation_token.cancelled() => return,
2874 guard = self.submission_gate.write() => guard,
2875 };
2876
2877 if self.ws_client.connection_epoch() != connection_epoch
2878 || self.ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
2879 {
2880 return;
2881 }
2882
2883 let result = tokio::select! {
2884 () = self.cancellation_token.cancelled() => return,
2885 result = self.http_client.get_next_nonce(
2886 credential.account_index(),
2887 credential.api_key_index(),
2888 ) => result,
2889 };
2890
2891 match result {
2892 Ok(response) => {
2893 if self.ws_client.connection_epoch() != connection_epoch {
2894 return;
2895 }
2896 self.dispatch.nonce_manager.refresh(
2897 credential.account_index(),
2898 credential.api_key_index(),
2899 response.nonce,
2900 );
2901 self.ready_connection_epoch
2902 .store(connection_epoch, Ordering::Release);
2903 log::info!(
2904 "Recovered Lighter nonce refresh: account_index={}, \
2905 connection_epoch={connection_epoch}, next_nonce={}",
2906 credential.account_index(),
2907 response.nonce,
2908 );
2909 return;
2910 }
2911 Err(e) => {
2912 log::warn!(
2913 "Failed to retry Lighter nonce refresh for connection epoch \
2914 {connection_epoch}: {e}",
2915 );
2916 }
2917 }
2918
2919 retry_delay = retry_delay
2920 .saturating_mul(2)
2921 .min(NONCE_REFRESH_RETRY_MAX_DELAY);
2922 }
2923 };
2924
2925 if let Err(e) = pending_tasks.spawn(future) {
2926 log::debug!("Skipping Lighter nonce refresh retry during shutdown: {e}");
2927 }
2928 }
2929}
2930
2931fn auth_token_rotation_channels(account_index: i64) -> [LighterWsChannel; 6] {
2932 [
2933 LighterWsChannel::AccountAllOrders(account_index),
2934 LighterWsChannel::AccountAllTrades(account_index),
2935 LighterWsChannel::AccountAllPositions(account_index),
2936 LighterWsChannel::AccountAllAssets(account_index),
2937 LighterWsChannel::AccountTx(account_index),
2938 LighterWsChannel::UserStats(account_index),
2939 ]
2940}
2941
2942async fn refresh_auth_token_until_rotated<MintToken, Subscribe, SubscribeFuture>(
2943 credential: &Credential,
2944 channels: &[LighterWsChannel],
2945 cancellation_token: &CancellationToken,
2946 backoff: AuthTokenRefreshBackoff,
2947 mut mint_token: MintToken,
2948 mut subscribe: Subscribe,
2949) -> AuthTokenRefreshOutcome
2950where
2951 MintToken: FnMut(&Credential) -> anyhow::Result<SecretString>,
2952 Subscribe: FnMut(LighterWsChannel, SecretString) -> SubscribeFuture,
2953 SubscribeFuture: Future<Output = Result<(), crate::websocket::error::LighterWsError>>,
2954{
2955 let retry_started = tokio::time::Instant::now();
2956 let mut retry_delay = backoff.initial_delay.min(backoff.max_delay);
2957 let mut attempt = 1_u32;
2958
2959 loop {
2960 match rotate_auth_token_once(credential, channels, &mut mint_token, &mut subscribe).await {
2961 Ok(()) => {
2962 log::debug!(
2963 "Lighter auth-token rotated for account_index={}, attempts={attempt}",
2964 credential.account_index(),
2965 );
2966 return AuthTokenRefreshOutcome::Rotated;
2967 }
2968 Err(e) => {
2969 log::warn!("Lighter auth-token rotation attempt {attempt} failed: {e:#}");
2970 }
2971 }
2972
2973 let Some(remaining) = backoff.window.checked_sub(retry_started.elapsed()) else {
2974 log::error!(
2975 "Lighter auth-token rotation retry window exhausted: account_index={}, attempts={attempt}",
2976 credential.account_index(),
2977 );
2978 return AuthTokenRefreshOutcome::Exhausted;
2979 };
2980
2981 if remaining.as_nanos() == 0 {
2982 log::error!(
2983 "Lighter auth-token rotation retry window exhausted: account_index={}, attempts={attempt}",
2984 credential.account_index(),
2985 );
2986 return AuthTokenRefreshOutcome::Exhausted;
2987 }
2988
2989 let delay = retry_delay.min(remaining);
2990 log::warn!(
2991 "Retrying Lighter auth-token rotation in {:.3}s: account_index={}, attempts={attempt}",
2992 delay.as_secs_f64(),
2993 credential.account_index(),
2994 );
2995
2996 if !sleep_or_auth_token_refresh_cancelled(delay, cancellation_token).await {
2997 return AuthTokenRefreshOutcome::Cancelled;
2998 }
2999
3000 retry_delay = next_auth_token_refresh_retry_delay(retry_delay, backoff.max_delay);
3001 attempt = attempt.saturating_add(1);
3002 }
3003}
3004
3005async fn rotate_auth_token_once<MintToken, Subscribe, SubscribeFuture>(
3006 credential: &Credential,
3007 channels: &[LighterWsChannel],
3008 mint_token: &mut MintToken,
3009 subscribe: &mut Subscribe,
3010) -> anyhow::Result<()>
3011where
3012 MintToken: FnMut(&Credential) -> anyhow::Result<SecretString>,
3013 Subscribe: FnMut(LighterWsChannel, SecretString) -> SubscribeFuture,
3014 SubscribeFuture: Future<Output = Result<(), crate::websocket::error::LighterWsError>>,
3015{
3016 let token =
3017 mint_token(credential).context("failed to mint Lighter auth token during rotation")?;
3018 let mut first_error = None;
3019
3020 for channel in channels {
3021 if let Err(e) = subscribe(channel.clone(), token.clone()).await {
3022 log::debug!("Lighter auth-token rotation: re-subscribe failed for {channel:?}: {e}",);
3023 first_error.get_or_insert_with(|| format!("{channel:?}: {e}"));
3024 }
3025 }
3026
3027 if let Some(error) = first_error {
3028 anyhow::bail!("failed to re-subscribe Lighter account channels: {error}");
3029 }
3030
3031 Ok(())
3032}
3033
3034async fn sleep_or_auth_token_refresh_cancelled(
3035 duration: Duration,
3036 cancellation_token: &CancellationToken,
3037) -> bool {
3038 tokio::select! {
3039 () = cancellation_token.cancelled() => false,
3040 () = tokio::time::sleep(duration) => true,
3041 }
3042}
3043
3044fn auth_token_refresh_next_delay(outcome: AuthTokenRefreshOutcome) -> Option<Duration> {
3045 match outcome {
3046 AuthTokenRefreshOutcome::Rotated => Some(AUTH_TOKEN_REFRESH_INTERVAL),
3047 AuthTokenRefreshOutcome::Cancelled => None,
3048 AuthTokenRefreshOutcome::Exhausted => Some(AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY),
3049 }
3050}
3051
3052fn next_auth_token_refresh_retry_delay(current: Duration, max: Duration) -> Duration {
3053 current.checked_mul(2).unwrap_or(max).min(max)
3054}
3055
3056#[derive(Debug)]
3057struct ReservedTxContext {
3058 context: TxContext,
3059 send_reservation: TxSendReservation,
3060}
3061
3062#[derive(Debug, Clone)]
3063struct TxSendSequencer {
3064 state: Arc<Mutex<TxSendSequencerState>>,
3065 version: Arc<AtomicU64>,
3066 changed: tokio::sync::watch::Sender<u64>,
3067}
3068
3069impl TxSendSequencer {
3070 fn new() -> Self {
3071 let (changed, _) = tokio::sync::watch::channel(0);
3072 Self {
3073 state: Arc::new(Mutex::new(TxSendSequencerState::default())),
3074 version: Arc::new(AtomicU64::new(0)),
3075 changed,
3076 }
3077 }
3078
3079 fn reserve(
3080 &self,
3081 account_index: i64,
3082 api_key_index: u8,
3083 allocate_nonce: impl FnOnce() -> Result<i64, NonceError>,
3084 ) -> Result<TxSendReservation, NonceError> {
3085 let key = TxSendKey {
3086 account_index,
3087 api_key_index,
3088 };
3089
3090 let mut state = self.state.lock();
3091 let nonce = allocate_nonce()?;
3092 *state
3093 .pending
3094 .entry(key)
3095 .or_default()
3096 .entry(nonce)
3097 .or_default() += 1;
3098 drop(state);
3099 self.notify_waiters();
3100 Ok(TxSendReservation {
3101 sequencer: self.clone(),
3102 key,
3103 nonce,
3104 released: false,
3105 nonce_guard: None,
3106 connection_epoch: 0,
3107 })
3108 }
3109
3110 async fn wait_for_turn(&self, key: TxSendKey, nonce: i64) {
3111 let mut changed = self.changed.subscribe();
3112
3113 loop {
3114 if self.ready_to_send(key, nonce) {
3115 return;
3116 }
3117
3118 if changed.changed().await.is_err() {
3119 tokio::task::yield_now().await;
3120 }
3121 }
3122 }
3123
3124 fn release(&self, key: TxSendKey, nonce: i64) {
3125 let mut state = self.state.lock();
3126
3127 let should_notify = if let Some(pending) = state.pending.get_mut(&key) {
3128 let removed = if let Some(count) = pending.get_mut(&nonce) {
3129 *count -= 1;
3130 if *count == 0 {
3131 pending.remove(&nonce);
3132 }
3133
3134 true
3135 } else {
3136 false
3137 };
3138
3139 if pending.is_empty() {
3140 state.pending.remove(&key);
3141 }
3142 removed
3143 } else {
3144 false
3145 };
3146 drop(state);
3147
3148 if should_notify {
3149 self.notify_waiters();
3150 }
3151 }
3152
3153 fn ready_to_send(&self, key: TxSendKey, nonce: i64) -> bool {
3154 let state = self.state.lock();
3155 state
3156 .pending
3157 .get(&key)
3158 .and_then(|pending| pending.first_key_value())
3159 .is_none_or(|(first, _)| *first >= nonce)
3160 }
3161
3162 fn notify_waiters(&self) {
3163 let version = self.version.fetch_add(1, Ordering::AcqRel) + 1;
3164 let _ = self.changed.send(version);
3165 }
3166}
3167
3168#[derive(Debug, Default)]
3169struct TxSendSequencerState {
3170 pending: BTreeMap<TxSendKey, BTreeMap<i64, usize>>,
3171}
3172
3173#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
3174struct TxSendKey {
3175 account_index: i64,
3176 api_key_index: u8,
3177}
3178
3179#[derive(Debug)]
3180struct TxSendReservation {
3181 sequencer: TxSendSequencer,
3182 key: TxSendKey,
3183 nonce: i64,
3184 released: bool,
3185 nonce_guard: Option<tokio::sync::OwnedRwLockReadGuard<()>>,
3186 connection_epoch: u64,
3187}
3188
3189impl TxSendReservation {
3190 fn with_nonce_ownership(
3191 mut self,
3192 nonce_guard: tokio::sync::OwnedRwLockReadGuard<()>,
3193 connection_epoch: u64,
3194 ) -> Self {
3195 self.nonce_guard = Some(nonce_guard);
3196 self.connection_epoch = connection_epoch;
3197 self
3198 }
3199
3200 async fn wait_for_turn(&self) {
3201 self.sequencer.wait_for_turn(self.key, self.nonce).await;
3202 }
3203
3204 fn release(&mut self) {
3205 if self.released {
3206 return;
3207 }
3208
3209 self.sequencer.release(self.key, self.nonce);
3210 self.nonce_guard.take();
3211 self.released = true;
3212 }
3213}
3214
3215impl Drop for TxSendReservation {
3216 fn drop(&mut self) {
3217 self.release();
3218 }
3219}
3220
3221#[cfg(test)]
3222async fn wait_for_tx_send_reservations(reservations: &[&TxSendReservation]) {
3223 let Some(first) = reservations.first() else {
3224 return;
3225 };
3226
3227 debug_assert!(
3228 reservations
3229 .iter()
3230 .all(|reservation| reservation.key == first.key),
3231 "batch send reservations must share one nonce stream",
3232 );
3233
3234 let nonce = reservations
3235 .iter()
3236 .map(|reservation| reservation.nonce)
3237 .min()
3238 .expect("reservations is non-empty");
3239 first.sequencer.wait_for_turn(first.key, nonce).await;
3240}
3241
3242struct PreparedCreateOrder {
3243 order: OrderAny,
3244 client_order_index: i64,
3245 tx_info: Box<serde_json::value::RawValue>,
3246 nonce: i64,
3247 api_key_index: u8,
3248 tx_hash: String,
3249 send_reservation: TxSendReservation,
3250}
3251
3252struct CreateOrderPlan {
3253 order: OrderAny,
3254 market_index: i64,
3255 base_amount: i64,
3256 price: u32,
3257 order_type: u8,
3258 time_in_force: u8,
3259 trigger_price: u32,
3260 order_expiry: i64,
3261 attributes: L2TxAttributes,
3262}
3263
3264impl CreateOrderPlan {
3265 fn order_info(&self, client_order_index: i64) -> OrderInfo {
3266 OrderInfo {
3267 market_index: self.market_index,
3268 client_order_index,
3269 base_amount: self.base_amount,
3270 price: self.price,
3271 is_ask: self.order.order_side() == OrderSide::Sell,
3272 order_type: self.order_type,
3273 time_in_force: self.time_in_force,
3274 reduce_only: self.order.is_reduce_only(),
3275 trigger_price: self.trigger_price,
3276 order_expiry: self.order_expiry,
3277 }
3278 }
3279}
3280
3281struct CreateGroupPlan {
3282 grouping_type: LighterGroupingType,
3283 orders: Vec<CreateOrderPlan>,
3284}
3285
3286struct PreparedCancelOrder {
3287 command_id: Option<UUID4>,
3288 client_order_id: ClientOrderId,
3289 strategy_id: StrategyId,
3290 instrument_id: InstrumentId,
3291 venue_order_id: Option<VenueOrderId>,
3292 tx_info: Box<serde_json::value::RawValue>,
3293 nonce: i64,
3294 api_key_index: u8,
3295 tx_hash: String,
3296 send_reservation: TxSendReservation,
3297}
3298
3299struct CancelOrderPlan {
3300 command_id: Option<UUID4>,
3301 client_order_id: ClientOrderId,
3302 strategy_id: StrategyId,
3303 instrument_id: InstrumentId,
3304 venue_order_id: Option<VenueOrderId>,
3305 market_index: i64,
3306 venue_index: i64,
3307}
3308
3309#[derive(Clone)]
3310struct FanoutDispatchContext {
3311 clock: &'static AtomicTime,
3312 chain_id: u32,
3313 integrator_account_index: Option<u64>,
3314 emitter: ExecutionEventEmitter,
3315 credential: Credential,
3316 http_client: LighterHttpClient,
3317 registry: Arc<MarketRegistry>,
3318 ws_client: LighterWebSocketClient,
3319 tx_rate_limiter: Arc<LighterTxRateLimiter>,
3320 tx_send_sequencer: TxSendSequencer,
3321 nonce_submission_gate: Arc<tokio::sync::RwLock<()>>,
3322 nonce_ready_connection_epoch: Arc<AtomicU64>,
3323 dispatch: WsDispatchState,
3324 nonce_recovery_inflight: Arc<AtomicBool>,
3325 pending_tasks: TaskSpawner,
3326}
3327
3328impl FanoutDispatchContext {
3329 async fn refresh_scheduled_cancel(&self, timeout_secs: u64) -> anyhow::Result<()> {
3330 let deadline = self
3331 .clock
3332 .get_time_ms()
3333 .checked_add(timeout_secs * 1_000)
3334 .and_then(|deadline| i64::try_from(deadline).ok())
3335 .context("scheduled cancellation deadline overflows milliseconds")?;
3336 let cancellation = self.pending_tasks.cancellation_token().child_token();
3337 let _guard = cancellation.clone().drop_guard();
3338 tokio::time::timeout(
3339 SCHEDULED_CANCEL_REQUEST_TIMEOUT,
3340 self.send_cancel_all(
3341 LighterCancelAllTimeInForce::Scheduled,
3342 deadline,
3343 true,
3344 cancellation,
3345 ),
3346 )
3347 .await
3348 .context("scheduled cancellation request timed out")??;
3349 Ok(())
3350 }
3351
3352 async fn send_cancel_all(
3353 &self,
3354 time_in_force: LighterCancelAllTimeInForce,
3355 scheduled_time_ms: i64,
3356 require_execution: bool,
3357 cancellation: CancellationToken,
3358 ) -> anyhow::Result<String> {
3359 let credential = &self.credential;
3360 let k = fresh_k()?;
3361 let ReservedTxContext {
3362 context,
3363 mut send_reservation,
3364 } = self.build_tx_context()?;
3365 let mut rollback_guard =
3366 TxDispatchGuard::new(self.dispatch.clone(), credential, None, context.nonce);
3367
3368 let tx = CancelAllOrdersTxInfo {
3369 context,
3370 time_in_force: time_in_force as u8,
3371 scheduled_time_ms,
3372 skip_nonce: 0,
3373 };
3374
3375 let signed = sign_tx(&tx, self.chain_id, &credential.private_key()?, k)?;
3376 let tx_info = TxInfoJson::cancel_all_orders(&tx, &signed);
3377 let tx_info = serde_json::value::RawValue::from_string(tx_info)
3378 .context("failed to wrap signed Lighter account cancellation JSON")?;
3379 let tx_hash = signed.tx_hash_hex();
3380 let connection_epoch = send_reservation.connection_epoch;
3381 let ws_client = self.ws_client.clone();
3382 let dispatch = self.dispatch.clone();
3383 let credential = credential.clone();
3384 let clock = self.clock;
3385 let tx_rate_limiter = self.tx_rate_limiter.clone();
3386 let dispatched_hash = tx_hash.clone();
3387 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3388
3389 self.pending_tasks
3390 .spawn(async move {
3391 {
3392 let ready = async {
3393 send_reservation.wait_for_turn().await;
3394 await_tx_quota(&tx_rate_limiter).await;
3395 };
3396
3397 tokio::pin!(ready);
3398 tokio::select! {
3399 biased;
3400 () = cancellation.cancelled() => {
3401 if matches!(dispatch.nonce_manager.ack_failure_if_latest(
3403 context.account_index, context.api_key_index, context.nonce,
3404 ), Ok(true)) {
3405 rollback_guard.disarm();
3406 return;
3407 }
3408 ready.await;
3409 }
3410 () = &mut ready => {},
3411 }
3412 }
3413
3414 dispatch.enqueue_pending_sendtx(PendingSendTx {
3415 batch_id: None,
3416 connection_epoch,
3417 kind: PendingSendTxKind::Other,
3418 submitted_at: clock.get_time_ns(),
3419 nonce: context.nonce,
3420 api_key_index: context.api_key_index,
3421 tx_hash: dispatched_hash.clone(),
3422 });
3423
3424 rollback_guard.disarm();
3425 let response = ws_client.send_tx_on_connection(
3426 LighterTxType::CancelAllOrders as u8, tx_info, connection_epoch,
3427 ).await;
3428
3429 if let Err(e) = &response {
3430 let failure =
3431 classify_lighter_ws_command_failure("cancel_all_orders_account", e);
3432 log::warn!(
3433 "Lighter account cancellation transaction {dispatched_hash} dispatch failed, nonce={}: {failure:?}; diagnostic={e}",
3434 context.nonce,
3435 );
3436
3437 if !matches!(failure, CommandFailure::Ambiguous(_)) {
3438 dispatch.remove_pending_sendtx_by_hash(connection_epoch, &dispatched_hash);
3439 rollback_tx_dispatch(&dispatch, &credential, None, context.nonce);
3440 }
3441 }
3442
3443 send_reservation.release();
3444 let _ = response_tx.send(response);
3445 })
3446 .context("failed to spawn Lighter account cancellation")?;
3447
3448 let response = response_rx.await.with_context(|| {
3449 format!("Lighter account cancellation transaction {tx_hash} dispatch outcome unknown after session shutdown")
3450 })?;
3451
3452 response.with_context(|| {
3453 format!("failed to submit Lighter account cancellation transaction {tx_hash}")
3454 })?;
3455
3456 let confirmation = async {
3457 loop {
3458 match self.http_client.get_tx(tx_hash.clone()).await {
3459 Ok(tx) => {
3460 anyhow::ensure!(
3461 tx.status != LighterTxStatus::Failed,
3462 "Lighter account cancellation transaction {tx_hash} failed: {}",
3463 tx.event_info,
3464 );
3465
3466 if require_execution && tx.status != LighterTxStatus::Executed {
3467 tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY).await;
3468 continue;
3469 }
3470
3471 let event = parse_acked_tx_event(&tx.event_info).with_context(|| {
3472 format!("failed to parse Lighter account cancellation transaction {tx_hash} event")
3473 })?;
3474
3475 anyhow::ensure!(
3476 event.app_error.is_empty(),
3477 "Lighter account cancellation transaction {tx_hash} failed: {}",
3478 event.app_error,
3479 );
3480
3481 anyhow::ensure!(
3482 !require_execution
3483 || (tx_hash_matches(&tx.hash, &tx_hash)
3484 && tx.tx_type == LighterTxType::CancelAllOrders as u8
3485 && tx.account_index == self.credential.account_index()
3486 && tx.api_key_index == self.credential.api_key_index()
3487 && tx.nonce == context.nonce),
3488 "Lighter scheduled cancellation transaction identity mismatch",
3489 );
3490 return Ok(tx_hash.clone());
3491 }
3492 Err(LighterHttpError::Venue { code: 21500, .. }) => {}
3493 Err(e) => {
3494 return Err(e).with_context(|| {
3495 format!("failed to look up Lighter account cancellation transaction {tx_hash}")
3496 });
3497 }
3498 }
3499
3500 tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY).await;
3501 }
3502 };
3503
3504 tokio::time::timeout(
3505 if require_execution {
3506 SCHEDULED_CANCEL_REQUEST_TIMEOUT
3507 } else {
3508 ACCOUNT_CANCEL_CONFIRM_TIMEOUT
3509 },
3510 confirmation,
3511 )
3512 .await
3513 .with_context(|| {
3514 format!("Lighter account cancellation transaction {tx_hash} confirmation timed out")
3515 })?
3516 }
3517
3518 async fn dispatch_batch_plans(&self, plans: Vec<BatchOrderPlan>) {
3519 for chunk in plans.chunks(LIGHTER_MAX_BATCH_TX) {
3520 let mut batch = Vec::with_capacity(chunk.len());
3521 let mut remaining = chunk.iter().peekable();
3522 let mut retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
3523
3524 while let Some(plan) = remaining.peek() {
3525 match self.sign_batch_plan(plan) {
3526 Ok(prepared) => {
3527 batch.push(prepared);
3528 retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
3529 }
3530 Err(e) if should_retry_batch_signing(&e) => {
3531 self.send_batch(std::mem::take(&mut batch)).await;
3532 tokio::time::sleep(retry_delay).await;
3533 retry_delay = (retry_delay * 2).min(NONCE_REFRESH_RETRY_MAX_DELAY);
3534 continue;
3535 }
3536 Err(e) => self.reject_batch_plan(plan, &format!("{e:#}")),
3537 }
3538
3539 remaining.next();
3540 }
3541
3542 self.send_batch(batch).await;
3543 }
3544 }
3545
3546 fn probe_context(&self) -> AckedOrderProbeContext {
3547 AckedOrderProbeContext {
3548 http_client: self.http_client.clone(),
3549 registry: Arc::clone(&self.registry),
3550 credential: self.credential.clone(),
3551 dispatch: self.dispatch.clone(),
3552 account_id: self.emitter.account_id(),
3553 clock: self.clock,
3554 emitter: self.emitter.clone(),
3555 connection_epoch: self.ws_client.connection_epoch_atomic(),
3556 cancellation_token: self.pending_tasks.cancellation_token(),
3557 pending_tasks: self.pending_tasks.clone(),
3558 }
3559 }
3560
3561 fn sign_batch_plan(&self, plan: &BatchOrderPlan) -> anyhow::Result<PreparedBatchTx> {
3562 match plan {
3563 BatchOrderPlan::Create(plan) => self.sign_create_order(plan).map(PreparedBatchTx::from),
3564 BatchOrderPlan::CreateGroup(plan) => self.sign_create_group(plan),
3565 BatchOrderPlan::Modify(plan) => {
3566 let k = fresh_k()?;
3567 let reserved = self.build_tx_context()?;
3568 self.sign_modify_order(plan, reserved, k)
3569 .map(PreparedBatchTx::from)
3570 }
3571 BatchOrderPlan::Cancel(plan, emit) => self
3572 .sign_cancel_order(plan)
3573 .map(|prepared| PreparedBatchTx::from((prepared, *emit))),
3574 }
3575 }
3576
3577 fn reject_batch_plan(&self, plan: &BatchOrderPlan, error: &str) {
3578 match plan {
3579 BatchOrderPlan::CreateGroup(plan) => {
3580 for leg in &plan.orders {
3581 self.emitter.emit_order_denied(
3582 &leg.order,
3583 &format!("Lighter grouped submit failed: {error}"),
3584 );
3585 }
3586 }
3587 BatchOrderPlan::Create(plan) => {
3588 let reason = OrderDeniedReason::SubmitFailed {
3589 detail: format!("Lighter submit_order_list failed: {error}"),
3590 };
3591
3592 self.emitter
3593 .emit_order_denied(&plan.order, &reason.to_string());
3594 }
3595 BatchOrderPlan::Modify(plan) => {
3596 let mut events = self.dispatch.order_events.lock();
3597
3598 if self
3599 .dispatch
3600 .clear_pending_modify_if_command(&plan.client_order_id, plan.command_id)
3601 {
3602 self.emitter.emit_order_modify_denied_event(
3603 plan.strategy_id,
3604 plan.instrument_id,
3605 plan.client_order_id,
3606 plan.venue_order_id,
3607 &format!("Lighter modify_order failed: {error}"),
3608 self.clock.get_time_ns(),
3609 Some(plan.command_id),
3610 );
3611
3612 if let Some(deferred) = events.remove(&plan.client_order_id) {
3613 flush_modify_events(
3614 plan.client_order_id,
3615 deferred,
3616 &self.dispatch,
3617 &self.emitter,
3618 &mut events,
3619 );
3620 }
3621 }
3622 }
3623 BatchOrderPlan::Cancel(plan, _) => {
3624 let Some(command_id) = plan.command_id else {
3625 return;
3626 };
3627
3628 self.dispatch
3629 .clear_pending_cancel_if_command(&plan.client_order_id, command_id);
3630 let reason = format!("Lighter cancel_order failed: {error}");
3631
3632 self.emitter.emit_order_cancel_denied_event(
3633 plan.strategy_id,
3634 plan.instrument_id,
3635 plan.client_order_id,
3636 plan.venue_order_id,
3637 &reason,
3638 self.clock.get_time_ns(),
3639 Some(command_id),
3640 );
3641 }
3642 }
3643 }
3644
3645 fn build_tx_context(&self) -> anyhow::Result<ReservedTxContext> {
3646 let connection_epoch = self.ws_client.connection_epoch();
3647 anyhow::ensure!(
3648 self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
3649 TxPreparationError::NonceRefreshPending(connection_epoch),
3650 );
3651 let nonce_guard = Arc::clone(&self.nonce_submission_gate)
3652 .try_read_owned()
3653 .context("Lighter nonce refresh is in progress")?;
3654 anyhow::ensure!(
3655 self.ws_client.connection_epoch() == connection_epoch
3656 && self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
3657 TxPreparationError::StateChanged,
3658 );
3659
3660 let send_reservation = match self.tx_send_sequencer.reserve(
3661 self.credential.account_index(),
3662 self.credential.api_key_index(),
3663 || {
3664 self.dispatch.nonce_manager.next_nonce(
3665 self.credential.account_index(),
3666 self.credential.api_key_index(),
3667 )
3668 },
3669 ) {
3670 Ok(reservation) => reservation,
3671 Err(e @ NonceError::SkipWindowExhausted { .. }) => {
3672 self.spawn_nonce_window_recovery();
3673 let reason = format!("failed to allocate Lighter nonce: {e}");
3674 return Err(anyhow::Error::new(e).context(reason));
3675 }
3676 Err(e) => anyhow::bail!("failed to allocate Lighter nonce: {e}"),
3677 };
3678
3679 let nonce = send_reservation.nonce;
3680
3681 let now_ns = self.clock.get_time_ns().as_u64() as i64;
3682 let expired_at = (now_ns / 1_000_000).saturating_add(DEFAULT_TX_EXPIRY_MS);
3683 let send_reservation = send_reservation.with_nonce_ownership(nonce_guard, connection_epoch);
3684
3685 Ok(ReservedTxContext {
3686 context: TxContext {
3687 account_index: self.credential.account_index(),
3688 api_key_index: self.credential.api_key_index(),
3689 nonce,
3690 expired_at,
3691 },
3692 send_reservation,
3693 })
3694 }
3695
3696 fn spawn_nonce_window_recovery(&self) {
3697 if self.nonce_recovery_inflight.swap(true, Ordering::AcqRel) {
3698 return;
3699 }
3700
3701 let inflight = Arc::clone(&self.nonce_recovery_inflight);
3702 let http_client = self.http_client.clone();
3703 let dispatch = self.dispatch.clone();
3704 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
3705 let account_index = self.credential.account_index();
3706 let api_key_index = self.credential.api_key_index();
3707
3708 let future = async move {
3709 let _refresh_guard = nonce_submission_gate.write().await;
3710 let result = http_client
3711 .get_next_nonce(account_index, api_key_index)
3712 .await;
3713 inflight.store(false, Ordering::Release);
3714
3715 match result {
3716 Ok(response) => {
3717 let _ = dispatch.nonce_manager.sync_from_venue(
3718 account_index,
3719 api_key_index,
3720 response.nonce,
3721 );
3722 log::debug!(
3723 "Resynced Lighter nonce baseline after skip-window exhaustion: \
3724 account_index={account_index}, api_key_index={api_key_index}, \
3725 next_nonce={}",
3726 response.nonce,
3727 );
3728 }
3729 Err(e) => {
3730 log::error!("Failed to resync Lighter nonce after skip-window exhaustion: {e}");
3731 }
3732 }
3733 };
3734
3735 if let Err(e) = self.pending_tasks.spawn(future) {
3736 self.nonce_recovery_inflight.store(false, Ordering::Release);
3737 log::warn!("Skipping Lighter nonce recovery after shutdown began: {e}");
3738 }
3739 }
3740
3741 fn sign_create_order(&self, plan: &CreateOrderPlan) -> anyhow::Result<PreparedCreateOrder> {
3742 let k = fresh_k()?;
3743 let cloid = plan.order.client_order_id();
3744 let client_order_index = self.dispatch.register_create_identity(&plan.order)?;
3745
3746 let ReservedTxContext {
3747 context,
3748 send_reservation,
3749 } = match self.build_tx_context() {
3750 Ok(context) => context,
3751 Err(e) => {
3752 self.dispatch.forget_cloid(client_order_index);
3753 self.dispatch.forget_order_identity(&cloid);
3754 return Err(e);
3755 }
3756 };
3757
3758 let nonce = context.nonce;
3759 let api_key_index = context.api_key_index;
3760 let mut rollback_guard = TxDispatchGuard::new(
3761 self.dispatch.clone(),
3762 &self.credential,
3763 Some(client_order_index),
3764 nonce,
3765 )
3766 .with_order_identity(cloid);
3767
3768 let tx = CreateOrderTxInfo {
3769 context,
3770 order: plan.order_info(client_order_index),
3771 attributes: plan.attributes,
3772 };
3773
3774 let signed = sign_tx(&tx, self.chain_id, &self.credential.private_key()?, k)?;
3775
3776 let tx_info =
3777 serde_json::value::RawValue::from_string(TxInfoJson::create_order(&tx, &signed))
3778 .context("failed to wrap signed Lighter tx_info JSON")?;
3779
3780 rollback_guard.disarm();
3781
3782 Ok(PreparedCreateOrder {
3783 order: plan.order.clone(),
3784 client_order_index,
3785 tx_info,
3786 nonce,
3787 api_key_index,
3788 tx_hash: signed.tx_hash_hex(),
3789 send_reservation,
3790 })
3791 }
3792
3793 fn sign_create_group(&self, plan: &CreateGroupPlan) -> anyhow::Result<PreparedBatchTx> {
3794 let k = fresh_k()?;
3795 let reserved = self.build_tx_context()?;
3796 let mut guards = Vec::with_capacity(plan.orders.len());
3797 let mut orders = Vec::with_capacity(plan.orders.len());
3798 let mut bodies = Vec::with_capacity(plan.orders.len());
3799
3800 let mut nonce_guard = TxDispatchGuard::new(
3801 self.dispatch.clone(),
3802 &self.credential,
3803 None,
3804 reserved.context.nonce,
3805 );
3806
3807 for leg in &plan.orders {
3808 let index = self.dispatch.register_create_identity(&leg.order)?;
3809 guards.push(
3810 TxDispatchGuard::new(
3811 self.dispatch.clone(),
3812 &self.credential,
3813 Some(index),
3814 reserved.context.nonce,
3815 )
3816 .with_order_identity(leg.order.client_order_id()),
3817 );
3818 bodies.push(leg.order_info(index));
3819 orders.push((leg.order.clone(), index));
3820 }
3821
3822 let tx = CreateGroupedOrdersTxInfo {
3823 context: reserved.context,
3824 grouping_type: plan.grouping_type,
3825 orders: bodies,
3826 attributes: plan.orders[0].attributes,
3827 };
3828
3829 let signed = sign_tx(&tx, self.chain_id, &self.credential.private_key()?, k)?;
3830 let tx_info = serde_json::value::RawValue::from_string(TxInfoJson::create_grouped_orders(
3831 &tx, &signed,
3832 ))
3833 .context("failed to wrap signed Lighter grouped tx_info JSON")?;
3834
3835 for guard in &mut guards {
3836 guard.disarm();
3837 }
3838
3839 nonce_guard.disarm();
3840 Ok(PreparedBatchTx {
3841 tx_type: LighterTxType::CreateGroupedOrders as u8,
3842 tx_info,
3843 pending: PendingSendTx {
3844 batch_id: None,
3845 connection_epoch: reserved.send_reservation.connection_epoch,
3846 kind: PendingSendTxKind::CreateGroup { orders },
3847 submitted_at: UnixNanos::default(),
3848 nonce: reserved.context.nonce,
3849 api_key_index: reserved.context.api_key_index,
3850 tx_hash: signed.tx_hash_hex(),
3851 },
3852 send_reservation: reserved.send_reservation,
3853 })
3854 }
3855
3856 async fn send_create_order(&self, prepared: PreparedCreateOrder) {
3857 let PreparedCreateOrder {
3858 order,
3859 client_order_index,
3860 tx_info,
3861 nonce,
3862 api_key_index,
3863 tx_hash,
3864 mut send_reservation,
3865 } = prepared;
3866 let connection_epoch = send_reservation.connection_epoch;
3867 let client_order_id = order.client_order_id();
3868
3869 self.emitter.emit_order_submitted(&order);
3870 log::debug!("Lighter submit_order: queueing CreateOrder tx for {client_order_id}");
3871 send_reservation.wait_for_turn().await;
3872 await_tx_quota(&self.tx_rate_limiter).await;
3873 self.dispatch.enqueue_pending_sendtx(PendingSendTx {
3874 batch_id: None,
3875 connection_epoch,
3876 kind: PendingSendTxKind::Create {
3877 order: Box::new(order.clone()),
3878 client_order_index,
3879 },
3880 submitted_at: self.clock.get_time_ns(),
3881 nonce,
3882 api_key_index,
3883 tx_hash,
3884 });
3885
3886 if let Err(e) = self
3887 .ws_client
3888 .send_tx_on_connection(LighterTxType::CreateOrder as u8, tx_info, connection_epoch)
3889 .await
3890 {
3891 let failure = classify_lighter_ws_command_failure("submit_order", &e);
3892 let reason = command_failure_reason(&failure);
3893 if matches!(&failure, CommandFailure::Ambiguous(_)) {
3894 log::warn!(
3895 "Lighter submit_order dispatch outcome unknown for {client_order_id}: {reason}; \
3896 retaining pending state for venue reconciliation; diagnostic={e:?}",
3897 );
3898 } else {
3899 log::warn!("{reason} for {client_order_id}; diagnostic={e:?}");
3900 self.dispatch
3901 .remove_pending_sendtx_by_nonce(connection_epoch, nonce);
3902 rollback_tx_dispatch_create(
3903 &self.dispatch,
3904 &self.credential,
3905 Some(client_order_index),
3906 &client_order_id,
3907 nonce,
3908 );
3909 self.emitter
3910 .emit_order_rejected(&order, reason, self.clock.get_time_ns(), false);
3911 }
3912 }
3913 send_reservation.release();
3914 }
3915
3916 fn sign_modify_order(
3917 &self,
3918 plan: &ModifyOrderPlan,
3919 reserved: ReservedTxContext,
3920 k: Scalar,
3921 ) -> anyhow::Result<PreparedModifyOrder> {
3922 let ReservedTxContext {
3923 context,
3924 send_reservation,
3925 } = reserved;
3926
3927 let captured_nonce = context.nonce;
3928 let captured_api_key_index = context.api_key_index;
3929
3930 let mut rollback_guard = TxDispatchGuard::new(
3931 self.dispatch.clone(),
3932 &self.credential,
3933 None,
3934 captured_nonce,
3935 );
3936
3937 let tx = ModifyOrderTxInfo {
3938 context,
3939 market_index: plan.market_index,
3940 index: plan.venue_index,
3941 base_amount: plan.base_amount,
3942 price: plan.price,
3943 trigger_price: plan.trigger_price,
3944 attributes: integrator_attributes(self.integrator_account_index),
3945 };
3946
3947 let signed = sign_tx(&tx, self.chain_id, &self.credential.private_key()?, k)?;
3948
3949 let tx_info_str = TxInfoJson::modify_order(&tx, &signed);
3950 let tx_info = serde_json::value::RawValue::from_string(tx_info_str)
3951 .context("failed to wrap signed Lighter modify tx_info JSON")?;
3952
3953 self.dispatch.set_pending_modify_nonce(
3954 &plan.client_order_id,
3955 plan.command_id,
3956 send_reservation.connection_epoch,
3957 captured_nonce,
3958 );
3959 rollback_guard.disarm();
3960
3961 Ok(PreparedModifyOrder {
3962 command_id: plan.command_id,
3963 client_order_id: plan.client_order_id,
3964 strategy_id: plan.strategy_id,
3965 instrument_id: plan.instrument_id,
3966 venue_order_id: plan.venue_order_id,
3967 tx_info,
3968 nonce: captured_nonce,
3969 api_key_index: captured_api_key_index,
3970 tx_hash: signed.tx_hash_hex(),
3971 send_reservation,
3972 })
3973 }
3974
3975 fn sign_cancel_order(&self, plan: &CancelOrderPlan) -> anyhow::Result<PreparedCancelOrder> {
3976 let k = fresh_k()?;
3977 let ReservedTxContext {
3978 context,
3979 send_reservation,
3980 } = self.build_tx_context()?;
3981
3982 let nonce = context.nonce;
3983 let api_key_index = context.api_key_index;
3984 let mut rollback_guard =
3985 TxDispatchGuard::new(self.dispatch.clone(), &self.credential, None, nonce);
3986
3987 let tx = CancelOrderTxInfo {
3988 context,
3989 market_index: plan.market_index,
3990 index: plan.venue_index,
3991 skip_nonce: 0,
3992 };
3993
3994 let signed = sign_tx(&tx, self.chain_id, &self.credential.private_key()?, k)?;
3995
3996 let tx_info =
3997 serde_json::value::RawValue::from_string(TxInfoJson::cancel_order(&tx, &signed))
3998 .context("failed to wrap signed Lighter cancel tx_info JSON")?;
3999
4000 rollback_guard.disarm();
4001
4002 Ok(PreparedCancelOrder {
4003 command_id: plan.command_id,
4004 client_order_id: plan.client_order_id,
4005 strategy_id: plan.strategy_id,
4006 instrument_id: plan.instrument_id,
4007 venue_order_id: plan.venue_order_id,
4008 tx_info,
4009 nonce,
4010 api_key_index,
4011 tx_hash: signed.tx_hash_hex(),
4012 send_reservation,
4013 })
4014 }
4015
4016 async fn send_cancel_order(
4017 &self,
4018 mut prepared: PreparedCancelOrder,
4019 emit_cancel_rejected: bool,
4020 ) {
4021 let connection_epoch = prepared.send_reservation.connection_epoch;
4022 prepared.send_reservation.wait_for_turn().await;
4023 await_tx_quota(&self.tx_rate_limiter).await;
4024 self.enqueue_cancel_order(&prepared, emit_cancel_rejected);
4025 let result = self
4026 .ws_client
4027 .send_tx_on_connection(
4028 LighterTxType::CancelOrder as u8,
4029 prepared.tx_info.clone(),
4030 connection_epoch,
4031 )
4032 .await;
4033
4034 match result {
4035 Ok(()) => {
4036 if let Some(command_id) = prepared.command_id {
4037 self.dispatch.promote_pending_cancel(
4038 &prepared.client_order_id,
4039 command_id,
4040 connection_epoch,
4041 prepared.nonce,
4042 emit_cancel_rejected,
4043 );
4044 }
4045 }
4046 Err(e) => self.handle_cancel_send_failure(&prepared, &e, emit_cancel_rejected),
4047 }
4048
4049 prepared.send_reservation.release();
4050 }
4051
4052 async fn send_batch(&self, mut batch: Vec<PreparedBatchTx>) {
4053 let mut batch_ids = Vec::new();
4054
4055 while !batch.is_empty() {
4056 let count = batch
4059 .windows(2)
4060 .position(|pair| {
4061 pair[1].pending.nonce != pair[0].pending.nonce + 1
4062 || pair[1].send_reservation.connection_epoch
4063 != pair[0].send_reservation.connection_epoch
4064 })
4065 .map_or(batch.len(), |index| index + 1);
4066
4067 let group: Vec<_> = batch.drain(..count).collect();
4068 let id = self.send_batch_group(group).await;
4069 batch_ids.push(id);
4070 }
4071
4072 let wait = async {
4074 while self.dispatch.pending_sendtx.lock().iter().any(|pending| {
4075 pending
4076 .batch_id
4077 .as_ref()
4078 .is_some_and(|id| batch_ids.contains(id))
4079 }) {
4080 tokio::time::sleep(Duration::from_millis(10)).await;
4081 }
4082 };
4083
4084 if tokio::time::timeout(BATCH_ACK_TIMEOUT, wait).await.is_err() {
4085 log::warn!("Lighter batch acknowledgement timed out: ids={batch_ids:?}");
4086 }
4087 }
4088
4089 async fn send_batch_group(&self, mut group: Vec<PreparedBatchTx>) -> String {
4090 let connection_epoch = group[0].send_reservation.connection_epoch;
4091 let id = format!("{TX_BATCH_ID_PREFIX}{}", group[0].pending.tx_hash);
4092 group[0].send_reservation.wait_for_turn().await;
4093 await_tx_quota(&self.tx_rate_limiter).await;
4094
4095 let data = crate::websocket::messages::LighterWsSendTxBatch {
4096 id: id.clone(),
4097 tx_types: serde_json::to_string(
4098 &group
4099 .iter()
4100 .map(|prepared| prepared.tx_type)
4101 .collect::<Vec<_>>(),
4102 )
4103 .expect("transaction types serialize"),
4104 tx_infos: serde_json::to_string(
4105 &group
4106 .iter()
4107 .map(|prepared| prepared.tx_info.get())
4108 .collect::<Vec<_>>(),
4109 )
4110 .expect("transaction strings serialize"),
4111 };
4112
4113 for prepared in &mut group {
4114 prepared.pending.batch_id = Some(id.clone());
4115 prepared.pending.submitted_at = self.clock.get_time_ns();
4116 self.enqueue_batch_tx(prepared);
4117 }
4118
4119 let result = self
4120 .ws_client
4121 .send_tx_batch_on_connection(data, connection_epoch)
4122 .await;
4123
4124 match result {
4125 Ok(()) => {
4126 for prepared in &group {
4127 promote_cancel_submission(&self.dispatch, &prepared.pending);
4128 }
4129 }
4130 Err(e) => {
4131 for prepared in group.iter().rev() {
4132 self.handle_batch_send_failure(prepared, &e);
4133 }
4134 }
4135 }
4136
4137 for prepared in &mut group {
4138 prepared.send_reservation.release();
4139 }
4140
4141 id
4142 }
4143
4144 fn enqueue_batch_tx(&self, prepared: &PreparedBatchTx) {
4145 for (order, _) in prepared.pending.kind.creates() {
4146 self.emitter.emit_order_submitted(order);
4147 }
4148
4149 self.dispatch
4150 .enqueue_pending_sendtx(prepared.pending.clone());
4151 if matches!(prepared.pending.kind, PendingSendTxKind::Modify { .. }) {
4152 spawn_acked_order_probe(&prepared.pending, self.probe_context());
4153 }
4154 }
4155
4156 fn handle_batch_send_failure(&self, prepared: &PreparedBatchTx, error: &LighterWsError) {
4157 let action = match prepared.pending.kind {
4158 PendingSendTxKind::Create { .. } | PendingSendTxKind::CreateGroup { .. } => {
4159 "submit_order"
4160 }
4161 PendingSendTxKind::Modify { .. } => "modify_order",
4162 PendingSendTxKind::Cancel { .. } => "cancel_order",
4163 PendingSendTxKind::Other => {
4164 unreachable!("order batches contain only order transactions")
4165 }
4166 };
4167
4168 let failure = classify_lighter_ws_command_failure(action, error);
4169 let reason = command_failure_reason(&failure);
4170
4171 if !matches!(failure, CommandFailure::NotSent(_)) {
4172 promote_cancel_submission(&self.dispatch, &prepared.pending);
4173 }
4174
4175 if matches!(failure, CommandFailure::Ambiguous(_)) {
4176 log::warn!("{reason}; retaining batch transaction for reconciliation");
4177 return;
4178 }
4179
4180 if let Some(pending) = self.dispatch.remove_pending_sendtx_by_hash(
4181 prepared.pending.connection_epoch,
4182 &prepared.pending.tx_hash,
4183 ) {
4184 reject_pending_sendtx(
4185 &self.dispatch,
4186 &self.emitter,
4187 Some(self.credential.account_index()),
4188 &pending,
4189 &failure,
4190 self.clock.get_time_ns(),
4191 false,
4192 );
4193 }
4194 }
4195
4196 fn enqueue_cancel_order(&self, prepared: &PreparedCancelOrder, emit_cancel_rejected: bool) {
4197 self.dispatch.enqueue_pending_sendtx(PendingSendTx {
4198 batch_id: None,
4199 connection_epoch: prepared.send_reservation.connection_epoch,
4200 kind: PendingSendTxKind::Cancel {
4201 command_id: prepared.command_id,
4202 emit_cancel_rejected,
4203 strategy_id: prepared.strategy_id,
4204 instrument_id: prepared.instrument_id,
4205 client_order_id: prepared.client_order_id,
4206 venue_order_id: prepared.venue_order_id,
4207 },
4208 submitted_at: self.clock.get_time_ns(),
4209 nonce: prepared.nonce,
4210 api_key_index: prepared.api_key_index,
4211 tx_hash: prepared.tx_hash.clone(),
4212 });
4213 }
4214
4215 fn handle_cancel_send_failure(
4216 &self,
4217 prepared: &PreparedCancelOrder,
4218 error: &LighterWsError,
4219 emit_cancel_rejected: bool,
4220 ) {
4221 let failure = classify_lighter_ws_command_failure("cancel_order", error);
4222 let reason = command_failure_reason(&failure);
4223
4224 if !matches!(failure, CommandFailure::NotSent(_))
4225 && let Some(command_id) = prepared.command_id
4226 {
4227 self.dispatch.promote_pending_cancel(
4228 &prepared.client_order_id,
4229 command_id,
4230 prepared.send_reservation.connection_epoch,
4231 prepared.nonce,
4232 emit_cancel_rejected,
4233 );
4234 }
4235
4236 if matches!(failure, CommandFailure::Ambiguous(_)) {
4237 log::warn!(
4238 "Lighter cancel_order outcome unknown for {}: {reason}; retaining pending state for venue reconciliation",
4239 prepared.client_order_id
4240 );
4241 return;
4242 }
4243
4244 log::warn!(
4245 "{reason} for {}; diagnostic={error:?}",
4246 prepared.client_order_id
4247 );
4248
4249 self.dispatch.remove_pending_sendtx_by_nonce(
4250 prepared.send_reservation.connection_epoch,
4251 prepared.nonce,
4252 );
4253 rollback_tx_dispatch(&self.dispatch, &self.credential, None, prepared.nonce);
4254
4255 let Some(command_id) = prepared.command_id else {
4256 return;
4257 };
4258
4259 self.dispatch
4260 .clear_pending_cancel_if_command(&prepared.client_order_id, command_id);
4261
4262 match &failure {
4263 CommandFailure::NotSent(reason) => self.emitter.emit_order_cancel_denied_event(
4264 prepared.strategy_id,
4265 prepared.instrument_id,
4266 prepared.client_order_id,
4267 prepared.venue_order_id,
4268 reason,
4269 self.clock.get_time_ns(),
4270 Some(command_id),
4271 ),
4272 CommandFailure::VenueRejected(reason) => self.emitter.emit_order_cancel_rejected_event(
4273 prepared.strategy_id,
4274 prepared.instrument_id,
4275 prepared.client_order_id,
4276 prepared.venue_order_id,
4277 reason,
4278 self.clock.get_time_ns(),
4279 Some(command_id),
4280 ),
4281 _ => {}
4282 }
4283 }
4284}
4285
4286fn should_retry_batch_signing(error: &anyhow::Error) -> bool {
4287 error.is::<TxPreparationError>()
4288 || error.is::<tokio::sync::TryLockError>()
4289 || matches!(
4290 error.downcast_ref::<NonceError>(),
4291 Some(NonceError::SkipWindowExhausted { .. })
4292 )
4293}
4294
4295#[derive(Debug, thiserror::Error)]
4296enum TxPreparationError {
4297 #[error("Lighter nonce refresh is pending for connection epoch {0}")]
4298 NonceRefreshPending(u64),
4299 #[error("Lighter connection or nonce state changed during transaction preparation")]
4300 StateChanged,
4301}
4302
4303struct PreparedModifyOrder {
4304 command_id: UUID4,
4305 client_order_id: ClientOrderId,
4306 strategy_id: StrategyId,
4307 instrument_id: InstrumentId,
4308 venue_order_id: Option<VenueOrderId>,
4309 tx_info: Box<serde_json::value::RawValue>,
4310 nonce: i64,
4311 api_key_index: u8,
4312 tx_hash: String,
4313 send_reservation: TxSendReservation,
4314}
4315
4316struct ModifyOrderPlan {
4317 command_id: UUID4,
4318 client_order_id: ClientOrderId,
4319 strategy_id: StrategyId,
4320 instrument_id: InstrumentId,
4321 venue_order_id: Option<VenueOrderId>,
4322 market_index: i64,
4323 venue_index: i64,
4324 base_amount: i64,
4325 price: u32,
4326 trigger_price: u32,
4327}
4328
4329enum BatchOrderPlan {
4330 Create(Box<CreateOrderPlan>),
4331 CreateGroup(CreateGroupPlan),
4332 Modify(ModifyOrderPlan),
4333 Cancel(CancelOrderPlan, bool),
4334}
4335
4336struct PreparedBatchTx {
4337 tx_type: u8,
4338 tx_info: Box<serde_json::value::RawValue>,
4339 pending: PendingSendTx,
4340 send_reservation: TxSendReservation,
4341}
4342
4343impl From<PreparedCreateOrder> for PreparedBatchTx {
4344 fn from(prepared: PreparedCreateOrder) -> Self {
4345 Self {
4346 tx_type: LighterTxType::CreateOrder as u8,
4347 tx_info: prepared.tx_info,
4348 pending: PendingSendTx {
4349 batch_id: None,
4350 connection_epoch: prepared.send_reservation.connection_epoch,
4351 kind: PendingSendTxKind::Create {
4352 order: Box::new(prepared.order),
4353 client_order_index: prepared.client_order_index,
4354 },
4355 submitted_at: UnixNanos::default(),
4356 nonce: prepared.nonce,
4357 api_key_index: prepared.api_key_index,
4358 tx_hash: prepared.tx_hash,
4359 },
4360 send_reservation: prepared.send_reservation,
4361 }
4362 }
4363}
4364
4365impl From<PreparedModifyOrder> for PreparedBatchTx {
4366 fn from(prepared: PreparedModifyOrder) -> Self {
4367 Self {
4368 tx_type: LighterTxType::ModifyOrder as u8,
4369 tx_info: prepared.tx_info,
4370 pending: PendingSendTx {
4371 batch_id: None,
4372 connection_epoch: prepared.send_reservation.connection_epoch,
4373 kind: PendingSendTxKind::Modify {
4374 command_id: prepared.command_id,
4375 client_order_id: prepared.client_order_id,
4376 strategy_id: prepared.strategy_id,
4377 instrument_id: prepared.instrument_id,
4378 venue_order_id: prepared.venue_order_id,
4379 },
4380 submitted_at: UnixNanos::default(),
4381 nonce: prepared.nonce,
4382 api_key_index: prepared.api_key_index,
4383 tx_hash: prepared.tx_hash,
4384 },
4385 send_reservation: prepared.send_reservation,
4386 }
4387 }
4388}
4389
4390impl From<(PreparedCancelOrder, bool)> for PreparedBatchTx {
4391 fn from((prepared, emit_cancel_rejected): (PreparedCancelOrder, bool)) -> Self {
4392 Self {
4393 tx_type: LighterTxType::CancelOrder as u8,
4394 tx_info: prepared.tx_info,
4395 pending: PendingSendTx {
4396 batch_id: None,
4397 connection_epoch: prepared.send_reservation.connection_epoch,
4398 kind: PendingSendTxKind::Cancel {
4399 command_id: prepared.command_id,
4400 emit_cancel_rejected,
4401 client_order_id: prepared.client_order_id,
4402 strategy_id: prepared.strategy_id,
4403 instrument_id: prepared.instrument_id,
4404 venue_order_id: prepared.venue_order_id,
4405 },
4406 submitted_at: UnixNanos::default(),
4407 nonce: prepared.nonce,
4408 api_key_index: prepared.api_key_index,
4409 tx_hash: prepared.tx_hash,
4410 },
4411 send_reservation: prepared.send_reservation,
4412 }
4413 }
4414}
4415
4416struct PreparedIntegratorApproval {
4417 tx_info: String,
4418 nonce: i64,
4419 api_key_index: u8,
4420 approval_expiry: i64,
4421 send_reservation: TxSendReservation,
4422}
4423
4424#[derive(Debug, PartialEq, Eq)]
4428enum TierCrossCheck {
4429 AboveTier { documented: u32 },
4430 RaiseHint { documented: u32 },
4431}
4432
4433fn tier_quota_report(
4438 tier: LighterAccountTier,
4439 rest_quota_per_min: Option<u32>,
4440 standard_rest: u32,
4441) -> (u32, Option<TierCrossCheck>) {
4442 let configured = rest_quota_per_min.filter(|&n| n > 0);
4443 let active_rest = configured.unwrap_or(standard_rest);
4444 let cross_check = match (tier.documented_rest_quota_per_min(), configured) {
4445 (Some(documented), Some(configured)) if configured > documented => {
4446 Some(TierCrossCheck::AboveTier { documented })
4447 }
4448 (Some(documented), None) if documented > standard_rest => {
4449 Some(TierCrossCheck::RaiseHint { documented })
4450 }
4451 _ => None,
4452 };
4453 (active_rest, cross_check)
4454}
4455
4456struct TxDispatchGuard {
4457 dispatch: WsDispatchState,
4458 account_index: i64,
4459 api_key_index: u8,
4460 client_order_index: Option<i64>,
4461 client_order_id: Option<ClientOrderId>,
4462 nonce: i64,
4463 armed: bool,
4464}
4465
4466impl TxDispatchGuard {
4467 fn new(
4468 dispatch: WsDispatchState,
4469 credential: &Credential,
4470 client_order_index: Option<i64>,
4471 nonce: i64,
4472 ) -> Self {
4473 Self {
4474 dispatch,
4475 account_index: credential.account_index(),
4476 api_key_index: credential.api_key_index(),
4477 client_order_index,
4478 client_order_id: None,
4479 nonce,
4480 armed: true,
4481 }
4482 }
4483
4484 fn with_order_identity(mut self, client_order_id: ClientOrderId) -> Self {
4485 self.client_order_id = Some(client_order_id);
4486 self
4487 }
4488
4489 fn disarm(&mut self) {
4490 self.armed = false;
4491 }
4492}
4493
4494impl Drop for TxDispatchGuard {
4495 fn drop(&mut self) {
4496 if self.armed {
4497 rollback_tx_dispatch_indices(
4498 &self.dispatch,
4499 self.account_index,
4500 self.api_key_index,
4501 self.client_order_index,
4502 self.client_order_id.as_ref(),
4503 self.nonce,
4504 );
4505 }
4506 }
4507}
4508
4509fn handle_send_tx_ack_for_connection(
4514 dispatch: &WsDispatchState,
4515 account_index: Option<i64>,
4516 connection_epoch: u64,
4517 code: i64,
4518 tx_hash: Option<&str>,
4519) -> Option<PendingSendTx> {
4520 let popped = match tx_hash {
4521 Some(hash) => {
4522 let matched = dispatch.remove_pending_sendtx_by_hash(connection_epoch, hash);
4523 if matched.is_none() {
4524 log::warn!("Lighter sendTx ack unmatched: tx_hash={hash} code={code}");
4525 }
4526 matched
4527 }
4528 None => dispatch.pop_pending_sendtx_if_only(connection_epoch),
4529 };
4530
4531 if let (Some(pending), Some(account_index)) = (&popped, account_index) {
4532 let _ =
4533 dispatch
4534 .nonce_manager
4535 .ack_success(account_index, pending.api_key_index, pending.nonce);
4536 }
4537
4538 if let Some(pending) = &popped {
4539 promote_cancel_submission(dispatch, pending);
4540 }
4541
4542 log::debug!(
4543 "Lighter sendTx ack: code={code} tx_hash={tx_hash:?} popped_nonce={:?}",
4544 popped.as_ref().map(|p| p.nonce),
4545 );
4546
4547 popped
4548}
4549
4550fn promote_cancel_submission(dispatch: &WsDispatchState, pending: &PendingSendTx) {
4551 if let PendingSendTxKind::Cancel {
4552 command_id: Some(command_id),
4553 client_order_id,
4554 emit_cancel_rejected,
4555 ..
4556 } = pending.kind
4557 {
4558 dispatch.promote_pending_cancel(
4559 &client_order_id,
4560 command_id,
4561 pending.connection_epoch,
4562 pending.nonce,
4563 emit_cancel_rejected,
4564 );
4565 }
4566}
4567
4568#[expect(
4569 clippy::too_many_arguments,
4570 reason = "batch response shares the single-transaction rejection sink"
4571)]
4572fn handle_send_tx_batch_result(
4573 dispatch: &WsDispatchState,
4574 emitter: &ExecutionEventEmitter,
4575 account_index: Option<i64>,
4576 connection_epoch: u64,
4577 now: UnixNanos,
4578 id: &str,
4579 code: i64,
4580 message: &str,
4581 tx_hashes: &[String],
4582) -> (Vec<PendingSendTx>, bool) {
4583 let mut pending: Vec<_> = dispatch
4584 .pending_sendtx
4585 .lock()
4586 .iter()
4587 .filter(|pending| {
4588 pending.connection_epoch == connection_epoch && pending.batch_id.as_deref() == Some(id)
4589 })
4590 .cloned()
4591 .collect();
4592
4593 if pending.is_empty() {
4594 log::warn!("Ignoring unmatched Lighter batch response: id={id}");
4595 return (Vec::new(), false);
4596 }
4597
4598 if code == 200 {
4599 let acked = pending
4600 .iter()
4601 .filter(|pending| tx_hashes.contains(&pending.tx_hash))
4602 .filter_map(|pending| {
4603 handle_send_tx_ack_for_connection(
4604 dispatch,
4605 account_index,
4606 connection_epoch,
4607 code,
4608 Some(&pending.tx_hash),
4609 )
4610 })
4611 .collect();
4612
4613 if pending
4614 .iter()
4615 .any(|pending| !tx_hashes.contains(&pending.tx_hash))
4616 {
4617 log::warn!(
4618 "Incomplete Lighter batch acknowledgement: id={id}; retaining unmatched transactions for reconciliation"
4619 );
4620 }
4621
4622 return (acked, false);
4623 }
4624
4625 pending.sort_unstable_by_key(|pending| std::cmp::Reverse(pending.nonce));
4627 let mut resync = false;
4628 for pending in pending {
4629 resync |= handle_send_tx_rejection_for_connection(
4630 dispatch,
4631 emitter,
4632 account_index,
4633 connection_epoch,
4634 now,
4635 SendTxRejectionSource::Ack,
4636 Some(code),
4637 message,
4638 Some(&pending.tx_hash),
4639 );
4640 }
4641
4642 (Vec::new(), resync)
4643}
4644
4645fn spawn_acked_order_probe(pending: &PendingSendTx, context: AckedOrderProbeContext) {
4646 if let PendingSendTxKind::CreateGroup { orders } = &pending.kind {
4647 for (order, index) in orders {
4648 let mut leg = pending.clone();
4649
4650 leg.kind = PendingSendTxKind::Create {
4651 order: Box::new(order.clone()),
4652 client_order_index: *index,
4653 };
4654
4655 spawn_acked_order_probe(&leg, context.clone());
4656 }
4657
4658 return;
4659 }
4660
4661 let action = matches!(
4662 pending.kind,
4663 PendingSendTxKind::Modify { .. } | PendingSendTxKind::Cancel { .. }
4664 )
4665 .then(|| pending.clone());
4666 let probe = AckedOrderProbe::from_pending(pending);
4667 if action.is_none() && probe.is_none() {
4668 return;
4669 }
4670
4671 let pending_tasks = context.pending_tasks.clone();
4672
4673 let future = async move {
4674 tokio::select! {
4675 () = context.cancellation_token.cancelled() => return,
4676 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
4677 }
4678
4679 if let Some(pending) = action {
4680 let result = if let PendingSendTxKind::Cancel {
4681 client_order_id, ..
4682 } = pending.kind
4683 {
4684 let Some(token) = context.dispatch.pending_cancel_token(
4685 &client_order_id,
4686 pending.connection_epoch,
4687 pending.nonce,
4688 ) else {
4689 return;
4690 };
4691
4692 tokio::select! {
4693 biased;
4694 () = context.cancellation_token.cancelled() => return,
4695 () = token.cancelled() => return,
4696 result = probe_acked_action(&pending,&context) => result,
4697 }
4698 } else {
4699 probe_acked_action(&pending, &context).await
4700 };
4701
4702 if let Err(e) = result {
4703 log::warn!("Lighter acknowledged action probe failed: {e}");
4704 }
4705
4706 return;
4707 }
4708
4709 if let Some(probe) = probe
4710 && let Err(e) = probe_acked_create(&probe, &context).await
4711 {
4712 log::warn!("Lighter acknowledged order probe failed: {e:?}");
4713 }
4714 };
4715
4716 if let Err(e) = pending_tasks.spawn(future) {
4717 log::debug!("Skipping Lighter acknowledged-order probe during shutdown: {e}");
4718 }
4719}
4720
4721#[derive(Clone)]
4722struct AckedOrderProbeContext {
4723 http_client: LighterHttpClient,
4724 registry: Arc<MarketRegistry>,
4725 credential: Credential,
4726 dispatch: WsDispatchState,
4727 account_id: AccountId,
4728 clock: &'static AtomicTime,
4729 emitter: ExecutionEventEmitter,
4730 connection_epoch: Arc<AtomicU64>,
4731 cancellation_token: CancellationToken,
4732 pending_tasks: TaskSpawner,
4733}
4734
4735fn handle_account_tx(connection_epoch: u64, txs: &[LighterWsTx], context: &AckedOrderProbeContext) {
4736 if connection_epoch != context.connection_epoch.load(Ordering::Acquire) {
4737 return;
4738 }
4739
4740 for tx in txs {
4741 let Some(pending) = context.dispatch.pending_modify_by_hash(&tx.hash) else {
4742 continue;
4743 };
4744
4745 let PendingSendTxKind::Modify {
4746 client_order_id, ..
4747 } = pending.kind
4748 else {
4749 unreachable!("pending modification owns a modify transaction")
4750 };
4751
4752 let failure = match tx_application_error(tx.status, &tx.event_info) {
4753 Ok(failure) => failure,
4754 Err(e) => {
4755 log::warn!("Lighter transaction update unavailable: {e}");
4756 continue;
4757 }
4758 };
4759
4760 if failure.is_none() && tx.status != LighterTxStatus::Executed {
4761 continue;
4762 }
4763
4764 let _ = context.dispatch.nonce_manager.ack_success(
4765 context.credential.account_index(),
4766 pending.api_key_index,
4767 pending.nonce,
4768 );
4769
4770 if let Some(detail) = failure {
4771 let reason = format!("Lighter modify transaction failed: {detail}");
4772 reject_modify_order(
4773 &context.dispatch,
4774 &context.emitter,
4775 &pending,
4776 &CommandFailure::venue_rejected(&reason),
4777 context.clock.get_time_ns(),
4778 true,
4779 );
4780 continue;
4781 }
4782
4783 context
4784 .dispatch
4785 .set_pending_modify_executed(&client_order_id, &pending);
4786 let order = context
4787 .dispatch
4788 .order_events
4789 .lock()
4790 .get(&client_order_id)
4791 .and_then(|events| events.order.clone());
4792
4793 if let Some(order) = order {
4794 confirm_executed_modify(&order, context);
4795 }
4796 }
4797}
4798
4799fn confirm_executed_modify(order: &LighterOrder, context: &AckedOrderProbeContext) {
4800 let venue_order_id = VenueOrderId::new(&order.order_id);
4801
4802 let Some(cloid) = context
4803 .dispatch
4804 .resolve_client_order_index_for_venue(order.client_order_index, venue_order_id)
4805 else {
4806 return;
4807 };
4808
4809 let Some(pending) = context.dispatch.pending_modify_executed(&cloid) else {
4810 return;
4811 };
4812
4813 if let Err(e) = confirm_modify_order(&pending, context, order) {
4814 log::debug!("Lighter executed modification awaits matching order state: {e}");
4815 }
4816}
4817
4818async fn probe_acked_action(
4819 pending: &PendingSendTx,
4820 context: &AckedOrderProbeContext,
4821) -> anyhow::Result<()> {
4822 let (client_order_id, _) = acked_action_identity(pending)?;
4823 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
4824 if !acked_action_is_pending(pending, context) {
4825 return Ok(());
4826 }
4827
4828 match context.http_client.get_tx(pending.tx_hash.clone()).await {
4829 Ok(tx) => match resolve_acked_action(pending, context, tx).await {
4830 Ok(true) => return Ok(()),
4831 Ok(false) => {}
4832 Err(e) => log::warn!("Lighter action confirmation unavailable: {e}"),
4833 },
4834 Err(LighterHttpError::Venue { code: 21500, .. }) => {}
4835 Err(e) => log::warn!("Lighter acknowledged action transaction lookup failed: {e}"),
4836 }
4837
4838 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
4839 tokio::select! {
4840 () = context.cancellation_token.cancelled() => return Ok(()),
4841 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
4842 }
4843 }
4844 }
4845
4846 log::warn!(
4847 "Lighter acknowledged action outcome unresolved for {client_order_id}; retaining pending action for reconciliation"
4848 );
4849 Ok(())
4850}
4851
4852fn acked_action_identity(
4853 pending: &PendingSendTx,
4854) -> anyhow::Result<(ClientOrderId, LighterTxType)> {
4855 match pending.kind {
4856 PendingSendTxKind::Cancel {
4857 client_order_id, ..
4858 } => Ok((client_order_id, LighterTxType::CancelOrder)),
4859 PendingSendTxKind::Modify {
4860 client_order_id, ..
4861 } => Ok((client_order_id, LighterTxType::ModifyOrder)),
4862 _ => anyhow::bail!("acknowledged action requires a cancel or modify transaction"),
4863 }
4864}
4865
4866fn acked_action_is_pending(pending: &PendingSendTx, context: &AckedOrderProbeContext) -> bool {
4867 match pending.kind {
4868 PendingSendTxKind::Cancel {
4869 command_id: Some(command_id),
4870 client_order_id,
4871 ..
4872 } => {
4873 context.connection_epoch.load(Ordering::Acquire) == pending.connection_epoch
4874 && context.dispatch.pending_cancel_identity(&client_order_id)
4875 == Some((command_id, (pending.connection_epoch, pending.nonce)))
4876 }
4877 PendingSendTxKind::Modify {
4878 client_order_id, ..
4879 } => {
4880 context.dispatch.pending_modify_nonce(&client_order_id)
4881 == Some((pending.connection_epoch, pending.nonce))
4882 }
4883 _ => false,
4884 }
4885}
4886
4887async fn resolve_acked_action(
4888 pending: &PendingSendTx,
4889 context: &AckedOrderProbeContext,
4890 tx: LighterTx,
4891) -> anyhow::Result<bool> {
4892 let (_, tx_type) = acked_action_identity(pending)?;
4893 anyhow::ensure!(
4894 tx_hash_matches(&tx.hash, &pending.tx_hash)
4895 && tx.tx_type == tx_type as u8
4896 && tx.account_index == context.credential.account_index()
4897 && tx.api_key_index == pending.api_key_index
4898 && tx.nonce == pending.nonce,
4899 "Lighter transaction lookup did not match acknowledged action identity"
4900 );
4901
4902 let is_modify = tx_type == LighterTxType::ModifyOrder;
4903
4904 if !acked_action_is_pending(pending, context) {
4905 return Ok(true);
4906 }
4907
4908 if let Some(detail) = tx_application_error(tx.status, &tx.event_info)? {
4909 let action = if is_modify { "modify" } else { "cancel" };
4910 let reason = format!("Lighter {action} transaction failed: {detail}");
4911 log::warn!("{reason} for nonce={}", pending.nonce);
4912
4913 match pending.kind {
4914 PendingSendTxKind::Modify { .. } => reject_modify_order(
4915 &context.dispatch,
4916 &context.emitter,
4917 pending,
4918 &CommandFailure::venue_rejected(&reason),
4919 context.clock.get_time_ns(),
4920 true,
4921 ),
4922 PendingSendTxKind::Cancel { .. } => reject_cancel_order(
4923 &context.dispatch,
4924 &context.emitter,
4925 pending,
4926 &CommandFailure::venue_rejected(&reason),
4927 context.clock.get_time_ns(),
4928 ),
4929 _ => unreachable!("validated acknowledged action"),
4930 }
4931
4932 return Ok(true);
4934 }
4935
4936 if tx.status == LighterTxStatus::Executed {
4937 return recover_acked_action_report(pending, context).await;
4938 }
4939
4940 Ok(false)
4941}
4942
4943async fn recover_acked_action_report(
4944 pending: &PendingSendTx,
4945 context: &AckedOrderProbeContext,
4946) -> anyhow::Result<bool> {
4947 let (instrument_id, client_order_id, venue_order_id) = match pending.kind {
4948 PendingSendTxKind::Cancel {
4949 instrument_id,
4950 client_order_id,
4951 venue_order_id,
4952 ..
4953 }
4954 | PendingSendTxKind::Modify {
4955 instrument_id,
4956 client_order_id,
4957 venue_order_id,
4958 ..
4959 } => (instrument_id, client_order_id, venue_order_id),
4960 _ => return Ok(true),
4961 };
4962
4963 let lookup = OrderReportLookup::new(
4964 &context.registry,
4965 &context.dispatch,
4966 Some(instrument_id),
4967 Some(&client_order_id),
4968 venue_order_id.as_ref(),
4969 )?;
4970
4971 anyhow::ensure!(
4972 matches!(pending.kind, PendingSendTxKind::Modify { .. })
4973 || !context.dispatch.has_pending_modify(&client_order_id),
4974 "Lighter cancellation waits for modification confirmation"
4975 );
4976 let order = collect_order_while_pending(
4977 &context.http_client,
4978 &context.registry,
4979 &context.credential,
4980 &lookup,
4981 || acked_action_is_pending(pending, context),
4982 )
4983 .await?;
4984
4985 if !acked_action_is_pending(pending, context) {
4986 return Ok(true);
4987 }
4988
4989 let Some(order) = order else {
4990 return Ok(false);
4991 };
4992
4993 if matches!(pending.kind, PendingSendTxKind::Modify { .. }) {
4994 confirm_modify_order(pending, context, &order)?;
4995 return Ok(true);
4996 }
4997
4998 recover_report_fills(&order, context).await?;
4999
5000 let events = context.dispatch.order_events.lock();
5001 if !acked_action_is_pending(pending, context) {
5002 return Ok(true);
5003 }
5004
5005 if context
5006 .dispatch
5007 .terminals_pending
5008 .lock()
5009 .contains_key(&client_order_id)
5010 || events.get(&client_order_id).is_some_and(|deferred| {
5011 deferred
5012 .order
5013 .as_ref()
5014 .is_some_and(|order| order.status.is_terminal())
5015 })
5016 || context
5017 .dispatch
5018 .order_identity(&client_order_id)
5019 .is_some_and(|identity| position_fills_pending(&identity, &events, &context.dispatch))
5020 {
5021 return Ok(false);
5022 }
5023
5024 let report = lookup.parse(
5025 &order,
5026 &context.registry,
5027 context.account_id,
5028 context.clock.get_time_ns(),
5029 &context.dispatch,
5030 )?;
5031 let report = lookup.translate(&context.dispatch, report);
5032 if context
5033 .dispatch
5034 .order_identity(&client_order_id)
5035 .is_some_and(|identity| {
5036 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
5037 })
5038 {
5039 return Ok(false);
5040 }
5041
5042 let resolved = report.order_status.is_closed();
5043
5044 let current = if resolved {
5045 context.dispatch.clear_pending_cancel_if_nonce(
5046 &client_order_id,
5047 pending.connection_epoch,
5048 pending.nonce,
5049 )
5050 } else {
5051 context.dispatch.pending_cancel_nonce(&client_order_id)
5052 == Some((pending.connection_epoch, pending.nonce))
5053 };
5054
5055 if !current {
5056 return Ok(true);
5057 }
5058
5059 if resolved {
5060 context
5061 .dispatch
5062 .clear_pending_order_action(&client_order_id);
5063 }
5064
5065 let report = if report.order_status.is_closed() {
5066 report
5067 } else {
5068 context.dispatch.preserve_pending_order_status(report)
5069 };
5070
5071 context.emitter.send_order_status_report(report);
5072 Ok(resolved)
5073}
5074
5075fn confirm_modify_order(
5076 pending: &PendingSendTx,
5077 context: &AckedOrderProbeContext,
5078 order: &crate::http::models::LighterOrder,
5079) -> anyhow::Result<()> {
5080 anyhow::ensure!(
5081 order.status != LighterOrderStatus::Unknown,
5082 "Lighter modified order has an unknown status"
5083 );
5084 let (cloid, _) = acked_action_identity(pending)?;
5085 let mut events = context.dispatch.order_events.lock();
5086 if !acked_action_is_pending(pending, context) {
5087 return Ok(());
5088 }
5089
5090 let identity = context
5091 .dispatch
5092 .order_identity(&cloid)
5093 .context("missing modified order identity")?;
5094 let instrument = LIGHTER_INSTRUMENT_CACHE
5095 .get(&identity.instrument_id)
5096 .context("missing modified order instrument")?;
5097 let shape = lighter_order_shape(order, &instrument, identity.order_type)?;
5098 let mut requested_shape = shape.clone();
5099 requested_shape.quantity =
5100 quantity_from_decimal(order.initial_base_amount, instrument.size_precision())?;
5101 anyhow::ensure!(
5102 context.dispatch.pending_modify_shape(&cloid).as_ref() == Some(&requested_shape),
5103 "Lighter executed modification shape is not available yet"
5104 );
5105 identity.record_reduction(order.initial_base_amount - shape.quantity.as_decimal());
5106 let updated = parse_lighter_order_updated(
5107 order,
5108 &instrument,
5109 &identity,
5110 cloid,
5111 context.account_id,
5112 context.emitter.trader_id(),
5113 context.clock.get_time_ns(),
5114 )?;
5115 context.dispatch.store_snapshot(cloid, shape);
5116 identity.publish_update(updated.event_id);
5117 context
5118 .emitter
5119 .send_order_event(OrderEventAny::Updated(updated));
5120 context
5121 .dispatch
5122 .clear_pending_modify_if_nonce(&cloid, pending.connection_epoch, pending.nonce);
5123 let mut deferred = events.remove(&cloid).unwrap_or_default();
5124 if deferred.order.is_none() {
5125 deferred.store_order(order);
5126 }
5127
5128 flush_modify_events(
5129 cloid,
5130 deferred,
5131 &context.dispatch,
5132 &context.emitter,
5133 &mut events,
5134 );
5135 drop(events);
5136 spawn_child_recoveries(context);
5137 Ok(())
5138}
5139
5140fn flush_modify_events(
5141 cloid: ClientOrderId,
5142 deferred: DeferredOrderEvents,
5143 dispatch: &WsDispatchState,
5144 emitter: &ExecutionEventEmitter,
5145 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
5146) {
5147 if dispatch
5148 .order_identity(&cloid)
5149 .is_some_and(|identity| identity.child_fill_baseline().is_some())
5150 && let Some(order) = deferred
5151 .order
5152 .as_ref()
5153 .filter(|order| is_order_triggered(order))
5154 {
5155 dispatch_deferred_order(cloid, order, dispatch, emitter);
5156 }
5157
5158 events.insert(cloid, deferred);
5159 publish_child_recovery(cloid, events, dispatch, emitter);
5160}
5161
5162fn flush_order_terminal(
5163 cloid: ClientOrderId,
5164 deferred: DeferredOrderEvents,
5165 dispatch: &WsDispatchState,
5166 emitter: &ExecutionEventEmitter,
5167 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
5168) {
5169 for fill in deferred.fills {
5170 if dispatch.mark_trade_seen(fill.trade_id) {
5171 dispatch_lighter_fill(fill, emitter, dispatch, events);
5172 }
5173 }
5174
5175 let Some(order) = deferred.order else {
5176 return;
5177 };
5178
5179 dispatch_deferred_order(cloid, &order, dispatch, emitter);
5180}
5181
5182fn dispatch_deferred_order(
5183 cloid: ClientOrderId,
5184 order: &crate::http::models::LighterOrder,
5185 dispatch: &WsDispatchState,
5186 emitter: &ExecutionEventEmitter,
5187) {
5188 let Some(identity) = dispatch
5189 .order_identities
5190 .get(&cloid)
5191 .map(|entry| entry.value().clone())
5192 else {
5193 return;
5194 };
5195
5196 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id) else {
5197 return;
5198 };
5199
5200 let ts_init = get_atomic_clock_realtime().get_time_ns();
5201
5202 if order.status == crate::common::enums::LighterOrderStatus::Filled {
5203 dispatch_owned_order(
5204 order,
5205 &instrument,
5206 &identity,
5207 cloid,
5208 VenueOrderId::new(&order.order_id),
5209 dispatch,
5210 emitter,
5211 emitter.account_id(),
5212 emitter.trader_id(),
5213 ts_init,
5214 );
5215 return;
5216 }
5217
5218 let context = OpenFrameContext {
5219 accepted_already_emitted: dispatch.accepted_was_emitted(&cloid),
5220 triggered_already_emitted: dispatch.triggered_was_emitted(&cloid),
5221 shape_changed: false,
5222 };
5223
5224 match parse_lighter_order_event(
5225 order,
5226 &instrument,
5227 &identity,
5228 cloid,
5229 emitter.account_id(),
5230 emitter.trader_id(),
5231 context,
5232 ts_init,
5233 ) {
5234 Ok(Some(event)) => {
5235 let Some(event) = defer_terminal(order, event, &identity, cloid, dispatch) else {
5236 return;
5237 };
5238
5239 dispatch_tracked_order_event(
5240 event,
5241 cloid,
5242 VenueOrderId::new(&order.order_id),
5243 &identity,
5244 emitter.account_id(),
5245 emitter.trader_id(),
5246 emitter,
5247 dispatch,
5248 ts_init,
5249 );
5250 }
5251 Ok(None) => {}
5252 Err(e) => log::error!("Failed to parse deferred Lighter order for {cloid}: {e}"),
5253 }
5254}
5255
5256async fn probe_acked_create(
5257 probe: &AckedOrderProbe,
5258 context: &AckedOrderProbeContext,
5259) -> anyhow::Result<()> {
5260 let AckedOrderProbe {
5261 order,
5262 client_order_index,
5263 connection_epoch,
5264 nonce,
5265 ..
5266 } = probe;
5267
5268 let client_order_id = order.client_order_id();
5269 let mut transaction_executed = false;
5270
5271 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
5272 if context.connection_epoch.load(Ordering::Acquire) != *connection_epoch {
5273 log::warn!(
5274 "Lighter acknowledged create outcome unresolved after reconnect for {client_order_id}; retaining identity for reconciliation",
5275 );
5276 return Ok(());
5277 }
5278
5279 if !context.dispatch.create_submission_is_pending(
5280 &client_order_id,
5281 *client_order_index,
5282 *nonce,
5283 ) {
5284 return Ok(());
5285 }
5286
5287 if probe_acked_create_report(probe, context, attempt).await {
5288 return Ok(());
5289 }
5290
5291 match probe_acked_create_tx(probe, context, attempt).await? {
5292 ControlFlow::Break(()) => return Ok(()),
5293 ControlFlow::Continue(executed) => transaction_executed |= executed,
5294 }
5295
5296 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
5297 tokio::select! {
5298 () = context.cancellation_token.cancelled() => return Ok(()),
5299 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
5300 }
5301 }
5302 }
5303
5304 if transaction_executed {
5305 let _ = context.dispatch.confirm_create_submission(
5306 &client_order_id,
5307 *client_order_index,
5308 *nonce,
5309 );
5310 log::warn!(
5311 "Lighter acknowledged create transaction executed without a queryable order for {client_order_id}; retaining identity for reconciliation",
5312 );
5313 } else {
5314 log::warn!(
5315 "Lighter acknowledged create outcome unresolved after {ACKED_ORDER_PROBE_ATTEMPTS} transaction lookups for {client_order_id}; retaining identity for reconciliation",
5316 );
5317 }
5318 Ok(())
5319}
5320
5321async fn probe_acked_create_report(
5322 probe: &AckedOrderProbe,
5323 context: &AckedOrderProbeContext,
5324 attempt: usize,
5325) -> bool {
5326 let AckedOrderProbe {
5327 order,
5328 client_order_index,
5329 nonce,
5330 ..
5331 } = probe;
5332
5333 let client_order_id = order.client_order_id();
5334
5335 let report = match lookup_create_order_status_report(
5336 &context.http_client,
5337 &context.registry,
5338 &context.credential,
5339 context.account_id,
5340 order.instrument_id(),
5341 client_order_id,
5342 *client_order_index,
5343 *nonce,
5344 &context.dispatch,
5345 context.clock,
5346 )
5347 .await
5348 {
5349 Ok(report) => report,
5350 Err(e) => {
5351 log::warn!(
5352 "Lighter acknowledged create order lookup failed for {client_order_id} on attempt {attempt}: {e:?}",
5353 );
5354 None
5355 }
5356 };
5357
5358 if let Some(report) = report {
5359 let _events = context.dispatch.order_events.lock();
5360 if context
5361 .dispatch
5362 .order_identity(&client_order_id)
5363 .is_some_and(|identity| {
5364 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
5365 })
5366 {
5367 return false;
5368 }
5369
5370 if context.dispatch.observe_create_submission(
5371 &client_order_id,
5372 *client_order_index,
5373 *nonce,
5374 report.venue_order_id,
5375 ) {
5376 context.dispatch.seed_accepted_from_report(&report);
5377 context.emitter.send_order_status_report(report);
5378 }
5379
5380 return true;
5381 }
5382
5383 false
5384}
5385
5386async fn probe_acked_create_tx(
5387 probe: &AckedOrderProbe,
5388 context: &AckedOrderProbeContext,
5389 attempt: usize,
5390) -> anyhow::Result<ControlFlow<(), bool>> {
5391 let AckedOrderProbe {
5392 order,
5393 client_order_index,
5394 connection_epoch,
5395 nonce,
5396 api_key_index,
5397 tx_hash,
5398 } = probe;
5399
5400 let client_order_id = order.client_order_id();
5401
5402 let tx = match context.http_client.get_tx(tx_hash.clone()).await {
5403 Ok(tx) => tx,
5404 Err(LighterHttpError::Venue { code: 21500, .. }) => {
5405 return Ok(ControlFlow::Continue(false));
5406 }
5407 Err(e) => {
5408 log::warn!(
5409 "Lighter acknowledged create transaction lookup failed for {client_order_id} on attempt {attempt}: {e}",
5410 );
5411 return Ok(ControlFlow::Continue(false));
5412 }
5413 };
5414
5415 validate_acked_create_tx(
5416 &tx,
5417 context.credential.account_index(),
5418 *api_key_index,
5419 *client_order_index,
5420 *nonce,
5421 tx_hash,
5422 )?;
5423
5424 let event = parse_acked_tx_event(&tx.event_info).unwrap_or_else(|e| {
5425 log::warn!(
5426 "Lighter create transaction carried invalid event_info for {client_order_id}: {e}",
5427 );
5428 AckedTxEvent::default()
5429 });
5430
5431 if tx.status == LighterTxStatus::Failed || !event.app_error.is_empty() {
5432 let detail = acked_tx_error_detail(&event.app_error);
5433 let reason = format!(
5434 "Lighter sequencer rejected acknowledged create transaction {tx_hash}: {detail}",
5435 );
5436 reject_create_order(
5437 &context.dispatch,
5438 &context.emitter,
5439 order,
5440 *client_order_index,
5441 *nonce,
5442 &reason,
5443 context.clock.get_time_ns(),
5444 lighter_reason_indicates_post_only_rejection(detail),
5445 Some((&context.connection_epoch, *connection_epoch)),
5446 );
5447 return Ok(ControlFlow::Break(()));
5448 }
5449
5450 Ok(ControlFlow::Continue(
5451 tx.status == LighterTxStatus::Executed,
5452 ))
5453}
5454
5455#[derive(Deserialize)]
5456struct AckedCreateTxInfo {
5457 #[serde(rename = "ClientOrderIndex")]
5458 client_order_index: i64,
5459}
5460
5461#[derive(Deserialize)]
5462struct AckedCreateGroupTxInfo {
5463 #[serde(rename = "Orders")]
5464 orders: Vec<AckedCreateTxInfo>,
5465}
5466
5467#[derive(Default, Deserialize)]
5468struct AckedTxEvent {
5469 #[serde(default, rename = "ae")]
5470 app_error: String,
5471}
5472
5473fn validate_acked_create_tx(
5474 tx: &crate::http::models::LighterTx,
5475 account_index: i64,
5476 api_key_index: u8,
5477 client_order_index: i64,
5478 nonce: i64,
5479 tx_hash: &str,
5480) -> anyhow::Result<()> {
5481 anyhow::ensure!(
5482 tx_hash_matches(&tx.hash, tx_hash)
5483 && matches!(tx.tx_type, value if value == LighterTxType::CreateOrder as u8 || value == LighterTxType::CreateGroupedOrders as u8)
5484 && tx.account_index == account_index
5485 && tx.api_key_index == api_key_index
5486 && tx.nonce == nonce,
5487 "Lighter transaction lookup did not match acknowledged create identity",
5488 );
5489
5490 if tx.tx_type == LighterTxType::CreateGroupedOrders as u8 {
5491 let info: AckedCreateGroupTxInfo = serde_json::from_str(&tx.info)
5492 .context("failed to parse Lighter grouped transaction info")?;
5493 anyhow::ensure!(
5494 info.orders
5495 .iter()
5496 .any(|order| order.client_order_index == client_order_index),
5497 "Lighter transaction lookup did not include acknowledged client_order_index {client_order_index}"
5498 );
5499 return Ok(());
5500 }
5501
5502 let info: AckedCreateTxInfo = serde_json::from_str(&tx.info)
5503 .context("failed to parse Lighter create transaction info")?;
5504 anyhow::ensure!(
5505 info.client_order_index == client_order_index,
5506 "Lighter transaction lookup returned client_order_index {} for acknowledged create {client_order_index}",
5507 info.client_order_index,
5508 );
5509 Ok(())
5510}
5511
5512fn parse_acked_tx_event(event_info: &str) -> anyhow::Result<AckedTxEvent> {
5513 if event_info.trim().is_empty() {
5514 return Ok(AckedTxEvent::default());
5515 }
5516
5517 serde_json::from_str(event_info).context("failed to parse Lighter transaction event")
5518}
5519
5520fn tx_application_error(
5521 status: LighterTxStatus,
5522 event_info: &str,
5523) -> anyhow::Result<Option<String>> {
5524 let event = match parse_acked_tx_event(event_info) {
5525 Ok(event) => event,
5526 Err(e) if status == LighterTxStatus::Failed => {
5527 log::warn!("Lighter failed transaction carried invalid event_info: {e}");
5528 AckedTxEvent::default()
5529 }
5530 Err(e) => return Err(e),
5531 };
5532
5533 Ok(
5534 (status == LighterTxStatus::Failed || !event.app_error.is_empty())
5535 .then(|| acked_tx_error_detail(&event.app_error).to_owned()),
5536 )
5537}
5538
5539fn acked_tx_error_detail(app_error: &str) -> &str {
5540 if app_error.is_empty() {
5541 "transaction failed without an application error"
5542 } else {
5543 app_error
5544 }
5545}
5546
5547#[derive(Debug, Clone)]
5548struct AckedOrderProbe {
5549 order: Box<OrderAny>,
5550 client_order_index: i64,
5551 connection_epoch: u64,
5552 nonce: i64,
5553 api_key_index: u8,
5554 tx_hash: String,
5555}
5556
5557impl AckedOrderProbe {
5558 fn from_pending(pending: &PendingSendTx) -> Option<Self> {
5559 let PendingSendTxKind::Create {
5560 order,
5561 client_order_index,
5562 } = &pending.kind
5563 else {
5564 return None;
5565 };
5566
5567 Some(Self {
5568 order: order.clone(),
5569 client_order_index: *client_order_index,
5570 connection_epoch: pending.connection_epoch,
5571 nonce: pending.nonce,
5572 api_key_index: pending.api_key_index,
5573 tx_hash: pending.tx_hash.clone(),
5574 })
5575 }
5576}
5577
5578#[expect(
5587 clippy::too_many_arguments,
5588 reason = "shared terminal create transition keeps cleanup and event attribution together"
5589)]
5590fn reject_create_order(
5591 dispatch: &WsDispatchState,
5592 emitter: &ExecutionEventEmitter,
5593 order: &OrderAny,
5594 client_order_index: i64,
5595 nonce: i64,
5596 reason: &str,
5597 now: UnixNanos,
5598 due_post_only: bool,
5599 connection_epoch: Option<(&AtomicU64, u64)>,
5600) -> bool {
5601 let cloid = order.client_order_id();
5602 if !dispatch.reject_create_submission(&cloid, client_order_index, nonce, connection_epoch) {
5603 log::warn!(
5604 "Ignored stale Lighter create rejection for cloid={cloid} client_order_index={client_order_index} nonce={nonce}",
5605 );
5606 return false;
5607 }
5608 emitter.emit_order_rejected(order, reason, now, due_post_only);
5609 true
5610}
5611
5612#[expect(
5613 clippy::too_many_arguments,
5614 reason = "consumer-loop sink that flattens one SendTxRejected message without a wrapper struct"
5615)]
5616fn handle_send_tx_rejection_for_connection(
5617 dispatch: &WsDispatchState,
5618 emitter: &ExecutionEventEmitter,
5619 account_index: Option<i64>,
5620 connection_epoch: u64,
5621 now: UnixNanos,
5622 source: SendTxRejectionSource,
5623 code: Option<i64>,
5624 message: &str,
5625 tx_hash: Option<&str>,
5626) -> bool {
5627 let needs_nonce_resync = code == Some(LIGHTER_ERROR_CODE_INVALID_NONCE);
5628 let failure = venue_rejection_failure(code, message);
5629
5630 let pending = match tx_hash {
5631 Some(hash) => dispatch.remove_pending_sendtx_by_hash(connection_epoch, hash),
5632 None => match source {
5633 SendTxRejectionSource::Ack => dispatch.pop_pending_sendtx_if_only(connection_epoch),
5634 SendTxRejectionSource::BareError => dispatch.pop_pending_sendtx_if_only_within(
5635 connection_epoch,
5636 now,
5637 SENDTX_BARE_ERROR_WINDOW_MS,
5638 ),
5639 },
5640 };
5641 let Some(pending) = pending else {
5642 log::warn!(
5643 "Lighter sendTx rejection unattributed (source={source:?} code={code:?}): {message:?}",
5644 );
5645 return needs_nonce_resync;
5646 };
5647
5648 reject_pending_sendtx(
5649 dispatch,
5650 emitter,
5651 account_index,
5652 &pending,
5653 &failure,
5654 now,
5655 lighter_reason_indicates_post_only_rejection(message),
5656 );
5657 needs_nonce_resync
5658}
5659
5660fn reject_pending_sendtx(
5661 dispatch: &WsDispatchState,
5662 emitter: &ExecutionEventEmitter,
5663 account_index: Option<i64>,
5664 pending: &PendingSendTx,
5665 failure: &CommandFailure,
5666 now: UnixNanos,
5667 due_post_only: bool,
5668) {
5669 let reason = command_failure_reason(failure);
5670 log::warn!(
5671 "{reason}, nonce={} api_key_index={}",
5672 pending.nonce,
5673 pending.api_key_index
5674 );
5675
5676 if let Some(account_index) = account_index {
5677 let _ = dispatch.nonce_manager.ack_failure_if_latest(
5678 account_index,
5679 pending.api_key_index,
5680 pending.nonce,
5681 );
5682 }
5683
5684 for (order, client_order_index) in pending.kind.creates() {
5685 reject_create_order(
5686 dispatch,
5687 emitter,
5688 order,
5689 client_order_index,
5690 pending.nonce,
5691 reason,
5692 now,
5693 due_post_only,
5694 None,
5695 );
5696 }
5697
5698 match &pending.kind {
5699 PendingSendTxKind::Create { .. } | PendingSendTxKind::CreateGroup { .. } => {}
5700 PendingSendTxKind::Cancel { .. } => {
5701 reject_cancel_order(dispatch, emitter, pending, failure, now);
5702 }
5703 PendingSendTxKind::Modify { .. } => {
5704 reject_modify_order(dispatch, emitter, pending, failure, now, false);
5705 }
5706 PendingSendTxKind::Other => {}
5707 }
5708}
5709
5710fn reject_cancel_order(
5711 dispatch: &WsDispatchState,
5712 emitter: &ExecutionEventEmitter,
5713 pending: &PendingSendTx,
5714 failure: &CommandFailure,
5715 now: UnixNanos,
5716) {
5717 let PendingSendTxKind::Cancel {
5718 command_id: Some(command_id),
5719 strategy_id,
5720 instrument_id,
5721 client_order_id,
5722 venue_order_id,
5723 ..
5724 } = pending.kind
5725 else {
5726 return;
5727 };
5728
5729 dispatch.clear_pending_cancel_if_command(&client_order_id, command_id);
5730
5731 match failure {
5732 CommandFailure::NotSent(reason) => emitter.emit_order_cancel_denied_event(
5733 strategy_id,
5734 instrument_id,
5735 client_order_id,
5736 venue_order_id,
5737 reason,
5738 now,
5739 Some(command_id),
5740 ),
5741 CommandFailure::VenueRejected(reason) => emitter.emit_order_cancel_rejected_event(
5742 strategy_id,
5743 instrument_id,
5744 client_order_id,
5745 venue_order_id,
5746 reason,
5747 now,
5748 Some(command_id),
5749 ),
5750 _ => {}
5751 }
5752}
5753
5754fn reject_modify_order(
5755 dispatch: &WsDispatchState,
5756 emitter: &ExecutionEventEmitter,
5757 pending: &PendingSendTx,
5758 failure: &CommandFailure,
5759 now: UnixNanos,
5760 require_pending: bool,
5761) {
5762 let PendingSendTxKind::Modify {
5763 command_id,
5764 strategy_id,
5765 instrument_id,
5766 client_order_id,
5767 venue_order_id,
5768 } = pending.kind
5769 else {
5770 return;
5771 };
5772
5773 let mut events = dispatch.order_events.lock();
5774 let current = dispatch.clear_pending_modify_if_nonce(
5775 &client_order_id,
5776 pending.connection_epoch,
5777 pending.nonce,
5778 );
5779
5780 if require_pending && !current {
5781 return;
5782 }
5783
5784 match failure {
5785 CommandFailure::NotSent(reason) => emitter.emit_order_modify_denied_event(
5786 strategy_id,
5787 instrument_id,
5788 client_order_id,
5789 venue_order_id,
5790 reason,
5791 now,
5792 Some(command_id),
5793 ),
5794 CommandFailure::VenueRejected(reason) => emitter.emit_order_modify_rejected_event(
5795 strategy_id,
5796 instrument_id,
5797 client_order_id,
5798 venue_order_id,
5799 reason,
5800 now,
5801 Some(command_id),
5802 ),
5803 _ => {}
5804 }
5805
5806 if current && let Some(deferred) = events.remove(&client_order_id) {
5807 flush_modify_events(client_order_id, deferred, dispatch, emitter, &mut events);
5808 }
5809}
5810
5811fn lighter_reason_indicates_post_only_rejection(reason: &str) -> bool {
5812 let normalized: String = reason
5813 .chars()
5814 .filter_map(|ch| {
5815 if ch == '-' || ch == '_' || ch.is_whitespace() {
5816 None
5817 } else {
5818 Some(ch.to_ascii_lowercase())
5819 }
5820 })
5821 .collect();
5822
5823 normalized.contains("postonly") || normalized.contains("postwouldexecute")
5824}
5825
5826fn lighter_http_error_is_definite_api_rejection(error: &LighterHttpError) -> bool {
5827 match error {
5828 LighterHttpError::RateLimit { .. } | LighterHttpError::Venue { .. } => true,
5829 LighterHttpError::Http { status, .. } => *status < 500,
5830 LighterHttpError::Network(_)
5831 | LighterHttpError::HistoryIncomplete { .. }
5832 | LighterHttpError::Parse(_) => false,
5833 }
5834}
5835
5836fn rollback_tx_dispatch(
5837 dispatch: &WsDispatchState,
5838 credential: &Credential,
5839 client_order_index: Option<i64>,
5840 nonce: i64,
5841) {
5842 rollback_tx_dispatch_indices(
5843 dispatch,
5844 credential.account_index(),
5845 credential.api_key_index(),
5846 client_order_index,
5847 None,
5848 nonce,
5849 );
5850}
5851
5852fn rollback_tx_dispatch_create(
5853 dispatch: &WsDispatchState,
5854 credential: &Credential,
5855 client_order_index: Option<i64>,
5856 client_order_id: &ClientOrderId,
5857 nonce: i64,
5858) {
5859 rollback_tx_dispatch_indices(
5860 dispatch,
5861 credential.account_index(),
5862 credential.api_key_index(),
5863 client_order_index,
5864 Some(client_order_id),
5865 nonce,
5866 );
5867}
5868
5869fn rollback_tx_dispatch_indices(
5873 dispatch: &WsDispatchState,
5874 account_index: i64,
5875 api_key_index: u8,
5876 client_order_index: Option<i64>,
5877 client_order_id: Option<&ClientOrderId>,
5878 nonce: i64,
5879) {
5880 let _ = dispatch
5881 .nonce_manager
5882 .ack_failure_if_latest(account_index, api_key_index, nonce);
5883
5884 if let Some(client_order_index) = client_order_index {
5885 dispatch.forget_cloid(client_order_index);
5886 }
5887
5888 if let Some(cloid) = client_order_id {
5889 dispatch.forget_order_identity(cloid);
5890 }
5891}
5892
5893fn integrator_attributes(integrator_account_index: Option<u64>) -> L2TxAttributes {
5894 integrator_account_index.map_or_else(L2TxAttributes::default, |integrator_account_index| {
5895 L2TxAttributes {
5896 integrator_account_index,
5897 ..Default::default()
5898 }
5899 })
5900}
5901
5902fn command_failure_reason(failure: &CommandFailure) -> &str {
5903 match failure {
5904 CommandFailure::NotSent(reason)
5905 | CommandFailure::Ambiguous(reason)
5906 | CommandFailure::VenueRejected(reason) => reason,
5907 }
5908}
5909
5910fn warn_pending_sendtx_unknown(pending_sendtx: &[PendingSendTx], context: &str) {
5911 if pending_sendtx.is_empty() {
5912 return;
5913 }
5914
5915 log::warn!(
5916 "Discarded {} pending sendTx entries during {context}; order state recovers via \
5917 reconciliation",
5918 pending_sendtx.len(),
5919 );
5920
5921 for pending in pending_sendtx {
5922 for (order, _) in pending.kind.creates() {
5923 log::warn!(
5924 "Lighter sendTx outcome unknown during {context} for {}",
5925 order.client_order_id(),
5926 );
5927 }
5928 }
5929}
5930
5931fn classify_lighter_ws_command_failure(action: &str, error: &LighterWsError) -> CommandFailure {
5932 let fallback = format!("Lighter {action} dispatch failed");
5933 let reason = sanitize_strategy_reason(&format!("{fallback}: {error}"))
5934 .unwrap_or_else(|| fallback.clone());
5935
5936 match error {
5937 LighterWsError::SendTxOutcomeUnknown(_)
5938 | LighterWsError::Network(_)
5939 | LighterWsError::Parse(_)
5940 | LighterWsError::Transport(SendError::WriteTimeout | SendError::BrokenPipe(_)) => {
5941 CommandFailure::ambiguous(reason)
5942 }
5943 LighterWsError::Transport(
5944 SendError::InvalidInput(_)
5945 | SendError::BufferFull
5946 | SendError::Closed
5947 | SendError::Timeout
5948 | SendError::ConnectionChanged,
5949 )
5950 | LighterWsError::Authentication(_)
5951 | LighterWsError::Client(_) => CommandFailure::not_sent(reason),
5952 }
5953}
5954
5955fn venue_rejection_failure(code: Option<i64>, message: &str) -> CommandFailure {
5956 let clean_message = sanitize_strategy_reason(message);
5957 let reason = match (code, clean_message) {
5958 (Some(code), Some(message)) => format!("LIGHTER_{code}: {message}"),
5959 (Some(code), None) => format!("LIGHTER_{code}"),
5960 (None, Some(message)) => message,
5961 (None, None) => "Lighter venue rejected sendTx".to_string(),
5962 };
5963 let reason = sanitize_strategy_reason(&reason)
5964 .unwrap_or_else(|| "Lighter venue rejected sendTx".to_string());
5965 CommandFailure::venue_rejected(reason)
5966}
5967
5968fn sanitize_strategy_reason(input: &str) -> Option<String> {
5969 let mut output = String::new();
5970 let mut inside_markup = false;
5971 let mut pending_space = false;
5972 let mut char_count = 0;
5973 let mut chars = input.chars().peekable();
5974
5975 while let Some(ch) = chars.next() {
5976 if inside_markup {
5977 if ch == '>' {
5978 inside_markup = false;
5979 }
5980 continue;
5981 }
5982
5983 match ch {
5984 '<' if chars.peek().is_some_and(|next| {
5985 next.is_ascii_alphabetic() || matches!(next, '/' | '!' | '?')
5986 }) =>
5987 {
5988 inside_markup = true;
5989 pending_space = !output.is_empty();
5990 }
5991 _ if ch.is_control() || is_unicode_format_control(ch) || ch.is_whitespace() => {
5992 pending_space = !output.is_empty();
5993 }
5994 _ => {
5995 if pending_space && char_count < STRATEGY_REASON_MAX_CHARS {
5996 output.push(' ');
5997 char_count += 1;
5998 }
5999 pending_space = false;
6000
6001 if char_count == STRATEGY_REASON_MAX_CHARS {
6002 break;
6003 }
6004 output.push(ch);
6005 char_count += 1;
6006 }
6007 }
6008 }
6009
6010 let output = output.trim().to_string();
6011 (!output.is_empty()).then_some(output)
6012}
6013
6014fn is_unicode_format_control(ch: char) -> bool {
6015 matches!(
6016 ch,
6017 '\u{00ad}'
6018 | '\u{061c}'
6019 | '\u{06dd}'
6020 | '\u{070f}'
6021 | '\u{08e2}'
6022 | '\u{180e}'
6023 | '\u{feff}'
6024 | '\u{fff9}'..='\u{fffb}'
6025 | '\u{0600}'..='\u{0605}'
6026 | '\u{0890}'..='\u{0891}'
6027 | '\u{200b}'..='\u{200f}'
6028 | '\u{202a}'..='\u{202e}'
6029 | '\u{2060}'..='\u{206f}'
6030 | '\u{110bd}'
6031 | '\u{110cd}'
6032 | '\u{13430}'..='\u{1343f}'
6033 | '\u{1bca0}'..='\u{1bca3}'
6034 | '\u{1d173}'..='\u{1d17a}'
6035 | '\u{e0001}'
6036 | '\u{e0020}'..='\u{e007f}'
6037 )
6038}
6039
6040fn format_between_timestamps(
6045 start: Option<UnixNanos>,
6046 end: Option<UnixNanos>,
6047 ts_now: UnixNanos,
6048) -> Option<String> {
6049 let (start, end) = match (start, end) {
6050 (None, None) => return None,
6051 (Some(s), Some(e)) => (s, e),
6052 (Some(s), None) => (s, ts_now),
6053 (None, Some(e)) => (UnixNanos::from(0), e),
6054 };
6055 let start_secs = start.as_u64() / 1_000_000_000;
6056 let end_secs = end.as_u64() / 1_000_000_000;
6057 Some(format!("{start_secs}-{end_secs}"))
6058}
6059
6060#[async_trait(?Send)]
6061impl ExecutionClient for LighterExecutionClient {
6062 fn is_connected(&self) -> bool {
6063 self.core.is_connected()
6064 }
6065
6066 fn client_id(&self) -> ClientId {
6067 self.core.client_id
6068 }
6069
6070 fn account_id(&self) -> AccountId {
6071 self.core.account_id
6072 }
6073
6074 fn venue(&self) -> Venue {
6075 self.core.venue
6076 }
6077
6078 fn oms_type(&self) -> OmsType {
6079 self.core.oms_type
6080 }
6081
6082 fn get_account(&self) -> Option<AccountAny> {
6083 self.core.cache().account_owned(&self.core.account_id)
6084 }
6085
6086 fn generate_account_state(
6087 &self,
6088 balances: Vec<AccountBalance>,
6089 margins: Vec<MarginBalance>,
6090 reported: bool,
6091 ts_event: UnixNanos,
6092 info: Option<Params>,
6093 ) -> anyhow::Result<()> {
6094 self.emitter
6095 .emit_account_state(balances, margins, reported, ts_event, info);
6096 Ok(())
6097 }
6098
6099 fn start(&mut self) -> anyhow::Result<()> {
6100 let sender = get_exec_event_sender();
6101 self.emitter.set_sender(sender);
6102 self.subscribe_applied_fills();
6103
6104 if self.core.is_started() {
6105 return Ok(());
6106 }
6107
6108 self.core.set_started();
6109
6110 log::info!(
6111 "Started Lighter execution client: client_id={}, account_id={}, environment={:?}, has_credentials={}",
6112 self.core.client_id,
6113 self.core.account_id,
6114 self.config.environment,
6115 self.has_credentials(),
6116 );
6117
6118 Ok(())
6119 }
6120
6121 fn stop(&mut self) -> anyhow::Result<()> {
6122 if self.core.is_stopped() && self.core.is_disconnected() && self.session_tasks_finished() {
6123 return Ok(());
6124 }
6125
6126 log::info!("Stopping Lighter execution client {}", self.core.client_id);
6127
6128 self.begin_session_shutdown();
6129
6130 self.core.set_stopped();
6131
6132 log::info!("Lighter execution client stopped");
6133 Ok(())
6134 }
6135
6136 fn reset(&mut self) -> anyhow::Result<()> {
6137 log::debug!("Resetting Lighter execution client {}", self.core.client_id);
6138 self.begin_session_shutdown();
6139 Ok(())
6140 }
6141
6142 fn dispose(&mut self) -> anyhow::Result<()> {
6143 log::debug!("Disposing Lighter execution client {}", self.core.client_id);
6144 self.stop()
6145 }
6146
6147 async fn connect(&mut self) -> anyhow::Result<()> {
6148 if self.core.is_connected() && self.pending_tasks.is_open() {
6149 return Ok(());
6150 }
6151
6152 if !self.has_credentials() {
6156 anyhow::bail!(
6157 "Lighter execution client requires credentials; \
6158 set private_key, account_index, and api_key_index in the config \
6159 or use the deployment-specific credential environment variables"
6160 );
6161 }
6162
6163 log::info!(
6164 "Connecting Lighter execution client {}",
6165 self.core.client_id
6166 );
6167
6168 if !self.session_tasks_finished() || !self.pending_tasks.is_open() {
6171 self.begin_session_shutdown();
6172 self.finish_session_shutdown().await?;
6173 }
6174
6175 if !self.pending_tasks.is_open() {
6176 self.pending_tasks
6177 .start_generation()
6178 .map_err(|e| anyhow::anyhow!("Failed to start Lighter task generation: {e}"))?;
6179 self.cancellation_token = self.pending_tasks.cancellation_token();
6180 }
6181
6182 let ws_client = self.ws_client.clone();
6183 let setup_guard = TaskGroupGuard::new(&[&self.pending_tasks], move || {
6184 ws_client.begin_shutdown();
6185 });
6186 self.auth_refresh_notify = Arc::new(tokio::sync::Notify::new());
6187 self.scheduled_cancel_notify = Arc::new(tokio::sync::Notify::new());
6188
6189 self.dispatch.account_streams_ready.reset();
6195 self.dispatch.clear_position_cache();
6196 self.dispatch.clear_account_state_cache();
6197
6198 self.ensure_instruments_initialized_async().await?;
6199 self.refresh_nonce().await?;
6200
6201 self.nonce_ready_connection_epoch
6203 .store(self.ws_client.connection_epoch(), Ordering::Release);
6204 let account_detail = self.fetch_account_detail().await;
6205
6206 if let Some(detail) = &account_detail {
6207 self.log_account_tier(detail);
6208
6209 let _ = tokio::time::timeout(
6210 REFERRAL_ATTRIBUTION_TIMEOUT,
6211 self.apply_referral_attribution(detail),
6212 )
6213 .await;
6214 }
6215
6216 if let Err(e) = self.submit_integrator_auto_approval().await {
6217 let is_unapproved = e.chain().any(|cause| {
6222 matches!(
6223 cause.downcast_ref::<LighterHttpError>(),
6224 Some(LighterHttpError::Venue { code: 21149, .. }),
6225 )
6226 });
6227
6228 if is_unapproved {
6229 self.nonce_ready_connection_epoch
6230 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
6231 return Err(e.context(
6232 "Lighter account is not integrator-approved (venue 21149); \
6233 orders cannot be placed",
6234 ));
6235 }
6236
6237 log::warn!("Lighter integrator approval failed; continuing startup: {e:?}");
6238 }
6239
6240 self.nonce_ready_connection_epoch
6241 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
6242
6243 if let Err(e) = self.sync_nonce_from_venue().await {
6244 log::debug!(
6245 "Failed to sync Lighter nonce after integrator approval; continuing startup: {e:?}"
6246 );
6247 }
6248
6249 self.subscribe_applied_fills();
6250
6251 if let Err(e) = self.spawn_ws_consumer().await {
6252 self.begin_session_shutdown();
6253 return match self.finish_session_shutdown().await {
6254 Ok(()) => Err(e),
6255 Err(shutdown_error) => Err(anyhow::anyhow!(
6256 "{e}; failed to roll back partial Lighter connection: {shutdown_error}"
6257 )),
6258 };
6259 }
6260 self.nonce_ready_connection_epoch
6261 .store(self.ws_client.connection_epoch(), Ordering::Release);
6262
6263 let ready = async {
6264 self.await_account_streams_ready(30.0).await?;
6265 self.start_scheduled_cancel().await
6266 }
6267 .await;
6268
6269 if let Err(e) = ready {
6270 log::warn!("Connect failed after WS started, tearing down: {e}");
6271 self.begin_session_shutdown();
6272
6273 if let Err(shutdown_error) = self.finish_session_shutdown().await {
6274 log::warn!("Failed to finish partial Lighter connection: {shutdown_error}");
6275 }
6276
6277 return Err(e);
6278 }
6279
6280 setup_guard.disarm();
6281 self.core.set_connected();
6282
6283 log::info!("Connected: client_id={}", self.core.client_id);
6284 Ok(())
6285 }
6286
6287 async fn disconnect(&mut self) -> anyhow::Result<()> {
6288 if self.core.is_disconnected() && self.session_tasks_finished() {
6289 return Ok(());
6290 }
6291
6292 log::info!(
6293 "Disconnecting Lighter execution client {}",
6294 self.core.client_id
6295 );
6296
6297 self.begin_session_shutdown();
6298 let tasks_result = self.finish_session_shutdown().await;
6299
6300 self.core.set_disconnected();
6301
6302 log::info!("Disconnected: client_id={}", self.core.client_id);
6303 tasks_result
6304 }
6305
6306 fn submit_order(&self, cmd: SubmitOrder) -> anyhow::Result<()> {
6307 let credential = self.credential.as_ref().ok_or_else(|| {
6308 anyhow::anyhow!("Lighter execution client cannot submit without credentials")
6309 })?;
6310
6311 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
6312
6313 if order.is_closed() {
6314 log::warn!("Cannot submit closed order {}", order.client_order_id());
6315 return Ok(());
6316 }
6317
6318 let cached_instrument = self
6319 .core
6320 .cache()
6321 .instrument(&order.instrument_id())
6322 .cloned();
6323
6324 if let Some(reason) = local_submit_denial_reason(&order, cached_instrument.as_ref()) {
6325 self.emitter.emit_order_denied(&order, &reason.to_string());
6326 return Ok(());
6327 }
6328
6329 let plan = match self.prepare_create_order_plan(&order, cmd.params.as_ref()) {
6330 Ok(plan) => plan,
6331 Err(e) => {
6332 let reason = OrderDeniedReason::ValidationFailed {
6333 detail: format!("Lighter submit_order failed: {e}"),
6334 };
6335 self.emitter.emit_order_denied(&order, &reason.to_string());
6336 return Ok(());
6337 }
6338 };
6339
6340 if let Err(e) = self.dispatch_create_order_plan(&plan, credential) {
6341 let reason = OrderDeniedReason::SubmitFailed {
6342 detail: format!("Lighter submit_order failed: {e:#}"),
6343 };
6344 self.emitter.emit_order_denied(&order, &reason.to_string());
6345 }
6346
6347 Ok(())
6348 }
6349
6350 fn submit_order_list(&self, cmd: SubmitOrderList) -> anyhow::Result<()> {
6351 let credential = self.credential.as_ref().ok_or_else(|| {
6352 anyhow::anyhow!("Lighter execution client cannot submit without credentials")
6353 })?;
6354
6355 if cmd.order_list.client_order_ids.is_empty() {
6356 log::debug!("submit_order_list called with empty order list");
6357 return Ok(());
6358 }
6359
6360 let orders = self.core.get_orders_for_list(&cmd.order_list)?;
6361
6362 if orders.iter().any(is_grouped_order) {
6363 let result = self
6364 .prepare_create_group_plan(&orders, cmd.params.as_ref())
6365 .and_then(|plan| {
6366 let context = self.fanout_dispatch_context(credential)?;
6367 self.spawn_task("submit_grouped_orders", async move {
6368 context
6369 .dispatch_batch_plans(vec![BatchOrderPlan::CreateGroup(plan)])
6370 .await;
6371 Ok(())
6372 });
6373
6374 Ok(())
6375 });
6376
6377 if let Err(e) = result {
6378 let reason = grouped_submit_denial_reason(&e);
6379 for order in &orders {
6380 self.emitter.emit_order_denied(order, &reason);
6381 }
6382 }
6383 return Ok(());
6384 }
6385
6386 let mut plans = Vec::with_capacity(orders.len());
6387
6388 for order in orders {
6389 if order.is_closed() {
6390 log::warn!("Cannot submit closed order {}", order.client_order_id());
6391 continue;
6392 }
6393
6394 let cached_instrument = self
6395 .core
6396 .cache()
6397 .instrument(&order.instrument_id())
6398 .cloned();
6399
6400 if let Some(reason) = local_submit_denial_reason(&order, cached_instrument.as_ref()) {
6401 self.emitter.emit_order_denied(&order, &reason.to_string());
6402 continue;
6403 }
6404
6405 match self.prepare_create_order_plan(&order, cmd.params.as_ref()) {
6406 Ok(plan) => plans.push(plan),
6407 Err(e) => {
6408 let reason = OrderDeniedReason::ValidationFailed {
6409 detail: format!("Lighter submit_order_list failed: {e}"),
6410 }
6411 .to_string();
6412
6413 self.emitter.emit_order_denied(&order, &reason);
6414 }
6415 }
6416 }
6417
6418 if plans.is_empty() {
6419 log::warn!(
6420 "Lighter submit_order_list: no supported orders to dispatch for {}",
6421 cmd.order_list.id,
6422 );
6423 return Ok(());
6424 }
6425
6426 let context = self.fanout_dispatch_context(credential)?;
6427 self.spawn_task("submit_order_list", async move {
6428 context
6429 .dispatch_batch_plans(
6430 plans
6431 .into_iter()
6432 .map(|plan| BatchOrderPlan::Create(Box::new(plan)))
6433 .collect(),
6434 )
6435 .await;
6436 Ok(())
6437 });
6438
6439 Ok(())
6440 }
6441
6442 fn modify_order(&self, cmd: ModifyOrder) -> anyhow::Result<()> {
6443 let credential = self
6444 .credential
6445 .as_ref()
6446 .ok_or_else(|| {
6447 anyhow::anyhow!("Lighter execution client cannot modify without credentials")
6448 })
6449 .inspect_err(|e| self.deny_modify_commands(std::slice::from_ref(&cmd), e))?;
6450
6451 self.dispatch_signed_modify_order(&cmd, credential);
6452 Ok(())
6453 }
6454
6455 fn batch_modify_orders(&self, cmd: BatchModifyOrders) -> anyhow::Result<()> {
6456 let credential = self
6457 .credential
6458 .as_ref()
6459 .ok_or_else(|| {
6460 anyhow::anyhow!("Lighter execution client cannot modify without credentials")
6461 })
6462 .inspect_err(|e| self.deny_modify_commands(&cmd.modifies, e))?;
6463 let context = self
6464 .fanout_dispatch_context(credential)
6465 .inspect_err(|e| self.deny_modify_commands(&cmd.modifies, e))?;
6466 let mut plans = Vec::with_capacity(cmd.modifies.len());
6467 for modify in &cmd.modifies {
6468 if !self.admit_modify_order(modify) {
6469 continue;
6470 }
6471
6472 match self.prepare_modify_order_plan(modify) {
6473 Ok(plan) => plans.push(BatchOrderPlan::Modify(plan)),
6474 Err(e) => {
6475 let mut events = self.dispatch.order_events.lock();
6476 self.dispatch.clear_pending_modify_if_command(
6477 &modify.client_order_id,
6478 modify.command_id,
6479 );
6480 self.emitter.emit_order_modify_denied_event(
6481 modify.strategy_id,
6482 modify.instrument_id,
6483 modify.client_order_id,
6484 modify.venue_order_id,
6485 &format!("Lighter modify_order failed: {e}"),
6486 self.clock.get_time_ns(),
6487 Some(modify.command_id),
6488 );
6489
6490 if let Some(deferred) = events.remove(&modify.client_order_id) {
6491 flush_modify_events(
6492 modify.client_order_id,
6493 deferred,
6494 &self.dispatch,
6495 &self.emitter,
6496 &mut events,
6497 );
6498 }
6499 }
6500 }
6501 }
6502
6503 if !plans.is_empty() {
6504 self.spawn_task("batch_modify_orders", async move {
6505 context.dispatch_batch_plans(plans).await;
6506 Ok(())
6507 });
6508 }
6509
6510 Ok(())
6511 }
6512
6513 fn cancel_order(&self, cmd: CancelOrder) -> anyhow::Result<()> {
6514 let credential = self
6515 .credential
6516 .as_ref()
6517 .ok_or_else(|| {
6518 anyhow::anyhow!("Lighter execution client cannot cancel without credentials")
6519 })
6520 .inspect_err(|e| self.deny_cancel_commands(std::slice::from_ref(&cmd), e))?;
6521
6522 if let Some(plans) = self.prepare_oco_cancel_plans(&cmd) {
6523 let context = self.fanout_dispatch_context(credential).inspect_err(|e| {
6524 self.deny_cancel_commands(std::slice::from_ref(&cmd), e);
6525 })?;
6526
6527 let plans = plans
6528 .into_iter()
6529 .map(|plan| {
6530 let emit = self.can_emit_order_cancel_rejected(&plan.client_order_id);
6531 if let Some(command_id) = plan.command_id {
6532 self.dispatch.register_cancel_command(
6533 plan.client_order_id,
6534 command_id,
6535 emit,
6536 );
6537 }
6538
6539 BatchOrderPlan::Cancel(plan, emit)
6540 })
6541 .collect();
6542
6543 self.spawn_task("cancel_oco_orders", async move {
6544 context.dispatch_batch_plans(plans).await;
6545 Ok(())
6546 });
6547
6548 return Ok(());
6549 }
6550
6551 self.dispatch_signed_cancel_order(&cmd, credential);
6552 Ok(())
6553 }
6554
6555 fn cancel_all_orders(&self, cmd: CancelAllOrders) -> anyhow::Result<()> {
6556 let cancels: Vec<_> = self
6559 .core
6560 .cache()
6561 .orders_open(
6562 None,
6563 Some(&cmd.instrument_id),
6564 None,
6565 Some(&self.core.account_id),
6566 cmd.order_side,
6567 )
6568 .into_iter()
6569 .map(|order| {
6570 cancel_order_from_cancel_all(&cmd, order.client_order_id(), order.strategy_id())
6571 })
6572 .collect();
6573
6574 if cancels.is_empty() {
6575 return Ok(());
6576 }
6577
6578 self.dispatch_cancel_orders(&cancels)
6579 }
6580
6581 fn batch_cancel_orders(&self, cmd: BatchCancelOrders) -> anyhow::Result<()> {
6582 self.dispatch_cancel_orders(&cmd.cancels)
6583 }
6584
6585 fn query_account(&self, _cmd: QueryAccount) -> anyhow::Result<()> {
6586 let cached = self.dispatch.snapshot_account_state();
6592 match cached {
6593 Some(state) => {
6594 log::debug!("Lighter query_account replaying cached AccountState");
6595 self.emitter.send_account_state(state);
6596 }
6597 None => {
6598 log::warn!(
6599 "Lighter query_account: no AccountState cached yet \
6600 (account_all_assets stream has not pushed since connect)",
6601 );
6602 }
6603 }
6604 Ok(())
6605 }
6606
6607 fn query_order(&self, cmd: QueryOrder) -> anyhow::Result<()> {
6608 let credential = self
6609 .credential
6610 .as_ref()
6611 .ok_or_else(|| anyhow::anyhow!("Lighter query_order requires credentials"))?
6612 .clone();
6613 let registry = Arc::clone(&self.registry);
6614 let http_client = self.http_client.clone();
6615 let emitter = self.emitter.clone();
6616 let core_account_id = self.core.account_id;
6617 let dispatch = self.dispatch.clone();
6618 let clock = self.clock;
6619
6620 if let Some(order) = self.core.cache().order_owned(&cmd.client_order_id) {
6621 self.dispatch.seed_order_restoration(order);
6622 }
6623
6624 let _ = order_shape_pending(&self.core, &self.dispatch, &cmd.client_order_id);
6625 let child_context = self.child_probe_context();
6626 self.spawn_task("query_order", async move {
6627 let lookup = OrderReportLookup::new(
6628 ®istry,
6629 &dispatch,
6630 Some(cmd.instrument_id),
6631 Some(&cmd.client_order_id),
6632 cmd.venue_order_id.as_ref(),
6633 )?;
6634
6635 let report = if let Some(order) =
6636 collect_order(&http_client, ®istry, &credential, &lookup).await?
6637 {
6638 if !dispatch.restore_cached_order(
6639 VenueOrderId::new(&order.order_id),
6640 order.client_order_index,
6641 cmd.instrument_id,
6642 ) {
6643 return Ok(());
6644 }
6645
6646 if let Some(context) = &child_context {
6647 recover_report_fills(&order, context).await?;
6648 }
6649
6650 let parsed = lookup.parse(
6651 &order,
6652 ®istry,
6653 core_account_id,
6654 clock.get_time_ns(),
6655 &dispatch,
6656 )?;
6657 let report = lookup.finish(&dispatch, parsed);
6658 if dispatch
6659 .order_identity(&cmd.client_order_id)
6660 .is_some_and(|identity| identity.child_fill_baseline().is_some())
6661 {
6662 if let Some(context) = &child_context {
6663 spawn_child_recovery(cmd.client_order_id, context);
6664 }
6665
6666 return Ok(());
6667 }
6668
6669 Some((order.client_order_index, report))
6670 } else {
6671 None
6672 };
6673
6674 dispatch_query_order_report(report, &cmd, &dispatch, &emitter);
6675 Ok(())
6676 });
6677 Ok(())
6678 }
6679
6680 fn generate_order_status_report_task(
6681 &self,
6682 cmd: &GenerateOrderStatusReport,
6683 ) -> Option<ExecutionReportTask<Option<OrderStatusReport>>> {
6684 let credential = self.credential.clone();
6685 let http = self.http_client.clone();
6686 let registry = Arc::clone(&self.registry);
6687 let account_id = self.core.account_id;
6688 let clock = self.clock;
6689 let core = self.core.clone();
6690 let dispatch = self.dispatch.clone();
6691 let cancellation_token = self.cancellation_token.clone();
6692 let command = cmd.clone();
6693 let child_context = self.child_probe_context();
6694 let collection_context = child_context.clone();
6695 let lookup_registry = Arc::clone(®istry);
6696
6697 let lookup = OrderReportLookup::new(
6698 ®istry,
6699 &dispatch,
6700 cmd.instrument_id,
6701 cmd.client_order_id.as_ref(),
6702 cmd.venue_order_id.as_ref(),
6703 );
6704 Some(ExecutionReportTask::new(
6705 async move {
6706 let Some(credential) = credential else {
6707 log::warn!("Lighter generate_order_status_report: no credentials");
6708 return Ok(None);
6709 };
6710
6711 if command.client_order_id.is_none() && command.venue_order_id.is_none() {
6712 log::warn!(
6713 "Lighter generate_order_status_report: must supply client_order_id or venue_order_id"
6714 );
6715 return Ok(None);
6716 }
6717
6718 let lookup = lookup?;
6719 let order = collect_order(&http, &lookup_registry, &credential, &lookup).await?;
6720 if let Some(order) = &order
6721 && order.status == LighterOrderStatus::Filled
6722 && let Some(context) = &collection_context
6723 {
6724 recover_report_fills(order, context).await?;
6725 }
6726
6727 Ok(order.map(|order| (lookup, order)))
6728 },
6729 move |collected| {
6730 anyhow::ensure!(
6731 !cancellation_token.is_cancelled(),
6732 "Lighter report session stopped"
6733 );
6734
6735 let report = collected
6736 .map(
6737 |(lookup, order)| -> anyhow::Result<Option<OrderStatusReport>> {
6738 let parsed = lookup.parse(
6739 &order,
6740 ®istry,
6741 account_id,
6742 clock.get_time_ns(),
6743 &dispatch,
6744 )?;
6745 let report = &parsed.0;
6746 let pending = core
6747 .cache()
6748 .client_order_id(&report.venue_order_id)
6749 .copied()
6750 .is_some_and(|cloid| dispatch.has_pending_modify(&cloid));
6751
6752 if !pending {
6753 restore_reconciled_order(
6754 &core,
6755 &dispatch,
6756 report.venue_order_id,
6757 order.client_order_index,
6758 report.order_status.is_closed(),
6759 );
6760 dispatch.record_order_reduction(
6761 order.client_order_index,
6762 report.venue_order_id,
6763 order.initial_base_amount - report.quantity.as_decimal(),
6764 );
6765 }
6766
6767 let report = lookup.finish(&dispatch, parsed);
6768 if is_unowned_child_cancel(&dispatch, order.client_order_index, &report)
6769 {
6770 return Ok(None);
6771 }
6772
6773 if let Some(context) = &child_context {
6774 request_child_recovery(&core, context, &report);
6775 }
6776
6777 ensure_order_report_ready(&core, &dispatch, &report)?;
6778 Ok(Some(report))
6779 },
6780 )
6781 .transpose()?
6782 .flatten();
6783
6784 if let Some(report) = &report {
6785 dispatch.seed_accepted_from_report(report);
6786 }
6787
6788 Ok(report)
6789 },
6790 ))
6791 }
6792
6793 fn generate_order_status_reports_task(
6794 &self,
6795 cmd: &GenerateOrderStatusReports,
6796 ) -> Option<ExecutionReportTask<Vec<OrderStatusReport>>> {
6797 let client = self.report_client();
6798 let core = self.core.clone();
6799 let dispatch = self.dispatch.clone();
6800 let cancellation_token = self.cancellation_token.clone();
6801 let child_context = self.child_probe_context();
6802 let command = cmd.clone();
6803 let collection_command = command.clone();
6804 let collection_context = child_context.clone();
6805 Some(ExecutionReportTask::new(
6806 async move {
6807 let sweep = client
6808 .collect_order_status_reports(&collection_command)
6809 .await;
6810
6811 if let Some(context) = &collection_context {
6812 recover_filled_reports(&sweep, context).await;
6813 }
6814
6815 Ok(sweep)
6816 },
6817 move |sweep| {
6818 anyhow::ensure!(
6819 !cancellation_token.is_cancelled(),
6820 "Lighter report session stopped"
6821 );
6822
6823 if let Some(context) = &child_context {
6824 request_child_report_recoveries(&core, context, &sweep);
6825 }
6826
6827 finish_order_reports(&core, &dispatch, sweep, &command, None)
6828 },
6829 ))
6830 }
6831
6832 fn generate_fill_reports_task(
6833 &self,
6834 cmd: &GenerateFillReports,
6835 ) -> Option<ExecutionReportTask<Vec<FillReport>>> {
6836 let client = self.report_client();
6837 let instruments = self.report_instruments(cmd.instrument_id);
6838 let core = self.core.clone();
6839 let dispatch = self.dispatch.clone();
6840 let cancellation_token = self.cancellation_token.clone();
6841 let child_context = self.child_probe_context();
6842 let command = cmd.clone();
6843 let cmd_log_level = cmd.log_receipt_level;
6844 Some(ExecutionReportTask::new(
6845 async move {
6846 let mut markets = BTreeSet::new();
6847 let result = client
6848 .collect_fill_reports(&command, &instruments, &mut markets)
6849 .await;
6850 Ok((markets, result))
6851 },
6852 move |(markets, result)| {
6853 anyhow::ensure!(
6854 !cancellation_token.is_cancelled(),
6855 "Lighter report session stopped"
6856 );
6857 let reports = finish_fill_reports(&core, &dispatch, markets, result)?.reports;
6858
6859 if let Some(context) = &child_context {
6860 spawn_child_recoveries(context);
6861 }
6862
6863 Self::log_report_receipt(reports.len(), "FillReport", cmd_log_level);
6864 Ok(reports)
6865 },
6866 ))
6867 }
6868
6869 fn generate_position_status_reports_task(
6870 &self,
6871 cmd: &GeneratePositionStatusReports,
6872 ) -> Option<ExecutionReportTask<Vec<PositionStatusReport>>> {
6873 let core = self.core.clone();
6874 let dispatch = self.dispatch.clone();
6875 let cancellation_token = self.cancellation_token.clone();
6876 let registry = Arc::clone(&self.registry);
6877 let command = cmd.clone();
6878 Some(ExecutionReportTask::new(async { Ok(()) }, move |()| {
6879 anyhow::ensure!(
6880 !cancellation_token.is_cancelled(),
6881 "Lighter report session stopped"
6882 );
6883 let (reports, complete, _) =
6884 cached_position_reports(&core, &dispatch, ®istry, &command)?;
6885 anyhow::ensure!(
6886 complete,
6887 "Lighter position snapshot does not cover the requested instrument scope"
6888 );
6889 Self::log_report_receipt(
6890 reports.len(),
6891 "PositionStatusReport",
6892 command.log_receipt_level,
6893 );
6894 Ok(reports)
6895 }))
6896 }
6897
6898 async fn generate_order_status_report(
6899 &self,
6900 cmd: &GenerateOrderStatusReport,
6901 ) -> anyhow::Result<Option<OrderStatusReport>> {
6902 let Some(credential) = &self.credential else {
6903 log::warn!("Lighter generate_order_status_report: no credentials");
6904 return Ok(None);
6905 };
6906
6907 if cmd.client_order_id.is_none() && cmd.venue_order_id.is_none() {
6908 log::warn!(
6909 "Lighter generate_order_status_report: must supply client_order_id or venue_order_id",
6910 );
6911 return Ok(None);
6912 }
6913
6914 let lookup = OrderReportLookup::new(
6915 &self.registry,
6916 &self.dispatch,
6917 cmd.instrument_id,
6918 cmd.client_order_id.as_ref(),
6919 cmd.venue_order_id.as_ref(),
6920 )?;
6921
6922 let report = if let Some(order) =
6923 collect_order(&self.http_client, &self.registry, credential, &lookup).await?
6924 {
6925 if order.status == LighterOrderStatus::Filled
6926 && let Some(context) = self.child_probe_context()
6927 {
6928 recover_report_fills(&order, &context).await?;
6929 }
6930
6931 let parsed = lookup.parse(
6932 &order,
6933 &self.registry,
6934 self.core.account_id,
6935 self.clock.get_time_ns(),
6936 &self.dispatch,
6937 )?;
6938
6939 if !self
6940 .core
6941 .cache()
6942 .client_order_id(&parsed.0.venue_order_id)
6943 .is_some_and(|cloid| self.dispatch.has_pending_modify(cloid))
6944 {
6945 restore_reconciled_order(
6946 &self.core,
6947 &self.dispatch,
6948 parsed.0.venue_order_id,
6949 order.client_order_index,
6950 parsed.0.order_status.is_closed(),
6951 );
6952 self.dispatch.record_order_reduction(
6953 order.client_order_index,
6954 parsed.0.venue_order_id,
6955 order.initial_base_amount - parsed.0.quantity.as_decimal(),
6956 );
6957 }
6958
6959 let report = lookup.finish(&self.dispatch, parsed);
6960 if is_unowned_child_cancel(&self.dispatch, order.client_order_index, &report) {
6961 return Ok(None);
6962 }
6963
6964 if let Some(context) = self.child_probe_context() {
6965 request_child_recovery(&self.core, &context, &report);
6966 }
6967
6968 ensure_order_report_ready(&self.core, &self.dispatch, &report)?;
6969 Some(report)
6970 } else {
6971 None
6972 };
6973
6974 if let Some(report) = &report {
6975 self.dispatch.seed_accepted_from_report(report);
6976 }
6977
6978 Ok(report)
6979 }
6980
6981 async fn generate_order_status_reports(
6982 &self,
6983 cmd: &GenerateOrderStatusReports,
6984 ) -> anyhow::Result<Vec<OrderStatusReport>> {
6985 self.collect_order_reports(cmd, None).await
6986 }
6987
6988 async fn generate_fill_reports(
6989 &self,
6990 cmd: GenerateFillReports,
6991 ) -> anyhow::Result<Vec<FillReport>> {
6992 let reports = self.paginate_fill_reports(&cmd).await?.reports;
6993 Self::log_report_receipt(reports.len(), "FillReport", cmd.log_receipt_level);
6994 Ok(reports)
6995 }
6996
6997 async fn generate_position_status_reports(
6998 &self,
6999 cmd: &GeneratePositionStatusReports,
7000 ) -> anyhow::Result<Vec<PositionStatusReport>> {
7001 let (reports, complete, _) = self.cached_position_reports(cmd)?;
7002 anyhow::ensure!(
7003 complete,
7004 "Lighter position snapshot does not cover the requested instrument scope",
7005 );
7006 Self::log_report_receipt(reports.len(), "PositionStatusReport", cmd.log_receipt_level);
7007 Ok(reports)
7008 }
7009
7010 async fn generate_mass_status(
7011 &self,
7012 lookback_mins: Option<u64>,
7013 ) -> anyhow::Result<Option<ExecutionMassStatus>> {
7014 let ts_init = self.clock.get_time_ns();
7015
7016 let lookback_start = lookback_mins
7019 .map(DurationNanos::try_from_mins)
7020 .transpose()?
7021 .map(|lookback| ts_init.saturating_sub(lookback));
7022
7023 let mut order_cmd = GenerateOrderStatusReports::new(
7028 UUID4::new(),
7029 ts_init,
7030 false,
7031 None,
7032 lookback_start,
7033 None,
7034 None,
7035 None,
7036 );
7037 order_cmd.log_receipt_level = LogLevel::Debug;
7038 let fill_cmd = GenerateFillReports::new(
7039 UUID4::new(),
7040 ts_init,
7041 None,
7042 None,
7043 lookback_start,
7044 None,
7045 None,
7046 None,
7047 );
7048 let position_cmd =
7049 GeneratePositionStatusReports::new(UUID4::new(), ts_init, None, None, None, None, None);
7050
7051 let (mut fill_reports, fill_reports_complete, withheld_fills) =
7052 fill_reports_for_mass_status(self.paginate_fill_reports(&fill_cmd).await)
7053 .context("Lighter fill reconciliation failed")?;
7054 Self::log_report_receipt(fill_reports.len(), "FillReport", fill_cmd.log_receipt_level);
7055
7056 let mut withheld_instruments = withheld_fills;
7059 let order_result = self
7060 .collect_order_reports(&order_cmd, Some(&fill_reports))
7061 .await;
7062
7063 let (mut order_reports, mut reports_complete) = match order_result {
7064 Ok(reports) => (reports, true),
7065 Err(e) => {
7066 withheld_instruments.extend(withheld_order_instruments(&e));
7067 log::warn!(
7068 "Lighter order report generation failed: {}",
7069 scrub_auth(&format!("{e:#}")),
7070 );
7071 let partial = partial_order_reports(&e);
7072 let reports = if partial.is_empty() {
7073 let mut active_cmd = GenerateOrderStatusReports::new(
7074 UUID4::new(),
7075 ts_init,
7076 true,
7077 None,
7078 None,
7079 None,
7080 None,
7081 None,
7082 );
7083 active_cmd.log_receipt_level = LogLevel::Debug;
7084 let active_result = self
7085 .collect_order_reports(&active_cmd, Some(&fill_reports))
7086 .await;
7087
7088 match active_result {
7089 Ok(reports) => reports,
7090 Err(e) => {
7091 withheld_instruments.extend(withheld_order_instruments(&e));
7092 log::warn!(
7093 "Lighter active order report generation incomplete: {}",
7094 scrub_auth(&format!("{e:#}")),
7095 );
7096 partial_order_reports(&e)
7097 }
7098 }
7099 } else {
7100 partial
7101 };
7102 (reports, false)
7103 }
7104 };
7105
7106 reports_complete &= fill_reports_complete;
7107 let mut reported_orders: AHashSet<VenueOrderId> = order_reports
7108 .iter()
7109 .map(|report| report.venue_order_id)
7110 .collect();
7111 let mut fill_markets: Vec<i64> = fill_reports
7112 .iter()
7113 .filter(|report| !reported_orders.contains(&report.venue_order_id))
7114 .filter_map(|report| self.registry.market_index(&report.instrument_id))
7115 .collect::<AHashSet<_>>()
7116 .into_iter()
7117 .collect();
7118 fill_markets.sort_unstable();
7119
7120 for market_index in fill_markets {
7121 let Some(instrument_id) = self.registry.instrument_id(market_index) else {
7122 reports_complete = false;
7123 continue;
7124 };
7125 let mut order_cmd = GenerateOrderStatusReports::new(
7126 UUID4::new(),
7127 ts_init,
7128 false,
7129 Some(instrument_id),
7130 lookback_start,
7131 None,
7132 None,
7133 None,
7134 );
7135 order_cmd.log_receipt_level = LogLevel::Debug;
7136 let order_result = self
7137 .collect_order_reports(&order_cmd, Some(&fill_reports))
7138 .await;
7139 reports_complete &= order_result.is_ok();
7140 let reports = match order_result {
7141 Ok(reports) => reports,
7142 Err(e) => {
7143 withheld_instruments.extend(withheld_order_instruments(&e));
7144 log::warn!(
7145 "Lighter order report generation failed: {}",
7146 scrub_auth(&format!("{e:#}")),
7147 );
7148 partial_order_reports(&e)
7149 }
7150 };
7151 reported_orders.extend(reports.iter().map(|report| report.venue_order_id));
7152 order_reports.extend(reports);
7153 }
7154
7155 if fill_reports
7156 .iter()
7157 .any(|report| !reported_orders.contains(&report.venue_order_id))
7158 {
7159 reports_complete = false;
7160 }
7161
7162 for fill_report in &mut fill_reports {
7163 if let Some(client_order_id) = order_reports
7164 .iter()
7165 .find(|order_report| order_report.venue_order_id == fill_report.venue_order_id)
7166 .and_then(|order_report| order_report.client_order_id)
7167 {
7168 fill_report.client_order_id = Some(client_order_id);
7169 }
7170 }
7171
7172 self.publish_terminal_fill_bundles(&mut order_reports, &fill_reports)?;
7173
7174 let position_result = self.cached_position_reports(&position_cmd);
7175
7176 let (mut position_reports, mut position_coverage) = match position_result {
7177 Ok((reports, complete, coverage)) => {
7178 reports_complete &= complete;
7179 (reports, coverage)
7180 }
7181 Err(e) => {
7182 reports_complete = false;
7183 log::warn!("Lighter position report generation failed: {e:#}");
7184 (Vec::new(), None)
7185 }
7186 };
7187
7188 position_reports.retain(|report| !withheld_instruments.contains(&report.instrument_id));
7189
7190 if let Some(skipped) = &mut position_coverage {
7191 skipped.extend(
7192 withheld_instruments
7193 .iter()
7194 .filter_map(|id| self.registry.market_index(id)),
7195 );
7196 }
7197
7198 Self::log_report_receipt(
7199 position_reports.len(),
7200 "PositionStatusReport",
7201 position_cmd.log_receipt_level,
7202 );
7203
7204 if lookback_start.is_some() {
7205 let touched_instruments: AHashSet<InstrumentId> = order_reports
7206 .iter()
7207 .map(|report| report.instrument_id)
7208 .chain(fill_reports.iter().map(|report| report.instrument_id))
7209 .collect();
7210 let position_instruments: AHashSet<InstrumentId> = position_reports
7211 .iter()
7212 .map(|report| report.instrument_id)
7213 .collect();
7214 let cache = self.core.cache();
7215 let mut touched_instruments: Vec<InstrumentId> =
7216 touched_instruments.into_iter().collect();
7217 touched_instruments.sort_unstable();
7218
7219 for instrument_id in touched_instruments {
7220 let Some(instrument) = cache.instrument(&instrument_id) else {
7221 reports_complete = false;
7222 continue;
7223 };
7224
7225 if !matches!(instrument, InstrumentAny::CryptoPerpetual(_)) {
7226 continue;
7227 }
7228 let Some(market_id) = self.registry.market_index(&instrument_id) else {
7229 reports_complete = false;
7230 continue;
7231 };
7232
7233 if !position_coverage
7234 .as_ref()
7235 .is_some_and(|skipped| !skipped.contains(&market_id))
7236 {
7237 reports_complete = false;
7238 continue;
7239 }
7240
7241 if position_instruments.contains(&instrument_id) {
7242 continue;
7243 }
7244
7245 position_reports.push(PositionStatusReport::new(
7246 self.core.account_id,
7247 instrument_id,
7248 PositionSide::Flat,
7249 Quantity::zero(instrument.size_precision()),
7250 ts_init,
7251 ts_init,
7252 Some(UUID4::new()),
7253 None,
7254 None,
7255 ));
7256 }
7257 }
7258
7259 let mut mass_status = ExecutionMassStatus::new(
7260 self.core.client_id,
7261 self.core.account_id,
7262 self.core.venue,
7263 ts_init,
7264 None,
7265 );
7266 mass_status.set_report_window(lookback_start, reports_complete);
7267 mass_status.add_order_reports(order_reports);
7268 mass_status.add_fill_reports(fill_reports);
7269 mass_status.add_position_reports(position_reports);
7270
7271 log::debug!(
7272 "Generated Lighter mass status: {} orders, {} fills, {} positions",
7273 mass_status.order_reports().len(),
7274 mass_status.fill_reports().len(),
7275 mass_status.position_reports().len(),
7276 );
7277
7278 Ok(Some(mass_status))
7279 }
7280}
7281
7282#[derive(Debug)]
7283struct AppliedFillSubscription(TypedHandler<OrderEventAny>);
7284
7285impl Drop for AppliedFillSubscription {
7286 fn drop(&mut self) {
7287 msgbus::unsubscribe_order_events(ORDER_EVENTS_PATTERN.into(), &self.0);
7288 }
7289}
7290
7291struct FillSweep {
7295 reports: Vec<FillReport>,
7296 covers_window: bool,
7297 withheld_instruments: AHashSet<InstrumentId>,
7298}
7299
7300#[derive(Debug, thiserror::Error)]
7301#[error("incomplete Lighter order reports: {detail}")]
7302struct IncompleteOrderReports {
7303 reports: Vec<OrderStatusReport>,
7304 withheld_instruments: AHashSet<InstrumentId>,
7305 detail: String,
7306}
7307
7308fn incomplete_order_reports(
7309 reports: Vec<OrderStatusReport>,
7310 detail: impl Into<String>,
7311 withheld_instruments: AHashSet<InstrumentId>,
7312) -> anyhow::Error {
7313 anyhow::Error::new(IncompleteOrderReports {
7314 reports,
7315 withheld_instruments,
7316 detail: detail.into(),
7317 })
7318}
7319
7320fn partial_order_reports(error: &anyhow::Error) -> Vec<OrderStatusReport> {
7321 error
7322 .downcast_ref::<IncompleteOrderReports>()
7323 .map_or_default(|incomplete| incomplete.reports.clone())
7324}
7325
7326fn withheld_order_instruments(error: &anyhow::Error) -> AHashSet<InstrumentId> {
7327 error
7328 .downcast_ref::<IncompleteOrderReports>()
7329 .map_or_default(|incomplete| incomplete.withheld_instruments.clone())
7330}
7331
7332fn is_commission_error(error: &anyhow::Error) -> bool {
7333 error
7334 .chain()
7335 .any(|cause| cause.downcast_ref::<LighterCommissionError>().is_some())
7336}
7337
7338fn fill_reports_for_mass_status(
7339 result: anyhow::Result<FillSweep>,
7340) -> anyhow::Result<(Vec<FillReport>, bool, AHashSet<InstrumentId>)> {
7341 match result {
7342 Ok(sweep) => Ok((
7343 sweep.reports,
7344 sweep.covers_window,
7345 sweep.withheld_instruments,
7346 )),
7347 Err(e) if is_commission_error(&e) => Err(e),
7348 Err(e) => {
7349 log::warn!(
7350 "Lighter fill report generation failed: {}",
7351 scrub_auth(&format!("{e:#}")),
7352 );
7353 Ok((Vec::new(), false, AHashSet::new()))
7354 }
7355 }
7356}
7357
7358impl LighterExecutionClient {
7359 async fn collect_order_reports(
7360 &self,
7361 cmd: &GenerateOrderStatusReports,
7362 fills: Option<&[FillReport]>,
7363 ) -> anyhow::Result<Vec<OrderStatusReport>> {
7364 let sweep = self.report_client().collect_order_status_reports(cmd).await;
7365 if let Some(context) = self.child_probe_context() {
7366 request_child_report_recoveries(&self.core, &context, &sweep);
7367 if fills.is_none() {
7368 recover_filled_reports(&sweep, &context).await;
7369 }
7370 }
7371
7372 finish_order_reports(&self.core, &self.dispatch, sweep, cmd, fills)
7373 }
7374
7375 fn publish_terminal_fill_bundles(
7376 &self,
7377 orders: &mut Vec<OrderStatusReport>,
7378 fills: &[FillReport],
7379 ) -> anyhow::Result<()> {
7380 let mut bundled = AHashSet::new();
7381
7382 for report in orders.iter() {
7383 if !matches!(
7384 report.order_status,
7385 OrderStatus::Canceled | OrderStatus::Expired
7386 ) || report
7387 .client_order_id
7388 .and_then(|id| self.dispatch.order_identity(&id))
7389 .is_none()
7390 || !order_report_ready(&self.core, &self.dispatch, report, fills)
7391 || !fills_for_order(report, fills).any(|fill| fill.ts_event > report.ts_last)
7392 {
7393 continue;
7394 }
7395
7396 let paired = fills_for_order(report, fills).cloned().collect();
7398 let bundle = ExecutionReport::OrderWithFills(Box::new(report.clone()), paired);
7399 self.emitter.try_send_execution_report(bundle)?;
7400 bundled.insert(report.venue_order_id);
7401 }
7402
7403 orders.retain(|report| !bundled.contains(&report.venue_order_id));
7404 Ok(())
7405 }
7406
7407 fn prepare_oco_cancel_plans(&self, cmd: &CancelOrder) -> Option<Vec<CancelOrderPlan>> {
7408 let cache = self.core.cache();
7409 let order = cache.order(&cmd.client_order_id)?;
7410 if order.contingency_type() != Some(ContingencyType::Oco) || order.is_closed() {
7411 return None;
7412 }
7413
7414 let [peer_id] = order.linked_order_ids()? else {
7415 return None;
7416 };
7417
7418 if *peer_id == order.client_order_id() {
7419 return None;
7420 }
7421
7422 let peer = cache.order(peer_id)?;
7423 if peer.is_closed()
7424 || peer.instrument_id() != order.instrument_id()
7425 || peer.strategy_id() != order.strategy_id()
7426 || peer.account_id() != order.account_id()
7427 || validate_oco_links(&order, &peer, order.parent_order_id()).is_err()
7428 {
7429 return None;
7430 }
7431
7432 let original = self.prepare_cancel_order_plan(cmd).ok()?;
7433
7434 let expected_venue = order.venue_order_id().or_else(|| {
7435 self.dispatch
7436 .lookup_venue_order_id(&order.client_order_id())
7437 });
7438
7439 if original.venue_order_id != expected_venue {
7440 return None;
7441 }
7442
7443 let venue_order_id = peer.venue_order_id().or_else(|| {
7444 self.dispatch
7445 .order_identity(peer_id)
7446 .and_then(|identity| identity.venue_order_id())
7447 });
7448
7449 let venue_index = match venue_order_id {
7451 Some(venue) => venue.as_str().parse().ok()?,
7452 None => {
7453 let identity = self.dispatch.order_identity(peer_id)?;
7454 let original = self.dispatch.order_identity(&order.client_order_id())?;
7455 if !identity.shares_submission(&original) {
7456 return None;
7457 }
7458
7459 identity.client_order_index
7460 }
7461 };
7462
7463 Some(vec![
7464 original,
7465 CancelOrderPlan {
7466 command_id: None,
7467 client_order_id: *peer_id,
7468 strategy_id: peer.strategy_id(),
7469 instrument_id: peer.instrument_id(),
7470 venue_order_id,
7471 market_index: self.registry.market_index(&peer.instrument_id())?,
7472 venue_index,
7473 },
7474 ])
7475 }
7476
7477 fn child_probe_context(&self) -> Option<AckedOrderProbeContext> {
7478 let credential = self.credential.as_ref()?;
7479 self.fanout_dispatch_context(credential)
7480 .ok()
7481 .map(|context| context.probe_context())
7482 }
7483
7484 fn report_client(&self) -> LighterReportClient {
7485 LighterReportClient {
7486 http_client: self.http_client.clone(),
7487 credential: self.credential.clone(),
7488 account_id: self.core.account_id,
7489 registry: Arc::clone(&self.registry),
7490 clock: self.clock,
7491 markets: self.dispatch.active_markets.clone(),
7492 dispatch: self.dispatch.clone(),
7493 }
7494 }
7495
7496 fn report_instruments(
7497 &self,
7498 instrument_id: Option<InstrumentId>,
7499 ) -> AHashMap<InstrumentId, InstrumentAny> {
7500 let cache = self.core.cache();
7501 match instrument_id {
7502 Some(instrument_id) => cache
7503 .instrument(&instrument_id)
7504 .map(|instrument| (instrument_id, instrument.clone()))
7505 .into_iter()
7506 .collect(),
7507 None => cache
7508 .instruments(&self.core.venue, None)
7509 .into_iter()
7510 .map(|instrument| (instrument.id(), instrument.clone()))
7511 .collect(),
7512 }
7513 }
7514
7515 fn cached_position_reports(
7516 &self,
7517 cmd: &GeneratePositionStatusReports,
7518 ) -> anyhow::Result<(Vec<PositionStatusReport>, bool, Option<AHashSet<i64>>)> {
7519 cached_position_reports(&self.core, &self.dispatch, &self.registry, cmd)
7520 }
7521
7522 async fn paginate_fill_reports(&self, cmd: &GenerateFillReports) -> anyhow::Result<FillSweep> {
7523 let client = self.report_client();
7524 let instruments = self.report_instruments(cmd.instrument_id);
7525 let mut markets = BTreeSet::new();
7526 let result = client
7527 .collect_fill_reports(cmd, &instruments, &mut markets)
7528 .await;
7529 let result = finish_fill_reports(&self.core, &self.dispatch, markets, result);
7530
7531 if let Some(context) = self.child_probe_context() {
7532 spawn_child_recoveries(&context);
7533 }
7534
7535 result
7536 }
7537}
7538
7539struct LighterReportClient {
7540 http_client: LighterHttpClient,
7541 credential: Option<Credential>,
7542 account_id: AccountId,
7543 registry: Arc<MarketRegistry>,
7544 clock: &'static AtomicTime,
7545 markets: Arc<dashmap::DashSet<i64>>,
7546 dispatch: WsDispatchState,
7547}
7548
7549#[derive(Default)]
7550struct OrderReportSweep {
7551 orders: Vec<(i64, OrderStatusReport)>,
7552 reductions: AHashMap<VenueOrderId, Decimal>,
7553 markets: BTreeSet<i64>,
7554 error: Option<anyhow::Error>,
7555 partial: bool,
7556}
7557
7558impl LighterReportClient {
7559 async fn collect_order_status_reports(
7560 &self,
7561 cmd: &GenerateOrderStatusReports,
7562 ) -> OrderReportSweep {
7563 let mut sweep = OrderReportSweep {
7564 markets: self.markets.iter().map(|market| *market).collect(),
7565 ..Default::default()
7566 };
7567
7568 sweep.error = self.collect_orders(cmd, &mut sweep).await.err();
7569 sweep
7570 }
7571
7572 async fn collect_orders(
7573 &self,
7574 cmd: &GenerateOrderStatusReports,
7575 sweep: &mut OrderReportSweep,
7576 ) -> anyhow::Result<()> {
7577 let Some(credential) = &self.credential else {
7578 log::warn!("Lighter generate_order_status_reports: no credentials");
7579
7580 return Ok(());
7581 };
7582
7583 let auth = build_auth_token_for(credential)
7584 .context("failed to mint Lighter auth token for report fetch")?;
7585 let ts_init = self.clock.get_time_ns();
7586
7587 if cmd.instrument_id.is_none() && sweep.markets.is_empty() {
7596 collect_active_markets_from_inactive_orders(
7597 &self.http_client,
7598 &mut sweep.markets,
7599 credential,
7600 &auth,
7601 format_between_timestamps(cmd.start, cmd.end, ts_init),
7602 )
7603 .await?;
7604 }
7605
7606 sweep
7607 .markets
7608 .extend(self.markets.iter().map(|market| *market));
7609
7610 let market_indices = match cmd.instrument_id {
7611 Some(id) => match self.registry.market_index(&id) {
7612 Some(idx) => vec![idx],
7613 None => {
7614 anyhow::bail!("no Lighter market_index for order report instrument {id}",);
7615 }
7616 },
7617 None => sweep.markets.iter().copied().collect::<Vec<_>>(),
7618 };
7619
7620 if market_indices.is_empty() {
7621 log::debug!(
7622 "Lighter generate_order_status_reports: no active markets yet; returning empty",
7623 );
7624
7625 return Ok(());
7626 }
7627
7628 sweep.partial = true;
7629 self.collect_orders_active(market_indices, credential, &auth, ts_init, sweep)
7630 .await?;
7631
7632 if !cmd.open_only {
7640 self.collect_orders_inactive(cmd, credential, &auth, ts_init, sweep)
7641 .await?;
7642 }
7643
7644 Ok(())
7645 }
7646
7647 async fn collect_orders_active(
7648 &self,
7649 market_indices: Vec<i64>,
7650 credential: &Credential,
7651 auth: &SecretString,
7652 ts_init: UnixNanos,
7653 sweep: &mut OrderReportSweep,
7654 ) -> anyhow::Result<()> {
7655 let mut active_errors = Vec::new();
7656
7657 for market_index in market_indices {
7663 let query = Zeroizing::new(LighterAccountActiveOrdersQuery {
7664 authorization: None,
7665 auth: Some(auth.clone()),
7666 account_index: credential.account_index(),
7667 market_id: market_index,
7668 });
7669
7670 let active = match self.http_client.get_account_active_orders(&query).await {
7671 Ok(response) => response,
7672 Err(e) => {
7673 let detail = format!(
7674 "failed to fetch Lighter active orders for market_index={market_index}: {}",
7675 scrub_auth(&format!("{e:#}")),
7676 );
7677 log::warn!("{detail}",);
7678 active_errors.push(detail);
7679 continue;
7680 }
7681 };
7682
7683 for order in &active.orders {
7684 sweep.markets.insert(order.market_index);
7685
7686 let Some(report) = parse_http_order_to_report(
7687 order,
7688 &self.registry,
7689 self.account_id,
7690 ts_init,
7691 &self.dispatch,
7692 ) else {
7693 let detail = format!(
7694 "failed to parse Lighter active order {} for market_index={market_index}",
7695 order.order_id,
7696 );
7697 log::warn!("{detail}");
7698 active_errors.push(detail);
7699 continue;
7700 };
7701
7702 let reduced_qty = order.initial_base_amount - report.quantity.as_decimal();
7703 if reduced_qty > Decimal::ZERO {
7704 sweep.reductions.insert(report.venue_order_id, reduced_qty);
7705 }
7706
7707 sweep.orders.push((order.client_order_index, report));
7708 }
7709 }
7710
7711 if !active_errors.is_empty() {
7712 anyhow::bail!(active_errors.join("; "));
7713 }
7714
7715 Ok(())
7716 }
7717
7718 async fn collect_orders_inactive(
7719 &self,
7720 cmd: &GenerateOrderStatusReports,
7721 credential: &Credential,
7722 auth: &SecretString,
7723 ts_init: UnixNanos,
7724 sweep: &mut OrderReportSweep,
7725 ) -> anyhow::Result<()> {
7726 sweep
7727 .markets
7728 .extend(self.markets.iter().map(|market| *market));
7729
7730 let inactive_markets: Vec<i64> = match cmd.instrument_id {
7731 Some(id) => self.registry.market_index(&id).map_or_default(|m| vec![m]),
7732 None => sweep.markets.iter().copied().collect::<Vec<_>>(),
7733 };
7734
7735 let between_timestamps = format_between_timestamps(cmd.start, cmd.end, ts_init);
7736 let mut inactive_errors = Vec::new();
7737
7738 for market_id in inactive_markets {
7739 let mut cursor: Option<String> = None;
7740 let mut seen_cursors = AHashSet::new();
7741 let mut pages = 0_usize;
7742 loop {
7743 pages += 1;
7744 if pages > MAX_RECONCILIATION_PAGES {
7745 anyhow::bail!(
7746 "Lighter inactive-order reconciliation exceeded {MAX_RECONCILIATION_PAGES} pages for market_index={market_id}",
7747 );
7748 }
7749
7750 let query = Zeroizing::new(LighterAccountInactiveOrdersQuery {
7751 authorization: None,
7752 auth: Some(auth.clone()),
7753 account_index: credential.account_index(),
7754 market_id: Some(market_id),
7755 ask_filter: None,
7756 between_timestamps: between_timestamps.clone(),
7757 cursor: cursor.clone(),
7758 limit: LIGHTER_REST_PAGE_SIZE,
7759 });
7760
7761 let inactive = self.http_client.get_account_inactive_orders(&query).await.map_err(|e| {
7762 anyhow::anyhow!(
7763 "failed to fetch Lighter inactive orders for market_index={market_id}: {}",
7764 scrub_auth(&format!("{e:#}")),
7765 )
7766 })?;
7767
7768 for order in &inactive.orders {
7769 let Some(report) = parse_http_order_to_report(
7770 order,
7771 &self.registry,
7772 self.account_id,
7773 ts_init,
7774 &self.dispatch,
7775 ) else {
7776 let detail = format!(
7777 "failed to parse Lighter inactive order {} for market_index={market_id}",
7778 order.order_id,
7779 );
7780 log::warn!("{detail}");
7781 inactive_errors.push(detail);
7782 continue;
7783 };
7784
7785 if cmd.start.is_some_and(|start| report.ts_last < start)
7786 || cmd.end.is_some_and(|end| report.ts_last > end)
7787 {
7788 continue;
7789 }
7790
7791 sweep.markets.insert(order.market_index);
7792 sweep.orders.push((order.client_order_index, report));
7793 }
7794
7795 let Some(next) = inactive.next_cursor else {
7796 break;
7797 };
7798
7799 if next.is_empty() {
7800 break;
7801 }
7802
7803 anyhow::ensure!(
7804 seen_cursors.insert(next.clone()),
7805 "Lighter inactive-order reconciliation repeated cursor `{next}` for market_index={market_id}",
7806 );
7807 cursor = Some(next);
7808 }
7809 }
7810
7811 if !inactive_errors.is_empty() {
7812 anyhow::bail!(inactive_errors.join("; "));
7813 }
7814
7815 Ok(())
7816 }
7817
7818 async fn collect_fill_reports(
7819 &self,
7820 cmd: &GenerateFillReports,
7821 instruments: &AHashMap<InstrumentId, InstrumentAny>,
7822 markets: &mut BTreeSet<i64>,
7823 ) -> anyhow::Result<FillSweep> {
7824 let Some(credential) = &self.credential else {
7825 log::warn!("Lighter generate_fill_reports: no credentials");
7826 return Ok(FillSweep {
7827 reports: Vec::new(),
7828 covers_window: true,
7829 withheld_instruments: AHashSet::new(),
7830 });
7831 };
7832
7833 collect_fill_reports(
7834 &self.http_client,
7835 &self.registry,
7836 credential,
7837 self.account_id,
7838 self.clock,
7839 cmd,
7840 instruments,
7841 markets,
7842 None,
7843 )
7844 .await
7845 }
7846}
7847
7848#[expect(
7849 clippy::too_many_arguments,
7850 reason = "ordinary and child-scoped recovery share authenticated fill pagination"
7851)]
7852async fn collect_fill_reports(
7853 http_client: &LighterHttpClient,
7854 registry: &Arc<MarketRegistry>,
7855 credential: &Credential,
7856 account_id: AccountId,
7857 clock: &AtomicTime,
7858 cmd: &GenerateFillReports,
7859 instruments: &AHashMap<InstrumentId, InstrumentAny>,
7860 markets: &mut BTreeSet<i64>,
7861 order_index: Option<i64>,
7862) -> anyhow::Result<FillSweep> {
7863 let market_id = match cmd.instrument_id {
7864 Some(instrument_id) => Some(registry.market_index(&instrument_id).ok_or_else(|| {
7865 anyhow::anyhow!("no Lighter market_index for fill instrument {instrument_id}",)
7866 })?),
7867 None => None,
7868 };
7869
7870 let auth = build_auth_token_for(credential)
7871 .context("failed to mint Lighter auth token for fill fetch")?;
7872
7873 let ts_init = clock.get_time_ns();
7874 let mut reports = Vec::new();
7875 let mut cursor: Option<String> = None;
7876 let mut seen_cursors = AHashSet::new();
7877 let mut seen_in_call = AHashSet::new();
7878 let mut pages = 0_usize;
7879 let mut oldest_served: Option<UnixNanos> = None;
7880 let mut covers_window = true;
7881
7882 loop {
7883 pages += 1;
7884 anyhow::ensure!(
7885 pages <= MAX_RECONCILIATION_PAGES,
7886 "Lighter fill reconciliation exceeded {MAX_RECONCILIATION_PAGES} pages",
7887 );
7888
7889 let query = Zeroizing::new(LighterTradesQuery {
7890 authorization: None,
7891 auth: Some(auth.clone()),
7892 market_id,
7893 account_index: Some(credential.account_index()),
7894 order_index,
7895 order_index_str: None,
7896 integrator_account_index: None,
7897 sort_by: LighterTradeSortBy::Timestamp,
7898 sort_dir: Some(LighterSortDirection::Desc),
7899 cursor: cursor.clone(),
7900 from_timestamp: None,
7903 ask_filter: None,
7904 role: None,
7905 trade_type: None,
7906 limit: LIGHTER_REST_PAGE_SIZE,
7907 aggregate: None,
7908 });
7909
7910 let response = match http_client.get_trades(&query).await {
7911 Ok(response) => response,
7912 Err(e) => {
7913 log::warn!(
7917 "Lighter get_trades failed (market_id={:?}, account_index={}, cursor={:?}): {}",
7918 query.market_id,
7919 credential.account_index(),
7920 cursor,
7921 scrub_auth(&format!("{e:#}")),
7922 );
7923 return Err(anyhow::Error::new(e).context("failed to fetch Lighter fills"));
7924 }
7925 };
7926
7927 for trade in &response.trades {
7928 let Some(report) = parse_scoped_fill_report(
7929 trade,
7930 registry,
7931 instruments,
7932 credential.account_index(),
7933 account_id,
7934 cmd,
7935 ts_init,
7936 )?
7937 else {
7938 continue;
7939 };
7940
7941 if !seen_in_call.insert(report.trade_id) {
7942 log::debug!(
7943 "Lighter duplicate trade {} ignored within HTTP fill pagination",
7944 report.trade_id,
7945 );
7946 continue;
7947 }
7948
7949 markets.insert(trade.market_id);
7950 reports.push(report);
7951 }
7952
7953 let page_oldest = response
7954 .trades
7955 .iter()
7956 .filter_map(|trade| u64::try_from(trade.timestamp).ok())
7957 .map(|timestamp_ms| UnixNanos::from(timestamp_ms.saturating_mul(1_000_000)))
7958 .min();
7959
7960 if let Some(page_oldest) = page_oldest {
7961 oldest_served = Some(oldest_served.map_or(page_oldest, |ts| ts.min(page_oldest)));
7962 }
7963
7964 let reached_start_boundary = cmd
7965 .start
7966 .is_some_and(|start| page_oldest.is_some_and(|oldest| oldest < start));
7967
7968 if reached_start_boundary {
7969 break;
7970 }
7971
7972 match response.next_cursor {
7973 Some(next) if !next.is_empty() => {
7974 anyhow::ensure!(
7975 seen_cursors.insert(next.clone()),
7976 "Lighter fill reconciliation repeated cursor `{next}`",
7977 );
7978 cursor = Some(next);
7979 }
7980 _ => {
7981 if let (Some(start), Some(oldest)) = (cmd.start, oldest_served) {
7987 covers_window = false;
7988
7989 log::warn!(
7990 "Lighter fill reports do not cover {} to {}: trade pagination exhausted before the requested start; the venue `export` endpoint serves full history",
7991 unix_nanos_to_iso8601(start),
7992 unix_nanos_to_iso8601(oldest),
7993 );
7994 }
7995
7996 break;
7997 }
7998 }
7999 }
8000
8001 Ok(FillSweep {
8002 reports,
8003 covers_window,
8004 withheld_instruments: AHashSet::new(),
8005 })
8006}
8007
8008fn parse_scoped_fill_report(
8009 trade: &LighterTrade,
8010 registry: &MarketRegistry,
8011 instruments: &AHashMap<InstrumentId, InstrumentAny>,
8012 account_index: i64,
8013 account_id: AccountId,
8014 cmd: &GenerateFillReports,
8015 ts_init: UnixNanos,
8016) -> anyhow::Result<Option<FillReport>> {
8017 let instrument_id = registry.instrument_id(trade.market_id).with_context(|| {
8018 format!(
8019 "no Lighter instrument registered for fill market_index={}",
8020 trade.market_id
8021 )
8022 })?;
8023
8024 let instrument = instruments
8025 .get(&instrument_id)
8026 .with_context(|| format!("Lighter fill instrument {instrument_id} missing from cache"))?;
8027
8028 let Some(report) = parse_ws_fill_report(trade, account_index, instrument, account_id, ts_init)
8029 .context("failed to parse Lighter fill report")?
8030 else {
8031 return Ok(None);
8032 };
8033
8034 if cmd.start.is_some_and(|start| report.ts_event < start)
8035 || cmd.end.is_some_and(|end| report.ts_event > end)
8036 {
8037 return Ok(None);
8038 }
8039
8040 Ok(Some(report))
8041}
8042
8043fn cached_position_reports(
8044 core: &ExecutionClientCore,
8045 dispatch: &WsDispatchState,
8046 registry: &Arc<MarketRegistry>,
8047 cmd: &GeneratePositionStatusReports,
8048) -> anyhow::Result<(Vec<PositionStatusReport>, bool, Option<AHashSet<i64>>)> {
8049 let (mut reports, mut coverage) = dispatch.snapshot_positions_with_coverage();
8053 let pending = execution_instruments_pending(core, dispatch);
8054 reports.retain(|report| !pending.contains(&report.instrument_id));
8055 if let Some(skipped) = &mut coverage {
8056 skipped.extend(pending.iter().filter_map(|id| registry.market_index(id)));
8057 }
8058
8059 let complete = match cmd.instrument_id {
8060 Some(instrument_id) => {
8061 let market_id = registry.market_index(&instrument_id).ok_or_else(|| {
8062 anyhow::anyhow!(
8063 "no Lighter market_index for position report instrument {instrument_id}",
8064 )
8065 })?;
8066
8067 reports.retain(|report| report.instrument_id == instrument_id);
8068 !pending.contains(&instrument_id)
8069 && coverage
8070 .as_ref()
8071 .is_some_and(|skipped| !skipped.contains(&market_id))
8072 }
8073 None => pending.is_empty() && coverage.as_ref().is_some_and(|skipped| skipped.is_empty()),
8074 };
8075
8076 Ok((reports, complete, coverage))
8077}
8078
8079fn execution_instruments_pending(
8080 core: &ExecutionClientCore,
8081 dispatch: &WsDispatchState,
8082) -> AHashSet<InstrumentId> {
8083 let events = dispatch.order_events.lock();
8084 let cache = core.cache();
8085 let mut pending = AHashSet::new();
8086 dispatch.any_order_identity(|cloid, identity, retired| {
8087 if (identity.has_inferred_fills()
8088 && cache.order(cloid).map_or(!retired, |order| {
8089 !order.is_closed() || inferred_reconciliation_trade_ids(&order).is_err()
8090 }))
8091 || dispatch.has_pending_modify(cloid)
8092 || dispatch.terminals_pending.lock().contains_key(cloid)
8093 || events.get(cloid).is_some_and(|deferred| {
8094 !deferred.fills.is_empty()
8095 || deferred
8096 .order
8097 .as_ref()
8098 .is_some_and(|order| order.status.is_terminal())
8099 || deferred.recovery_epoch.is_some()
8100 })
8101 || identity.publication_pending(cache.order(cloid).as_deref())
8102 {
8103 pending.insert(identity.instrument_id);
8104 }
8105
8106 false
8107 });
8108
8109 pending
8110}
8111
8112fn dispatch_query_order_report(
8113 report: Option<(i64, OrderStatusReport)>,
8114 cmd: &QueryOrder,
8115 dispatch: &WsDispatchState,
8116 emitter: &ExecutionEventEmitter,
8117) {
8118 let events = dispatch.order_events.lock();
8119 if dispatch
8120 .terminals_pending
8121 .lock()
8122 .contains_key(&cmd.client_order_id)
8123 || events.get(&cmd.client_order_id).is_some_and(|deferred| {
8124 deferred
8125 .order
8126 .as_ref()
8127 .is_some_and(|order| order.status.is_terminal())
8128 })
8129 || dispatch
8130 .order_identity(&cmd.client_order_id)
8131 .is_some_and(|identity| {
8132 position_fills_pending(&identity, &events, dispatch)
8133 || dispatch.any_order_identity(|_, other, _| {
8134 other.strategy_id == identity.strategy_id
8135 && other.instrument_id == identity.instrument_id
8136 && other.has_publication()
8137 })
8138 })
8139 || dispatch.has_pending_modify(&cmd.client_order_id)
8140 || dispatch
8141 .order_identity(&cmd.client_order_id)
8142 .is_some_and(|identity| identity.has_publication())
8143 {
8144 return;
8145 }
8146
8147 match report {
8148 Some((index, report)) => {
8149 if is_unowned_child_cancel(dispatch, index, &report) {
8150 return;
8151 }
8152
8153 if dispatch
8154 .order_identity(&cmd.client_order_id)
8155 .is_some_and(|identity| !identity.report_fills_covered(&report, &[]))
8156 {
8157 return;
8158 }
8159
8160 log::debug!(
8161 "Lighter query_order returning report for {}",
8162 cmd.client_order_id
8163 );
8164 dispatch.seed_accepted_from_report(&report);
8165 emitter.send_order_status_report(report);
8166 }
8167 None => {
8168 log::warn!(
8169 "Lighter query_order: no order found for {}",
8170 cmd.client_order_id,
8171 );
8172 }
8173 }
8174}
8175
8176fn finish_order_reports(
8177 core: &ExecutionClientCore,
8178 dispatch: &WsDispatchState,
8179 sweep: OrderReportSweep,
8180 cmd: &GenerateOrderStatusReports,
8181 fills: Option<&[FillReport]>,
8182) -> anyhow::Result<Vec<OrderStatusReport>> {
8183 for market in sweep.markets {
8184 dispatch.note_active_market(market);
8185 }
8186
8187 let mut withheld = AHashSet::new();
8188 let mut reports = Vec::with_capacity(sweep.orders.len());
8189 for (client_order_index, report) in sweep.orders {
8190 let pending_cloid = core
8191 .cache()
8192 .client_order_id(&report.venue_order_id)
8193 .copied()
8194 .or_else(|| {
8195 dispatch
8196 .resolve_client_order_index_for_venue(client_order_index, report.venue_order_id)
8197 });
8198
8199 if pending_cloid
8200 .as_ref()
8201 .is_some_and(|cloid| order_shape_pending(core, dispatch, cloid))
8202 {
8203 withheld.insert(report.instrument_id);
8204 continue;
8205 }
8206
8207 restore_reconciled_order(
8208 core,
8209 dispatch,
8210 report.venue_order_id,
8211 client_order_index,
8212 report.order_status.is_closed(),
8213 );
8214
8215 if let Some(reduced_qty) = sweep.reductions.get(&report.venue_order_id) {
8216 dispatch.record_order_reduction(
8217 client_order_index,
8218 report.venue_order_id,
8219 *reduced_qty,
8220 );
8221 }
8222
8223 let report = dispatch.translate_order_cloid(report);
8224 let report = dispatch.preserve_pending_order_status(report);
8225 if is_unowned_child_cancel(dispatch, client_order_index, &report) {
8226 continue;
8227 }
8228
8229 if report
8230 .client_order_id
8231 .as_ref()
8232 .is_some_and(|cloid| order_shape_pending(core, dispatch, cloid))
8233 {
8234 withheld.insert(report.instrument_id);
8235 continue;
8236 }
8237
8238 if !order_report_collectible(core, dispatch, &report, fills) {
8239 withheld.insert(report.instrument_id);
8240 continue;
8241 }
8242
8243 dispatch.seed_accepted_from_report(&report);
8244 reports.push(report);
8245 }
8246
8247 match sweep.error {
8248 Some(e) if sweep.partial => {
8249 return Err(incomplete_order_reports(reports, e.to_string(), withheld));
8250 }
8251 Some(e) => return Err(e),
8252 None if !withheld.is_empty() => {
8253 return Err(incomplete_order_reports(
8254 reports,
8255 "Lighter execution confirmation is pending".to_string(),
8256 withheld,
8257 ));
8258 }
8259 None => {}
8260 }
8261
8262 LighterExecutionClient::log_report_receipt(
8263 reports.len(),
8264 "OrderStatusReport",
8265 cmd.log_receipt_level,
8266 );
8267 Ok(reports)
8268}
8269
8270fn is_unowned_child_cancel(
8271 dispatch: &WsDispatchState,
8272 client_order_index: i64,
8273 report: &OrderStatusReport,
8274) -> bool {
8275 report.order_status == OrderStatus::Canceled
8277 && report.cancel_reason.as_deref() == LighterOrderStatus::CanceledChild.as_cancel_reason()
8278 && report.parent_order_id.is_some()
8279 && report.quantity.is_zero()
8280 && report.filled_qty.is_zero()
8281 && dispatch
8282 .resolve_client_order_index_for_venue(client_order_index, report.venue_order_id)
8283 .is_none()
8284}
8285
8286fn finish_fill_reports(
8287 core: &ExecutionClientCore,
8288 dispatch: &WsDispatchState,
8289 markets: BTreeSet<i64>,
8290 result: anyhow::Result<FillSweep>,
8291) -> anyhow::Result<FillSweep> {
8292 for market in markets {
8293 dispatch.note_active_market(market);
8294 }
8295
8296 let mut sweep = result?;
8297 let mut reports = Vec::with_capacity(sweep.reports.len());
8298
8299 for mut report in sweep.reports {
8300 defer_reconciled_fill(core, dispatch, &mut report);
8301
8302 let report = dispatch.translate_fill_cloid(report);
8305 if report.client_order_id.as_ref().is_some_and(|cloid| {
8306 order_shape_pending(core, dispatch, cloid)
8307 || (dispatch
8308 .order_identity(cloid)
8309 .is_some_and(|identity| identity.child_fill_baseline().is_some())
8310 && core
8311 .cache()
8312 .order(cloid)
8313 .is_none_or(|order| !order.is_closed()))
8314 }) {
8315 sweep.covers_window = false;
8316 sweep.withheld_instruments.insert(report.instrument_id);
8317 continue;
8318 }
8319
8320 if matches!(
8321 dispatch.trade_source(report.trade_id),
8322 Some(TradeDedupSource::Live)
8323 ) {
8324 log::debug!(
8325 "Lighter trade {} ignored in HTTP fill reports after live delivery",
8326 report.trade_id
8327 );
8328 continue;
8329 }
8330
8331 reports.push(report);
8332 }
8333
8334 sweep.reports = reports;
8335 Ok(sweep)
8336}
8337
8338fn defer_reconciled_fill(
8339 core: &ExecutionClientCore,
8340 dispatch: &WsDispatchState,
8341 report: &mut FillReport,
8342) {
8343 let Some((cloid, identity)) = restore_fill_identity(core, dispatch, report) else {
8344 return;
8345 };
8346
8347 report.client_order_id = Some(cloid);
8348
8349 if identity.has_inferred_fills() {
8350 return;
8351 }
8352
8353 let Some(fill) = child_fill_from_report(report, &identity, core.trader_id) else {
8354 return;
8355 };
8356
8357 if core
8358 .cache()
8359 .order(&cloid)
8360 .is_some_and(|order| order.is_duplicate_fill(&fill))
8361 {
8362 return;
8363 }
8364
8365 let mut events = dispatch.order_events.lock();
8366 let baseline = identity.child_fill_baseline();
8367 let new_fill = baseline
8368 .as_ref()
8369 .is_none_or(|baseline| !baseline.trade_ids.contains(&fill.trade_id));
8370 if new_fill
8371 && (baseline.is_some()
8372 || dispatch.has_pending_modify(&cloid)
8373 || position_fills_pending(&identity, &events, dispatch))
8374 {
8375 dispatch.defer_fill(events.entry(cloid).or_default(), fill);
8376 }
8377}
8378
8379fn restore_fill_identity(
8380 core: &ExecutionClientCore,
8381 dispatch: &WsDispatchState,
8382 report: &FillReport,
8383) -> Option<(ClientOrderId, OrderIdentity)> {
8384 let cached_order = {
8385 let cache = core.cache();
8386 cache
8387 .client_order_id(&report.venue_order_id)
8388 .and_then(|cloid| cache.order_owned(cloid))
8389 .filter(|order| {
8390 !order.strategy_id().is_external()
8391 && order.instrument_id() == report.instrument_id
8392 && order.account_id() == Some(report.account_id)
8393 })
8394 }?;
8395
8396 let cloid = cached_order.client_order_id();
8397 if dispatch.order_identity(&cloid).is_none() {
8398 let index = report
8399 .client_order_id
8400 .and_then(|id| id.as_str().parse::<i64>().ok())
8401 .unwrap_or(0);
8402 dispatch.seed_order_restoration(cached_order);
8403 dispatch.restore_cached_order(report.venue_order_id, index, report.instrument_id);
8404 }
8405
8406 dispatch
8407 .order_identity(&cloid)
8408 .map(|identity| (cloid, identity))
8409}
8410
8411fn ensure_order_report_ready(
8412 core: &ExecutionClientCore,
8413 dispatch: &WsDispatchState,
8414 report: &OrderStatusReport,
8415) -> anyhow::Result<()> {
8416 anyhow::ensure!(
8417 order_report_collectible(core, dispatch, report, None),
8418 "Lighter order report deferred until execution state is applied",
8419 );
8420 Ok(())
8421}
8422
8423fn order_report_collectible(
8424 core: &ExecutionClientCore,
8425 dispatch: &WsDispatchState,
8426 report: &OrderStatusReport,
8427 fills: Option<&[FillReport]>,
8428) -> bool {
8429 if fills.is_none()
8431 && matches!(
8432 report.order_status,
8433 OrderStatus::Canceled | OrderStatus::Expired
8434 )
8435 {
8436 order_report_shape_ready(core, dispatch, report)
8437 } else {
8438 order_report_ready(core, dispatch, report, fills.unwrap_or_default())
8439 }
8440}
8441
8442fn order_report_ready(
8443 core: &ExecutionClientCore,
8444 dispatch: &WsDispatchState,
8445 report: &OrderStatusReport,
8446 fills: &[FillReport],
8447) -> bool {
8448 order_report_shape_ready(core, dispatch, report)
8449 && report
8450 .client_order_id
8451 .and_then(|cloid| dispatch.order_identity(&cloid))
8452 .is_none_or(|identity| identity.report_fills_covered(report, fills))
8453}
8454
8455fn order_report_shape_ready(
8456 core: &ExecutionClientCore,
8457 dispatch: &WsDispatchState,
8458 report: &OrderStatusReport,
8459) -> bool {
8460 let Some(cloid) = report.client_order_id else {
8461 return true;
8462 };
8463
8464 let pending = order_shape_pending(core, dispatch, &cloid);
8465
8466 if let Some(identity) = dispatch.order_identity(&cloid)
8467 && identity.child_fill_baseline().is_some()
8468 && core
8469 .cache()
8470 .order(&cloid)
8471 .is_none_or(|order| report.filled_qty != order.filled_qty())
8472 {
8473 return false;
8474 }
8475
8476 if !pending {
8477 return true;
8478 }
8479
8480 report.order_status == OrderStatus::PendingUpdate
8481 && core.cache().order(&cloid).is_some_and(|order| {
8482 order.quantity() == report.quantity
8483 && order.price() == report.price
8484 && order.trigger_price() == report.trigger_price
8485 && order.filled_qty() == report.filled_qty
8486 })
8487}
8488
8489fn order_shape_pending(
8490 core: &ExecutionClientCore,
8491 dispatch: &WsDispatchState,
8492 cloid: &ClientOrderId,
8493) -> bool {
8494 let events = dispatch.order_events.lock();
8495 if dispatch.has_pending_modify(cloid) {
8496 return true;
8497 }
8498
8499 let Some(identity) = dispatch.order_identity(cloid) else {
8500 return false;
8501 };
8502
8503 if let Some(order) = core.cache().order(cloid) {
8504 identity.sync_fills(&order);
8505 }
8506
8507 if identity.has_inferred_fills() {
8508 return true;
8509 }
8510
8511 if dispatch.terminals_pending.lock().contains_key(cloid) {
8512 return true;
8513 }
8514
8515 if position_fills_pending(&identity, &events, dispatch) {
8516 return true;
8517 }
8518
8519 let cache = core.cache();
8520
8521 if dispatch.any_order_identity(|id, other, _| {
8522 other.strategy_id == identity.strategy_id
8523 && other.instrument_id == identity.instrument_id
8524 && other.publication_pending(cache.order(id).as_deref())
8525 }) {
8526 return true;
8527 }
8528
8529 let order = cache.order(cloid);
8530 let pending = identity.publication_pending(order.as_deref());
8531 if !pending && order.as_ref().is_some_and(|order| order.is_closed()) {
8532 dispatch.retire_order_identity(cloid);
8533 }
8534
8535 pending
8536}
8537
8538fn restore_reconciled_order(
8539 core: &ExecutionClientCore,
8540 dispatch: &WsDispatchState,
8541 venue_order_id: VenueOrderId,
8542 client_order_index: i64,
8543 terminal: bool,
8544) {
8545 let cached_order = {
8546 let cache = core.cache();
8547
8548 let Some(cloid) = cache.client_order_id(&venue_order_id).copied() else {
8549 return;
8550 };
8551
8552 let Some(order) = cache.order_owned(&cloid) else {
8553 log::warn!(
8554 "Ignoring reconciled Lighter order missing from cache: cloid={cloid}, venue_order_id={venue_order_id}",
8555 );
8556 return;
8557 };
8558
8559 order
8560 };
8561
8562 if cached_order.instrument_id().venue != core.venue
8563 || cached_order.account_id() != Some(core.account_id)
8564 {
8565 log::warn!(
8566 "Ignoring reconciled Lighter order outside this client: cloid={}, venue_order_id={venue_order_id}",
8567 cached_order.client_order_id(),
8568 );
8569 return;
8570 }
8571
8572 if order_shape_pending(core, dispatch, &cached_order.client_order_id()) {
8573 return;
8574 }
8575
8576 let terminal =
8577 terminal && (cached_order.parent_order_id().is_none() || cached_order.is_closed());
8578
8579 if let Err(e) = dispatch.restore_reconciled_order(
8580 &cached_order,
8581 client_order_index,
8582 venue_order_id,
8583 terminal,
8584 ) {
8585 log::warn!(
8586 "Ignoring conflicting Lighter reconciliation identity: cloid={}, venue_order_id={venue_order_id}, client_order_index={}, error={e}",
8587 cached_order.client_order_id(),
8588 client_order_index,
8589 );
8590 }
8591}
8592
8593fn grouped_submit_denial_reason(error: &anyhow::Error) -> String {
8594 match error.downcast_ref::<OrderDeniedReason>() {
8595 Some(reason) => reason.to_string(),
8596 None => OrderDeniedReason::UnsupportedOrderList {
8597 detail: format!("Lighter grouped submit failed: {error}"),
8598 }
8599 .to_string(),
8600 }
8601}
8602
8603fn local_submit_denial_reason(
8604 order: &OrderAny,
8605 instrument: Option<&InstrumentAny>,
8606) -> Option<anyhow::Error> {
8607 if instrument.is_none() {
8608 return Some(anyhow::anyhow!(
8609 OrderDeniedReason::InstrumentNotFound {
8610 instrument_id: order.instrument_id(),
8611 }
8612 .to_string(),
8613 ));
8614 }
8615
8616 if !is_lighter_supported_order_type(order.order_type()) {
8617 return Some(anyhow::anyhow!(unsupported_lighter_order_type_reason(
8618 order.order_type()
8619 )));
8620 }
8621
8622 if is_lighter_limit_style_order(order.order_type()) && order.price().is_none() {
8623 return Some(anyhow::anyhow!(
8624 OrderDeniedReason::ValidationFailed {
8625 detail: "Lighter limit-style orders require a limit price".to_string(),
8626 }
8627 .to_string(),
8628 ));
8629 }
8630
8631 if order.is_quote_quantity() {
8632 return Some(anyhow::anyhow!(
8633 OrderDeniedReason::unsupported_quote_quantity()
8634 ));
8635 }
8636
8637 if order.display_qty().is_some() {
8638 return Some(anyhow::anyhow!(
8639 OrderDeniedReason::ValidationFailed {
8640 detail: "Lighter orders do not support display_qty iceberg instructions"
8641 .to_string(),
8642 }
8643 .to_string(),
8644 ));
8645 }
8646
8647 if is_lighter_spot_order(order, instrument) && is_lighter_conditional_order(order.order_type())
8648 {
8649 let denied = OrderDeniedReason::UnsupportedOrderType {
8650 order_type: order.order_type(),
8651 };
8652
8653 return Some(anyhow::anyhow!(
8654 "{denied}; Lighter spot markets do not support conditional orders",
8655 ));
8656 }
8657
8658 nautilus_to_lighter_tif(
8659 order.order_type(),
8660 order.time_in_force(),
8661 order.is_post_only(),
8662 )
8663 .err()
8664 .map(|e| {
8665 let denied = OrderDeniedReason::UnsupportedTimeInForce(order.time_in_force());
8666 anyhow::anyhow!("{denied}; {e}")
8667 })
8668}
8669
8670fn unsupported_lighter_order_type_reason(order_type: OrderType) -> String {
8671 let denied = OrderDeniedReason::UnsupportedOrderType { order_type };
8672 format!(
8673 "{denied}; Lighter supports MARKET, LIMIT, STOP_MARKET, STOP_LIMIT, MARKET_IF_TOUCHED, and LIMIT_IF_TOUCHED",
8674 )
8675}
8676
8677fn is_grouped_order(order: &OrderAny) -> bool {
8678 order.contingency_type().is_some() || order.parent_order_id().is_some()
8679}
8680
8681fn grouped_order_indices(orders: &[OrderAny]) -> anyhow::Result<(LighterGroupingType, Vec<usize>)> {
8682 anyhow::ensure!(
8683 (2..=3).contains(&orders.len()),
8684 "Lighter groups require two or three orders"
8685 );
8686 let first = &orders[0];
8687 anyhow::ensure!(
8688 orders
8689 .iter()
8690 .enumerate()
8691 .all(|(index, order)| orders[..index]
8692 .iter()
8693 .all(|other| other.client_order_id() != order.client_order_id())),
8694 "Lighter groups require unique client order IDs"
8695 );
8696 anyhow::ensure!(
8697 orders.iter().all(|order| !order.is_closed()
8698 && order.instrument_id() == first.instrument_id()
8699 && order.strategy_id() == first.strategy_id()
8700 && order.order_list_id() == first.order_list_id()),
8701 "Lighter groups require open orders from one instrument, strategy, and order list"
8702 );
8703 anyhow::ensure!(
8704 !orders
8705 .iter()
8706 .any(|order| order.contingency_type() == Some(ContingencyType::Ouo)),
8707 "Lighter native groups do not support Nautilus OUO quantity updates"
8708 );
8709
8710 let parents: Vec<_> = orders
8711 .iter()
8712 .enumerate()
8713 .filter(|(_, order)| {
8714 order.contingency_type() == Some(ContingencyType::Oto)
8715 && order.parent_order_id().is_none()
8716 })
8717 .map(|(index, _)| index)
8718 .collect();
8719
8720 if parents.is_empty() {
8721 anyhow::ensure!(
8722 orders.len() == 2,
8723 "Lighter OCO requires exactly two siblings"
8724 );
8725 validate_oco_links(&orders[0], &orders[1], None)?;
8726 return Ok((LighterGroupingType::OneCancelsTheOther, vec![0, 1]));
8727 }
8728
8729 anyhow::ensure!(
8730 parents.len() == 1,
8731 "Lighter OTO groups require exactly one parent"
8732 );
8733 let parent_index = parents[0];
8734 let parent = &orders[parent_index];
8735 anyhow::ensure!(
8736 matches!(parent.order_type(), OrderType::Market | OrderType::Limit),
8737 "Lighter OTO parents must be MARKET or LIMIT orders"
8738 );
8739 let children: Vec<_> = orders
8740 .iter()
8741 .enumerate()
8742 .filter(|(index, _)| *index != parent_index)
8743 .collect();
8744 let child_ids: Vec<_> = children
8745 .iter()
8746 .map(|(_, order)| order.client_order_id())
8747 .collect();
8748 let links = parent
8749 .linked_order_ids()
8750 .context("Lighter OTO parent requires linked children")?;
8751 anyhow::ensure!(
8752 links.len() == child_ids.len() && child_ids.iter().all(|id| links.contains(id)),
8753 "Lighter OTO parent links must identify every child exactly once"
8754 );
8755 anyhow::ensure!(
8756 children
8757 .iter()
8758 .all(|(_, order)| order.parent_order_id() == Some(parent.client_order_id())),
8759 "Lighter OTO children must reference their parent"
8760 );
8761
8762 let grouping_type = if children.len() == 1 {
8763 anyhow::ensure!(
8764 children[0].1.contingency_type().is_none()
8765 && children[0].1.linked_order_ids().is_none(),
8766 "Lighter OTO child must not have a sibling contingency"
8767 );
8768 LighterGroupingType::OneTriggersTheOther
8769 } else {
8770 validate_oco_links(children[0].1, children[1].1, Some(parent.client_order_id()))?;
8771 LighterGroupingType::OneTriggersOneCancelsTheOther
8772 };
8773
8774 Ok((
8775 grouping_type,
8776 std::iter::once(parent_index)
8777 .chain(children.iter().map(|(index, _)| *index))
8778 .collect(),
8779 ))
8780}
8781
8782fn validate_oco_links(
8783 first: &OrderAny,
8784 second: &OrderAny,
8785 parent: Option<ClientOrderId>,
8786) -> anyhow::Result<()> {
8787 let orders = [first, second];
8788 for (index, order) in orders.iter().enumerate() {
8789 anyhow::ensure!(
8790 order.contingency_type() == Some(ContingencyType::Oco)
8791 && order.parent_order_id() == parent,
8792 "Lighter OCO siblings must have matching OCO relationships"
8793 );
8794 anyhow::ensure!(
8795 order
8796 .linked_order_ids()
8797 .is_some_and(|links| links == [orders[1 - index].client_order_id()]),
8798 "Lighter OCO siblings must link only to each other"
8799 );
8800 }
8801
8802 Ok(())
8803}
8804
8805fn prepare_group_bodies(
8806 grouping_type: LighterGroupingType,
8807 plans: &mut [CreateOrderPlan],
8808 use_gtd: bool,
8809) -> anyhow::Result<()> {
8810 let has_parent = grouping_type != LighterGroupingType::OneCancelsTheOther;
8811 let parent_quantity = plans[0].order.quantity();
8812 let child_start = usize::from(has_parent);
8813
8814 let default_expiry = plans
8815 .iter()
8816 .find(|plan| {
8817 plan.order_expiry != 0 && !(plan.order.time_in_force() == TimeInForce::Gtd && use_gtd)
8818 })
8819 .map_or(plans[child_start].order_expiry, |plan| plan.order_expiry);
8820
8821 for plan in plans.iter_mut() {
8822 if plan.order_expiry != 0 && !(plan.order.time_in_force() == TimeInForce::Gtd && use_gtd) {
8823 plan.order_expiry = default_expiry;
8824 }
8825 }
8826
8827 let expiry = plans[child_start].order_expiry;
8828 let parent_side = plans[0].order.order_side();
8829 for child in &mut plans[child_start..] {
8830 anyhow::ensure!(
8831 is_lighter_conditional_order(child.order.order_type()) && child.order.is_reduce_only(),
8832 "Lighter group children require reduce-only SL/TP orders"
8833 );
8834 anyhow::ensure!(
8835 child.order.quantity() == parent_quantity,
8836 "Lighter group quantities must match before parent-fill sizing"
8837 );
8838 anyhow::ensure!(
8839 (child.order.order_side() != parent_side) == has_parent,
8840 "Lighter OTO children must oppose their parent; OCO siblings must have the same side"
8841 );
8842 anyhow::ensure!(
8843 child.order_expiry == expiry && expiry != 0,
8844 "Lighter group children require matching nonzero expiries"
8845 );
8846
8847 if has_parent {
8848 child.base_amount = 0;
8849 }
8850 }
8851
8852 anyhow::ensure!(
8853 plans[0].order_expiry == 0 || plans[0].order_expiry == expiry,
8854 "Lighter parent and child expiries must match"
8855 );
8856
8857 if plans.len() - child_start == 2 {
8858 let stop_count = plans[child_start..]
8859 .iter()
8860 .filter(|plan| {
8861 matches!(
8862 plan.order.order_type(),
8863 OrderType::StopMarket | OrderType::StopLimit
8864 )
8865 })
8866 .count();
8867
8868 anyhow::ensure!(
8869 stop_count == 1,
8870 "Lighter OCO requires one stop-loss and one take-profit"
8871 );
8872 }
8873
8874 Ok(())
8875}
8876
8877fn is_lighter_spot_order(order: &OrderAny, instrument: Option<&InstrumentAny>) -> bool {
8878 instrument.is_some_and(|instrument| matches!(instrument, InstrumentAny::CurrencyPair(_)))
8879 || product_type_from_instrument_id(&order.instrument_id()) == Some(LighterProductType::Spot)
8880}
8881
8882fn is_lighter_supported_order_type(order_type: OrderType) -> bool {
8883 matches!(
8884 order_type,
8885 OrderType::Market
8886 | OrderType::Limit
8887 | OrderType::StopMarket
8888 | OrderType::StopLimit
8889 | OrderType::MarketIfTouched
8890 | OrderType::LimitIfTouched
8891 )
8892}
8893
8894fn is_lighter_limit_style_order(order_type: OrderType) -> bool {
8895 matches!(
8896 order_type,
8897 OrderType::Limit | OrderType::StopLimit | OrderType::LimitIfTouched
8898 )
8899}
8900
8901fn is_lighter_conditional_order(order_type: OrderType) -> bool {
8902 matches!(
8903 order_type,
8904 OrderType::StopMarket
8905 | OrderType::StopLimit
8906 | OrderType::MarketIfTouched
8907 | OrderType::LimitIfTouched
8908 )
8909}
8910
8911async fn collect_active_markets_from_inactive_orders(
8912 http_client: &LighterHttpClient,
8913 markets: &mut BTreeSet<i64>,
8914 credential: &Credential,
8915 auth: &SecretString,
8916 between_timestamps: Option<String>,
8917) -> anyhow::Result<()> {
8918 let mut cursor: Option<String> = None;
8919 let mut seen_cursors = AHashSet::new();
8920 let mut orders_seen = 0_usize;
8921 let mut pages = 0_usize;
8922 loop {
8923 pages += 1;
8924 anyhow::ensure!(
8925 pages <= MAX_RECONCILIATION_PAGES,
8926 "Lighter active-market seed exceeded {MAX_RECONCILIATION_PAGES} pages",
8927 );
8928
8929 let query = Zeroizing::new(LighterAccountInactiveOrdersQuery {
8930 authorization: None,
8931 auth: Some(auth.clone()),
8932 account_index: credential.account_index(),
8933 market_id: None,
8934 ask_filter: None,
8935 between_timestamps: between_timestamps.clone(),
8936 cursor: cursor.clone(),
8937 limit: LIGHTER_REST_PAGE_SIZE,
8938 });
8939 let response = http_client
8940 .get_account_inactive_orders(&query)
8941 .await
8942 .context("failed to seed Lighter active markets from inactive orders")?;
8943
8944 for order in &response.orders {
8945 markets.insert(order.market_index);
8946 orders_seen += 1;
8947 }
8948
8949 match response.next_cursor {
8950 Some(next) if !next.is_empty() => {
8951 anyhow::ensure!(
8952 seen_cursors.insert(next.clone()),
8953 "Lighter active-market seed repeated cursor `{next}`",
8954 );
8955 cursor = Some(next);
8956 }
8957 _ => break,
8958 }
8959 }
8960
8961 if orders_seen > 0 {
8962 log::debug!("Seeded Lighter active markets from {orders_seen} inactive order report(s)");
8963 }
8964
8965 Ok(())
8966}
8967
8968fn cancel_order_from_cancel_all(
8969 cmd: &CancelAllOrders,
8970 client_order_id: ClientOrderId,
8971 strategy_id: StrategyId,
8972) -> CancelOrder {
8973 CancelOrder {
8974 trader_id: cmd.trader_id,
8975 client_id: cmd.client_id,
8976 strategy_id,
8977 instrument_id: cmd.instrument_id,
8978 client_order_id,
8979 venue_order_id: None,
8980 command_id: cmd.command_id,
8981 ts_init: cmd.ts_init,
8982 params: cmd.params.clone(),
8983 correlation_id: cmd.correlation_id,
8984 causation_id: cmd.causation_id,
8985 }
8986}
8987
8988fn validate_order_amount(
8989 instrument: &InstrumentAny,
8990 quantity: Quantity,
8991 price_ticks: u32,
8992 price_precision: u8,
8993) -> anyhow::Result<()> {
8994 if let Some(min_quantity) = instrument.min_quantity() {
8995 anyhow::ensure!(
8996 quantity >= min_quantity,
8997 "quantity `{quantity}` below Lighter min_base_amount `{min_quantity}` for {}",
8998 instrument.id(),
8999 );
9000 }
9001
9002 if let Some(min_notional) = instrument.min_notional() {
9003 let price = decimal_from_ticks(price_ticks, price_precision);
9004 let notional = quantity.as_decimal() * price;
9005 anyhow::ensure!(
9006 notional >= min_notional.as_decimal(),
9007 "order notional `{notional}` below Lighter min_quote_amount `{}` for {}",
9008 min_notional.as_decimal(),
9009 instrument.id(),
9010 );
9011 }
9012
9013 Ok(())
9014}
9015
9016fn decimal_from_ticks(ticks: u32, decimals: u8) -> Decimal {
9017 Decimal::from(ticks) / Decimal::from(10_i64.pow(u32::from(decimals)))
9018}
9019
9020fn dispatch_lighter_order(
9024 order: &crate::http::models::LighterOrder,
9025 dispatch: &WsDispatchState,
9026 emitter: &ExecutionEventEmitter,
9027 registry: &Arc<MarketRegistry>,
9028 account_id: AccountId,
9029 trader_id: TraderId,
9030 ts_init: UnixNanos,
9031) {
9032 if order.status == LighterOrderStatus::Unknown {
9033 log::warn!(
9034 "Lighter order frame skipped: unknown order status, order_id={}, market_index={}",
9035 order.order_id,
9036 order.market_index,
9037 );
9038 return;
9039 }
9040
9041 let mut events = dispatch.order_events.lock();
9042
9043 let instrument_id = match registry.instrument_id(order.market_index) {
9044 Some(id) => id,
9045 None => {
9046 log::debug!(
9047 "Lighter order frame dropped: no instrument for market_index={}",
9048 order.market_index,
9049 );
9050 return;
9051 }
9052 };
9053
9054 dispatch.note_active_market(order.market_index);
9055
9056 let instrument = match LIGHTER_INSTRUMENT_CACHE.get(&instrument_id) {
9057 Some(inst) => inst.value().clone(),
9058 None => {
9059 log::debug!("Lighter order frame dropped: instrument {instrument_id} not in cache",);
9060 return;
9061 }
9062 };
9063
9064 let venue_order_id = VenueOrderId::new(order.order_id.as_str());
9065 if !dispatch.restore_cached_order(venue_order_id, order.client_order_index, instrument_id) {
9066 return;
9067 }
9068
9069 let resolved_cloid =
9070 dispatch.resolve_live_order_cloid(order.client_order_id.as_str(), venue_order_id);
9071
9072 let identity = resolved_cloid.and_then(|cid| {
9073 dispatch
9074 .order_identities
9075 .get(&cid)
9076 .map(|entry| (cid, entry.value().clone()))
9077 });
9078
9079 let Some((cloid, identity)) = identity else {
9080 dispatch_unowned_order(order, &instrument, dispatch, emitter, account_id, ts_init);
9081 return;
9082 };
9083
9084 if identity.instrument_id != instrument_id {
9085 return;
9086 }
9087
9088 if defer_owned_order(order, &identity, cloid, &mut events, dispatch, emitter) {
9089 return;
9090 }
9091
9092 dispatch_owned_order(
9093 order,
9094 &instrument,
9095 &identity,
9096 cloid,
9097 venue_order_id,
9098 dispatch,
9099 emitter,
9100 account_id,
9101 trader_id,
9102 ts_init,
9103 );
9104}
9105
9106fn defer_owned_order(
9107 order: &LighterOrder,
9108 identity: &OrderIdentity,
9109 cloid: ClientOrderId,
9110 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
9111 dispatch: &WsDispatchState,
9112 emitter: &ExecutionEventEmitter,
9113) -> bool {
9114 if dispatch.has_pending_modify(&cloid) {
9115 events.entry(cloid).or_default().store_order(order);
9116 return true;
9117 }
9118
9119 if identity.child_fill_baseline().is_some() && !identity.has_inferred_fills() {
9120 if identity.has_child_publication() {
9121 return true;
9122 }
9123
9124 let deferred = events.entry(cloid).or_default();
9125
9126 let had_terminal = deferred
9127 .order
9128 .as_ref()
9129 .is_some_and(|prior| prior.status.is_terminal());
9130
9131 deferred.store_order(order);
9132 let terminal = order.status.is_terminal();
9133
9134 if terminal || had_terminal {
9135 publish_child_recovery(cloid, events, dispatch, emitter);
9136 return true;
9137 }
9138 }
9139
9140 if position_fills_pending(identity, events, dispatch) && order.status.is_terminal() {
9141 events.entry(cloid).or_default().store_order(order);
9142 return true;
9143 }
9144
9145 false
9146}
9147
9148fn dispatch_unowned_order(
9149 order: &LighterOrder,
9150 instrument: &InstrumentAny,
9151 dispatch: &WsDispatchState,
9152 emitter: &ExecutionEventEmitter,
9153 account_id: AccountId,
9154 ts_init: UnixNanos,
9155) {
9156 match parse_ws_order_status_report(order, instrument, account_id, ts_init) {
9157 Ok(mut report) => {
9158 report = dispatch.apply_order_contingencies(order, report);
9159 report = dispatch.translate_order_cloid(report);
9160 report = dispatch.preserve_pending_order_status(report);
9161 if is_unowned_child_cancel(dispatch, order.client_order_index, &report) {
9162 return;
9163 }
9164
9165 if report
9166 .client_order_id
9167 .and_then(|id| dispatch.order_identity(&id))
9168 .is_some_and(|identity| {
9169 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
9170 })
9171 {
9172 return;
9173 }
9174
9175 if let Some(cloid) = &report.client_order_id {
9176 dispatch.venue_id_map.insert(*cloid, report.venue_order_id);
9177 }
9178
9179 if report.order_status.is_closed() {
9180 evict_terminal_mappings(&report, &dispatch.venue_id_map);
9181 }
9182
9183 log::debug!(
9184 "Lighter OrderStatusReport: voi={} status={:?} cloid={:?}",
9185 report.venue_order_id,
9186 report.order_status,
9187 report.client_order_id,
9188 );
9189 emitter.send_order_status_report(report);
9190 }
9191 Err(e) => {
9192 log::error!(
9193 "Failed to parse Lighter order status report: error={e}, order_id={}",
9194 order.order_id,
9195 );
9196 }
9197 }
9198}
9199
9200#[expect(
9201 clippy::too_many_arguments,
9202 reason = "tracked order publication uses the same identity and emitter context as stream dispatch"
9203)]
9204fn dispatch_owned_order(
9205 order: &crate::http::models::LighterOrder,
9206 instrument: &InstrumentAny,
9207 identity: &OrderIdentity,
9208 cloid: ClientOrderId,
9209 venue_order_id: VenueOrderId,
9210 dispatch: &WsDispatchState,
9211 emitter: &ExecutionEventEmitter,
9212 account_id: AccountId,
9213 trader_id: TraderId,
9214 ts_init: UnixNanos,
9215) {
9216 dispatch.venue_id_map.insert(cloid, venue_order_id);
9217
9218 let is_live_status = matches!(
9223 order.status,
9224 crate::common::enums::LighterOrderStatus::Pending
9225 | crate::common::enums::LighterOrderStatus::Open,
9226 );
9227 let pending_action = dispatch.pending_order_action(&cloid);
9228 if is_live_status && pending_action == Some(PendingOrderAction::Cancel) {
9229 log::debug!(
9230 "Deferring Lighter {:?} frame while cancel is pending for {cloid}",
9231 order.status,
9232 );
9233 return;
9234 }
9235
9236 let current_shape = match lighter_order_shape(order, instrument, identity.order_type) {
9237 Ok(shape) => shape,
9238 Err(e) => {
9239 log::error!(
9240 "Failed to compute Lighter order shape: error={e}, voi={venue_order_id}, cloid={cloid}",
9241 );
9242 return;
9243 }
9244 };
9245
9246 let prior_shape = dispatch.snapshot_for(&cloid);
9247 identity.record_reduction(order.initial_base_amount - current_shape.quantity.as_decimal());
9248 let shape_changed = prior_shape.map_or_else(
9249 || current_shape.quantity.as_decimal() < order.initial_base_amount,
9250 |previous| {
9251 identity.normalize_shape(previous) != identity.normalize_shape(current_shape.clone())
9252 },
9253 );
9254
9255 if shape_changed
9256 && !dispatch.accepted_was_emitted(&cloid)
9257 && (is_live_status || order.status == crate::common::enums::LighterOrderStatus::Filled)
9258 {
9259 ensure_accepted_emitted(
9260 cloid,
9261 venue_order_id,
9262 identity,
9263 account_id,
9264 trader_id,
9265 emitter,
9266 dispatch,
9267 ts_init,
9268 );
9269 }
9270
9271 let fresh_trigger = is_order_triggered(order)
9272 && TRIGGERABLE_ORDER_TYPES.contains(&identity.order_type)
9273 && !dispatch.triggered_was_emitted(&cloid);
9274
9275 if pending_action == Some(PendingOrderAction::Modify) && fresh_trigger && !shape_changed {
9276 log::debug!("Deferring Lighter trigger while modify confirmation is pending for {cloid}",);
9277 return;
9278 }
9279
9280 let open_ctx = OpenFrameContext {
9281 accepted_already_emitted: dispatch.accepted_was_emitted(&cloid),
9282 triggered_already_emitted: dispatch.triggered_was_emitted(&cloid),
9283 shape_changed,
9284 };
9285
9286 let event_opt = match parse_lighter_order_event(
9287 order, instrument, identity, cloid, account_id, trader_id, open_ctx, ts_init,
9288 ) {
9289 Ok(event) => event,
9290 Err(e) => {
9291 log::error!(
9292 "Failed to parse Lighter order event: error={e}, voi={venue_order_id}, cloid={cloid}"
9293 );
9294 return;
9295 }
9296 };
9297
9298 dispatch.store_snapshot(cloid, current_shape);
9299 if let Some(event) = event_opt {
9300 let Some(event) = defer_terminal(order, event, identity, cloid, dispatch) else {
9301 return;
9302 };
9303
9304 dispatch_tracked_order_event(
9305 event,
9306 cloid,
9307 venue_order_id,
9308 identity,
9309 account_id,
9310 trader_id,
9311 emitter,
9312 dispatch,
9313 ts_init,
9314 );
9315 }
9316
9317 if order.status == crate::common::enums::LighterOrderStatus::Filled {
9318 dispatch.clear_pending_order_action(&cloid);
9319 dispatch.retire_order_identity(&cloid);
9320 }
9321}
9322
9323#[expect(
9327 clippy::too_many_arguments,
9328 reason = "consumption-loop dispatch threads identity and emitter context"
9329)]
9330fn dispatch_lighter_trade(
9331 trade: &crate::http::models::LighterTrade,
9332 dispatch: &WsDispatchState,
9333 emitter: &ExecutionEventEmitter,
9334 registry: &Arc<MarketRegistry>,
9335 account_id: AccountId,
9336 trader_id: TraderId,
9337 account_index: Option<i64>,
9338 ts_init: UnixNanos,
9339) {
9340 let mut events = dispatch.order_events.lock();
9341
9342 let Some(account_index) = account_index else {
9343 log::debug!("Lighter trade frame dropped: no credential / account_index available",);
9344 return;
9345 };
9346
9347 let Some(instrument_id) = registry.instrument_id(trade.market_id) else {
9348 log::debug!(
9349 "Lighter trade frame dropped: no instrument for market_id={}",
9350 trade.market_id,
9351 );
9352 return;
9353 };
9354
9355 dispatch.note_active_market(trade.market_id);
9356
9357 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE
9358 .get(&instrument_id)
9359 .map(|instrument| instrument.value().clone())
9360 else {
9361 log::debug!("Lighter trade frame dropped: instrument {instrument_id} not in cache",);
9362 return;
9363 };
9364
9365 let user_is_bidder = trade.bid_account_id == account_index;
9366 let user_is_asker = trade.ask_account_id == account_index;
9367 if !user_is_bidder && !user_is_asker {
9368 return;
9371 }
9372
9373 let trade_id = match parse_lighter_trade_id(trade) {
9376 Ok(id) => id,
9377 Err(e) => {
9378 log::error!("Lighter trade has invalid trade_id: {e}");
9379 return;
9380 }
9381 };
9382
9383 let (raw_client_id, client_order_index, venue_order_id) = if user_is_bidder {
9384 (
9385 trade
9386 .bid_client_id_str
9387 .as_deref()
9388 .map_or_else(|| trade.bid_client_id.to_string(), str::to_string),
9389 trade.bid_client_id,
9390 trade.bid_id_str.as_deref().map_or_else(
9391 || VenueOrderId::new(trade.bid_id.to_string()),
9392 VenueOrderId::new,
9393 ),
9394 )
9395 } else {
9396 (
9397 trade
9398 .ask_client_id_str
9399 .as_deref()
9400 .map_or_else(|| trade.ask_client_id.to_string(), str::to_string),
9401 trade.ask_client_id,
9402 trade.ask_id_str.as_deref().map_or_else(
9403 || VenueOrderId::new(trade.ask_id.to_string()),
9404 VenueOrderId::new,
9405 ),
9406 )
9407 };
9408
9409 if !dispatch.restore_cached_order(venue_order_id, client_order_index, instrument_id) {
9410 return;
9411 }
9412
9413 let resolved_cloid = dispatch.resolve_live_trade_cloid(raw_client_id.as_str(), venue_order_id);
9414 let identity = resolved_cloid.and_then(|cid| {
9415 dispatch
9416 .order_identity(&cid)
9417 .map(|identity| (cid, identity))
9418 });
9419
9420 let Some((cloid, identity)) = identity else {
9421 match parse_ws_fill_report(trade, account_index, &instrument, account_id, ts_init) {
9422 Ok(Some(mut report)) => {
9423 report = dispatch.translate_fill_cloid(report);
9424 log::debug!(
9425 "Lighter FillReport: voi={} qty={} px={} liq={:?} cloid={:?}",
9426 report.venue_order_id,
9427 report.last_qty,
9428 report.last_px,
9429 report.liquidity_side,
9430 report.client_order_id,
9431 );
9432
9433 if dispatch.mark_trade_seen(trade_id) {
9434 emitter.send_fill_report(report);
9435 }
9436 }
9437 Ok(None) => {}
9438 Err(e) => {
9439 log::error!("Failed to parse Lighter fill report: error={e}, trade_id={trade_id}",);
9440 }
9441 }
9442
9443 return;
9444 };
9445
9446 if identity.instrument_id != instrument_id {
9447 return;
9448 }
9449
9450 dispatch_owned_trade(
9451 trade,
9452 &instrument,
9453 &identity,
9454 cloid,
9455 venue_order_id,
9456 &mut events,
9457 dispatch,
9458 emitter,
9459 account_id,
9460 trader_id,
9461 account_index,
9462 trade_id,
9463 ts_init,
9464 );
9465}
9466
9467#[expect(
9468 clippy::too_many_arguments,
9469 reason = "tracked trade publication uses the stream identity and emitter context"
9470)]
9471fn dispatch_owned_trade(
9472 trade: &crate::http::models::LighterTrade,
9473 instrument: &InstrumentAny,
9474 identity: &OrderIdentity,
9475 cloid: ClientOrderId,
9476 venue_order_id: VenueOrderId,
9477 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
9478 dispatch: &WsDispatchState,
9479 emitter: &ExecutionEventEmitter,
9480 account_id: AccountId,
9481 trader_id: TraderId,
9482 account_index: i64,
9483 trade_id: TradeId,
9484 ts_init: UnixNanos,
9485) {
9486 if identity.has_inferred_fills() {
9487 log::warn!(
9488 "Withholding Lighter trade {trade_id} for {cloid}: inferred fills require reconciliation"
9489 );
9490 return;
9491 }
9492
9493 match parse_lighter_order_filled(
9494 trade,
9495 instrument,
9496 identity,
9497 cloid,
9498 account_id,
9499 trader_id,
9500 account_index,
9501 ts_init,
9502 ) {
9503 Ok(Some(filled)) => {
9504 ensure_accepted_emitted(
9505 cloid,
9506 venue_order_id,
9507 identity,
9508 account_id,
9509 trader_id,
9510 emitter,
9511 dispatch,
9512 ts_init,
9513 );
9514
9515 log::debug!(
9516 "Lighter OrderFilled: voi={} qty={} px={} liq={:?} cloid={cloid}",
9517 filled.venue_order_id,
9518 filled.last_qty,
9519 filled.last_px,
9520 filled.liquidity_side,
9521 );
9522
9523 if let Some(baseline) = identity.child_fill_baseline() {
9524 if baseline.trade_ids.contains(&trade_id) {
9525 return;
9526 }
9527
9528 dispatch.defer_fill(events.entry(cloid).or_default(), filled);
9529 publish_child_recovery(cloid, events, dispatch, emitter);
9530 } else if dispatch.has_pending_modify(&cloid)
9531 || position_fills_pending(identity, events, dispatch)
9532 {
9533 dispatch.defer_fill(events.entry(cloid).or_default(), filled);
9534 } else if dispatch.mark_trade_seen(trade_id) {
9535 dispatch_lighter_fill(filled, emitter, dispatch, events);
9536 }
9537 }
9538 Ok(None) => {}
9539 Err(e) => {
9540 log::error!("Failed to parse Lighter typed fill: error={e}, trade_id={trade_id}",);
9541 }
9542 }
9543}
9544
9545fn spawn_child_recoveries(context: &AckedOrderProbeContext) {
9546 let candidates = context
9547 .dispatch
9548 .order_events
9549 .lock()
9550 .iter()
9551 .filter_map(|(cloid, deferred)| (!deferred.fills.is_empty()).then_some(*cloid))
9552 .collect::<Vec<_>>();
9553
9554 for cloid in candidates {
9555 spawn_child_recovery(cloid, context);
9556 }
9557}
9558
9559fn request_child_report_recoveries(
9560 core: &ExecutionClientCore,
9561 context: &AckedOrderProbeContext,
9562 sweep: &OrderReportSweep,
9563) {
9564 for (index, report) in &sweep.orders {
9565 if !core
9566 .cache()
9567 .client_order_id(&report.venue_order_id)
9568 .is_some_and(|cloid| context.dispatch.has_pending_modify(cloid))
9569 {
9570 restore_reconciled_order(
9571 core,
9572 &context.dispatch,
9573 report.venue_order_id,
9574 *index,
9575 report.order_status.is_closed(),
9576 );
9577 }
9578
9579 request_child_recovery(core, context, report);
9580 }
9581}
9582
9583fn request_child_recovery(
9584 core: &ExecutionClientCore,
9585 context: &AckedOrderProbeContext,
9586 report: &OrderStatusReport,
9587) {
9588 let cache = core.cache();
9589
9590 let Some(cloid) = cache.client_order_id(&report.venue_order_id).copied() else {
9591 return;
9592 };
9593
9594 let Some(order) = cache.order(&cloid) else {
9595 return;
9596 };
9597
9598 if order.parent_order_id().is_some()
9599 && !order.is_closed()
9600 && (report.order_status.is_closed() || report.filled_qty != order.filled_qty())
9601 {
9602 spawn_child_recovery(cloid, context);
9603 }
9604}
9605
9606fn spawn_child_recovery(cloid: ClientOrderId, context: &AckedOrderProbeContext) {
9607 let epoch = context.connection_epoch.load(Ordering::Acquire);
9608 {
9609 let mut events = context.dispatch.order_events.lock();
9610
9611 let Some(identity) = context.dispatch.order_identity(&cloid) else {
9612 return;
9613 };
9614
9615 if identity.child_fill_baseline().is_none()
9616 || identity.has_inferred_fills()
9617 || identity.has_child_publication()
9618 || context.dispatch.has_pending_modify(&cloid)
9619 {
9620 return;
9621 }
9622
9623 let deferred = events.entry(cloid).or_default();
9624 if deferred.recovery_epoch == Some(epoch) {
9625 return;
9626 }
9627
9628 deferred.recovery_epoch = Some(epoch);
9629 }
9630
9631 let context = context.clone();
9632 let tasks = context.pending_tasks.clone();
9633 let dispatch = context.dispatch.clone();
9634
9635 let future = async move {
9636 for attempt in 0..ACKED_ORDER_PROBE_ATTEMPTS {
9637 if context.connection_epoch.load(Ordering::Acquire) != epoch {
9638 break;
9639 }
9640
9641 let result = tokio::select! {
9642 () = context.cancellation_token.cancelled() => break,
9643 result = recover_child_order(cloid, epoch, &context) => result,
9644 };
9645
9646 match result {
9647 Ok(true) => break,
9648 Ok(false) => {}
9649 Err(e) => log::warn!(
9650 "Lighter child recovery unavailable for {cloid}: {}",
9651 scrub_auth(&format!("{e:#}"))
9652 ),
9653 }
9654
9655 if attempt + 1 < ACKED_ORDER_PROBE_ATTEMPTS {
9656 tokio::select! {
9657 () = context.cancellation_token.cancelled() => break,
9658 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {},
9659 }
9660 }
9661 }
9662
9663 let mut events = context.dispatch.order_events.lock();
9664 if let Some(deferred) = events.get_mut(&cloid)
9665 && deferred.recovery_epoch == Some(epoch)
9666 {
9667 deferred.recovery_epoch = None;
9668 }
9669 };
9670
9671 if let Err(e) = tasks.spawn(future) {
9672 if let Some(deferred) = dispatch.order_events.lock().get_mut(&cloid) {
9673 deferred.recovery_epoch = None;
9674 }
9675
9676 log::debug!("Skipping Lighter child recovery during shutdown: {e}");
9677 }
9678}
9679
9680async fn recover_child_order(
9681 cloid: ClientOrderId,
9682 epoch: u64,
9683 context: &AckedOrderProbeContext,
9684) -> anyhow::Result<bool> {
9685 let Some(identity) = context.dispatch.order_identity(&cloid) else {
9686 return Ok(true);
9687 };
9688
9689 if context.dispatch.has_pending_modify(&cloid)
9690 || identity.has_inferred_fills()
9691 || identity.has_child_publication()
9692 {
9693 return Ok(true);
9694 }
9695
9696 let Some(venue_order_id) = identity.venue_order_id() else {
9697 return Ok(false);
9698 };
9699
9700 let lookup = OrderReportLookup::new(
9701 &context.registry,
9702 &context.dispatch,
9703 Some(identity.instrument_id),
9704 Some(&cloid),
9705 Some(&venue_order_id),
9706 )?;
9707
9708 let Some(order) = collect_order(
9709 &context.http_client,
9710 &context.registry,
9711 &context.credential,
9712 &lookup,
9713 )
9714 .await?
9715 else {
9716 return Ok(false);
9717 };
9718
9719 anyhow::ensure!(
9720 order.owner_account_index == context.credential.account_index()
9721 && context.registry.instrument_id(order.market_index) == Some(identity.instrument_id)
9722 && identity.matches_venue_order_id(VenueOrderId::new(&order.order_id)),
9723 "Lighter child recovery ownership mismatch"
9724 );
9725 let instrument = LIGHTER_INSTRUMENT_CACHE
9726 .get(&identity.instrument_id)
9727 .context("missing child recovery instrument")?
9728 .value()
9729 .clone();
9730 let report = parse_ws_order_status_report(
9731 &order,
9732 &instrument,
9733 context.account_id,
9734 context.clock.get_time_ns(),
9735 )?;
9736
9737 if !report.order_status.is_closed() {
9738 return Ok(false);
9739 }
9740
9741 let mut instruments = AHashMap::new();
9742 instruments.insert(identity.instrument_id, instrument.clone());
9743
9744 let command = GenerateFillReports::new(
9745 UUID4::new(),
9746 context.clock.get_time_ns(),
9747 Some(identity.instrument_id),
9748 Some(venue_order_id),
9749 None,
9750 None,
9751 None,
9752 None,
9753 );
9754 let sweep = collect_fill_reports(
9755 &context.http_client,
9756 &context.registry,
9757 &context.credential,
9758 context.account_id,
9759 context.clock,
9760 &command,
9761 &instruments,
9762 &mut BTreeSet::new(),
9763 Some(order.order_index),
9764 )
9765 .await?;
9766 let mut events = context.dispatch.order_events.lock();
9767 if context.connection_epoch.load(Ordering::Acquire) != epoch
9768 || context.dispatch.has_pending_modify(&cloid)
9769 || identity.has_inferred_fills()
9770 || identity.has_child_publication()
9771 {
9772 return Ok(true);
9773 }
9774
9775 let deferred = events.entry(cloid).or_default();
9776 deferred.store_order(&order);
9777
9778 for mut fill in sweep.reports {
9779 fill.client_order_id = Some(cloid);
9780 if fill.venue_order_id != venue_order_id {
9781 continue;
9782 }
9783
9784 if let Some(fill) = child_fill_from_report(&fill, &identity, context.emitter.trader_id()) {
9785 context.dispatch.defer_fill(deferred, fill);
9786 }
9787 }
9788
9789 publish_child_recovery(cloid, &mut events, &context.dispatch, &context.emitter);
9790 Ok(identity.has_child_publication())
9791}
9792
9793fn child_fill_from_report(
9794 report: &FillReport,
9795 identity: &OrderIdentity,
9796 trader_id: TraderId,
9797) -> Option<OrderFilled> {
9798 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id)?;
9799 let cloid = report.client_order_id.filter(|_| {
9800 identity.matches_venue_order_id(report.venue_order_id)
9801 && identity.instrument_id == report.instrument_id
9802 })?;
9803
9804 Some(OrderFilled::new(
9805 trader_id,
9806 identity.strategy_id,
9807 identity.instrument_id,
9808 cloid,
9809 report.venue_order_id,
9810 report.account_id,
9811 report.trade_id,
9812 report.order_side,
9813 identity.order_type,
9814 report.last_qty,
9815 report.last_px,
9816 instrument.quote_currency(),
9817 report.liquidity_side,
9818 UUID4::new(),
9819 report.ts_event,
9820 report.ts_init,
9821 false,
9822 None,
9823 Some(report.commission),
9824 None,
9825 ))
9826}
9827
9828fn publish_child_recovery(
9829 cloid: ClientOrderId,
9830 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
9831 dispatch: &WsDispatchState,
9832 emitter: &ExecutionEventEmitter,
9833) {
9834 let Some(identity) = dispatch.order_identity(&cloid) else {
9835 return;
9836 };
9837
9838 if position_modify_pending(&identity, dispatch) {
9839 return;
9840 }
9841
9842 let Some((ids, children)) = collect_child_recoveries(&identity, events, dispatch, emitter)
9843 else {
9844 return;
9845 };
9846
9847 let mut fills = Vec::new();
9848 let mut terminals = Vec::new();
9849
9850 for id in ids {
9851 let Some(mut deferred) = events.remove(&id) else {
9852 continue;
9853 };
9854
9855 fills.extend(deferred.fills.drain(..).map(|fill| {
9856 let sequence = deferred.fill_sequence.remove(&fill.trade_id).unwrap_or(0);
9857 (fill, sequence)
9858 }));
9859
9860 terminals.push((id, deferred));
9861 }
9862
9863 publish_child_updates(&children, &terminals, emitter);
9864
9865 fills.sort_by_key(|(fill, sequence)| (fill.ts_event, *sequence));
9866 for (fill, _) in fills {
9867 if dispatch.mark_trade_seen(fill.trade_id) {
9868 dispatch_lighter_fill(fill, emitter, dispatch, events);
9869 }
9870 }
9871
9872 for child in children {
9873 let id = child.cloid;
9874 let bundle = ExecutionReport::OrderWithFills(Box::new(child.report.clone()), Vec::new());
9875
9876 if let Err(e) = emitter.try_send_execution_report(bundle) {
9877 log::warn!("Lighter child report publication failed for {id}: {e}");
9878
9879 let Some(index) = terminals.iter().position(|(cid, _)| *cid == id) else {
9880 continue;
9881 };
9882
9883 let (_, deferred) = terminals.swap_remove(index);
9884 events.insert(id, deferred);
9885
9886 continue;
9887 }
9888
9889 child.identity.publish_child(&child.report, child.trade_ids);
9890 dispatch.retire_order_identity(&id);
9891 }
9892
9893 for (id, deferred) in terminals {
9894 if dispatch
9895 .order_identity(&id)
9896 .is_some_and(|identity| identity.child_fill_baseline().is_none())
9897 {
9898 flush_order_terminal(id, deferred, dispatch, emitter, events);
9899 }
9900 }
9901}
9902
9903struct ChildRecovery {
9904 cloid: ClientOrderId,
9905 identity: OrderIdentity,
9906 report: OrderStatusReport,
9907 trade_ids: Vec<TradeId>,
9908}
9909
9910fn collect_child_recoveries(
9911 identity: &OrderIdentity,
9912 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
9913 dispatch: &WsDispatchState,
9914 emitter: &ExecutionEventEmitter,
9915) -> Option<(Vec<ClientOrderId>, Vec<ChildRecovery>)> {
9916 let mut children = Vec::new();
9917 let mut ids = Vec::new();
9918
9919 for (id, deferred) in events {
9920 let Some(other) = dispatch.order_identity(id) else {
9921 continue;
9922 };
9923
9924 if other.strategy_id != identity.strategy_id
9925 || other.instrument_id != identity.instrument_id
9926 {
9927 continue;
9928 }
9929
9930 if dispatch.has_pending_modify(id) {
9931 return None;
9932 }
9933
9934 if other.child_fill_baseline().is_none() {
9935 ids.push(*id);
9936 continue;
9937 }
9938
9939 if deferred.fills.is_empty()
9940 && deferred
9941 .order
9942 .as_ref()
9943 .is_none_or(|order| !order.status.is_terminal())
9944 {
9945 continue;
9946 }
9947
9948 let report = child_recovery_report(*id, events, dispatch, emitter)?;
9949 let trade_ids = deferred.fills.iter().map(|fill| fill.trade_id).collect();
9950 children.push(ChildRecovery {
9951 cloid: *id,
9952 identity: other,
9953 report,
9954 trade_ids,
9955 });
9956
9957 ids.push(*id);
9958 }
9959
9960 Some((ids, children))
9961}
9962
9963fn publish_child_updates(
9964 children: &[ChildRecovery],
9965 terminals: &[(ClientOrderId, DeferredOrderEvents)],
9966 emitter: &ExecutionEventEmitter,
9967) {
9968 for child in children {
9969 if child.trade_ids.is_empty() {
9970 continue;
9971 }
9972
9973 let Some(order) = terminals
9974 .iter()
9975 .find(|(cid, _)| *cid == child.cloid)
9976 .and_then(|(_, deferred)| deferred.order.as_ref())
9977 else {
9978 continue;
9979 };
9980
9981 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE.get(&child.identity.instrument_id) else {
9982 continue;
9983 };
9984
9985 if let Ok(updated) = parse_lighter_order_updated(
9986 order,
9987 &instrument,
9988 &child.identity,
9989 child.cloid,
9990 emitter.account_id(),
9991 emitter.trader_id(),
9992 child.report.ts_init,
9993 ) {
9994 emitter.send_order_event(OrderEventAny::Updated(updated));
9995 }
9996 }
9997}
9998
9999fn position_fills_pending(
10000 identity: &OrderIdentity,
10001 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
10002 dispatch: &WsDispatchState,
10003) -> bool {
10004 position_modify_pending(identity, dispatch)
10005 || events.iter().any(|(id, deferred)| {
10006 !deferred.fills.is_empty()
10007 && dispatch.order_identity(id).is_some_and(|other| {
10008 other.strategy_id == identity.strategy_id
10009 && other.instrument_id == identity.instrument_id
10010 })
10011 })
10012}
10013
10014fn position_modify_pending(identity: &OrderIdentity, dispatch: &WsDispatchState) -> bool {
10015 dispatch.order_identities.iter().any(|entry| {
10016 entry.value().strategy_id == identity.strategy_id
10017 && entry.value().instrument_id == identity.instrument_id
10018 && dispatch.has_pending_modify(entry.key())
10019 })
10020}
10021
10022fn child_recovery_report(
10023 cloid: ClientOrderId,
10024 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
10025 dispatch: &WsDispatchState,
10026 emitter: &ExecutionEventEmitter,
10027) -> Option<OrderStatusReport> {
10028 let identity = dispatch.order_identity(&cloid)?;
10029 if identity.has_child_publication() || identity.has_inferred_fills() {
10030 return None;
10031 }
10032
10033 let baseline = identity.child_fill_baseline()?;
10034 let deferred = events.get(&cloid)?;
10035 let order = deferred.order.as_ref()?;
10036 if !identity.matches_venue_order_id(VenueOrderId::new(&order.order_id)) {
10037 return None;
10038 }
10039
10040 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id)?;
10041 let report = parse_ws_order_status_report(
10042 order,
10043 &instrument,
10044 emitter.account_id(),
10045 get_atomic_clock_realtime().get_time_ns(),
10046 )
10047 .ok()?;
10048 let report = dispatch.translate_order_cloid(dispatch.apply_order_contingencies(order, report));
10049 if !report.order_status.is_closed() || report.filled_qty > report.quantity {
10050 return None;
10051 }
10052
10053 let mut total = baseline.quantity.as_decimal();
10054 for fill in &deferred.fills {
10055 if baseline.trade_ids.contains(&fill.trade_id) {
10056 continue;
10057 }
10058
10059 if fill.venue_order_id != report.venue_order_id
10060 || fill.instrument_id != report.instrument_id
10061 || fill.account_id != report.account_id
10062 || Some(fill.order_side) != report.order_side
10063 || fill.commission.is_none()
10064 {
10065 return None;
10066 }
10067
10068 total += fill.last_qty.as_decimal();
10069 }
10070
10071 (total == report.filled_qty.as_decimal()).then_some(report)
10072}
10073
10074fn dispatch_lighter_fill(
10075 filled: OrderFilled,
10076 emitter: &ExecutionEventEmitter,
10077 dispatch: &WsDispatchState,
10078 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
10079) {
10080 let cloid = filled.client_order_id;
10081 if let Some(identity) = dispatch.order_identity(&cloid)
10082 && !identity.publish_fill(&filled)
10083 {
10084 return;
10085 }
10086
10087 ensure_fill_triggered(&filled, emitter, dispatch);
10088 emitter.send_order_event(OrderEventAny::Filled(filled));
10089 release_terminal(cloid, false, dispatch, emitter, events);
10090}
10091
10092fn defer_terminal(
10093 order: &LighterOrder,
10094 event: ParsedOrderEvent,
10095 identity: &OrderIdentity,
10096 cloid: ClientOrderId,
10097 dispatch: &WsDispatchState,
10098) -> Option<ParsedOrderEvent> {
10099 if !matches!(
10100 event,
10101 ParsedOrderEvent::Canceled(_) | ParsedOrderEvent::Expired(_)
10102 ) {
10103 return Some(event);
10104 }
10105
10106 let mut pending = dispatch.terminals_pending.lock();
10107 if pending.contains_key(&cloid) {
10108 return None;
10109 }
10110
10111 if order.filled_base_amount <= identity.filled_qty() {
10112 return Some(event);
10113 }
10114
10115 pending.insert(
10116 cloid,
10117 PendingTerminal {
10118 order: order.clone(),
10119 event,
10120 retry_at: tokio::time::Instant::now() + TERMINAL_FILL_DELAY,
10121 attempts: 0,
10122 },
10123 );
10124
10125 None
10126}
10127
10128fn release_terminal(
10129 cloid: ClientOrderId,
10130 force: bool,
10131 dispatch: &WsDispatchState,
10132 emitter: &ExecutionEventEmitter,
10133 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
10134) {
10135 let Some(identity) = dispatch.order_identity(&cloid) else {
10136 return;
10137 };
10138
10139 let terminal = {
10140 let mut pending = dispatch.terminals_pending.lock();
10141
10142 let Some(terminal) = pending.get(&cloid) else {
10143 return;
10144 };
10145
10146 if (identity.child_fill_baseline().is_some() && !identity.has_inferred_fills())
10147 || dispatch.has_pending_modify(&cloid)
10148 || position_fills_pending(&identity, events, dispatch)
10149 {
10150 return;
10151 }
10152
10153 if !force && identity.filled_qty() < terminal.order.filled_base_amount {
10154 return;
10155 }
10156
10157 pending.remove(&cloid).expect("pending terminal exists")
10158 };
10159
10160 let missing_qty = terminal.order.filled_base_amount - identity.filled_qty();
10161 if missing_qty > Decimal::ZERO {
10162 log::error!(
10163 "Lighter terminal fill recovery exhausted for {cloid}, venue_order_id={}, missing quantity={missing_qty}; releasing terminal for reconciliation",
10164 terminal.order.order_id,
10165 );
10166 }
10167
10168 dispatch_tracked_order_event(
10169 terminal.event,
10170 cloid,
10171 VenueOrderId::new(&terminal.order.order_id),
10172 &identity,
10173 emitter.account_id(),
10174 emitter.trader_id(),
10175 emitter,
10176 dispatch,
10177 get_atomic_clock_realtime().get_time_ns(),
10178 );
10179}
10180
10181async fn run_terminal_recovery(
10182 context: AckedOrderProbeContext,
10183 producers_stopped: CancellationToken,
10184) {
10185 let mut interval = tokio::time::interval(Duration::from_millis(100));
10186 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
10187 let mut immediate = false;
10188
10189 loop {
10190 tokio::select! {
10191 biased;
10192 () = context.cancellation_token.cancelled() => break,
10193 () = context.dispatch.terminals_recover.notified() => immediate = true,
10194 _ = interval.tick() => {},
10195 }
10196
10197 tokio::select! {
10198 biased;
10199 () = context.cancellation_token.cancelled() => break,
10200 () = context.dispatch.terminals_recover.notified() => {
10201 immediate = true;
10202 continue;
10203 }
10204 () = recover_terminals(&context, immediate) => {},
10205 }
10206 immediate = false;
10207 }
10208
10209 producers_stopped.cancelled().await;
10210
10211 let drain = async {
10212 for _ in 0..ACKED_ORDER_PROBE_ATTEMPTS {
10213 recover_terminals(&context, true).await;
10214 if context.dispatch.terminals_pending.lock().is_empty() {
10215 break;
10216 }
10217 }
10218 };
10219
10220 let _ = tokio::time::timeout(TERMINAL_SHUTDOWN_TIMEOUT, drain).await;
10221 let events = context.dispatch.order_events.lock();
10222 let remaining = context
10223 .dispatch
10224 .terminals_pending
10225 .lock()
10226 .keys()
10227 .copied()
10228 .collect::<Vec<_>>();
10229
10230 for cloid in remaining {
10231 release_terminal(cloid, true, &context.dispatch, &context.emitter, &events);
10232 }
10233}
10234
10235async fn recover_terminals(context: &AckedOrderProbeContext, immediate: bool) {
10236 let now = tokio::time::Instant::now();
10237 let candidates = context
10238 .dispatch
10239 .terminals_pending
10240 .lock()
10241 .iter()
10242 .filter(|(_, terminal)| immediate || terminal.retry_at <= now)
10243 .map(|(cloid, terminal)| {
10244 (
10245 *cloid,
10246 terminal.order.clone(),
10247 terminal.attempts >= ACKED_ORDER_PROBE_ATTEMPTS,
10248 )
10249 })
10250 .collect::<Vec<_>>();
10251
10252 stream::iter(candidates)
10253 .for_each_concurrent(4, |(cloid, order, exhausted)| async move {
10254 {
10255 let events = context.dispatch.order_events.lock();
10256 release_terminal(
10257 cloid,
10258 exhausted,
10259 &context.dispatch,
10260 &context.emitter,
10261 &events,
10262 );
10263
10264 if exhausted {
10265 return;
10266 }
10267
10268 let mut pending = context.dispatch.terminals_pending.lock();
10269
10270 let Some(terminal) = pending.get_mut(&cloid) else {
10271 return;
10272 };
10273
10274 terminal.attempts += 1;
10275 }
10276
10277 let result = tokio::time::timeout(
10278 TERMINAL_RECOVERY_TIMEOUT,
10279 recover_terminal_trades(order.market_index, order.order_index, context),
10280 )
10281 .await;
10282
10283 match result {
10284 Ok(Ok(())) => {}
10285 Ok(Err(e)) => log::warn!(
10286 "Lighter terminal trade lookup failed for {cloid}: {}",
10287 scrub_auth(&format!("{e:#}"))
10288 ),
10289 Err(_) => log::warn!("Lighter terminal trade lookup timed out for {cloid}"),
10290 }
10291
10292 let events = context.dispatch.order_events.lock();
10293 release_terminal(cloid, false, &context.dispatch, &context.emitter, &events);
10294
10295 let exhausted = {
10296 let mut pending = context.dispatch.terminals_pending.lock();
10297
10298 let Some(terminal) = pending.get_mut(&cloid) else {
10299 return;
10300 };
10301
10302 terminal.retry_at = tokio::time::Instant::now() + TERMINAL_FILL_DELAY;
10303 terminal.attempts >= ACKED_ORDER_PROBE_ATTEMPTS
10304 };
10305
10306 if exhausted {
10307 release_terminal(cloid, true, &context.dispatch, &context.emitter, &events);
10308 }
10309 })
10310 .await;
10311}
10312
10313async fn recover_filled_reports(sweep: &OrderReportSweep, context: &AckedOrderProbeContext) {
10314 for (index, report) in &sweep.orders {
10315 if report.order_status != OrderStatus::Filled
10316 || !context.dispatch.restore_cached_order(
10317 report.venue_order_id,
10318 *index,
10319 report.instrument_id,
10320 )
10321 {
10322 continue;
10323 }
10324
10325 let Some(cloid) = context
10326 .dispatch
10327 .resolve_client_order_index_for_venue(*index, report.venue_order_id)
10328 else {
10329 continue;
10330 };
10331
10332 let Some(identity) = context.dispatch.order_identity(&cloid) else {
10333 continue;
10334 };
10335
10336 if identity.child_fill_baseline().is_some()
10337 || identity.has_inferred_fills()
10338 || report.filled_qty.as_decimal() <= identity.filled_qty()
10339 {
10340 continue;
10341 }
10342
10343 let Some(market_index) = context.registry.market_index(&report.instrument_id) else {
10344 continue;
10345 };
10346
10347 let result = async {
10348 let order_index = report
10349 .venue_order_id
10350 .as_str()
10351 .parse::<i64>()
10352 .context("invalid Lighter filled order index")?;
10353 tokio::time::timeout(
10354 TERMINAL_RECOVERY_TIMEOUT,
10355 recover_terminal_trades(market_index, order_index, context),
10356 )
10357 .await
10358 .context("Lighter filled-order trade lookup timed out")?
10359 }
10360 .await;
10361
10362 if let Err(e) = result {
10363 log::warn!(
10364 "Lighter filled-order recovery unavailable for {cloid}: {}",
10365 scrub_auth(&format!("{e:#}"))
10366 );
10367 }
10368 }
10369}
10370
10371async fn recover_report_fills(
10372 order: &LighterOrder,
10373 context: &AckedOrderProbeContext,
10374) -> anyhow::Result<()> {
10375 if !order.status.is_terminal() {
10376 return Ok(());
10377 }
10378
10379 let venue = VenueOrderId::new(&order.order_id);
10380
10381 let Some(instrument_id) = context.registry.instrument_id(order.market_index) else {
10382 return Ok(());
10383 };
10384
10385 if !context
10386 .dispatch
10387 .restore_cached_order(venue, order.client_order_index, instrument_id)
10388 {
10389 return Ok(());
10390 }
10391
10392 let Some(cloid) = context
10393 .dispatch
10394 .resolve_client_order_index_for_venue(order.client_order_index, venue)
10395 else {
10396 return Ok(());
10397 };
10398
10399 let missing = (order.status == LighterOrderStatus::Filled
10400 || context.dispatch.order_identities.contains_key(&cloid))
10401 && context
10402 .dispatch
10403 .order_identity(&cloid)
10404 .is_some_and(|identity| {
10405 order.filled_base_amount > identity.filled_qty()
10406 && (order.status != LighterOrderStatus::Filled
10407 || !identity.has_inferred_fills())
10408 });
10409
10410 if !missing {
10411 return Ok(());
10412 }
10413
10414 if order.status == LighterOrderStatus::Filled {
10415 tokio::time::timeout(
10416 TERMINAL_RECOVERY_TIMEOUT,
10417 recover_terminal_trades(order.market_index, order.order_index, context),
10418 )
10419 .await
10420 .context("Lighter filled-order trade lookup timed out")??;
10421 } else {
10422 dispatch_lighter_order(
10423 order,
10424 &context.dispatch,
10425 &context.emitter,
10426 &context.registry,
10427 context.account_id,
10428 context.emitter.trader_id(),
10429 context.clock.get_time_ns(),
10430 );
10431 }
10432
10433 Ok(())
10434}
10435
10436async fn recover_terminal_trades(
10437 market_index: i64,
10438 order_index: i64,
10439 context: &AckedOrderProbeContext,
10440) -> anyhow::Result<()> {
10441 let auth = build_auth_token_for(&context.credential)?;
10442
10443 let mut query = Zeroizing::new(LighterTradesQuery {
10444 authorization: None,
10445 auth: Some(auth),
10446 market_id: Some(market_index),
10447 account_index: Some(context.credential.account_index()),
10448 order_index: Some(order_index),
10449 order_index_str: None,
10450 integrator_account_index: None,
10451 sort_by: LighterTradeSortBy::Timestamp,
10452 sort_dir: Some(LighterSortDirection::Desc),
10453 cursor: None,
10454 from_timestamp: None,
10455 ask_filter: None,
10456 role: None,
10457 trade_type: None,
10458 limit: LIGHTER_REST_PAGE_SIZE,
10459 aggregate: None,
10460 });
10461
10462 let mut cursors = AHashSet::new();
10463 let mut trades = Vec::new();
10464
10465 for page in 0..MAX_RECONCILIATION_PAGES {
10466 let response = context.http_client.get_trades(&query).await?;
10467 for trade in response.trades {
10468 let account = context.credential.account_index();
10469 anyhow::ensure!(
10470 trade.market_id == market_index
10471 && ((trade.bid_account_id == account && trade.bid_id == order_index)
10472 || (trade.ask_account_id == account && trade.ask_id == order_index)),
10473 "Lighter terminal recovery returned a trade outside the requested order"
10474 );
10475 trades.push(trade);
10476 }
10477
10478 match response.next_cursor.filter(|cursor| !cursor.is_empty()) {
10479 Some(cursor) => {
10480 anyhow::ensure!(
10481 cursors.insert(cursor.clone()),
10482 "Lighter terminal recovery repeated cursor"
10483 );
10484 anyhow::ensure!(
10485 page + 1 < MAX_RECONCILIATION_PAGES,
10486 "Lighter terminal recovery exceeded page limit"
10487 );
10488 query.cursor = Some(cursor);
10489 }
10490 None => break,
10491 }
10492 }
10493
10494 trades.sort_by_key(|trade| (trade.timestamp, trade.trade_id));
10495 for trade in trades {
10496 dispatch_lighter_trade(
10497 &trade,
10498 &context.dispatch,
10499 &context.emitter,
10500 &context.registry,
10501 context.account_id,
10502 context.emitter.trader_id(),
10503 Some(context.credential.account_index()),
10504 context.clock.get_time_ns(),
10505 );
10506 }
10507
10508 Ok(())
10509}
10510
10511fn ensure_fill_triggered(
10512 filled: &OrderFilled,
10513 emitter: &ExecutionEventEmitter,
10514 dispatch: &WsDispatchState,
10515) {
10516 if TRIGGERABLE_ORDER_TYPES.contains(&filled.order_type)
10517 && dispatch.mark_triggered_emitted(filled.client_order_id)
10518 {
10519 let triggered = OrderTriggered::new(
10520 filled.trader_id,
10521 filled.strategy_id,
10522 filled.instrument_id,
10523 filled.client_order_id,
10524 UUID4::new(),
10525 filled.ts_event,
10526 filled.ts_init,
10527 false,
10528 Some(filled.venue_order_id),
10529 Some(filled.account_id),
10530 );
10531 emitter.send_order_event(OrderEventAny::Triggered(triggered));
10532 }
10533}
10534
10535#[expect(
10539 clippy::too_many_arguments,
10540 reason = "shared cleanup point across the typed-event variants"
10541)]
10542#[expect(
10543 clippy::needless_pass_by_value,
10544 reason = "event is destructured into typed OrderEventAny variants that consume the payload"
10545)]
10546fn dispatch_tracked_order_event(
10547 event: ParsedOrderEvent,
10548 cloid: ClientOrderId,
10549 venue_order_id: VenueOrderId,
10550 identity: &OrderIdentity,
10551 account_id: AccountId,
10552 trader_id: TraderId,
10553 emitter: &ExecutionEventEmitter,
10554 dispatch: &WsDispatchState,
10555 ts_init: UnixNanos,
10556) {
10557 let is_terminal;
10558
10559 match event {
10560 ParsedOrderEvent::Accepted(e) => {
10561 if !dispatch.claim_accepted_emission(&cloid) {
10562 log::debug!("Skipping duplicate OrderAccepted for {cloid}");
10563 return;
10564 }
10565
10566 is_terminal = false;
10567 emitter.send_order_event(OrderEventAny::Accepted(e));
10568 }
10569 ParsedOrderEvent::Triggered(e) => {
10570 if !dispatch.mark_triggered_emitted(cloid) {
10571 log::debug!("Skipping duplicate OrderTriggered for {cloid}");
10572 return;
10573 }
10574
10575 ensure_accepted_emitted(
10576 cloid,
10577 venue_order_id,
10578 identity,
10579 account_id,
10580 trader_id,
10581 emitter,
10582 dispatch,
10583 ts_init,
10584 );
10585 is_terminal = false;
10586 emitter.send_order_event(OrderEventAny::Triggered(e));
10587 }
10588 ParsedOrderEvent::Updated(e) => {
10589 dispatch.clear_pending_modify_if_update(&cloid, e.quantity, e.price, e.trigger_price);
10594 is_terminal = false;
10595 identity.publish_update(e.event_id);
10596 emitter.send_order_event(OrderEventAny::Updated(e));
10597 }
10598 ParsedOrderEvent::UpdatedThenTriggered { updated, triggered } => {
10599 if !dispatch.mark_triggered_emitted(cloid) {
10600 log::debug!("Skipping duplicate OrderTriggered for {cloid}");
10601 return;
10602 }
10603
10604 ensure_accepted_emitted(
10605 cloid,
10606 venue_order_id,
10607 identity,
10608 account_id,
10609 trader_id,
10610 emitter,
10611 dispatch,
10612 ts_init,
10613 );
10614 dispatch.clear_pending_modify_if_update(
10615 &cloid,
10616 updated.quantity,
10617 updated.price,
10618 updated.trigger_price,
10619 );
10620 is_terminal = false;
10621 identity.publish_update(updated.event_id);
10622 emitter.send_order_event(OrderEventAny::Updated(updated));
10623 emitter.send_order_event(OrderEventAny::Triggered(triggered));
10624 }
10625 ParsedOrderEvent::Canceled(e) => {
10626 ensure_accepted_emitted(
10627 cloid,
10628 venue_order_id,
10629 identity,
10630 account_id,
10631 trader_id,
10632 emitter,
10633 dispatch,
10634 ts_init,
10635 );
10636 is_terminal = true;
10637 emitter.send_order_event(OrderEventAny::Canceled(e));
10638 }
10639 ParsedOrderEvent::Expired(e) => {
10640 ensure_accepted_emitted(
10641 cloid,
10642 venue_order_id,
10643 identity,
10644 account_id,
10645 trader_id,
10646 emitter,
10647 dispatch,
10648 ts_init,
10649 );
10650 is_terminal = true;
10651 emitter.send_order_event(OrderEventAny::Expired(e));
10652 }
10653 ParsedOrderEvent::Rejected(e) => {
10654 is_terminal = true;
10655 emitter.send_order_event(OrderEventAny::Rejected(e));
10656 }
10657 }
10658
10659 if is_terminal {
10660 dispatch.clear_pending_order_action(&cloid);
10661 dispatch.retire_order_identity(&cloid);
10662 }
10663}
10664
10665#[expect(
10668 clippy::too_many_arguments,
10669 reason = "synthesized events need the full identity context to populate the event"
10670)]
10671fn ensure_accepted_emitted(
10672 cloid: ClientOrderId,
10673 venue_order_id: VenueOrderId,
10674 identity: &OrderIdentity,
10675 account_id: AccountId,
10676 trader_id: TraderId,
10677 emitter: &ExecutionEventEmitter,
10678 dispatch: &WsDispatchState,
10679 ts_init: UnixNanos,
10680) {
10681 if !dispatch.claim_accepted_emission(&cloid) {
10682 return;
10683 }
10684
10685 let accepted = OrderAccepted::new(
10686 trader_id,
10687 identity.strategy_id,
10688 identity.instrument_id,
10689 cloid,
10690 venue_order_id,
10691 account_id,
10692 UUID4::new(),
10693 ts_init,
10694 ts_init,
10695 false,
10696 );
10697 emitter.send_order_event(OrderEventAny::Accepted(accepted));
10698}
10699
10700#[cfg(test)]
10701#[path = "../tests/integration/exec_client.rs"]
10702mod exec_client_tests;
10703
10704#[cfg(test)]
10705mod tests {
10706 use std::{
10707 cell::RefCell,
10708 rc::Rc,
10709 sync::{Arc, atomic::AtomicUsize},
10710 };
10711
10712 use axum::{
10713 Router,
10714 extract::Query,
10715 http::HeaderMap,
10716 response::IntoResponse,
10717 routing::{get, post},
10718 };
10719 use nautilus_common::{
10720 cache::Cache,
10721 clock::{Clock, VirtualClock},
10722 factories::OrderFactory,
10723 live::runner::replace_exec_event_sender,
10724 messages::{ExecutionEvent, ExecutionReport as EngineExecutionReport},
10725 testing::wait_until_async,
10726 };
10727 use nautilus_execution::engine::ExecutionEngine;
10728 use nautilus_live::manager::{ExecutionManager, ExecutionManagerConfig};
10729 use nautilus_model::{
10730 accounts::MarginAccount,
10731 data::QuoteTick,
10732 enums::{ContingencyType, LiquiditySide, OrderSide, OrderStatus, TimeInForce},
10733 events::{
10734 AccountState, OrderCanceled, OrderEventAny, OrderPendingCancel, OrderSubmitted,
10735 OrderTriggered,
10736 },
10737 identifiers::{
10738 InstrumentId, OrderListId, PositionId, StrategyId, Symbol, TradeId, TraderId,
10739 VenueOrderId,
10740 },
10741 instruments::CryptoPerpetual,
10742 orders::{LimitOrder, OrderList, OrderTestBuilder, stubs::TestOrderEventStubs},
10743 position::Position,
10744 types::{Currency, Money, Price},
10745 };
10746 use rstest::rstest;
10747
10748 use super::*;
10749 use crate::{
10750 common::{
10751 consts::{
10752 LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
10753 LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX, LIGHTER_VENUE,
10754 },
10755 enums::{LighterDeployment, LighterEnvironment, LighterProductType},
10756 },
10757 http::models::{LighterNextNonce, LighterOrders, LighterTx},
10758 signing::tx::TX_HASH_BYTES,
10759 };
10760
10761 const TEST_PRIVATE_KEY: &str =
10762 "0b8e0f63c24d8baacd9d29ad4e9a4b73c4a8d2bb8b16dc4fa9d7c2e1d3a8b1f0e8d3a4c5b6e7f001";
10763 const TEST_ACCOUNT_INDEX: u64 = 12345;
10764 const TEST_ACCOUNT_INDEX_I64: i64 = 12345;
10765 const TEST_API_KEY_INDEX: u8 = 5;
10766 const TEST_NEXT_NONCE: i64 = 42;
10767 const TEST_MARKET_INDEX: i64 = 0;
10768 const TEST_ORDER_NONCE: i64 = 281_474_720_725_346;
10769 const TEST_SUBMISSION_NONCE: i64 = 2_042;
10770
10771 fn handle_send_tx_ack(
10772 dispatch: &WsDispatchState,
10773 account_index: Option<i64>,
10774 code: i64,
10775 tx_hash: Option<&str>,
10776 ) -> Option<PendingSendTx> {
10777 handle_send_tx_ack_for_connection(dispatch, account_index, 0, code, tx_hash)
10778 }
10779
10780 #[expect(
10781 clippy::too_many_arguments,
10782 reason = "test wrapper preserves the established rejection call shape"
10783 )]
10784 fn handle_send_tx_rejection(
10785 dispatch: &WsDispatchState,
10786 emitter: &ExecutionEventEmitter,
10787 account_index: Option<i64>,
10788 now: UnixNanos,
10789 source: SendTxRejectionSource,
10790 code: Option<i64>,
10791 message: &str,
10792 tx_hash: Option<&str>,
10793 ) -> bool {
10794 handle_send_tx_rejection_for_connection(
10795 dispatch,
10796 emitter,
10797 account_index,
10798 0,
10799 now,
10800 source,
10801 code,
10802 message,
10803 tx_hash,
10804 )
10805 }
10806
10807 fn trader_id() -> TraderId {
10808 TraderId::from("TRADER-001")
10809 }
10810
10811 fn client_id() -> ClientId {
10812 ClientId::from("LIGHTER")
10813 }
10814
10815 fn account_id() -> AccountId {
10816 AccountId::from("LIGHTER-001")
10817 }
10818
10819 fn strategy_id() -> StrategyId {
10820 StrategyId::from("S-001")
10821 }
10822
10823 fn test_credential() -> Credential {
10824 Credential::new(TEST_API_KEY_INDEX, TEST_PRIVATE_KEY, TEST_ACCOUNT_INDEX).unwrap()
10825 }
10826
10827 fn test_config() -> LighterExecutionClientConfig {
10828 test_config_with_use_gtd(true)
10829 }
10830
10831 fn test_config_with_use_gtd(use_gtd: bool) -> LighterExecutionClientConfig {
10832 LighterExecutionClientConfig {
10833 account_id: account_id(),
10834 account_index: Some(TEST_ACCOUNT_INDEX),
10835 api_key_index: Some(TEST_API_KEY_INDEX),
10836 private_key: Some(TEST_PRIVATE_KEY.into()),
10837 base_url_http: Some("http://127.0.0.1:1".to_string()),
10838 base_url_ws: Some("ws://127.0.0.1:1/stream".to_string()),
10839 proxy_url: None,
10840 environment: LighterEnvironment::Testnet,
10841 deployment: LighterDeployment::Lighter,
10842 venue: None,
10843 http_timeout_secs: 1,
10844 ws_timeout_secs: 1,
10845 market_order_slippage_bps: 50,
10846 rest_quota_per_min: None,
10847 sendtx_quota_per_min: None,
10848 transport_backend: Default::default(),
10849 use_gtd,
10850 scheduled_cancel_timeout_secs: None,
10851 }
10852 }
10853
10854 #[rstest]
10855 fn format_between_timestamps_uses_lighter_seconds_range() {
10856 let start = UnixNanos::from(1_700_000_000_123_456_789);
10857 let end = UnixNanos::from(1_700_003_600_987_654_321);
10858 let now = UnixNanos::from(1_700_007_200_000_000_000);
10859
10860 assert_eq!(
10861 format_between_timestamps(Some(start), Some(end), now),
10862 Some("1700000000-1700003600".to_string()),
10863 );
10864 assert_eq!(
10865 format_between_timestamps(Some(start), None, now),
10866 Some("1700000000-1700007200".to_string()),
10867 );
10868 assert_eq!(
10869 format_between_timestamps(None, Some(end), now),
10870 Some("0-1700003600".to_string()),
10871 );
10872 assert_eq!(format_between_timestamps(None, None, now), None);
10873 }
10874
10875 #[rstest]
10876 #[case::write_timeout(LighterWsError::Transport(SendError::WriteTimeout), "Ambiguous")]
10877 #[case::broken_pipe(
10878 LighterWsError::Transport(SendError::BrokenPipe("writer closed".to_string())),
10879 "Ambiguous",
10880 )]
10881 #[case::handler_result_lost(
10882 LighterWsError::SendTxOutcomeUnknown("result sender dropped".to_string()),
10883 "Ambiguous",
10884 )]
10885 #[case::network(LighterWsError::Network("disconnected".to_string()), "Ambiguous")]
10886 #[case::parse(LighterWsError::Parse("invalid ack".to_string()), "Ambiguous")]
10887 #[case::invalid_input(
10888 LighterWsError::Transport(SendError::InvalidInput("invalid payload".to_string())),
10889 "NotSent",
10890 )]
10891 #[case::buffer_full(LighterWsError::Transport(SendError::BufferFull), "NotSent")]
10892 #[case::closed(LighterWsError::Transport(SendError::Closed), "NotSent")]
10893 #[case::connection_changed(LighterWsError::Transport(SendError::ConnectionChanged), "NotSent")]
10894 #[case::wait_timeout(LighterWsError::Transport(SendError::Timeout), "NotSent")]
10895 #[case::authentication(
10896 LighterWsError::Authentication("invalid token".to_string()),
10897 "NotSent",
10898 )]
10899 #[case::client_unavailable(
10900 LighterWsError::Client("handler unavailable".to_string()),
10901 "NotSent",
10902 )]
10903 fn ws_command_failure_classifies_delivery_evidence(
10904 #[case] error: LighterWsError,
10905 #[case] expected: &str,
10906 ) {
10907 let failure = classify_lighter_ws_command_failure("submit_order", &error);
10908
10909 let actual = match &failure {
10910 CommandFailure::NotSent(_) => "NotSent",
10911 CommandFailure::Ambiguous(_) => "Ambiguous",
10912 CommandFailure::VenueRejected(_) => "VenueRejected",
10913 };
10914
10915 assert_eq!(actual, expected);
10916 assert!(command_failure_reason(&failure).contains("submit_order"));
10917 }
10918
10919 #[rstest]
10920 fn venue_rejection_reason_is_clean_and_bounded() {
10921 let oversized = "x".repeat(STRATEGY_REASON_MAX_CHARS + 100);
10922 let message = format!("<html>\n rejected\0 because {oversized}</html>");
10923
10924 let failure = venue_rejection_failure(Some(20_001), &message);
10925 let reason = command_failure_reason(&failure);
10926
10927 assert!(matches!(&failure, CommandFailure::VenueRejected(_)));
10928 assert!(!reason.contains('<'));
10929 assert!(!reason.contains('\0'));
10930 assert!(!reason.contains(" "));
10931 assert_eq!(reason.chars().count(), STRATEGY_REASON_MAX_CHARS);
10932 assert!(reason.starts_with("LIGHTER_20001: rejected because "));
10933 }
10934
10935 #[rstest]
10936 fn venue_rejection_reason_uses_stable_fallback_for_empty_markup() {
10937 let failure = venue_rejection_failure(Some(20_001), "<html></html>\n");
10938
10939 assert_eq!(
10940 failure,
10941 CommandFailure::VenueRejected("LIGHTER_20001".to_string()),
10942 );
10943 }
10944
10945 #[rstest]
10946 fn venue_rejection_reason_preserves_comparison_and_removes_format_controls() {
10947 let failure = venue_rejection_failure(
10948 Some(20_001),
10949 "price must be < 100\u{202e}\u{2066}\u{200b}<strong>now</strong>",
10950 );
10951
10952 assert_eq!(
10953 failure,
10954 CommandFailure::VenueRejected("LIGHTER_20001: price must be < 100 now".to_string(),),
10955 );
10956 }
10957
10958 #[rstest]
10959 fn unsupported_order_type_reason_keeps_lighter_capabilities() {
10960 assert_eq!(
10961 unsupported_lighter_order_type_reason(OrderType::MarketToLimit),
10962 "UNSUPPORTED_ORDER_TYPE: MARKET_TO_LIMIT; Lighter supports MARKET, LIMIT, STOP_MARKET, STOP_LIMIT, MARKET_IF_TOUCHED, and LIMIT_IF_TOUCHED",
10963 );
10964 }
10965
10966 #[rstest]
10967 fn mass_status_propagates_commission_error_and_downgrades_other_fill_errors() {
10968 let e = anyhow::Error::new(LighterCommissionError::new("invalid precision"))
10969 .context("failed to parse Lighter fill report");
10970
10971 let commission_error = fill_reports_for_mass_status(Err(e))
10972 .expect_err("commission construction must fail mass status");
10973 let ordinary_error = fill_reports_for_mass_status(Err(anyhow::anyhow!("transport failed")))
10974 .expect("ordinary source errors make mass status incomplete");
10975
10976 let complete = fill_reports_for_mass_status(Ok(FillSweep {
10977 reports: Vec::new(),
10978 covers_window: true,
10979 withheld_instruments: AHashSet::new(),
10980 }))
10981 .expect("complete fill sweep");
10982
10983 assert!(is_commission_error(&commission_error));
10984 assert!(ordinary_error.0.is_empty());
10985 assert!(!ordinary_error.1);
10986 assert!(complete.0.is_empty());
10987 assert!(complete.1);
10988 }
10989
10990 fn create_execution_client() -> (
10991 LighterExecutionClient,
10992 Rc<RefCell<Cache>>,
10993 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
10994 ) {
10995 create_execution_client_with_config(test_config())
10996 }
10997
10998 #[rstest]
10999 #[case(
11000 "proxy_url",
11001 "127.0.0.1:8080",
11002 "Lighter proxy_url must be an absolute http:// or https:// URL with a host"
11003 )]
11004 #[case(
11005 "base_url_ws",
11006 "wss://example.com/stream?readonly=true",
11007 "Lighter execution WebSocket URL must not set readonly=true; it does not disable trading"
11008 )]
11009 #[case(
11010 "venue",
11011 "LIGHTER-ALT",
11012 "Lighter execution venue must not contain '-'; use '_' so the account ID issuer can match"
11013 )]
11014 fn execution_constructor_rejects_config_footguns(
11015 #[case] field: &str,
11016 #[case] value: &str,
11017 #[case] expected: &str,
11018 ) {
11019 let (client, _, _) = create_execution_client();
11020 let mut config = test_config();
11021 match field {
11022 "proxy_url" => config.proxy_url = Some(value.into()),
11023 "base_url_ws" => config.base_url_ws = Some(value.to_string()),
11024 "venue" => config.venue = Some(Venue::from(value)),
11025 _ => unreachable!(),
11026 }
11027
11028 assert_eq!(
11029 LighterExecutionClient::new(client.core, config)
11030 .unwrap_err()
11031 .to_string(),
11032 expected
11033 );
11034 }
11035
11036 #[rstest]
11037 #[case(0, true)]
11038 #[case(9_999, true)]
11039 #[case(10_000, false)]
11040 #[case(u32::MAX, false)]
11041 fn market_order_slippage_constructor_validates_range(#[case] value: u32, #[case] valid: bool) {
11042 let (client, _, _) = create_execution_client();
11043 let mut config = test_config();
11044 config.market_order_slippage_bps = value;
11045 let result = LighterExecutionClient::new(client.core, config);
11046
11047 if valid {
11048 assert_eq!(result.unwrap().config().market_order_slippage_bps, value);
11049 } else {
11050 assert_eq!(
11051 result.unwrap_err().to_string(),
11052 "Lighter market_order_slippage_bps must be in [0, 10000)"
11053 );
11054 }
11055 }
11056
11057 #[rstest]
11058 #[case(419)]
11059 #[case(1_295_941)]
11060 #[case(u64::MAX)]
11061 fn scheduled_cancel_constructor_rejects_invalid_timeout(#[case] timeout: u64) {
11062 let (client, _, _) = create_execution_client();
11063 let mut config = test_config();
11064 config.scheduled_cancel_timeout_secs = Some(timeout);
11065 let error = LighterExecutionClient::new(client.core, config).unwrap_err();
11066 assert_eq!(
11067 error.to_string(),
11068 "scheduled_cancel_timeout_secs must be in [420, 1295940]"
11069 );
11070 }
11071
11072 #[tokio::test]
11073 async fn scheduled_cancel_timeout_preserves_non_tail_nonce() {
11074 let (mut client, _, _) = create_execution_client();
11075 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
11076 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
11077 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
11078 let context = client
11079 .fanout_dispatch_context(client.credential.as_ref().unwrap())
11080 .unwrap();
11081
11082 let later = {
11083 let renewal = context.refresh_scheduled_cancel(420);
11084 tokio::pin!(renewal);
11085 tokio::select! {
11086 result = &mut renewal => panic!("renewal must wait for quota: {result:?}"),
11087 () = tokio::time::sleep(Duration::from_millis(20)) => {},
11088 }
11089 let later = context.build_tx_context().unwrap();
11090 assert_eq!(later.context.nonce, TEST_NEXT_NONCE + 1);
11091 assert!(
11092 tokio::time::timeout(Duration::from_millis(50), &mut renewal)
11093 .await
11094 .is_err()
11095 );
11096 later
11097 };
11098
11099 assert!(
11100 tokio::time::timeout(
11101 Duration::from_millis(150),
11102 later.send_reservation.wait_for_turn()
11103 )
11104 .await
11105 .is_err()
11106 );
11107 assert!(!client.pending_tasks.all_finished());
11108 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11109 assert_eq!(
11110 client
11111 .dispatch
11112 .nonce_manager
11113 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
11114 Some(TEST_NEXT_NONCE + 1)
11115 );
11116 drop(later);
11117 client.begin_session_shutdown();
11118 client.finish_session_shutdown().await.unwrap();
11119 assert!(client.pending_tasks.is_empty());
11120 }
11121
11122 #[tokio::test]
11123 async fn scheduled_cancel_timeout_cancels_quota_wait_and_releases_nonce() {
11124 let (mut client, _, _) = create_execution_client();
11125 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
11126 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
11127 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
11128 let context = client
11129 .fanout_dispatch_context(client.credential.as_ref().unwrap())
11130 .unwrap();
11131 assert!(
11132 tokio::time::timeout(
11133 Duration::from_millis(50),
11134 context.refresh_scheduled_cancel(420)
11135 )
11136 .await
11137 .is_err()
11138 );
11139 tokio::time::timeout(Duration::from_secs(2), async {
11140 while !client.pending_tasks.all_finished() {
11141 tokio::task::yield_now().await;
11142 }
11143 })
11144 .await
11145 .unwrap();
11146
11147 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11148 assert_eq!(
11149 client
11150 .dispatch
11151 .nonce_manager
11152 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11153 .unwrap(),
11154 TEST_NEXT_NONCE
11155 );
11156 client.begin_session_shutdown();
11157 client.finish_session_shutdown().await.unwrap();
11158 assert!(client.pending_tasks.is_empty());
11159 }
11160
11161 #[rstest]
11162 #[tokio::test]
11163 async fn account_cancellation_dropped_during_quota_wait_is_retained_until_shutdown() {
11164 let (mut client, _, _) = create_execution_client();
11165 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
11166 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
11167 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
11168 {
11169 let pending = client.cancel_all_orders_account();
11170 tokio::pin!(pending);
11171 tokio::select! {
11172 result = &mut pending => panic!("control must wait for quota: {result:?}"),
11173 () = tokio::time::sleep(Duration::from_millis(50)) => {}
11174 }
11175 }
11176
11177 assert!(!client.pending_tasks.is_empty());
11178 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11179
11180 client.begin_session_shutdown();
11181 client.finish_session_shutdown().await.unwrap();
11182
11183 assert!(client.pending_tasks.is_empty());
11184 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11185 assert_eq!(
11186 client
11187 .dispatch
11188 .nonce_manager
11189 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11190 .unwrap(),
11191 TEST_NEXT_NONCE,
11192 );
11193 }
11194
11195 #[rstest]
11196 #[tokio::test]
11197 async fn account_cancellation_spawn_rejection_releases_latest_nonce() {
11198 let (client, _, _) = create_execution_client();
11199 client.pending_tasks.begin_shutdown();
11200
11201 let error = client.cancel_all_orders_account().await.unwrap_err();
11202
11203 assert_eq!(
11204 error.to_string(),
11205 "failed to spawn Lighter account cancellation"
11206 );
11207 assert!(client.pending_tasks.is_empty());
11208 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11209 assert_eq!(
11210 client
11211 .dispatch
11212 .nonce_manager
11213 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11214 .unwrap(),
11215 TEST_NEXT_NONCE,
11216 );
11217 }
11218
11219 #[rstest]
11220 #[tokio::test]
11221 async fn account_cancellation_ambiguous_error_identifies_pending_transaction() {
11222 let (mut client, _, _) = create_execution_client();
11223 client.ws_client.drop_next_send_tx_result_for_test().await;
11224
11225 let error = client.cancel_all_orders_account().await.unwrap_err();
11226 let pending = client.dispatch.take_pending_sendtx();
11227
11228 assert_eq!(pending.len(), 1);
11229 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
11230 assert_eq!(pending[0].api_key_index, TEST_API_KEY_INDEX);
11231 assert_eq!(
11232 error.to_string(),
11233 format!(
11234 "failed to submit Lighter account cancellation transaction {}",
11235 pending[0].tx_hash
11236 ),
11237 );
11238 assert!(matches!(
11239 error.downcast_ref::<LighterWsError>(),
11240 Some(LighterWsError::SendTxOutcomeUnknown(_)),
11241 ));
11242 assert_eq!(
11243 client
11244 .dispatch
11245 .nonce_manager
11246 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11247 .unwrap(),
11248 TEST_NEXT_NONCE + 1,
11249 );
11250 client.begin_session_shutdown();
11251 client.finish_session_shutdown().await.unwrap();
11252 }
11253
11254 #[rstest]
11255 #[case::definite_failure(false, 0, TEST_NEXT_NONCE + 1)]
11256 #[case::ambiguous_failure(true, 1, TEST_NEXT_NONCE + 2)]
11257 #[tokio::test]
11258 async fn account_cancellation_dropped_caller_preserves_dispatch_failure_handling(
11259 #[case] ambiguous: bool,
11260 #[case] pending_count: usize,
11261 #[case] next_nonce: i64,
11262 ) {
11263 let (mut client, _, _) = create_execution_client();
11264 if ambiguous {
11265 client.ws_client.drop_next_send_tx_result_for_test().await;
11266 }
11267
11268 let mut blocker = client.build_tx_context(&test_credential()).unwrap();
11269 {
11270 let pending = client.cancel_all_orders_account();
11271 tokio::pin!(pending);
11272 tokio::select! {
11273 result = &mut pending => panic!("control must wait for its turn: {result:?}"),
11274 () = tokio::time::sleep(Duration::from_millis(50)) => {}
11275 }
11276 }
11277
11278 assert!(!client.pending_tasks.is_empty());
11279 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11280 blocker.send_reservation.release();
11281 wait_until_async(
11282 || async { client.pending_tasks.is_empty() },
11283 Duration::from_secs(2),
11284 )
11285 .await;
11286
11287 assert_eq!(client.dispatch.pending_sendtx_len(), pending_count);
11288 assert_eq!(
11289 client
11290 .dispatch
11291 .nonce_manager
11292 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11293 .unwrap(),
11294 next_nonce,
11295 );
11296 client.begin_session_shutdown();
11297 client.finish_session_shutdown().await.unwrap();
11298 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11299 }
11300
11301 #[rstest]
11302 #[case::first_start(false)]
11303 #[case::already_started(true)]
11304 fn execution_sender_is_available_before_start_and_replaced_on_start(#[case] started: bool) {
11305 let (client, _cache, _rx) = create_execution_client();
11306 let (first_sender, mut first_receiver) = tokio::sync::mpsc::unbounded_channel();
11307 replace_exec_event_sender(first_sender);
11308 let mut client = LighterExecutionClient::new(client.core, test_config()).unwrap();
11309 let emitter = client.emitter.clone();
11310
11311 let submitted = OrderSubmitted::new(
11312 trader_id(),
11313 strategy_id(),
11314 InstrumentId::from("ETH-PERP.LIGHTER"),
11315 ClientOrderId::from("BEFORE-START"),
11316 account_id(),
11317 UUID4::new(),
11318 UnixNanos::from(17),
11319 UnixNanos::from(23),
11320 );
11321 emitter
11322 .try_send_order_event(OrderEventAny::Submitted(submitted.clone()))
11323 .unwrap();
11324
11325 if started {
11326 client.start().unwrap();
11327 }
11328
11329 let (second_sender, mut second_receiver) = tokio::sync::mpsc::unbounded_channel();
11330 replace_exec_event_sender(second_sender);
11331 client.start().unwrap();
11332
11333 let after_start = OrderSubmitted::new(
11334 trader_id(),
11335 strategy_id(),
11336 InstrumentId::from("BTC-PERP.LIGHTER"),
11337 ClientOrderId::from("AFTER-START"),
11338 account_id(),
11339 UUID4::new(),
11340 UnixNanos::from(29),
11341 UnixNanos::from(31),
11342 );
11343 emitter
11344 .try_send_order_event(OrderEventAny::Submitted(after_start.clone()))
11345 .unwrap();
11346
11347 for (receiver, expected) in [
11348 (&mut first_receiver, submitted),
11349 (&mut second_receiver, after_start),
11350 ] {
11351 let ExecutionEvent::Order(OrderEventAny::Submitted(actual)) =
11352 receiver.try_recv().unwrap()
11353 else {
11354 panic!("expected submitted event");
11355 };
11356
11357 assert_eq!(actual, expected);
11358 assert!(receiver.try_recv().is_err());
11359 }
11360 }
11361
11362 fn create_execution_client_with_config(
11363 config: LighterExecutionClientConfig,
11364 ) -> (
11365 LighterExecutionClient,
11366 Rc<RefCell<Cache>>,
11367 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
11368 ) {
11369 let cache = Rc::new(RefCell::new(Cache::default()));
11370 let venue = config.resolved_venue();
11371
11372 let core = ExecutionClientCore::new(
11373 trader_id(),
11374 client_id(),
11375 venue,
11376 OmsType::Netting,
11377 account_id(),
11378 AccountType::Margin,
11379 None,
11380 cache.clone(),
11381 );
11382
11383 let mut client = LighterExecutionClient::new(core, config).unwrap();
11384 client.dispatch.nonce_manager.refresh(
11385 TEST_ACCOUNT_INDEX_I64,
11386 TEST_API_KEY_INDEX,
11387 TEST_NEXT_NONCE,
11388 );
11389
11390 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
11391 client.emitter.set_sender(sender);
11392
11393 (client, cache, receiver)
11394 }
11395
11396 #[rstest]
11397 #[case::inline_deferred(false, true, false, false)]
11398 #[case::worker_deferred(true, true, false, false)]
11399 #[case::inline_absent(false, false, false, false)]
11400 #[case::worker_absent(true, false, false, false)]
11401 #[case::inline_account_orders_failure(false, true, true, false)]
11402 #[case::worker_account_orders_failure(true, true, true, false)]
11403 #[case::inline_ambiguous_account_orders(false, true, false, true)]
11404 #[case::worker_ambiguous_account_orders(true, true, false, true)]
11405 #[tokio::test]
11406 async fn single_order_lookup_distinguishes_deferred_report_from_absence(
11407 #[case] worker: bool,
11408 #[case] present: bool,
11409 #[case] account_orders_fails: bool,
11410 #[case] duplicate: bool,
11411 ) {
11412 let rig = dispatcher_rig("DEFERRED-PROBE");
11413 register_identity(&rig);
11414 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
11415 let venue_order_id = VenueOrderId::from(order.order_id.as_str());
11416
11417 if !duplicate {
11418 rig.dispatch.observe_create_submission(
11419 &rig.cloid,
11420 order.client_order_index,
11421 TEST_SUBMISSION_NONCE,
11422 venue_order_id,
11423 );
11424 }
11425
11426 rig.dispatch
11427 .order_identity(&rig.cloid)
11428 .unwrap()
11429 .publish_update(UUID4::new());
11430
11431 let response = LighterOrders {
11432 code: 200,
11433 message: None,
11434 next_cursor: None,
11435 orders: if duplicate {
11436 vec![order.clone(), order]
11437 } else if present {
11438 vec![order]
11439 } else {
11440 Vec::new()
11441 },
11442 };
11443
11444 let body = serde_json::to_string(&response).unwrap();
11445 let account_orders_calls = Arc::new(AtomicUsize::new(0));
11446 let active_calls = Arc::new(AtomicUsize::new(0));
11447 let inactive_calls = Arc::new(AtomicUsize::new(0));
11448
11449 let app = Router::new()
11450 .route(
11451 "/api/v1/accountOrders",
11452 get({
11453 let body = body.clone();
11454 let calls = Arc::clone(&account_orders_calls);
11455 move || {
11456 calls.fetch_add(1, Ordering::SeqCst);
11457 let body = body.clone();
11458 async move {
11459 if account_orders_fails {
11460 (axum::http::StatusCode::SERVICE_UNAVAILABLE, body).into_response()
11461 } else {
11462 (axum::http::StatusCode::OK, body).into_response()
11463 }
11464 }
11465 }
11466 }),
11467 )
11468 .route(
11469 "/api/v1/accountActiveOrders",
11470 get({
11471 let body = body.clone();
11472 let calls = Arc::clone(&active_calls);
11473 move || {
11474 calls.fetch_add(1, Ordering::SeqCst);
11475 let body = body.clone();
11476 async move { body }
11477 }
11478 }),
11479 )
11480 .route(
11481 "/api/v1/accountInactiveOrders",
11482 get({
11483 let calls = Arc::clone(&inactive_calls);
11484 move || {
11485 calls.fetch_add(1, Ordering::SeqCst);
11486 async move {
11487 axum::Json(LighterOrders {
11488 code: 200,
11489 message: None,
11490 next_cursor: None,
11491 orders: Vec::new(),
11492 })
11493 }
11494 }
11495 }),
11496 );
11497
11498 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
11499 let addr = listener.local_addr().unwrap();
11500 let server = tokio::spawn(async move {
11501 axum::serve(listener, app).await.unwrap();
11502 });
11503
11504 let config = LighterExecutionClientConfig {
11505 base_url_http: Some(format!("http://{addr}")),
11506 ..test_config()
11507 };
11508
11509 let (mut client, _cache, mut rx) = create_execution_client_with_config(config);
11510 client.registry = rig.registry;
11511 client.dispatch = rig.dispatch;
11512
11513 let command = GenerateOrderStatusReport::new(
11514 UUID4::new(),
11515 UnixNanos::from(1),
11516 Some(rig.instrument_id),
11517 Some(rig.cloid),
11518 if duplicate {
11519 None
11520 } else {
11521 Some(venue_order_id)
11522 },
11523 None,
11524 None,
11525 );
11526
11527 let result = if worker {
11528 let task = client.generate_order_status_report_task(&command).unwrap();
11529 task.collection.await;
11530 (task.finish)()
11531 } else {
11532 client.generate_order_status_report(&command).await
11533 };
11534
11535 server.abort();
11536 let _ = server.await;
11537
11538 assert_eq!(
11539 account_orders_calls.load(Ordering::SeqCst),
11540 if account_orders_fails { 4 } else { 1 }
11541 );
11542 assert_eq!(
11543 active_calls.load(Ordering::SeqCst),
11544 usize::from(!present || account_orders_fails)
11545 );
11546 assert_eq!(inactive_calls.load(Ordering::SeqCst), usize::from(!present));
11547
11548 if duplicate {
11549 let error = result.unwrap_err();
11550 assert!(
11551 error
11552 .to_string()
11553 .contains("ambiguous Lighter account-order lookup"),
11554 "unexpected error: {error:#}"
11555 );
11556 } else if present {
11557 assert_eq!(
11558 result.unwrap_err().to_string(),
11559 "Lighter order report deferred until execution state is applied"
11560 );
11561 } else {
11562 assert_eq!(result.unwrap(), None);
11563 }
11564
11565 assert!(
11566 client
11567 .dispatch
11568 .order_identity(&rig.cloid)
11569 .unwrap()
11570 .has_publication()
11571 );
11572 assert!(rx.try_recv().is_err());
11573 }
11574
11575 #[rstest]
11576 #[case::other_market(LighterOrderStatus::Open, 999, false, false, false, 0)]
11577 #[case::terminal(
11578 LighterOrderStatus::Canceled,
11579 TEST_MARKET_INDEX,
11580 false,
11581 false,
11582 false,
11583 0
11584 )]
11585 #[case::pending_flip(LighterOrderStatus::Open, TEST_MARKET_INDEX, true, false, false, 0)]
11586 #[case::direct_open(LighterOrderStatus::Open, TEST_MARKET_INDEX, false, true, false, 0)]
11587 #[case::known_terminal(LighterOrderStatus::Canceled, TEST_MARKET_INDEX, false, true, true, 0)]
11588 #[case::known_other_market(LighterOrderStatus::Open, 999, false, false, true, 1)]
11589 #[tokio::test]
11590 async fn client_order_lookup_filters_market_status_and_pending(
11591 #[case] status: LighterOrderStatus,
11592 #[case] market_index: i64,
11593 #[case] stop_pending_after_response: bool,
11594 #[case] expect_present: bool,
11595 #[case] known_venue: bool,
11596 #[case] expected_inactive_calls: usize,
11597 ) {
11598 let rig = dispatcher_rig("FILTERED-PROBE");
11599 register_identity(&rig);
11600 let mut order = dispatcher_test_order(&rig, status);
11601 order.market_index = market_index;
11602 let venue_order_id = known_venue.then(|| VenueOrderId::from(order.order_id.as_str()));
11603 rig.dispatch
11604 .order_identity(&rig.cloid)
11605 .unwrap()
11606 .publish_update(UUID4::new());
11607
11608 let account_orders_body = serde_json::to_string(&LighterOrders {
11609 code: 200,
11610 message: None,
11611 next_cursor: None,
11612 orders: vec![order],
11613 })
11614 .unwrap();
11615
11616 let empty_body = serde_json::to_string(&LighterOrders {
11617 code: 200,
11618 message: None,
11619 next_cursor: None,
11620 orders: Vec::new(),
11621 })
11622 .unwrap();
11623
11624 let account_orders_calls = Arc::new(AtomicUsize::new(0));
11625 let active_calls = Arc::new(AtomicUsize::new(0));
11626 let inactive_calls = Arc::new(AtomicUsize::new(0));
11627 let pending = Arc::new(AtomicBool::new(true));
11628
11629 let app = Router::new()
11630 .route(
11631 "/api/v1/accountOrders",
11632 get({
11633 let body = account_orders_body.clone();
11634 let calls = Arc::clone(&account_orders_calls);
11635 let pending = Arc::clone(&pending);
11636 move || {
11637 calls.fetch_add(1, Ordering::SeqCst);
11638 let body = body.clone();
11639 let pending = Arc::clone(&pending);
11640 async move {
11641 if stop_pending_after_response {
11642 pending.store(false, Ordering::Release);
11643 }
11644
11645 body
11646 }
11647 }
11648 }),
11649 )
11650 .route(
11651 "/api/v1/accountActiveOrders",
11652 get({
11653 let body = empty_body.clone();
11654 let calls = Arc::clone(&active_calls);
11655 move || {
11656 calls.fetch_add(1, Ordering::SeqCst);
11657 let body = body.clone();
11658 async move { body }
11659 }
11660 }),
11661 )
11662 .route(
11663 "/api/v1/accountInactiveOrders",
11664 get({
11665 let calls = Arc::clone(&inactive_calls);
11666 move || {
11667 calls.fetch_add(1, Ordering::SeqCst);
11668 let body = empty_body.clone();
11669 async move { body }
11670 }
11671 }),
11672 );
11673
11674 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
11675 let addr = listener.local_addr().unwrap();
11676 let server = tokio::spawn(async move {
11677 axum::serve(listener, app).await.unwrap();
11678 });
11679
11680 let config = LighterExecutionClientConfig {
11681 base_url_http: Some(format!("http://{addr}")),
11682 ..test_config()
11683 };
11684
11685 let (mut client, _cache, _rx) = create_execution_client_with_config(config);
11686 client.registry = rig.registry;
11687 client.dispatch = rig.dispatch;
11688 let credential = test_credential();
11689 let lookup = OrderReportLookup::new(
11690 &client.registry,
11691 &client.dispatch,
11692 Some(rig.instrument_id),
11693 Some(&rig.cloid),
11694 venue_order_id.as_ref(),
11695 )
11696 .unwrap();
11697 let result = collect_order_while_pending(
11698 &client.http_client,
11699 &client.registry,
11700 &credential,
11701 &lookup,
11702 || pending.load(Ordering::Acquire),
11703 )
11704 .await;
11705 server.abort();
11706 let _ = server.await;
11707
11708 assert_eq!(account_orders_calls.load(Ordering::SeqCst), 1);
11709 assert_eq!(
11710 active_calls.load(Ordering::SeqCst),
11711 usize::from(!expect_present && !stop_pending_after_response)
11712 );
11713 assert_eq!(
11714 inactive_calls.load(Ordering::SeqCst),
11715 expected_inactive_calls
11716 );
11717
11718 if expect_present {
11719 assert!(result.unwrap().is_some());
11720 } else {
11721 assert_eq!(result.unwrap(), None);
11722 }
11723 }
11724
11725 fn register_test_instrument(
11726 client: &LighterExecutionClient,
11727 cache: &Rc<RefCell<Cache>>,
11728 ) -> InstrumentId {
11729 let instrument_id =
11730 client
11731 .registry
11732 .insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
11733 let instrument = InstrumentAny::CryptoPerpetual(
11734 CryptoPerpetual::builder()
11735 .instrument_id(instrument_id)
11736 .raw_symbol(Symbol::new("ETH-PERP"))
11737 .base_currency(Currency::from("ETH"))
11738 .quote_currency(Currency::from("USDC"))
11739 .settlement_currency(Currency::from("USDC"))
11740 .is_inverse(false)
11741 .price_precision(2)
11742 .size_precision(4)
11743 .price_increment(Price::from("0.01"))
11744 .size_increment(Quantity::from("0.0001"))
11745 .min_notional(Money::from("10.000000 USDC"))
11746 .ts_event(UnixNanos::default())
11747 .ts_init(UnixNanos::default())
11748 .build()
11749 .unwrap(),
11750 );
11751
11752 cache.borrow_mut().add_instrument(instrument).unwrap();
11753
11754 instrument_id
11755 }
11756
11757 fn test_order_factory() -> OrderFactory {
11758 let clock = Rc::new(RefCell::new(VirtualClock::new()));
11759
11760 OrderFactory::new(
11761 trader_id(),
11762 strategy_id(),
11763 Some(0),
11764 Some(0),
11765 clock,
11766 false,
11767 false,
11768 )
11769 }
11770
11771 fn test_limit_order(
11772 factory: &mut OrderFactory,
11773 instrument_id: InstrumentId,
11774 client_order_id: &str,
11775 ) -> OrderAny {
11776 test_limit_order_with(
11777 factory,
11778 instrument_id,
11779 client_order_id,
11780 OrderSide::Buy,
11781 TimeInForce::Gtc,
11782 None,
11783 false,
11784 )
11785 }
11786
11787 fn test_limit_order_with(
11788 factory: &mut OrderFactory,
11789 instrument_id: InstrumentId,
11790 client_order_id: &str,
11791 side: OrderSide,
11792 tif: TimeInForce,
11793 expire_time: Option<UnixNanos>,
11794 reduce_only: bool,
11795 ) -> OrderAny {
11796 factory.limit(
11797 instrument_id,
11798 side,
11799 Quantity::from("0.1000"),
11800 Price::from("2361.31"),
11801 Some(tif),
11802 expire_time,
11803 Some(false),
11804 Some(reduce_only),
11805 None,
11806 None,
11807 None,
11808 None,
11809 None,
11810 None,
11811 None,
11812 Some(ClientOrderId::from(client_order_id)),
11813 )
11814 }
11815
11816 fn cache_order(cache: &Rc<RefCell<Cache>>, order: OrderAny) {
11817 cache
11818 .borrow_mut()
11819 .add_order(order, None, Some(client_id()), false)
11820 .unwrap();
11821 }
11822
11823 fn cache_accepted_order(
11824 cache: &Rc<RefCell<Cache>>,
11825 order: OrderAny,
11826 venue_order_id: VenueOrderId,
11827 client_id: Option<ClientId>,
11828 ) -> (InstrumentId, ClientOrderId) {
11829 let instrument_id = order.instrument_id();
11830 let client_order_id = order.client_order_id();
11831 cache
11832 .borrow_mut()
11833 .add_order(order, None, client_id, false)
11834 .unwrap();
11835
11836 let accepted = OrderEventAny::Accepted(OrderAccepted::new(
11837 trader_id(),
11838 strategy_id(),
11839 instrument_id,
11840 client_order_id,
11841 venue_order_id,
11842 account_id(),
11843 UUID4::new(),
11844 UnixNanos::default(),
11845 UnixNanos::default(),
11846 false,
11847 ));
11848 cache.borrow_mut().update_order(&accepted).unwrap();
11849
11850 (instrument_id, client_order_id)
11851 }
11852
11853 fn cache_pending_cancel_order(
11854 cache: &Rc<RefCell<Cache>>,
11855 order: OrderAny,
11856 venue_order_id: VenueOrderId,
11857 causation_id: Option<UUID4>,
11858 ) {
11859 let (instrument_id, client_order_id) =
11860 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
11861
11862 apply_pending_cancel(
11863 cache,
11864 instrument_id,
11865 client_order_id,
11866 venue_order_id,
11867 causation_id,
11868 );
11869 }
11870
11871 fn apply_pending_cancel(
11872 cache: &Rc<RefCell<Cache>>,
11873 instrument_id: InstrumentId,
11874 client_order_id: ClientOrderId,
11875 venue_order_id: VenueOrderId,
11876 causation_id: Option<UUID4>,
11877 ) {
11878 let mut pending_cancel = OrderPendingCancel::new(
11879 trader_id(),
11880 strategy_id(),
11881 instrument_id,
11882 client_order_id,
11883 Some(account_id()),
11884 UUID4::new(),
11885 UnixNanos::default(),
11886 UnixNanos::default(),
11887 false,
11888 Some(venue_order_id),
11889 );
11890 pending_cancel.causation_id = causation_id;
11891 cache
11892 .borrow_mut()
11893 .update_order(&OrderEventAny::PendingCancel(pending_cancel))
11894 .unwrap();
11895 }
11896
11897 fn submit_order_list_command(orders: &[OrderAny], order_list_id: &str) -> SubmitOrderList {
11898 let order_list = OrderList::new(
11899 OrderListId::from(order_list_id),
11900 orders[0].instrument_id(),
11901 strategy_id(),
11902 orders.iter().map(|order| order.client_order_id()).collect(),
11903 UnixNanos::default(),
11904 );
11905 let order_inits = orders
11906 .iter()
11907 .map(|order| order.init_event().clone())
11908 .collect();
11909
11910 SubmitOrderList::new(
11911 trader_id(),
11912 Some(client_id()),
11913 strategy_id(),
11914 order_list,
11915 order_inits,
11916 None,
11917 None,
11918 None,
11919 UUID4::new(),
11920 UnixNanos::default(),
11921 None,
11922 )
11923 }
11924
11925 fn test_contingent_limit_order(
11926 instrument_id: InstrumentId,
11927 client_order_id: &str,
11928 order_list_id: &str,
11929 linked_order_id: &str,
11930 ) -> OrderAny {
11931 OrderAny::Limit(LimitOrder::new(
11932 trader_id(),
11933 strategy_id(),
11934 instrument_id,
11935 ClientOrderId::from(client_order_id),
11936 OrderSide::Buy,
11937 Quantity::from("0.1000"),
11938 Price::from("2361.31"),
11939 TimeInForce::Gtc,
11940 None,
11941 false,
11942 false,
11943 false,
11944 None,
11945 None,
11946 None,
11947 Some(ContingencyType::Oco),
11948 Some(OrderListId::from(order_list_id)),
11949 Some(vec![ClientOrderId::from(linked_order_id)]),
11950 None,
11951 None,
11952 None,
11953 None,
11954 None,
11955 UUID4::new(),
11956 UnixNanos::default(),
11957 ))
11958 }
11959
11960 async fn recv_order_event(
11961 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
11962 ) -> OrderEventAny {
11963 let event = tokio::time::timeout(Duration::from_secs(2), rx.recv())
11964 .await
11965 .expect("timed out waiting for execution event")
11966 .expect("execution event channel closed");
11967
11968 match event {
11969 ExecutionEvent::Order(event) => event,
11970 event => panic!("expected order event, was {event:?}"),
11971 }
11972 }
11973
11974 async fn assert_modify_denied_reason(
11975 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
11976 reason_part: &str,
11977 ) {
11978 match recv_order_event(rx).await {
11979 OrderEventAny::ModifyDenied(event) => {
11980 assert!(
11981 event.reason.contains(reason_part),
11982 "expected modify denial containing `{reason_part}`, was `{}`",
11983 event.reason,
11984 );
11985 }
11986 event => panic!("expected modify denied event, was {event:?}"),
11987 }
11988 }
11989
11990 fn assert_nonce_reusable(dispatch: &WsDispatchState) {
11991 assert_eq!(
11992 dispatch
11993 .nonce_manager
11994 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
11995 Some(TEST_NEXT_NONCE - 1),
11996 );
11997 assert_eq!(
11998 dispatch
11999 .nonce_manager
12000 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12001 .unwrap(),
12002 TEST_NEXT_NONCE,
12003 );
12004 }
12005
12006 #[tokio::test]
12007 async fn auth_token_rotation_retries_failed_mint() {
12008 let credential = test_credential();
12009 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
12010 let cancellation_token = CancellationToken::new();
12011 let mint_attempts = Arc::new(AtomicUsize::new(0));
12012 let subscribe_attempts = Arc::new(AtomicUsize::new(0));
12013
12014 let outcome = tokio::time::timeout(
12015 Duration::from_secs(3),
12016 refresh_auth_token_until_rotated(
12017 &credential,
12018 &channels,
12019 &cancellation_token,
12020 AuthTokenRefreshBackoff {
12021 initial_delay: Duration::from_millis(10),
12022 max_delay: Duration::from_millis(20),
12023 window: Duration::from_secs(1),
12024 },
12025 {
12026 let mint_attempts = Arc::clone(&mint_attempts);
12027 move |_| {
12028 let attempt = mint_attempts.fetch_add(1, Ordering::AcqRel);
12029 if attempt == 0 {
12030 Err(anyhow::anyhow!("mint unavailable"))
12031 } else {
12032 Ok(format!("token-{attempt}").into())
12033 }
12034 }
12035 },
12036 {
12037 let subscribe_attempts = Arc::clone(&subscribe_attempts);
12038 move |_channel, _token| {
12039 let subscribe_attempts = Arc::clone(&subscribe_attempts);
12040 async move {
12041 subscribe_attempts.fetch_add(1, Ordering::AcqRel);
12042 Ok::<(), crate::websocket::error::LighterWsError>(())
12043 }
12044 }
12045 },
12046 ),
12047 )
12048 .await
12049 .expect("rotation retry must complete within the test window");
12050
12051 assert_eq!(outcome, AuthTokenRefreshOutcome::Rotated);
12052 assert_eq!(
12053 mint_attempts.load(Ordering::Acquire),
12054 2,
12055 "failed mint must retry before the next refresh interval",
12056 );
12057 assert_eq!(
12058 subscribe_attempts.load(Ordering::Acquire),
12059 channels.len(),
12060 "subscriptions must wait until token mint succeeds",
12061 );
12062 }
12063
12064 #[tokio::test]
12065 async fn auth_token_rotation_retries_failed_resubscribe() {
12066 let credential = test_credential();
12067 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
12068 let cancellation_token = CancellationToken::new();
12069 let mint_attempts = Arc::new(AtomicUsize::new(0));
12070 let subscribe_attempts = Arc::new(AtomicUsize::new(0));
12071
12072 let outcome = tokio::time::timeout(
12073 Duration::from_secs(3),
12074 refresh_auth_token_until_rotated(
12075 &credential,
12076 &channels,
12077 &cancellation_token,
12078 AuthTokenRefreshBackoff {
12079 initial_delay: Duration::from_millis(10),
12080 max_delay: Duration::from_millis(20),
12081 window: Duration::from_secs(1),
12082 },
12083 {
12084 let mint_attempts = Arc::clone(&mint_attempts);
12085 move |_| {
12086 let attempt = mint_attempts.fetch_add(1, Ordering::AcqRel);
12087 Ok(format!("token-{attempt}").into())
12088 }
12089 },
12090 {
12091 let subscribe_attempts = Arc::clone(&subscribe_attempts);
12092 move |_channel, _token| {
12093 let subscribe_attempts = Arc::clone(&subscribe_attempts);
12094 async move {
12095 let attempt = subscribe_attempts.fetch_add(1, Ordering::AcqRel);
12096 if attempt == 0 {
12097 Err(crate::websocket::error::LighterWsError::Client(
12098 "handler unavailable".to_string(),
12099 ))
12100 } else {
12101 Ok(())
12102 }
12103 }
12104 }
12105 },
12106 ),
12107 )
12108 .await
12109 .expect("rotation retry must complete within the test window");
12110
12111 assert_eq!(outcome, AuthTokenRefreshOutcome::Rotated);
12112 assert_eq!(
12113 mint_attempts.load(Ordering::Acquire),
12114 2,
12115 "failed resubscribe must trigger a fresh auth-token mint",
12116 );
12117 assert_eq!(
12118 subscribe_attempts.load(Ordering::Acquire),
12119 channels.len() * 2,
12120 "failed resubscribe must retry the private account-channel set",
12121 );
12122 }
12123
12124 #[tokio::test]
12125 async fn auth_token_rotation_exhausts_retry_window() {
12126 let credential = test_credential();
12127 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
12128 let cancellation_token = CancellationToken::new();
12129 let mint_attempts = Arc::new(AtomicUsize::new(0));
12130
12131 let refresh = refresh_auth_token_until_rotated(
12132 &credential,
12133 &channels,
12134 &cancellation_token,
12135 AuthTokenRefreshBackoff {
12136 initial_delay: Duration::from_millis(100),
12137 max_delay: Duration::from_millis(100),
12138 window: Duration::from_secs(1),
12139 },
12140 {
12141 let mint_attempts = Arc::clone(&mint_attempts);
12142 move |_| {
12143 mint_attempts.fetch_add(1, Ordering::AcqRel);
12144 Err(anyhow::anyhow!("mint unavailable"))
12145 }
12146 },
12147 |_channel, _token| async { Ok::<(), crate::websocket::error::LighterWsError>(()) },
12148 );
12149 let observe_retry = wait_until_async(
12150 || async { mint_attempts.load(Ordering::Acquire) > 1 },
12151 Duration::from_secs(2),
12152 );
12153
12154 let (outcome, ()) = tokio::time::timeout(Duration::from_secs(3), async {
12155 tokio::join!(refresh, observe_retry)
12156 })
12157 .await
12158 .expect("rotation retry exhaustion must complete within the test window");
12159
12160 assert_eq!(outcome, AuthTokenRefreshOutcome::Exhausted);
12161 assert!(
12162 mint_attempts.load(Ordering::Acquire) > 1,
12163 "persistent mint failure must retry until the window is exhausted",
12164 );
12165 }
12166
12167 #[tokio::test]
12168 async fn auth_token_rotation_cancels_during_retry_backoff() {
12169 let credential = test_credential();
12170 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
12171 let cancellation_token = CancellationToken::new();
12172 let mint_attempts = Arc::new(AtomicUsize::new(0));
12173
12174 let cancel = cancellation_token.clone();
12175 let cancel_after_first_attempt = wait_until_async(
12176 || async { mint_attempts.load(Ordering::Acquire) > 0 },
12177 Duration::from_secs(2),
12178 );
12179
12180 let refresh = refresh_auth_token_until_rotated(
12181 &credential,
12182 &channels,
12183 &cancellation_token,
12184 AuthTokenRefreshBackoff {
12185 initial_delay: Duration::from_secs(2),
12186 max_delay: Duration::from_secs(2),
12187 window: Duration::from_secs(5),
12188 },
12189 {
12190 let mint_attempts = Arc::clone(&mint_attempts);
12191 move |_| {
12192 mint_attempts.fetch_add(1, Ordering::AcqRel);
12193 Err(anyhow::anyhow!("mint unavailable"))
12194 }
12195 },
12196 |_channel, _token| async { Ok::<(), crate::websocket::error::LighterWsError>(()) },
12197 );
12198
12199 let cancel_task = async move {
12200 cancel_after_first_attempt.await;
12201 cancel.cancel();
12202 };
12203
12204 let (outcome, ()) = tokio::time::timeout(Duration::from_secs(6), async {
12205 tokio::join!(refresh, cancel_task)
12206 })
12207 .await
12208 .expect("rotation cancellation must complete within the test window");
12209
12210 assert_eq!(outcome, AuthTokenRefreshOutcome::Cancelled);
12211 assert_eq!(
12212 mint_attempts.load(Ordering::Acquire),
12213 1,
12214 "cancellation during backoff must stop before the next retry",
12215 );
12216 }
12217
12218 #[rstest]
12219 #[case::rotated(AuthTokenRefreshOutcome::Rotated, Some(AUTH_TOKEN_REFRESH_INTERVAL))]
12220 #[case::cancelled(AuthTokenRefreshOutcome::Cancelled, None)]
12221 #[case::exhausted(
12222 AuthTokenRefreshOutcome::Exhausted,
12223 Some(AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY)
12224 )]
12225 fn auth_token_refresh_next_delay_matches_outcome(
12226 #[case] outcome: AuthTokenRefreshOutcome,
12227 #[case] expected: Option<Duration>,
12228 ) {
12229 assert_eq!(auth_token_refresh_next_delay(outcome), expected);
12230 }
12231
12232 #[rstest]
12233 fn auth_token_rotation_channels_match_private_account_streams() {
12234 assert_eq!(
12235 auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64),
12236 [
12237 LighterWsChannel::AccountAllOrders(TEST_ACCOUNT_INDEX_I64),
12238 LighterWsChannel::AccountAllTrades(TEST_ACCOUNT_INDEX_I64),
12239 LighterWsChannel::AccountAllPositions(TEST_ACCOUNT_INDEX_I64),
12240 LighterWsChannel::AccountAllAssets(TEST_ACCOUNT_INDEX_I64),
12241 LighterWsChannel::AccountTx(TEST_ACCOUNT_INDEX_I64),
12242 LighterWsChannel::UserStats(TEST_ACCOUNT_INDEX_I64),
12243 ],
12244 );
12245 }
12246
12247 #[rstest]
12248 #[case::below_max(
12249 Duration::from_millis(10),
12250 Duration::from_millis(100),
12251 Duration::from_millis(20)
12252 )]
12253 #[case::at_max(
12254 Duration::from_millis(100),
12255 Duration::from_millis(100),
12256 Duration::from_millis(100)
12257 )]
12258 #[case::overflow_clamps(Duration::MAX, Duration::from_secs(300), Duration::from_secs(300))]
12259 fn next_auth_token_refresh_retry_delay_doubles_and_caps(
12260 #[case] current: Duration,
12261 #[case] max: Duration,
12262 #[case] expected: Duration,
12263 ) {
12264 assert_eq!(next_auth_token_refresh_retry_delay(current, max), expected);
12265 }
12266
12267 #[rstest]
12268 #[tokio::test]
12269 async fn auth_token_refresh_is_retained_until_session_shutdown() {
12270 let (mut client, _cache, _rx) = create_execution_client();
12271
12272 client
12273 .spawn_auth_token_refresh(test_credential())
12274 .expect("spawn auth token refresh");
12275 client
12276 .nonce_recovery_inflight
12277 .store(true, Ordering::Release);
12278
12279 assert!(
12280 client
12281 .auth_refresh_handle
12282 .as_ref()
12283 .is_some_and(|handle| !handle.is_finished()),
12284 );
12285
12286 client.begin_session_shutdown();
12287 client
12288 .finish_session_shutdown()
12289 .await
12290 .expect("session shutdown");
12291
12292 assert!(client.auth_refresh_handle.is_none());
12293 assert!(!client.nonce_recovery_inflight.load(Ordering::Acquire));
12294 }
12295
12296 #[rstest]
12297 #[tokio::test]
12298 async fn session_shutdown_rejects_tasks_registered_during_abort() {
12299 struct SpawnOnDrop {
12300 pending_tasks: TaskSpawner,
12301 rejected: Arc<AtomicBool>,
12302 polled: Arc<AtomicBool>,
12303 }
12304
12305 impl Drop for SpawnOnDrop {
12306 fn drop(&mut self) {
12307 let polled = Arc::clone(&self.polled);
12308 let result = self.pending_tasks.spawn(async move {
12309 polled.store(true, Ordering::Release);
12310 });
12311 self.rejected.store(result.is_err(), Ordering::Release);
12312 }
12313 }
12314
12315 let (mut client, _cache, _rx) = create_execution_client();
12316 let rejected = Arc::new(AtomicBool::new(false));
12317 let polled = Arc::new(AtomicBool::new(false));
12318 let pending_spawner = client.pending_tasks.spawner().expect("task spawner");
12319 let guard = SpawnOnDrop {
12320 pending_tasks: pending_spawner.clone(),
12321 rejected: Arc::clone(&rejected),
12322 polled: Arc::clone(&polled),
12323 };
12324
12325 pending_spawner
12326 .spawn(async move {
12327 let _guard = guard;
12328 std::future::pending::<()>().await;
12329 })
12330 .expect("parent task spawn");
12331
12332 client
12333 .finish_session_shutdown()
12334 .await
12335 .expect("session shutdown");
12336
12337 assert!(client.pending_tasks.is_empty());
12338 assert!(rejected.load(Ordering::Acquire));
12339 assert!(!polled.load(Ordering::Acquire));
12340 }
12341
12342 #[rstest]
12343 #[tokio::test]
12344 async fn session_shutdown_discards_pending_sendtx_before_epoch_reuse() {
12345 struct EnqueueOnDrop {
12346 dispatch: WsDispatchState,
12347 pending: Option<PendingSendTx>,
12348 }
12349
12350 impl Drop for EnqueueOnDrop {
12351 fn drop(&mut self) {
12352 self.dispatch
12353 .enqueue_pending_sendtx(self.pending.take().expect("pending sendTx"));
12354 }
12355 }
12356
12357 let (mut client, _cache, _rx) = create_execution_client();
12358 let old_epoch = client.ws_client.connection_epoch();
12359 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
12360 batch_id: None,
12361 connection_epoch: old_epoch,
12362 kind: PendingSendTxKind::Other,
12363 submitted_at: UnixNanos::default(),
12364 nonce: TEST_NEXT_NONCE,
12365 api_key_index: TEST_API_KEY_INDEX,
12366 tx_hash: "shutdown-pending".to_string(),
12367 });
12368 let guard = EnqueueOnDrop {
12369 dispatch: client.dispatch.clone(),
12370 pending: Some(PendingSendTx {
12371 batch_id: None,
12372 connection_epoch: old_epoch,
12373 kind: PendingSendTxKind::Other,
12374 submitted_at: UnixNanos::default(),
12375 nonce: TEST_NEXT_NONCE + 1,
12376 api_key_index: TEST_API_KEY_INDEX,
12377 tx_hash: "shutdown-late".to_string(),
12378 }),
12379 };
12380
12381 client
12382 .pending_tasks
12383 .spawn(async move {
12384 let _guard = guard;
12385 std::future::pending::<()>().await;
12386 })
12387 .expect("pending sendTx producer spawn");
12388 client
12389 .ws_stream_handle
12390 .insert(get_runtime().spawn(std::future::pending()));
12391
12392 client.begin_session_shutdown();
12393 client
12394 .finish_session_shutdown()
12395 .await
12396 .expect("session shutdown");
12397
12398 assert_eq!(client.ws_client.connection_epoch(), old_epoch);
12399 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12400 }
12401
12402 #[rstest]
12403 fn replacing_ws_client_preserves_cached_instruments() {
12404 let (mut client, cache, _rx) = create_execution_client();
12405 let instrument_id = register_test_instrument(&client, &cache);
12406 let instrument = cache
12407 .borrow()
12408 .instrument(&instrument_id)
12409 .expect("instrument")
12410 .clone();
12411 client
12412 .ws_client
12413 .cache_instrument(TEST_MARKET_INDEX, instrument.clone());
12414
12415 let _old_ws_client = client.take_ws_client();
12416
12417 let cached_instruments = client.ws_client.instruments_cache();
12418 assert_eq!(
12419 cached_instruments
12420 .get(&TEST_MARKET_INDEX)
12421 .expect("cached instrument")
12422 .value(),
12423 &instrument,
12424 );
12425 }
12426
12427 #[rstest]
12428 fn tx_send_sequencer_allocates_under_registration_lock() {
12429 let sequencer = TxSendSequencer::new();
12430
12431 let mut lower = sequencer
12432 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12433 assert!(
12434 sequencer.state.try_lock().is_none(),
12435 "allocation must be serialized with registration"
12436 );
12437 Ok(TEST_NEXT_NONCE)
12438 })
12439 .unwrap();
12440
12441 let higher = sequencer
12442 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12443 Ok(TEST_NEXT_NONCE + 1)
12444 })
12445 .unwrap();
12446
12447 assert!(!sequencer.ready_to_send(higher.key, higher.nonce));
12448 lower.release();
12449 assert!(sequencer.ready_to_send(higher.key, higher.nonce));
12450 }
12451
12452 #[rstest]
12453 fn tx_send_sequencer_retains_reused_nonce_reservation() {
12454 let (client, _cache, _rx) = create_execution_client();
12455 let context = client
12456 .fanout_dispatch_context(client.credential.as_ref().unwrap())
12457 .unwrap();
12458 let mut original = context.build_tx_context().unwrap();
12459 let nonce = original.context.nonce;
12460 rollback_tx_dispatch(
12461 &client.dispatch,
12462 client.credential.as_ref().unwrap(),
12463 None,
12464 nonce,
12465 );
12466 let mut replacement = context.build_tx_context().unwrap();
12467 let higher = context.build_tx_context().unwrap();
12468 original.send_reservation.release();
12469 assert_eq!(replacement.context.nonce, nonce);
12470 assert_eq!(higher.context.nonce, nonce + 1);
12471 assert!(
12472 !context
12473 .tx_send_sequencer
12474 .ready_to_send(higher.send_reservation.key, higher.context.nonce)
12475 );
12476 replacement.send_reservation.release();
12477 assert!(
12478 context
12479 .tx_send_sequencer
12480 .ready_to_send(higher.send_reservation.key, higher.context.nonce)
12481 );
12482 }
12483
12484 #[tokio::test]
12485 async fn tx_send_sequencer_blocks_higher_nonce_until_lower_batch_releases() {
12486 let sequencer = TxSendSequencer::new();
12487
12488 let mut lower_a = sequencer
12489 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12490 Ok(TEST_NEXT_NONCE)
12491 })
12492 .unwrap();
12493
12494 let mut lower_b = sequencer
12495 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12496 Ok(TEST_NEXT_NONCE + 1)
12497 })
12498 .unwrap();
12499
12500 let mut higher = sequencer
12501 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12502 Ok(TEST_NEXT_NONCE + 2)
12503 })
12504 .unwrap();
12505
12506 let (sent_tx, mut sent_rx) = tokio::sync::mpsc::unbounded_channel();
12507
12508 let higher_task = tokio::spawn(async move {
12509 higher.wait_for_turn().await;
12510 sent_tx.send(higher.nonce).unwrap();
12511 higher.release();
12512 });
12513
12514 assert!(
12515 tokio::time::timeout(Duration::from_millis(50), sent_rx.recv())
12516 .await
12517 .is_err(),
12518 "higher nonce must wait while the lower batch is pending",
12519 );
12520
12521 {
12522 let lower_reservations = [&lower_a, &lower_b];
12523 tokio::time::timeout(
12524 Duration::from_millis(50),
12525 wait_for_tx_send_reservations(&lower_reservations),
12526 )
12527 .await
12528 .expect("lower batch must already have the send turn");
12529 }
12530
12531 lower_a.release();
12532 lower_b.release();
12533
12534 let sent = tokio::time::timeout(Duration::from_secs(2), sent_rx.recv())
12535 .await
12536 .expect("higher nonce must proceed after lower batch releases")
12537 .expect("send channel must stay open");
12538 higher_task.await.unwrap();
12539
12540 assert_eq!(
12541 sent,
12542 TEST_NEXT_NONCE + 2,
12543 "higher nonce must send after the lower batch releases",
12544 );
12545 }
12546
12547 #[tokio::test]
12548 async fn tx_send_reservation_drop_releases_lower_nonce() {
12549 let sequencer = TxSendSequencer::new();
12550
12551 let lower = sequencer
12552 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12553 Ok(TEST_NEXT_NONCE)
12554 })
12555 .unwrap();
12556
12557 let mut higher = sequencer
12558 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12559 Ok(TEST_NEXT_NONCE + 1)
12560 })
12561 .unwrap();
12562
12563 let (sent_tx, mut sent_rx) = tokio::sync::mpsc::unbounded_channel();
12564
12565 let higher_task = tokio::spawn(async move {
12566 higher.wait_for_turn().await;
12567 sent_tx.send(higher.nonce).unwrap();
12568 higher.release();
12569 });
12570
12571 assert!(
12572 tokio::time::timeout(Duration::from_millis(50), sent_rx.recv())
12573 .await
12574 .is_err(),
12575 "higher nonce must wait while the lower reservation is pending",
12576 );
12577
12578 drop(lower);
12579
12580 let sent = tokio::time::timeout(Duration::from_secs(2), sent_rx.recv())
12581 .await
12582 .expect("higher nonce must proceed after lower reservation drops")
12583 .expect("send channel must stay open");
12584 higher_task.await.unwrap();
12585
12586 assert_eq!(
12587 sent,
12588 TEST_NEXT_NONCE + 1,
12589 "higher nonce must send after the lower reservation drops",
12590 );
12591 }
12592
12593 #[tokio::test]
12594 async fn tx_send_sequencer_keeps_nonce_streams_independent() {
12595 let sequencer = TxSendSequencer::new();
12596
12597 let _other_account = sequencer
12598 .reserve(TEST_ACCOUNT_INDEX_I64 + 1, TEST_API_KEY_INDEX, || {
12599 Ok(TEST_NEXT_NONCE)
12600 })
12601 .unwrap();
12602
12603 let _other_api_key = sequencer
12604 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX + 1, || {
12605 Ok(TEST_NEXT_NONCE)
12606 })
12607 .unwrap();
12608
12609 let mut current = sequencer
12610 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
12611 Ok(TEST_NEXT_NONCE + 10)
12612 })
12613 .unwrap();
12614
12615 tokio::time::timeout(Duration::from_millis(50), current.wait_for_turn())
12616 .await
12617 .expect("lower nonces for other keys must not block this key");
12618 current.release();
12619 }
12620
12621 #[rstest]
12622 fn tx_dispatch_guard_rolls_back_nonce_and_cloid_when_armed() {
12623 let dispatch = WsDispatchState::new();
12624 let credential = test_credential();
12625 dispatch
12626 .nonce_manager
12627 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
12628
12629 let cloid = ClientOrderId::from("O-GUARD-ARMED");
12630 let client_order_index = dispatch.derive_client_order_index(&cloid);
12631 dispatch.register_cloid(client_order_index, cloid).unwrap();
12632 let nonce = dispatch
12633 .nonce_manager
12634 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12635 .unwrap();
12636
12637 {
12638 let _guard = TxDispatchGuard::new(
12639 dispatch.clone(),
12640 &credential,
12641 Some(client_order_index),
12642 nonce,
12643 );
12644 }
12645
12646 assert_nonce_reusable(&dispatch);
12647 assert!(dispatch.cloid_map.get(&client_order_index).is_none());
12648 }
12649
12650 #[rstest]
12651 fn tx_dispatch_guard_rolls_back_nonce_without_cloid_when_armed() {
12652 let dispatch = WsDispatchState::new();
12653 let credential = test_credential();
12654 dispatch
12655 .nonce_manager
12656 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
12657 let nonce = dispatch
12658 .nonce_manager
12659 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12660 .unwrap();
12661
12662 {
12663 let _guard = TxDispatchGuard::new(dispatch.clone(), &credential, None, nonce);
12664 }
12665
12666 assert_nonce_reusable(&dispatch);
12667 assert!(dispatch.cloid_map.is_empty());
12668 }
12669
12670 #[rstest]
12671 fn tx_dispatch_guard_preserves_nonce_and_cloid_when_disarmed() {
12672 let dispatch = WsDispatchState::new();
12673 let credential = test_credential();
12674 dispatch
12675 .nonce_manager
12676 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
12677
12678 let cloid = ClientOrderId::from("O-GUARD-DISARMED");
12679 let client_order_index = dispatch.derive_client_order_index(&cloid);
12680 dispatch.register_cloid(client_order_index, cloid).unwrap();
12681 let nonce = dispatch
12682 .nonce_manager
12683 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12684 .unwrap();
12685
12686 {
12687 let mut guard = TxDispatchGuard::new(
12688 dispatch.clone(),
12689 &credential,
12690 Some(client_order_index),
12691 nonce,
12692 );
12693 guard.disarm();
12694 }
12695
12696 assert_eq!(
12697 dispatch
12698 .nonce_manager
12699 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12700 Some(TEST_NEXT_NONCE),
12701 );
12702 assert_eq!(
12703 dispatch
12704 .cloid_map
12705 .get(&client_order_index)
12706 .map(|entry| *entry.value()),
12707 Some(cloid),
12708 );
12709 }
12710
12711 #[tokio::test]
12712 async fn submit_order_send_failure_emits_submitted_then_rejected_and_rolls_back() {
12713 let (client, cache, mut rx) = create_execution_client();
12714 let instrument_id = register_test_instrument(&client, &cache);
12715 let mut factory = test_order_factory();
12716 let order = test_limit_order(&mut factory, instrument_id, "O-SUBMIT-FAIL");
12717 let client_order_index = client
12718 .dispatch
12719 .derive_client_order_index(&order.client_order_id());
12720 cache_order(&cache, order.clone());
12721
12722 let command = SubmitOrder::from_order(
12723 &order,
12724 trader_id(),
12725 Some(client_id()),
12726 None,
12727 UUID4::new(),
12728 UnixNanos::default(),
12729 );
12730 client.submit_order(command).unwrap();
12731
12732 let submitted = recv_order_event(&mut rx).await;
12733 let rejected = recv_order_event(&mut rx).await;
12734
12735 match submitted {
12736 OrderEventAny::Submitted(event) => {
12737 assert_eq!(event.client_order_id, order.client_order_id());
12738 assert_eq!(event.instrument_id, instrument_id);
12739 }
12740 event => panic!("expected submitted event, was {event:?}"),
12741 }
12742
12743 match rejected {
12744 OrderEventAny::Rejected(event) => {
12745 assert_eq!(event.client_order_id, order.client_order_id());
12746 assert_eq!(event.instrument_id, instrument_id);
12747 assert!(
12748 event
12749 .reason
12750 .contains("Lighter submit_order dispatch failed"),
12751 );
12752 assert!(event.reason.contains("handler unavailable"));
12753 }
12754 event => panic!("expected rejected event, was {event:?}"),
12755 }
12756
12757 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12758 assert_nonce_reusable(&client.dispatch);
12759 assert_eq!(
12760 client.dispatch.pending_sendtx_len(),
12761 0,
12762 "local-send-failure must remove the pending entry by nonce",
12763 );
12764 }
12765
12766 #[tokio::test]
12767 async fn submit_order_unknown_send_outcome_retains_pending_state() {
12768 let (client, cache, mut rx) = create_execution_client();
12769 let instrument_id = register_test_instrument(&client, &cache);
12770 let mut factory = test_order_factory();
12771 let order = test_limit_order(&mut factory, instrument_id, "O-SUBMIT-UNKNOWN");
12772 let client_order_index = client
12773 .dispatch
12774 .derive_client_order_index(&order.client_order_id());
12775 cache_order(&cache, order.clone());
12776 client.ws_client.drop_next_send_tx_result_for_test().await;
12777
12778 client
12779 .submit_order(SubmitOrder::from_order(
12780 &order,
12781 trader_id(),
12782 Some(client_id()),
12783 None,
12784 UUID4::new(),
12785 UnixNanos::default(),
12786 ))
12787 .unwrap();
12788
12789 assert!(matches!(
12790 recv_order_event(&mut rx).await,
12791 OrderEventAny::Submitted(_)
12792 ));
12793 assert!(
12794 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12795 .await
12796 .is_err(),
12797 "unknown outcome must not emit OrderRejected",
12798 );
12799 assert!(client.dispatch.cloid_map.contains_key(&client_order_index));
12800 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
12801 assert_eq!(
12802 client
12803 .dispatch
12804 .nonce_manager
12805 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12806 Some(TEST_NEXT_NONCE),
12807 );
12808
12809 let stale = client.dispatch.drain_pending_sendtx(0);
12810 assert_eq!(stale.len(), 1);
12811 assert!(matches!(stale[0].kind, PendingSendTxKind::Create { .. }));
12812 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12813 assert!(client.dispatch.cloid_map.contains_key(&client_order_index));
12814 assert!(
12815 client
12816 .dispatch
12817 .order_identity(&order.client_order_id())
12818 .is_some(),
12819 "reconnect draining must retain identity for reconciliation",
12820 );
12821 assert_eq!(
12822 client
12823 .dispatch
12824 .nonce_manager
12825 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12826 Some(TEST_NEXT_NONCE),
12827 );
12828 }
12829
12830 #[tokio::test]
12831 async fn submit_order_cloid_probe_exhaustion_denies_without_overwrite() {
12832 let (client, cache, mut rx) = create_execution_client();
12833 let instrument_id = register_test_instrument(&client, &cache);
12834 let mut factory = test_order_factory();
12835 let order = test_limit_order(&mut factory, instrument_id, "O-PROBE-EXHAUSTED");
12836 let cloid = order.client_order_id();
12837 let mut index = client.dispatch.derive_client_order_index(&cloid);
12838 let mut existing = Vec::new();
12839
12840 for attempt in 0..=crate::websocket::dispatch::CLOID_INDEX_PROBE_LIMIT {
12841 let existing_cloid = ClientOrderId::from(format!("EXISTING-{attempt}"));
12842 client.dispatch.cloid_map.insert(index, existing_cloid);
12843 existing.push((index, existing_cloid));
12844 index = if index == i64::from(i32::MAX) {
12845 0
12846 } else {
12847 index + 1
12848 };
12849 }
12850
12851 cache_order(&cache, order.clone());
12852
12853 client
12854 .submit_order(SubmitOrder::from_order(
12855 &order,
12856 trader_id(),
12857 Some(client_id()),
12858 None,
12859 UUID4::new(),
12860 UnixNanos::default(),
12861 ))
12862 .unwrap();
12863
12864 let event = recv_order_event(&mut rx).await;
12865 assert!(matches!(event, OrderEventAny::Denied(_)));
12866 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12867 assert!(client.dispatch.order_identity(&cloid).is_none());
12868
12869 for (existing_index, existing_cloid) in existing {
12870 assert_eq!(
12871 client
12872 .dispatch
12873 .cloid_map
12874 .get(&existing_index)
12875 .map(|entry| *entry.value()),
12876 Some(existing_cloid),
12877 );
12878 }
12879 }
12880
12881 #[tokio::test]
12882 async fn submit_sell_order_send_failure_dispatches_and_rolls_back() {
12883 let (client, cache, mut rx) = create_execution_client();
12886 let instrument_id = register_test_instrument(&client, &cache);
12887 let mut factory = test_order_factory();
12888 let order = test_limit_order_with(
12889 &mut factory,
12890 instrument_id,
12891 "O-SUBMIT-FAIL-SELL",
12892 OrderSide::Sell,
12893 TimeInForce::Gtc,
12894 None,
12895 false,
12896 );
12897 let client_order_index = client
12898 .dispatch
12899 .derive_client_order_index(&order.client_order_id());
12900 cache_order(&cache, order.clone());
12901
12902 let command = SubmitOrder::from_order(
12903 &order,
12904 trader_id(),
12905 Some(client_id()),
12906 None,
12907 UUID4::new(),
12908 UnixNanos::default(),
12909 );
12910 client.submit_order(command).unwrap();
12911
12912 let _submitted = recv_order_event(&mut rx).await;
12913 let rejected = recv_order_event(&mut rx).await;
12914
12915 match rejected {
12916 OrderEventAny::Rejected(event) => {
12917 assert_eq!(event.client_order_id, order.client_order_id());
12918 assert!(
12919 event
12920 .reason
12921 .contains("Lighter submit_order dispatch failed"),
12922 );
12923 }
12924 event => panic!("expected rejected event, was {event:?}"),
12925 }
12926
12927 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12928 assert_nonce_reusable(&client.dispatch);
12929 }
12930
12931 #[tokio::test]
12932 async fn submit_gtd_order_with_explicit_expiry_dispatches_and_rolls_back() {
12933 let (client, cache, mut rx) = create_execution_client();
12938 let instrument_id = register_test_instrument(&client, &cache);
12939 let mut factory = test_order_factory();
12940 let expiry = client.clock.get_time_ns() + DurationNanos::from_mins(10);
12941 let order = test_limit_order_with(
12942 &mut factory,
12943 instrument_id,
12944 "O-SUBMIT-FAIL-GTD",
12945 OrderSide::Buy,
12946 TimeInForce::Gtd,
12947 Some(expiry),
12948 false,
12949 );
12950 let client_order_index = client
12951 .dispatch
12952 .derive_client_order_index(&order.client_order_id());
12953 cache_order(&cache, order.clone());
12954
12955 let command = SubmitOrder::from_order(
12956 &order,
12957 trader_id(),
12958 Some(client_id()),
12959 None,
12960 UUID4::new(),
12961 UnixNanos::default(),
12962 );
12963 client.submit_order(command).unwrap();
12964
12965 let _submitted = recv_order_event(&mut rx).await;
12966 let rejected = recv_order_event(&mut rx).await;
12967
12968 match rejected {
12969 OrderEventAny::Rejected(event) => {
12970 assert!(
12971 event
12972 .reason
12973 .contains("Lighter submit_order dispatch failed"),
12974 );
12975 }
12976 event => panic!("expected rejected event, was {event:?}"),
12977 }
12978
12979 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12980 assert_nonce_reusable(&client.dispatch);
12981 }
12982
12983 #[tokio::test]
12984 async fn submit_gtd_order_with_short_expiry_is_denied_before_nonce_allocation() {
12985 let (client, cache, mut rx) = create_execution_client();
12986 let instrument_id = register_test_instrument(&client, &cache);
12987 let mut factory = test_order_factory();
12988 let expiry = UnixNanos::from(
12989 client
12990 .clock
12991 .get_time_ns()
12992 .as_u64()
12993 .saturating_add(60 * 1_000_000_000),
12994 );
12995 let order = test_limit_order_with(
12996 &mut factory,
12997 instrument_id,
12998 "O-SHORT-GTD",
12999 OrderSide::Buy,
13000 TimeInForce::Gtd,
13001 Some(expiry),
13002 false,
13003 );
13004 cache_order(&cache, order.clone());
13005
13006 let command = SubmitOrder::from_order(
13007 &order,
13008 trader_id(),
13009 Some(client_id()),
13010 None,
13011 UUID4::new(),
13012 UnixNanos::default(),
13013 );
13014 client.submit_order(command).unwrap();
13015
13016 let denied = recv_order_event(&mut rx).await;
13017 match denied {
13018 OrderEventAny::Denied(event) => {
13019 assert!(event.reason.contains("at least 5 minutes"));
13020 }
13021 event => panic!("expected denied event, was {event:?}"),
13022 }
13023
13024 assert_nonce_reusable(&client.dispatch);
13025 }
13026
13027 #[tokio::test]
13028 async fn submit_gtd_order_with_short_expiry_dispatches_when_use_gtd_false() {
13029 let (client, cache, mut rx) =
13034 create_execution_client_with_config(test_config_with_use_gtd(false));
13035 let instrument_id = register_test_instrument(&client, &cache);
13036 let mut factory = test_order_factory();
13037 let expiry = UnixNanos::from(
13038 client
13039 .clock
13040 .get_time_ns()
13041 .as_u64()
13042 .saturating_add(60 * 1_000_000_000),
13043 );
13044 let order = test_limit_order_with(
13045 &mut factory,
13046 instrument_id,
13047 "O-SHORT-GTD-MANAGED",
13048 OrderSide::Buy,
13049 TimeInForce::Gtd,
13050 Some(expiry),
13051 false,
13052 );
13053 let client_order_index = client
13054 .dispatch
13055 .derive_client_order_index(&order.client_order_id());
13056 cache_order(&cache, order.clone());
13057
13058 let command = SubmitOrder::from_order(
13059 &order,
13060 trader_id(),
13061 Some(client_id()),
13062 None,
13063 UUID4::new(),
13064 UnixNanos::default(),
13065 );
13066 client.submit_order(command).unwrap();
13067
13068 let _submitted = recv_order_event(&mut rx).await;
13069 let rejected = recv_order_event(&mut rx).await;
13070
13071 match rejected {
13072 OrderEventAny::Rejected(event) => {
13073 assert!(
13074 event
13075 .reason
13076 .contains("Lighter submit_order dispatch failed"),
13077 );
13078 }
13079 event => panic!("expected rejected event, was {event:?}"),
13080 }
13081
13082 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
13083 assert_nonce_reusable(&client.dispatch);
13084 }
13085
13086 #[rstest]
13087 fn prepare_managed_gtd_plan_uses_safe_venue_expiry() {
13088 let (client, cache, _rx) =
13089 create_execution_client_with_config(test_config_with_use_gtd(false));
13090 let instrument_id = register_test_instrument(&client, &cache);
13091 let mut factory = test_order_factory();
13092 let expiry = UnixNanos::from(
13093 client
13094 .clock
13095 .get_time_ns()
13096 .as_u64()
13097 .saturating_add(60 * 1_000_000_000),
13098 );
13099 let order = test_limit_order_with(
13100 &mut factory,
13101 instrument_id,
13102 "O-SHORT-GTD-PLAN",
13103 OrderSide::Buy,
13104 TimeInForce::Gtd,
13105 Some(expiry),
13106 false,
13107 );
13108
13109 let before_ms = (client.clock.get_time_ns().as_u64() / 1_000_000) as i64;
13110 let plan = client
13111 .prepare_create_order_plan(&order, None)
13112 .expect("managed GTD plan must validate");
13113 let after_ms = (client.clock.get_time_ns().as_u64() / 1_000_000) as i64;
13114
13115 assert!(
13116 (before_ms + crate::websocket::dispatch::ORDER_EXPIRY_DEFAULT_GTC_MS
13117 ..=after_ms + crate::websocket::dispatch::ORDER_EXPIRY_DEFAULT_GTC_MS)
13118 .contains(&plan.order_expiry)
13119 );
13120 assert_eq!(
13121 plan.time_in_force,
13122 crate::common::enums::LighterTimeInForce::GoodTillTime as u8,
13123 );
13124 }
13125
13126 #[tokio::test]
13127 async fn submit_reduce_only_order_dispatches_and_rolls_back() {
13128 let (client, cache, mut rx) = create_execution_client();
13130 let instrument_id = register_test_instrument(&client, &cache);
13131 let mut factory = test_order_factory();
13132 let order = test_limit_order_with(
13133 &mut factory,
13134 instrument_id,
13135 "O-SUBMIT-FAIL-REDUCE",
13136 OrderSide::Sell,
13137 TimeInForce::Gtc,
13138 None,
13139 true,
13140 );
13141 assert!(order.is_reduce_only());
13142 let client_order_index = client
13143 .dispatch
13144 .derive_client_order_index(&order.client_order_id());
13145 cache_order(&cache, order.clone());
13146
13147 let command = SubmitOrder::from_order(
13148 &order,
13149 trader_id(),
13150 Some(client_id()),
13151 None,
13152 UUID4::new(),
13153 UnixNanos::default(),
13154 );
13155 client.submit_order(command).unwrap();
13156
13157 let _submitted = recv_order_event(&mut rx).await;
13158 let rejected = recv_order_event(&mut rx).await;
13159
13160 match rejected {
13161 OrderEventAny::Rejected(event) => {
13162 assert!(
13163 event
13164 .reason
13165 .contains("Lighter submit_order dispatch failed"),
13166 );
13167 }
13168 event => panic!("expected rejected event, was {event:?}"),
13169 }
13170
13171 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
13172 assert_nonce_reusable(&client.dispatch);
13173 }
13174
13175 #[rstest]
13176 #[case::nonce_refresh(false)]
13177 #[case::submission_gate(true)]
13178 #[tokio::test]
13179 async fn batch_signing_retries_after_nonce_readiness_recovers(#[case] lock_submission: bool) {
13180 let (client, _cache, _rx) = create_execution_client();
13181 let credential = test_credential();
13182 let context = client.fanout_dispatch_context(&credential).unwrap();
13183 let connection_epoch = context.ws_client.connection_epoch();
13184 let guard = if lock_submission {
13185 Some(context.nonce_submission_gate.write().await)
13186 } else {
13187 context
13188 .nonce_ready_connection_epoch
13189 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
13190 None
13191 };
13192
13193 let error = context.build_tx_context().unwrap_err();
13194 drop(guard);
13195 context
13196 .nonce_ready_connection_epoch
13197 .store(connection_epoch, Ordering::Release);
13198 let reserved = context.build_tx_context().unwrap();
13199
13200 assert_eq!(reserved.context.nonce, TEST_NEXT_NONCE);
13201 assert!(should_retry_batch_signing(&error));
13202 assert_eq!(
13203 error.to_string(),
13204 if lock_submission {
13205 "Lighter nonce refresh is in progress".to_string()
13206 } else {
13207 format!("Lighter nonce refresh is pending for connection epoch {connection_epoch}")
13208 }
13209 );
13210 }
13211
13212 #[rstest]
13213 #[case::state_changed(TxPreparationError::StateChanged.into(), true)]
13214 #[case::skip_window(NonceError::SkipWindowExhausted {
13215 account_index: TEST_ACCOUNT_INDEX_I64,
13216 api_key_index: TEST_API_KEY_INDEX,
13217 outstanding: 17,
13218 skip_window: 16,
13219 }.into(), true)]
13220 #[case::uninitialized(NonceError::NotInitialized {
13221 account_index: TEST_ACCOUNT_INDEX_I64,
13222 api_key_index: TEST_API_KEY_INDEX,
13223 }.into(), false)]
13224 #[case::signing(anyhow::anyhow!("invalid signing key"), false)]
13225 fn batch_signing_retry_classifies_preparation_errors(
13226 #[case] error: anyhow::Error,
13227 #[case] expected: bool,
13228 ) {
13229 assert_eq!(should_retry_batch_signing(&error), expected);
13230 }
13231
13232 #[rstest]
13233 #[case::submit(false)]
13234 #[case::modify(true)]
13235 #[tokio::test]
13236 async fn batch_orders_wait_for_nonce_readiness(#[case] modify: bool) {
13237 let (client, cache, mut rx) = create_execution_client();
13238 let instrument_id = register_test_instrument(&client, &cache);
13239 let mut factory = test_order_factory();
13240 let orders: Vec<_> = ["O-READY-A", "O-READY-B"]
13241 .into_iter()
13242 .map(|id| test_limit_order(&mut factory, instrument_id, id))
13243 .collect();
13244
13245 for order in &orders {
13246 cache_order(&cache, order.clone());
13247 }
13248
13249 client.ws_client.drop_next_send_tx_result_for_test().await;
13250 client
13251 .nonce_ready_connection_epoch
13252 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
13253
13254 if modify {
13255 let modifies = orders
13256 .iter()
13257 .enumerate()
13258 .map(|(i, order)| {
13259 ModifyOrder::new(
13260 trader_id(),
13261 Some(client_id()),
13262 strategy_id(),
13263 instrument_id,
13264 order.client_order_id(),
13265 Some(VenueOrderId::from(format!("{}", 123 + i))),
13266 Some(Quantity::from("0.2000")),
13267 Some(Price::from("2362.00")),
13268 None,
13269 UUID4::new(),
13270 UnixNanos::default(),
13271 None,
13272 None,
13273 )
13274 })
13275 .collect();
13276
13277 client
13278 .batch_modify_orders(BatchModifyOrders::new(
13279 trader_id(),
13280 Some(client_id()),
13281 strategy_id(),
13282 instrument_id,
13283 modifies,
13284 UUID4::new(),
13285 UnixNanos::default(),
13286 None,
13287 None,
13288 ))
13289 .unwrap();
13290 } else {
13291 client
13292 .submit_order_list(submit_order_list_command(&orders, "OL-READY"))
13293 .unwrap();
13294 }
13295
13296 assert!(
13297 tokio::time::timeout(Duration::from_millis(100), rx.recv())
13298 .await
13299 .is_err()
13300 );
13301 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13302 assert_eq!(
13303 client
13304 .dispatch
13305 .nonce_manager
13306 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13307 Some(TEST_NEXT_NONCE - 1)
13308 );
13309 client
13310 .nonce_ready_connection_epoch
13311 .store(0, Ordering::Release);
13312 wait_until_async(
13313 || async { client.dispatch.pending_sendtx_len() == 2 },
13314 Duration::from_secs(2),
13315 )
13316 .await;
13317
13318 let pending = client.dispatch.pending_sendtx.lock().clone();
13319 assert_eq!(pending.len(), 2);
13320 assert_eq!(pending[0].batch_id, pending[1].batch_id);
13321 assert!(
13322 pending[0]
13323 .batch_id
13324 .as_ref()
13325 .unwrap()
13326 .starts_with(TX_BATCH_ID_PREFIX)
13327 );
13328
13329 for (i, (pending, order)) in pending.iter().zip(&orders).enumerate() {
13330 assert_eq!(pending.nonce, TEST_NEXT_NONCE + i as i64);
13331 assert_eq!(pending.api_key_index, TEST_API_KEY_INDEX);
13332 assert_eq!(pending.connection_epoch, 0);
13333
13334 if modify {
13335 assert!(
13336 matches!(&pending.kind, PendingSendTxKind::Modify { client_order_id, .. } if *client_order_id == order.client_order_id())
13337 );
13338 assert_eq!(
13339 client
13340 .dispatch
13341 .pending_order_action(&order.client_order_id()),
13342 Some(PendingOrderAction::Modify)
13343 );
13344 } else {
13345 assert!(
13346 matches!(&pending.kind, PendingSendTxKind::Create { order: pending_order, .. } if pending_order.client_order_id() == order.client_order_id())
13347 );
13348 let event = recv_order_event(&mut rx).await;
13349
13350 let OrderEventAny::Submitted(event) = event else {
13351 panic!("expected Submitted after nonce readiness")
13352 };
13353
13354 assert_eq!(event.client_order_id, order.client_order_id());
13355 }
13356 }
13357
13358 assert!(
13359 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13360 .await
13361 .is_err()
13362 );
13363 assert_eq!(
13364 client
13365 .dispatch
13366 .nonce_manager
13367 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13368 Some(TEST_NEXT_NONCE + 1)
13369 );
13370 }
13371
13372 #[tokio::test]
13373 async fn submit_order_list_send_failure_emits_submitted_then_rejected_and_rolls_back() {
13374 let (client, cache, mut rx) = create_execution_client();
13375 let instrument_id = register_test_instrument(&client, &cache);
13376 let mut factory = test_order_factory();
13377 let order_a = test_limit_order(&mut factory, instrument_id, "O-LIST-FAIL-A");
13378 let order_b = test_limit_order(&mut factory, instrument_id, "O-LIST-FAIL-B");
13379 let index_a = client
13380 .dispatch
13381 .derive_client_order_index(&order_a.client_order_id());
13382 let index_b = client
13383 .dispatch
13384 .derive_client_order_index(&order_b.client_order_id());
13385 cache_order(&cache, order_a.clone());
13386 cache_order(&cache, order_b.clone());
13387
13388 let command = submit_order_list_command(&[order_a.clone(), order_b.clone()], "OL-FAIL");
13389 client.submit_order_list(command).unwrap();
13390
13391 for expected in [order_a.client_order_id(), order_b.client_order_id()] {
13392 match recv_order_event(&mut rx).await {
13393 OrderEventAny::Submitted(e) => assert_eq!(e.client_order_id, expected),
13394 other => panic!("expected Submitted, was {other:?}"),
13395 }
13396 }
13397
13398 for expected in [order_b.client_order_id(), order_a.client_order_id()] {
13399 match recv_order_event(&mut rx).await {
13400 OrderEventAny::Rejected(e) => {
13401 assert_eq!(e.client_order_id, expected);
13402 assert!(e.reason.contains("Lighter submit_order dispatch failed"));
13403 }
13404 other => panic!("expected Rejected, was {other:?}"),
13405 }
13406 }
13407
13408 assert!(client.dispatch.cloid_map.get(&index_a).is_none());
13409 assert!(client.dispatch.cloid_map.get(&index_b).is_none());
13410 assert_nonce_reusable(&client.dispatch);
13411 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13412 }
13413
13414 #[rstest]
13415 #[case::spill(16)]
13416 #[case::three(31)]
13417 #[tokio::test]
13418 async fn submit_order_list_chunks_large_lists_on_send_failure(#[case] count: usize) {
13419 let (client, cache, mut rx) = create_execution_client();
13420 let instrument_id = register_test_instrument(&client, &cache);
13421 let mut factory = test_order_factory();
13422 let mut orders = Vec::new();
13423
13424 for i in 0..count {
13425 let order = test_limit_order(&mut factory, instrument_id, &format!("O-LIST-MAX-{i}"));
13426 cache_order(&cache, order.clone());
13427 orders.push(order);
13428 }
13429
13430 let command = submit_order_list_command(&orders, "OL-MAX");
13431 client.submit_order_list(command).unwrap();
13432
13433 for chunk in orders.chunks(LIGHTER_MAX_BATCH_TX) {
13434 for order in chunk {
13435 match recv_order_event(&mut rx).await {
13436 OrderEventAny::Submitted(e) => {
13437 assert_eq!(e.client_order_id, order.client_order_id());
13438 }
13439 other => panic!("expected Submitted, was {other:?}"),
13440 }
13441 }
13442
13443 for order in chunk.iter().rev() {
13444 match recv_order_event(&mut rx).await {
13445 OrderEventAny::Rejected(e) => {
13446 assert_eq!(e.client_order_id, order.client_order_id());
13447 assert!(e.reason.contains("Lighter submit_order dispatch failed"));
13448 }
13449 other => panic!("expected Rejected, was {other:?}"),
13450 }
13451 }
13452 }
13453
13454 assert!(
13455 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13456 .await
13457 .is_err(),
13458 "chunked send failure must not emit extra events",
13459 );
13460
13461 for order in &orders {
13462 let index = client
13463 .dispatch
13464 .derive_client_order_index(&order.client_order_id());
13465 assert!(client.dispatch.cloid_map.get(&index).is_none());
13466 }
13467
13468 assert_nonce_reusable(&client.dispatch);
13469 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13470 }
13471
13472 #[tokio::test]
13473 async fn submit_order_list_denies_unsupported_order_and_dispatches_supported() {
13474 let (client, cache, mut rx) = create_execution_client();
13475 let instrument_id = register_test_instrument(&client, &cache);
13476 let mut factory = test_order_factory();
13477 let valid = test_limit_order(&mut factory, instrument_id, "O-LIST-VALID");
13478 let unsupported = factory.limit(
13479 instrument_id,
13480 OrderSide::Buy,
13481 Quantity::from("0.1000"),
13482 Price::from("2361.31"),
13483 Some(TimeInForce::Gtc),
13484 None,
13485 Some(false),
13486 Some(false),
13487 None,
13488 Some(Quantity::from("0.0500")),
13489 None,
13490 None,
13491 None,
13492 None,
13493 None,
13494 Some(ClientOrderId::from("O-LIST-ICEBERG")),
13495 );
13496 let unsupported_index = client
13497 .dispatch
13498 .derive_client_order_index(&unsupported.client_order_id());
13499 cache_order(&cache, valid.clone());
13500 cache_order(&cache, unsupported.clone());
13501
13502 let command =
13503 submit_order_list_command(&[unsupported.clone(), valid.clone()], "OL-PARTIAL");
13504 client.submit_order_list(command).unwrap();
13505
13506 match recv_order_event(&mut rx).await {
13507 OrderEventAny::Denied(e) => {
13508 assert_eq!(e.client_order_id, unsupported.client_order_id());
13509 assert!(e.reason.contains("display_qty"));
13510 }
13511 other => panic!("expected Denied, was {other:?}"),
13512 }
13513
13514 match recv_order_event(&mut rx).await {
13515 OrderEventAny::Submitted(e) => assert_eq!(e.client_order_id, valid.client_order_id()),
13516 other => panic!("expected Submitted, was {other:?}"),
13517 }
13518
13519 match recv_order_event(&mut rx).await {
13520 OrderEventAny::Rejected(e) => {
13521 assert_eq!(e.client_order_id, valid.client_order_id());
13522 assert!(e.reason.contains("Lighter submit_order dispatch failed"),);
13523 }
13524 other => panic!("expected Rejected, was {other:?}"),
13525 }
13526
13527 assert!(client.dispatch.cloid_map.get(&unsupported_index).is_none());
13528 assert_nonce_reusable(&client.dispatch);
13529 }
13530
13531 #[rstest]
13532 #[case(LighterGroupingType::OneTriggersTheOther)]
13533 #[case(LighterGroupingType::OneCancelsTheOther)]
13534 #[case(LighterGroupingType::OneTriggersOneCancelsTheOther)]
13535 fn prepare_native_group_preserves_relationships_and_sizes_children(
13536 #[case] grouping: LighterGroupingType,
13537 ) {
13538 let (client, cache, _rx) = create_execution_client();
13539 let instrument = register_test_instrument(&client, &cache);
13540 let mut orders = test_native_group(instrument, grouping);
13541 orders.reverse();
13542 let plan = client.prepare_create_group_plan(&orders, None).unwrap();
13543 let context = client
13544 .fanout_dispatch_context(client.credential.as_ref().unwrap())
13545 .unwrap();
13546 let signed = context.sign_create_group(&plan).unwrap();
13547 let json: serde_json::Value = serde_json::from_str(signed.tx_info.get()).unwrap();
13548 let bodies = json["Orders"].as_array().unwrap();
13549 let has_parent = grouping != LighterGroupingType::OneCancelsTheOther;
13550 assert_eq!(plan.grouping_type, grouping);
13551 assert_eq!(signed.tx_type, LighterTxType::CreateGroupedOrders as u8);
13552 assert_eq!(bodies.len(), orders.len());
13553 assert_eq!(bodies[0]["BaseAmount"], 1_000);
13554
13555 for (index, (body, leg)) in bodies.iter().zip(&plan.orders).enumerate() {
13556 assert_eq!(body["MarketIndex"], TEST_MARKET_INDEX);
13557 assert_eq!(
13558 body["ClientOrderIndex"],
13559 client
13560 .dispatch
13561 .client_order_index(&leg.order.client_order_id())
13562 .unwrap()
13563 );
13564 assert_eq!(body["Price"], leg.price);
13565 assert_eq!(body["Type"], leg.order_type);
13566 assert_eq!(body["IsAsk"], u8::from(index > 0 || !has_parent));
13567 assert_eq!(body["ReduceOnly"], u8::from(index > 0 || !has_parent));
13568 assert_eq!(body["TimeInForce"], leg.time_in_force);
13569 assert_eq!(body["TriggerPrice"], leg.trigger_price);
13570 assert_eq!(body["OrderExpiry"], leg.order_expiry);
13571 assert_eq!(
13572 body["BaseAmount"],
13573 if has_parent && index > 0 { 0 } else { 1_000 }
13574 );
13575 assert_eq!(leg.order.quantity(), Quantity::from("0.1000"));
13576 }
13577
13578 assert_eq!(signed.pending.kind.creates().count(), orders.len());
13579 assert_eq!(
13580 client
13581 .dispatch
13582 .nonce_manager
13583 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13584 Some(TEST_NEXT_NONCE)
13585 );
13586 }
13587
13588 #[tokio::test]
13589 async fn submit_native_group_send_failure_rejects_every_leg_and_releases_one_nonce() {
13590 let (client, cache, mut rx) = create_execution_client();
13591 let instrument = register_test_instrument(&client, &cache);
13592 let orders = test_native_group(
13593 instrument,
13594 LighterGroupingType::OneTriggersOneCancelsTheOther,
13595 );
13596
13597 for order in &orders {
13598 cache_order(&cache, order.clone());
13599 }
13600
13601 client
13602 .submit_order_list(submit_order_list_command(&orders, "OL-GROUP"))
13603 .unwrap();
13604
13605 for order in &orders {
13606 match recv_order_event(&mut rx).await {
13607 OrderEventAny::Submitted(event) => {
13608 assert_eq!(event.client_order_id, order.client_order_id());
13609 }
13610 event => panic!("expected submitted event, was {event:?}"),
13611 }
13612 }
13613
13614 for order in &orders {
13615 match recv_order_event(&mut rx).await {
13616 OrderEventAny::Rejected(event) => {
13617 assert_eq!(event.client_order_id, order.client_order_id());
13618 assert!(
13619 event
13620 .reason
13621 .contains("Lighter submit_order dispatch failed")
13622 );
13623 }
13624 event => panic!("expected rejected event, was {event:?}"),
13625 }
13626
13627 assert!(
13628 client
13629 .dispatch
13630 .order_identity(&order.client_order_id())
13631 .is_none()
13632 );
13633 }
13634
13635 assert_eq!(client.dispatch.cloid_map.len(), 0);
13636 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13637 assert_nonce_reusable(&client.dispatch);
13638 }
13639
13640 #[tokio::test]
13641 async fn native_group_denies_default_bracket_without_dispatch() {
13642 let (client, cache, mut rx) = create_execution_client();
13643 let instrument = register_test_instrument(&client, &cache);
13644 let orders = test_order_factory()
13645 .bracket()
13646 .instrument_id(instrument)
13647 .order_side(OrderSide::Buy)
13648 .quantity(Quantity::from("0.1000"))
13649 .entry_order_type(OrderType::Limit)
13650 .entry_price(Price::from("2300.00"))
13651 .tp_price(Price::from("2400.00"))
13652 .sl_trigger_price(Price::from("2200.00"))
13653 .call();
13654
13655 for order in &orders {
13656 cache_order(&cache, order.clone());
13657 }
13658
13659 client
13660 .submit_order_list(submit_order_list_command(
13661 &orders,
13662 orders[0].order_list_id().unwrap().as_str(),
13663 ))
13664 .unwrap();
13665
13666 for order in &orders {
13667 match recv_order_event(&mut rx).await {
13668 OrderEventAny::Denied(event) => {
13669 assert_eq!(event.client_order_id, order.client_order_id());
13670 assert_eq!(
13671 event.reason.as_str(),
13672 "UNSUPPORTED_ORDER_LIST: Lighter grouped submit failed: Lighter native groups do not support Nautilus OUO quantity updates"
13673 );
13674 }
13675 event => panic!("expected denied event, was {event:?}"),
13676 }
13677 }
13678
13679 assert_eq!(client.dispatch.cloid_map.len(), 0);
13680 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13681 assert_nonce_reusable(&client.dispatch);
13682 }
13683
13684 #[rstest]
13685 #[case(LighterGroupingType::OneTriggersTheOther)]
13686 #[case(LighterGroupingType::OneCancelsTheOther)]
13687 #[case(LighterGroupingType::OneTriggersOneCancelsTheOther)]
13688 #[tokio::test]
13689 async fn native_group_denies_quote_quantity_without_dispatch(
13690 #[case] grouping: LighterGroupingType,
13691 #[values(false, true)] first_leg: bool,
13692 ) {
13693 let (client, cache, mut rx) = create_execution_client();
13694 let instrument = register_test_instrument(&client, &cache);
13695 let mut orders = test_native_group(instrument, grouping);
13696
13697 let index = if first_leg { 0 } else { orders.len() - 1 };
13698 let order = &orders[index];
13699 let mut builder = OrderTestBuilder::new(order.order_type());
13700 builder
13701 .trader_id(order.trader_id())
13702 .strategy_id(order.strategy_id())
13703 .instrument_id(instrument)
13704 .client_order_id(order.client_order_id())
13705 .side(order.order_side())
13706 .quantity(order.quantity())
13707 .price(order.price().unwrap())
13708 .reduce_only(order.is_reduce_only())
13709 .quote_quantity(true)
13710 .order_list_id(order.order_list_id().unwrap());
13711
13712 if let Some(price) = order.trigger_price() {
13713 builder.trigger_price(price);
13714 }
13715
13716 if let Some(parent) = order.parent_order_id() {
13717 builder.parent_order_id(parent);
13718 }
13719
13720 if let Some(contingency) = order.contingency_type() {
13721 builder.contingency_type(contingency);
13722 }
13723
13724 if let Some(linked) = order.linked_order_ids() {
13725 builder.linked_order_ids(linked.to_vec());
13726 }
13727
13728 orders[index] = builder.build();
13729
13730 for order in &orders {
13731 cache_order(&cache, order.clone());
13732 }
13733
13734 client
13735 .submit_order_list(submit_order_list_command(&orders, "OL-GROUP"))
13736 .unwrap();
13737
13738 for order in &orders {
13739 match recv_order_event(&mut rx).await {
13740 OrderEventAny::Denied(event) => {
13741 assert_eq!(event.client_order_id, order.client_order_id());
13742 assert_eq!(event.reason.as_str(), "UNSUPPORTED_QUOTE_QUANTITY");
13743 }
13744 event => panic!("expected denied event, was {event:?}"),
13745 }
13746 }
13747
13748 assert!(matches!(
13749 rx.try_recv(),
13750 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
13751 ));
13752 assert_eq!(client.dispatch.cloid_map.len(), 0);
13753 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13754 assert_nonce_reusable(&client.dispatch);
13755 }
13756
13757 #[rstest]
13758 #[case(true)]
13759 #[case(false)]
13760 fn native_group_preserves_explicit_gtd_expiries(#[case] matching: bool) {
13761 let (client, cache, _rx) = create_execution_client();
13762 let instrument = register_test_instrument(&client, &cache);
13763 let mut orders = test_native_group(instrument, LighterGroupingType::OneCancelsTheOther);
13764 let expiry = client.clock.get_time_ns() + DurationNanos::from_mins(10);
13765
13766 for (index, order) in orders.iter_mut().enumerate() {
13767 *order = OrderTestBuilder::new(order.order_type())
13768 .trader_id(order.trader_id())
13769 .strategy_id(order.strategy_id())
13770 .instrument_id(instrument)
13771 .client_order_id(order.client_order_id())
13772 .side(order.order_side())
13773 .quantity(order.quantity())
13774 .price(order.price().unwrap())
13775 .trigger_price(order.trigger_price().unwrap())
13776 .reduce_only(true)
13777 .order_list_id(order.order_list_id().unwrap())
13778 .contingency_type(ContingencyType::Oco)
13779 .linked_order_ids(order.linked_order_ids().unwrap().to_vec())
13780 .time_in_force(TimeInForce::Gtd)
13781 .expire_time(UnixNanos::from(
13782 expiry.as_u64() + u64::from(!matching) * index as u64 * 1_000_000_000,
13783 ))
13784 .build();
13785 }
13786
13787 let result = client.prepare_create_group_plan(&orders, None);
13788 if matching {
13789 let plan = result.unwrap();
13790 assert_eq!(
13791 plan.orders
13792 .iter()
13793 .map(|leg| leg.order_expiry)
13794 .collect::<Vec<_>>(),
13795 vec![expiry.as_u64() as i64 / 1_000_000; 2]
13796 );
13797 } else {
13798 assert_eq!(
13799 result.err().unwrap().to_string(),
13800 "Lighter group children require matching nonzero expiries"
13801 );
13802 }
13803
13804 assert_eq!(client.dispatch.cloid_map.len(), 0);
13805 }
13806
13807 #[rstest]
13808 fn native_group_denies_quantity_mismatch_without_registering_orders() {
13809 let (client, cache, _rx) = create_execution_client();
13810 let instrument = register_test_instrument(&client, &cache);
13811 let mut orders = test_native_group(
13812 instrument,
13813 LighterGroupingType::OneTriggersOneCancelsTheOther,
13814 );
13815 orders[1].set_quantity(Quantity::from("0.2000"));
13816 let error = client
13817 .prepare_create_group_plan(&orders, None)
13818 .err()
13819 .unwrap();
13820 assert_eq!(
13821 error.to_string(),
13822 "Lighter group quantities must match before parent-fill sizing"
13823 );
13824 assert_eq!(client.dispatch.cloid_map.len(), 0);
13825 }
13826
13827 #[rstest]
13828 fn native_group_denies_duplicate_client_ids_without_registering_orders() {
13829 let (client, cache, _rx) = create_execution_client();
13830 let instrument = register_test_instrument(&client, &cache);
13831 let mut orders = test_native_group(
13832 instrument,
13833 LighterGroupingType::OneTriggersOneCancelsTheOther,
13834 );
13835 orders[1] = orders[2].clone();
13836 let error = client
13837 .prepare_create_group_plan(&orders, None)
13838 .err()
13839 .unwrap();
13840 assert_eq!(
13841 error.to_string(),
13842 "Lighter groups require unique client order IDs"
13843 );
13844 assert_eq!(client.dispatch.cloid_map.len(), 0);
13845 }
13846
13847 #[rstest]
13848 fn native_group_reports_keep_client_links_and_held_quantities() {
13849 let (client, cache, _rx) = create_execution_client();
13850 let instrument = register_test_instrument(&client, &cache);
13851 let orders = test_native_group(
13852 instrument,
13853 LighterGroupingType::OneTriggersOneCancelsTheOther,
13854 );
13855 let plan = client.prepare_create_group_plan(&orders, None).unwrap();
13856 let context = client
13857 .fanout_dispatch_context(client.credential.as_ref().unwrap())
13858 .unwrap();
13859 let signed = context.sign_create_group(&plan).unwrap();
13860 context.enqueue_batch_tx(&signed);
13861 let venues = [
13862 VenueOrderId::from("281477001"),
13863 VenueOrderId::from("281477002"),
13864 VenueOrderId::from("281477003"),
13865 ];
13866 let indices: Vec<_> = signed
13867 .pending
13868 .kind
13869 .creates()
13870 .map(|(_, index)| index)
13871 .collect();
13872
13873 for (index, order) in orders.iter().enumerate() {
13874 assert!(client.dispatch.observe_create_submission(
13875 &order.client_order_id(),
13876 indices[index],
13877 signed.pending.nonce,
13878 venues[index]
13879 ));
13880 }
13881
13882 let mut parent = reconciliation_raw_order(indices[0], venues[0], LighterOrderStatus::Open);
13883 parent.to_trigger_order_id_0 = venues[1].to_string();
13884 parent.to_trigger_order_id_1 = venues[2].to_string();
13885 let parent_report = parse_http_order_to_report(
13886 &parent,
13887 &client.registry,
13888 account_id(),
13889 UnixNanos::from(7),
13890 &client.dispatch,
13891 )
13892 .unwrap();
13893 let parent_report = client.dispatch.translate_order_cloid(parent_report);
13894 let mut child =
13895 reconciliation_raw_order(indices[1], venues[1], LighterOrderStatus::Pending);
13896 child.order_type = LighterOrderKind::StopLossLimit;
13897 child.parent_order_id = venues[0].to_string();
13898 child.to_cancel_order_id_0 = venues[2].to_string();
13899 child.initial_base_amount = Decimal::ZERO;
13900 child.filled_base_amount = Decimal::ZERO;
13901 child.trigger_price = Decimal::from(2200);
13902 let child_report = parse_http_order_to_report(
13903 &child,
13904 &client.registry,
13905 account_id(),
13906 UnixNanos::from(8),
13907 &client.dispatch,
13908 )
13909 .unwrap();
13910 let child_report = client.dispatch.translate_order_cloid(child_report);
13911 client
13912 .dispatch
13913 .retire_order_identity(&orders[0].client_order_id());
13914 child.parent_order_id = "0".to_string();
13915 child.to_cancel_order_id_0 = "0".to_string();
13916 child.status = LighterOrderStatus::Open;
13917 child.trigger_status = LighterTriggerStatus::Na;
13918 child.trigger_price = Decimal::ZERO;
13919 child.initial_base_amount = Decimal::from_str("0.0400").unwrap();
13920 child.remaining_base_amount = child.initial_base_amount;
13921 let activated = parse_http_order_to_report(
13922 &child,
13923 &client.registry,
13924 account_id(),
13925 UnixNanos::from(9),
13926 &client.dispatch,
13927 )
13928 .unwrap();
13929 let activated = client.dispatch.translate_order_cloid(activated);
13930 assert_eq!(
13931 parent_report.client_order_id,
13932 Some(orders[0].client_order_id())
13933 );
13934 assert_eq!(parent_report.contingency_type, Some(ContingencyType::Oto));
13935 assert_eq!(
13936 parent_report.linked_order_ids,
13937 Some(vec![
13938 orders[1].client_order_id(),
13939 orders[2].client_order_id()
13940 ])
13941 );
13942 assert_eq!(parent_report.parent_order_id, None);
13943 assert_eq!(
13944 child_report.client_order_id,
13945 Some(orders[1].client_order_id())
13946 );
13947 assert_eq!(child_report.contingency_type, Some(ContingencyType::Oco));
13948 assert_eq!(
13949 child_report.linked_order_ids,
13950 Some(vec![orders[2].client_order_id()])
13951 );
13952 assert_eq!(
13953 child_report.parent_order_id,
13954 Some(orders[0].client_order_id())
13955 );
13956 assert_eq!(child_report.quantity, Quantity::from("0.1000"));
13957 assert_eq!(activated.order_type, OrderType::StopLimit);
13958 assert_eq!(activated.order_status, OrderStatus::Triggered);
13959 assert_eq!(activated.trigger_price, Some(Price::from("2200.00")));
13960 assert_eq!(activated.quantity, Quantity::from("0.0400"));
13961 assert_eq!(activated.parent_order_id, Some(orders[0].client_order_id()));
13962 assert_eq!(
13963 activated.linked_order_ids,
13964 Some(vec![orders[2].client_order_id()])
13965 );
13966 }
13967
13968 #[rstest]
13969 #[case::unactivated(LighterOrderStatus::CanceledChild, "0.0000", "0.0000", true, true)]
13970 #[case::activated(LighterOrderStatus::CanceledChild, "0.0400", "0.0000", true, false)]
13971 #[case::filled(LighterOrderStatus::CanceledChild, "0.0000", "0.0200", true, false)]
13972 #[case::no_parent(LighterOrderStatus::CanceledChild, "0.0000", "0.0000", false, false)]
13973 #[case::other_cancel(LighterOrderStatus::CanceledOco, "0.0000", "0.0000", true, false)]
13974 fn unowned_child_cancel_filters_only_unactivated_children(
13975 #[case] status: LighterOrderStatus,
13976 #[case] initial: &str,
13977 #[case] filled: &str,
13978 #[case] has_parent: bool,
13979 #[case] filtered: bool,
13980 ) {
13981 let (client, cache, mut rx) = create_execution_client();
13982 register_test_instrument(&client, &cache);
13983 let mut raw = reconciliation_raw_order(42, VenueOrderId::from("281477004"), status);
13984 raw.order_type = LighterOrderKind::StopLossLimit;
13985 raw.initial_base_amount = Decimal::from_str(initial).unwrap();
13986 raw.remaining_base_amount = Decimal::ZERO;
13987 raw.filled_base_amount = Decimal::from_str(filled).unwrap();
13988 raw.parent_order_id = if has_parent { "281477001" } else { "0" }.to_string();
13989 raw.to_cancel_order_id_0 = "281477005".to_string();
13990 let report = parse_http_order_to_report(
13991 &raw,
13992 &client.registry,
13993 account_id(),
13994 UnixNanos::from(2),
13995 &client.dispatch,
13996 )
13997 .unwrap();
13998
13999 let cmd = GenerateOrderStatusReports::new(
14000 UUID4::new(),
14001 UnixNanos::from(3),
14002 false,
14003 None,
14004 None,
14005 None,
14006 None,
14007 None,
14008 );
14009
14010 let reports = finish_order_reports(
14011 &client.core,
14012 &client.dispatch,
14013 OrderReportSweep {
14014 orders: vec![(raw.client_order_index, report)],
14015 ..Default::default()
14016 },
14017 &cmd,
14018 Some(&[]),
14019 )
14020 .unwrap();
14021
14022 dispatch_lighter_order(
14023 &raw,
14024 &client.dispatch,
14025 &client.emitter,
14026 &client.registry,
14027 account_id(),
14028 trader_id(),
14029 UnixNanos::from(4),
14030 );
14031 let events = drain_events(&mut rx);
14032 assert_eq!(reports.len(), usize::from(!filtered));
14033 assert_eq!(events.len(), usize::from(!filtered));
14034
14035 if !filtered {
14036 let report = &reports[0];
14037 assert_eq!(
14038 report.quantity.as_decimal(),
14039 Decimal::from_str(initial).unwrap()
14040 );
14041 assert_eq!(
14042 report.filled_qty.as_decimal(),
14043 Decimal::from_str(filled).unwrap()
14044 );
14045 assert_eq!(report.order_status, OrderStatus::Canceled);
14046 assert_eq!(report.cancel_reason.as_deref(), status.as_cancel_reason());
14047 assert_eq!(
14048 report.parent_order_id,
14049 has_parent.then(|| ClientOrderId::from("281477001"))
14050 );
14051 assert_eq!(
14052 report.linked_order_ids,
14053 Some(vec![ClientOrderId::from("281477005")])
14054 );
14055 }
14056 }
14057
14058 #[rstest]
14059 #[case::normal_index(42)]
14060 #[case::zero_index(0)]
14061 fn unowned_child_cancel_preserves_known_children(#[case] index: i64) {
14062 let (client, cache, _rx) = create_execution_client();
14063 let instrument = register_test_instrument(&client, &cache);
14064 let orders = test_native_group(
14065 instrument,
14066 LighterGroupingType::OneTriggersOneCancelsTheOther,
14067 );
14068 let child = orders[1].clone();
14069 let venue = VenueOrderId::from("281477004");
14070 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
14071 let mut raw = reconciliation_raw_order(index, venue, LighterOrderStatus::CanceledChild);
14072 raw.order_type = LighterOrderKind::StopLossLimit;
14073 raw.initial_base_amount = Decimal::ZERO;
14074 raw.remaining_base_amount = Decimal::ZERO;
14075 raw.parent_order_id = "281477001".to_string();
14076 raw.to_cancel_order_id_0 = "281477005".to_string();
14077 let report = parse_http_order_to_report(
14078 &raw,
14079 &client.registry,
14080 account_id(),
14081 UnixNanos::from(2),
14082 &client.dispatch,
14083 )
14084 .unwrap();
14085
14086 let cmd = GenerateOrderStatusReports::new(
14087 UUID4::new(),
14088 UnixNanos::from(3),
14089 false,
14090 None,
14091 None,
14092 None,
14093 None,
14094 None,
14095 );
14096
14097 let reports = finish_order_reports(
14098 &client.core,
14099 &client.dispatch,
14100 OrderReportSweep {
14101 orders: vec![(index, report)],
14102 ..Default::default()
14103 },
14104 &cmd,
14105 Some(&[]),
14106 )
14107 .unwrap();
14108
14109 assert_eq!(reports.len(), 1);
14110 assert_eq!(reports[0].venue_order_id, venue);
14111 assert_eq!(reports[0].order_status, OrderStatus::Canceled);
14112 assert_eq!(
14113 client
14114 .dispatch
14115 .resolve_client_order_index_for_venue(index, venue),
14116 Some(child.client_order_id()),
14117 );
14118
14119 if index != 0 {
14120 assert_eq!(reports[0].quantity, child.quantity());
14121 assert_eq!(reports[0].client_order_id, Some(child.client_order_id()));
14122 assert_eq!(
14123 reports[0].parent_order_id,
14124 Some(orders[0].client_order_id())
14125 );
14126 assert_eq!(
14127 reports[0].linked_order_ids,
14128 Some(vec![orders[2].client_order_id()])
14129 );
14130 }
14131 }
14132
14133 #[tokio::test]
14134 async fn native_group_canceled_children_do_not_reappear_after_restart() {
14135 let (client, cache, mut rx) = create_execution_client();
14136 let instrument_id = register_test_instrument(&client, &cache);
14137 let orders = test_native_group(
14138 instrument_id,
14139 LighterGroupingType::OneTriggersOneCancelsTheOther,
14140 );
14141 let venues = [
14142 VenueOrderId::from("281477001"),
14143 VenueOrderId::from("281477002"),
14144 VenueOrderId::from("281477003"),
14145 ];
14146 let history = native_group_canceled_orders(&orders, &venues);
14147 for (i, (order, raw)) in orders.iter().zip(&history).enumerate() {
14148 cache_accepted_order(&cache, order.clone(), venues[i], Some(client_id()));
14149 let cached = cache
14150 .borrow()
14151 .order_owned(&order.client_order_id())
14152 .unwrap();
14153 client
14154 .dispatch
14155 .restore_reconciled_order(&cached, raw.client_order_index, venues[i], false)
14156 .unwrap();
14157
14158 for _ in 0..2 {
14159 dispatch_lighter_order(
14160 raw,
14161 &client.dispatch,
14162 &client.emitter,
14163 &client.registry,
14164 account_id(),
14165 trader_id(),
14166 UnixNanos::from(2),
14167 );
14168 }
14169 }
14170
14171 let events = drain_events(&mut rx);
14172 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
14173 let mut engine = ExecutionEngine::new(clock, cache.clone(), None);
14174 let applied = Rc::new(RefCell::new(Vec::new()));
14175 let captured = applied.clone();
14176 let handler = TypedHandler::from(move |event: &OrderEventAny| {
14177 captured.borrow_mut().push(event.clone());
14178 });
14179
14180 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
14181 let subscription = AppliedFillSubscription(handler);
14182
14183 for event in events {
14184 match event {
14185 ExecutionEvent::Order(event) => engine.process(&event),
14186 ExecutionEvent::Report(report) => engine.reconcile_execution_report(&report),
14187 other => panic!("unexpected cancellation event: {other:?}"),
14188 }
14189 }
14190
14191 let applied = applied.take();
14192 assert_eq!(applied.len(), 3, "applied cancellation events: {applied:?}");
14193
14194 for (event, order) in applied.iter().zip(&orders) {
14195 assert_eq!(event.client_order_id(), order.client_order_id());
14196 assert!(matches!(event, OrderEventAny::Canceled(_)));
14197 assert_eq!(
14198 cache
14199 .borrow()
14200 .order(&order.client_order_id())
14201 .unwrap()
14202 .status(),
14203 OrderStatus::Canceled
14204 );
14205 }
14206
14207 drop(subscription);
14208
14209 let (mut fresh, fresh_cache, mut fresh_rx) = create_execution_client();
14210 register_test_instrument(&fresh, &fresh_cache);
14211 fresh.dispatch.note_active_market(TEST_MARKET_INDEX);
14212 fresh.dispatch.replace_position_snapshot(&[], &[], &[]);
14213
14214 for raw in &history[1..] {
14215 dispatch_lighter_order(
14216 raw,
14217 &fresh.dispatch,
14218 &fresh.emitter,
14219 &fresh.registry,
14220 account_id(),
14221 trader_id(),
14222 UnixNanos::from(3),
14223 );
14224 }
14225
14226 let app = Router::new()
14227 .route(
14228 "/api/v1/accountActiveOrders",
14229 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
14230 )
14231 .route(
14232 "/api/v1/accountInactiveOrders",
14233 get(move || {
14234 let history = history.clone();
14235 async move { axum::Json(serde_json::json!({"code":200,"orders":history})) }
14236 }),
14237 )
14238 .route(
14239 "/api/v1/trades",
14240 get(|| async { axum::Json(serde_json::json!({"code":200,"trades":[]})) }),
14241 );
14242
14243 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
14244 let addr = listener.local_addr().unwrap();
14245 let server = tokio::spawn(async move {
14246 axum::serve(listener, app).await.unwrap();
14247 });
14248
14249 fresh.http_client = LighterHttpClient::new(
14250 LighterEnvironment::Testnet,
14251 Some(format!("http://{addr}")),
14252 1,
14253 None,
14254 )
14255 .unwrap();
14256 let first = fresh.generate_mass_status(None).await.unwrap().unwrap();
14257 let second = fresh.generate_mass_status(None).await.unwrap().unwrap();
14258
14259 let parent_cmd = GenerateOrderStatusReport::new(
14260 UUID4::new(),
14261 UnixNanos::from(4),
14262 Some(instrument_id),
14263 None,
14264 Some(venues[0]),
14265 None,
14266 None,
14267 );
14268 let parent_report = fresh
14269 .generate_order_status_report(&parent_cmd)
14270 .await
14271 .unwrap()
14272 .unwrap();
14273
14274 for venue in &venues[1..] {
14275 let cmd = GenerateOrderStatusReport::new(
14276 UUID4::new(),
14277 UnixNanos::from(4),
14278 Some(instrument_id),
14279 None,
14280 Some(*venue),
14281 None,
14282 None,
14283 );
14284 assert_eq!(
14285 fresh.generate_order_status_report(&cmd).await.unwrap(),
14286 None
14287 );
14288 let task = fresh.generate_order_status_report_task(&cmd).unwrap();
14289 task.collection.await;
14290 assert_eq!((task.finish)().unwrap(), None);
14291 fresh
14292 .query_order(QueryOrder::new(
14293 trader_id(),
14294 Some(client_id()),
14295 strategy_id(),
14296 instrument_id,
14297 ClientOrderId::from(venue.as_str()),
14298 Some(*venue),
14299 UUID4::new(),
14300 UnixNanos::from(5),
14301 None,
14302 None,
14303 ))
14304 .unwrap();
14305 }
14306
14307 fresh.pending_tasks.begin_shutdown();
14308 fresh
14309 .pending_tasks
14310 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
14311 .await
14312 .unwrap();
14313 server.abort();
14314 let _ = server.await;
14315 let mut expected_parent = first.order_reports()[&venues[0]].clone();
14316 expected_parent.report_id = parent_report.report_id;
14317 expected_parent.ts_init = parent_report.ts_init;
14318 assert_eq!(parent_report, expected_parent);
14319
14320 for mass in [&first, &second] {
14321 assert!(mass.reports_complete());
14322 assert_eq!(mass.order_reports().len(), 1);
14323 let parent = &mass.order_reports()[&venues[0]];
14324 assert_eq!(parent.order_status, OrderStatus::Canceled);
14325 assert_eq!(parent.quantity, orders[0].quantity());
14326 assert_eq!(parent.filled_qty, Quantity::from("0.0000"));
14327 assert_eq!(
14328 parent.linked_order_ids,
14329 Some(vec![
14330 ClientOrderId::from("281477002"),
14331 ClientOrderId::from("281477003")
14332 ])
14333 );
14334 assert!(mass.fill_reports().is_empty());
14335 assert!(mass.position_reports().is_empty());
14336 }
14337
14338 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
14339 let mut engine = ExecutionEngine::new(clock.clone(), fresh_cache.clone(), None);
14340 engine.register_client(Box::new(fresh)).unwrap();
14341 let engine = Rc::new(RefCell::new(engine));
14342 let mut manager = ExecutionManager::new(
14343 clock,
14344 fresh_cache.clone(),
14345 ExecutionManagerConfig::default(),
14346 )
14347 .unwrap();
14348 let applied = Rc::new(RefCell::new(Vec::new()));
14349 let captured = applied.clone();
14350
14351 let handler = TypedHandler::from(move |event: &OrderEventAny| {
14352 if matches!(event, OrderEventAny::Canceled(_) | OrderEventAny::Filled(_)) {
14353 captured.borrow_mut().push(event.clone());
14354 }
14355 });
14356
14357 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
14358 let subscription = AppliedFillSubscription(handler);
14359 manager.reconcile_execution_mass_status(&first, &engine);
14360 manager.reconcile_execution_mass_status(&second, &engine);
14361 drop(subscription);
14362 let applied = applied.take();
14363 assert_eq!(applied.len(), 1, "fresh reconciliation events: {applied:?}");
14364
14365 let OrderEventAny::Canceled(canceled) = &applied[0] else {
14366 panic!("expected the parent cancellation");
14367 };
14368
14369 assert_eq!(
14370 canceled.client_order_id,
14371 ClientOrderId::from(venues[0].as_str())
14372 );
14373 assert_eq!(canceled.venue_order_id, Some(venues[0]));
14374 assert_eq!(
14375 fresh_cache
14376 .borrow()
14377 .orders_total_count(None, None, None, None, None),
14378 1
14379 );
14380 assert!(drain_events(&mut fresh_rx).is_empty());
14381 }
14382
14383 fn native_group_canceled_orders(
14384 orders: &[OrderAny],
14385 venues: &[VenueOrderId; 3],
14386 ) -> Vec<LighterOrder> {
14387 let mut history = Vec::new();
14388
14389 for (i, order) in orders.iter().enumerate() {
14390 let mut raw = reconciliation_raw_order(
14391 42 + i as i64,
14392 venues[i],
14393 LighterOrderStatus::CanceledChild,
14394 );
14395
14396 raw.initial_base_amount = Decimal::ZERO;
14397 raw.remaining_base_amount = Decimal::ZERO;
14398 raw.price = order.price().unwrap().as_decimal();
14399 raw.trigger_price = order
14400 .trigger_price()
14401 .map_or(Decimal::ZERO, |price| price.as_decimal());
14402 raw.is_ask = i > 0;
14403 raw.side = Some(if i == 0 {
14404 LighterOrderSide::Buy
14405 } else {
14406 LighterOrderSide::Sell
14407 });
14408
14409 raw.reduce_only = i > 0;
14410 if i == 0 {
14411 raw.status = LighterOrderStatus::Canceled;
14412 raw.initial_base_amount = order.quantity().as_decimal();
14413 raw.to_trigger_order_id_0 = venues[1].to_string();
14414 raw.to_trigger_order_id_1 = venues[2].to_string();
14415 } else {
14416 raw.order_type = if i == 1 {
14417 LighterOrderKind::StopLossLimit
14418 } else {
14419 LighterOrderKind::TakeProfitLimit
14420 };
14421
14422 raw.parent_order_id = venues[0].to_string();
14423 raw.to_cancel_order_id_0 = venues[3 - i].to_string();
14424 }
14425
14426 history.push(raw);
14427 }
14428
14429 history
14430 }
14431
14432 #[rstest]
14433 fn native_group_restores_held_child_before_first_stream_frame() {
14434 let (client, cache, mut rx) = create_execution_client();
14435 let instrument = register_test_instrument(&client, &cache);
14436 let orders = test_native_group(
14437 instrument,
14438 LighterGroupingType::OneTriggersOneCancelsTheOther,
14439 );
14440 let child = orders[1].clone();
14441 let venue = VenueOrderId::from("281477004");
14442 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
14443 let cached = cache
14444 .borrow()
14445 .order_owned(&child.client_order_id())
14446 .unwrap();
14447 client.dispatch.seed_order_restorations(vec![cached]);
14448 let (_, index) = forced_probed_index(child.client_order_id());
14449 let mut raw = reconciliation_raw_order(index, venue, LighterOrderStatus::Pending);
14450 raw.order_type = LighterOrderKind::StopLossLimit;
14451 raw.initial_base_amount = Decimal::ZERO;
14452 raw.filled_base_amount = Decimal::ZERO;
14453 raw.is_ask = true;
14454 raw.side = Some(LighterOrderSide::Sell);
14455 raw.reduce_only = true;
14456 raw.price = Decimal::from(2190);
14457 raw.trigger_price = Decimal::from(2200);
14458 dispatch_lighter_order(
14459 &raw,
14460 &client.dispatch,
14461 &client.emitter,
14462 &client.registry,
14463 account_id(),
14464 trader_id(),
14465 UnixNanos::from(1),
14466 );
14467 let events = drain_events(&mut rx);
14468 let report = parse_http_order_to_report(
14469 &raw,
14470 &client.registry,
14471 account_id(),
14472 UnixNanos::from(2),
14473 &client.dispatch,
14474 )
14475 .unwrap();
14476 let report = client.dispatch.translate_order_cloid(report);
14477 assert!(
14478 events.is_empty(),
14479 "restored pending frame must be a tracked replay: {events:?}"
14480 );
14481 assert_eq!(
14482 cache
14483 .borrow()
14484 .order(&child.client_order_id())
14485 .unwrap()
14486 .quantity(),
14487 Quantity::from("0.1000")
14488 );
14489 assert_eq!(
14490 client.dispatch.client_order_index(&child.client_order_id()),
14491 Some(index)
14492 );
14493 assert_eq!(report.client_order_id, Some(child.client_order_id()));
14494 assert_eq!(report.quantity, Quantity::from("0.1000"));
14495 assert_eq!(report.parent_order_id, Some(orders[0].client_order_id()));
14496 assert_eq!(
14497 report.linked_order_ids,
14498 Some(vec![orders[2].client_order_id()])
14499 );
14500 assert_eq!(report.contingency_type, Some(ContingencyType::Oco));
14501 assert_eq!(report.order_type, OrderType::StopLimit);
14502 assert_eq!(report.trigger_price, Some(Price::from("2200.00")));
14503 assert_eq!(report.price, Some(Price::from("2190.00")));
14504 assert_eq!(report.order_side, Some(OrderSide::Sell));
14505 assert_eq!(report.order_status, OrderStatus::Accepted);
14506 assert!(report.reduce_only);
14507 }
14508
14509 #[rstest]
14510 #[case(true)]
14511 #[case(false)]
14512 #[tokio::test]
14513 async fn native_oco_cancel_targets_peer_with_venue_or_shared_submission_index(
14514 #[case] acknowledged: bool,
14515 #[values(false, true)] reverse: bool,
14516 ) {
14517 let (client, cache, mut rx) = create_execution_client();
14518 let instrument = register_test_instrument(&client, &cache);
14519 let orders = test_native_group(
14520 instrument,
14521 LighterGroupingType::OneTriggersOneCancelsTheOther,
14522 );
14523 let group = client.prepare_create_group_plan(&orders, None).unwrap();
14524 let context = client
14525 .fanout_dispatch_context(client.credential.as_ref().unwrap())
14526 .unwrap();
14527 let mut signed = context.sign_create_group(&group).unwrap();
14528 client
14529 .dispatch
14530 .enqueue_pending_sendtx(signed.pending.clone());
14531 let first = VenueOrderId::from("281477101");
14532 let second = VenueOrderId::from("281477102");
14533 cache_accepted_order(&cache, orders[1].clone(), first, Some(client_id()));
14534 let first_index = client
14535 .dispatch
14536 .client_order_index(&orders[1].client_order_id())
14537 .unwrap();
14538 let second_index = client
14539 .dispatch
14540 .client_order_index(&orders[2].client_order_id())
14541 .unwrap();
14542 assert!(client.dispatch.observe_create_submission(
14543 &orders[1].client_order_id(),
14544 first_index,
14545 signed.pending.nonce,
14546 first
14547 ));
14548
14549 if acknowledged {
14550 cache_accepted_order(&cache, orders[2].clone(), second, Some(client_id()));
14551 assert!(client.dispatch.observe_create_submission(
14552 &orders[2].client_order_id(),
14553 second_index,
14554 signed.pending.nonce,
14555 second
14556 ));
14557 } else {
14558 cache_order(&cache, orders[2].clone());
14559 cache
14560 .borrow_mut()
14561 .update_order(&TestOrderEventStubs::submitted(&orders[2], account_id()))
14562 .unwrap();
14563 }
14564
14565 let command = CancelOrder::new(
14566 trader_id(),
14567 Some(client_id()),
14568 strategy_id(),
14569 instrument,
14570 orders[1].client_order_id(),
14571 None,
14572 UUID4::new(),
14573 UnixNanos::default(),
14574 None,
14575 None,
14576 );
14577 let plans = client.prepare_oco_cancel_plans(&command).unwrap();
14578 assert_eq!(plans.len(), 2);
14579
14580 for (index, plan) in plans.iter().enumerate() {
14581 assert_eq!(plan.client_order_id, orders[index + 1].client_order_id());
14582 assert_eq!(plan.strategy_id, strategy_id());
14583 assert_eq!(plan.instrument_id, instrument);
14584 assert_eq!(plan.market_index, TEST_MARKET_INDEX);
14585 }
14586
14587 assert_eq!(plans[0].command_id, Some(command.command_id));
14588 assert_eq!(plans[1].command_id, None);
14589 assert_eq!(plans[0].venue_order_id, Some(first));
14590 assert_eq!(plans[0].venue_index, 281_477_101);
14591 assert_eq!(plans[1].venue_order_id, acknowledged.then_some(second));
14592 assert_eq!(
14593 plans[1].venue_index,
14594 if acknowledged {
14595 281_477_102
14596 } else {
14597 second_index
14598 }
14599 );
14600
14601 for plan in plans {
14602 context.reject_batch_plan(&BatchOrderPlan::Cancel(plan, false), "signing failed");
14603 }
14604
14605 let OrderEventAny::CancelDenied(denied) = recv_order_event(&mut rx).await else {
14606 panic!("expected cancel denial")
14607 };
14608
14609 assert_eq!(denied.client_order_id, command.client_order_id);
14610 assert_eq!(denied.causation_id, Some(command.command_id));
14611 assert!(rx.try_recv().is_err());
14612
14613 let mut peer_command = command.clone();
14614 peer_command.client_order_id = orders[2].client_order_id();
14615 peer_command.command_id = UUID4::new();
14616 let mut cancels = vec![command.clone(), peer_command.clone()];
14617 if reverse {
14618 cancels.reverse();
14619 }
14620
14621 let batch = BatchCancelOrders::new(
14622 trader_id(),
14623 Some(client_id()),
14624 strategy_id(),
14625 instrument,
14626 cancels,
14627 UUID4::new(),
14628 UnixNanos::default(),
14629 None,
14630 None,
14631 );
14632 client.dispatch.remove_pending_sendtx_by_hash(
14633 signed.pending.connection_epoch,
14634 &signed.pending.tx_hash,
14635 );
14636 signed.send_reservation.release();
14637 client.batch_cancel_orders(batch).unwrap();
14638 let mut denied_ids = AHashSet::new();
14639
14640 for _ in 0..2 {
14641 let OrderEventAny::CancelDenied(denied) = recv_order_event(&mut rx).await else {
14642 panic!("expected cancel denial")
14643 };
14644
14645 let expected = if denied.client_order_id == command.client_order_id {
14646 command.command_id
14647 } else {
14648 assert_eq!(denied.client_order_id, peer_command.client_order_id);
14649 peer_command.command_id
14650 };
14651
14652 assert!(denied_ids.insert(denied.client_order_id));
14653 assert_eq!(denied.causation_id, Some(expected));
14654 }
14655
14656 assert_eq!(
14657 denied_ids,
14658 AHashSet::from_iter([command.client_order_id, peer_command.client_order_id])
14659 );
14660 assert!(rx.try_recv().is_err());
14661 let mut override_venue = command.clone();
14662 override_venue.venue_order_id = Some(VenueOrderId::from("281477103"));
14663 assert!(client.prepare_oco_cancel_plans(&override_venue).is_none());
14664
14665 if !acknowledged {
14666 client
14667 .dispatch
14668 .mark_order_submission(&orders[2].client_order_id(), signed.pending.nonce + 1);
14669 assert!(client.prepare_oco_cancel_plans(&command).is_none());
14670 }
14671 }
14672
14673 #[rstest]
14674 fn native_group_denies_held_child_modification() {
14675 let (client, cache, _rx) = create_execution_client();
14676 let instrument = register_test_instrument(&client, &cache);
14677 let orders = test_native_group(instrument, LighterGroupingType::OneTriggersTheOther);
14678 let child = orders[1].clone();
14679 let venue = VenueOrderId::from("281477004");
14680 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
14681 client.dispatch.store_snapshot(
14682 child.client_order_id(),
14683 OrderShapeSnapshot {
14684 quantity: Quantity::from("0.0000"),
14685 price: child.price(),
14686 trigger_price: child.trigger_price(),
14687 },
14688 );
14689
14690 let command = ModifyOrder::new(
14691 trader_id(),
14692 Some(client_id()),
14693 strategy_id(),
14694 instrument,
14695 child.client_order_id(),
14696 Some(venue),
14697 None,
14698 Some(Price::from("2180.00")),
14699 None,
14700 UUID4::new(),
14701 UnixNanos::default(),
14702 None,
14703 None,
14704 );
14705 let error = client.prepare_modify_order_plan(&command).err().unwrap();
14706 assert_eq!(
14707 error.to_string(),
14708 "Lighter cannot modify an OTO child while it waits for parent execution"
14709 );
14710 }
14711
14712 #[rstest]
14713 fn native_group_modifies_triggered_child_without_rearming_trigger() {
14714 let (client, cache, _rx) = create_execution_client();
14715 let instrument = register_test_instrument(&client, &cache);
14716 let orders = test_native_group(
14717 instrument,
14718 LighterGroupingType::OneTriggersOneCancelsTheOther,
14719 );
14720 let child = orders[2].clone();
14721 let venue = VenueOrderId::from("281477004");
14722 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
14723 client
14724 .dispatch
14725 .mark_triggered_emitted(child.client_order_id());
14726
14727 let command = ModifyOrder::new(
14728 trader_id(),
14729 Some(client_id()),
14730 strategy_id(),
14731 instrument,
14732 child.client_order_id(),
14733 Some(venue),
14734 None,
14735 Some(Price::from("2380.00")),
14736 None,
14737 UUID4::new(),
14738 UnixNanos::default(),
14739 None,
14740 None,
14741 );
14742 let plan = client.prepare_modify_order_plan(&command).unwrap();
14743 let mut rearm = command;
14744 rearm.trigger_price = Some(Price::from("2450.00"));
14745 let error = client.prepare_modify_order_plan(&rearm).err().unwrap();
14746 assert_eq!(plan.trigger_price, 0);
14747 assert_eq!(plan.price, 238_000);
14748 assert_eq!(plan.base_amount, 1_000);
14749 assert_eq!(
14750 error.to_string(),
14751 "Lighter cannot modify a trigger price after the order has triggered"
14752 );
14753 }
14754
14755 fn test_native_group(
14756 instrument_id: InstrumentId,
14757 grouping: LighterGroupingType,
14758 ) -> Vec<OrderAny> {
14759 let has_parent = grouping != LighterGroupingType::OneCancelsTheOther;
14760 let ids = [
14761 ClientOrderId::from("O-GROUP-ENTRY"),
14762 ClientOrderId::from("O-GROUP-SL"),
14763 ClientOrderId::from("O-GROUP-TP"),
14764 ];
14765 let mut orders = Vec::new();
14766 if has_parent {
14767 orders.push(
14768 OrderTestBuilder::new(OrderType::Limit)
14769 .trader_id(trader_id())
14770 .strategy_id(strategy_id())
14771 .instrument_id(instrument_id)
14772 .client_order_id(ids[0])
14773 .side(OrderSide::Buy)
14774 .quantity(Quantity::from("0.1000"))
14775 .price(Price::from("2300.00"))
14776 .order_list_id(OrderListId::from("OL-GROUP"))
14777 .contingency_type(ContingencyType::Oto)
14778 .linked_order_ids(if grouping == LighterGroupingType::OneTriggersTheOther {
14779 vec![ids[1]]
14780 } else {
14781 vec![ids[1], ids[2]]
14782 })
14783 .build(),
14784 );
14785 }
14786
14787 let count = if grouping == LighterGroupingType::OneTriggersTheOther {
14788 1
14789 } else {
14790 2
14791 };
14792
14793 let children = [
14794 (
14795 OrderType::StopLimit,
14796 Price::from("2190.00"),
14797 Price::from("2200.00"),
14798 ),
14799 (
14800 OrderType::LimitIfTouched,
14801 Price::from("2390.00"),
14802 Price::from("2400.00"),
14803 ),
14804 ];
14805
14806 for (index, (kind, price, trigger)) in children.into_iter().take(count).enumerate() {
14807 let mut builder = OrderTestBuilder::new(kind);
14808 builder
14809 .trader_id(trader_id())
14810 .strategy_id(strategy_id())
14811 .instrument_id(instrument_id)
14812 .client_order_id(ids[index + 1])
14813 .side(OrderSide::Sell)
14814 .quantity(Quantity::from("0.1000"))
14815 .price(price)
14816 .trigger_price(trigger)
14817 .reduce_only(true)
14818 .order_list_id(OrderListId::from("OL-GROUP"));
14819
14820 if has_parent {
14821 builder.parent_order_id(ids[0]);
14822 }
14823
14824 if count == 2 {
14825 builder
14826 .contingency_type(ContingencyType::Oco)
14827 .linked_order_ids(vec![ids[2 - index]]);
14828 }
14829
14830 orders.push(builder.build());
14831 }
14832
14833 orders
14834 }
14835
14836 #[tokio::test]
14837 async fn submit_order_list_plain_limit_oco_denies_all_without_dispatch() {
14838 let (client, cache, mut rx) = create_execution_client();
14839 let instrument_id = register_test_instrument(&client, &cache);
14840 let order_a_id = "O-LIST-OCO-A";
14841 let order_b_id = "O-LIST-OCO-B";
14842 let order_a = test_contingent_limit_order(instrument_id, order_a_id, "OL-OCO", order_b_id);
14843 let order_b = test_contingent_limit_order(instrument_id, order_b_id, "OL-OCO", order_a_id);
14844 cache_order(&cache, order_a.clone());
14845 cache_order(&cache, order_b.clone());
14846
14847 let command = submit_order_list_command(&[order_a.clone(), order_b.clone()], "OL-OCO");
14848 client.submit_order_list(command).unwrap();
14849
14850 for order in [&order_a, &order_b] {
14851 match recv_order_event(&mut rx).await {
14852 OrderEventAny::Denied(e) => {
14853 assert_eq!(e.client_order_id, order.client_order_id());
14854 assert!(e.reason.contains("reduce-only SL/TP orders"));
14855 }
14856 other => panic!("expected Denied, was {other:?}"),
14857 }
14858 }
14859
14860 assert!(
14861 tokio::time::timeout(Duration::from_millis(50), rx.recv())
14862 .await
14863 .is_err(),
14864 "grouped list denial must not emit extra events",
14865 );
14866 assert_nonce_reusable(&client.dispatch);
14867 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14868 }
14869
14870 #[tokio::test]
14871 async fn cancel_order_send_failure_emits_cancel_denied_and_rolls_back() {
14872 let (client, cache, mut rx) = create_execution_client();
14873 let instrument_id = register_test_instrument(&client, &cache);
14874 let mut factory = test_order_factory();
14875 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-FAIL");
14876 let client_order_id = order.client_order_id();
14877 let venue_order_id = VenueOrderId::from("123");
14878 cache_pending_cancel_order(&cache, order, venue_order_id, None);
14879
14880 let command = CancelOrder::new(
14881 trader_id(),
14882 Some(client_id()),
14883 strategy_id(),
14884 instrument_id,
14885 client_order_id,
14886 Some(venue_order_id),
14887 UUID4::new(),
14888 UnixNanos::default(),
14889 None,
14890 None,
14891 );
14892 client.cancel_order(command).unwrap();
14893
14894 let rejected = recv_order_event(&mut rx).await;
14895
14896 match rejected {
14897 OrderEventAny::CancelDenied(event) => {
14898 assert_eq!(event.client_order_id, client_order_id);
14899 assert_eq!(event.instrument_id, instrument_id);
14900 assert_eq!(event.venue_order_id, Some(venue_order_id));
14901 assert!(
14902 event
14903 .reason
14904 .contains("Lighter cancel_order dispatch failed"),
14905 );
14906 assert!(event.reason.contains("handler unavailable"));
14907 }
14908 event => panic!("expected cancel rejected event, was {event:?}"),
14909 }
14910
14911 assert_nonce_reusable(&client.dispatch);
14912 assert_eq!(
14913 client.dispatch.pending_sendtx_len(),
14914 0,
14915 "local-send-failure must remove the pending cancel entry",
14916 );
14917 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
14918 }
14919
14920 #[tokio::test]
14921 async fn cancel_order_unknown_send_outcome_retains_pending_state() {
14922 let (client, cache, mut rx) = create_execution_client();
14923 let instrument_id = register_test_instrument(&client, &cache);
14924 let mut factory = test_order_factory();
14925 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-UNKNOWN");
14926 let client_order_id = order.client_order_id();
14927 let venue_order_id = VenueOrderId::from("123");
14928 cache_pending_cancel_order(&cache, order, venue_order_id, None);
14929 client.ws_client.drop_next_send_tx_result_for_test().await;
14930
14931 client
14932 .cancel_order(CancelOrder::new(
14933 trader_id(),
14934 Some(client_id()),
14935 strategy_id(),
14936 instrument_id,
14937 client_order_id,
14938 Some(venue_order_id),
14939 UUID4::new(),
14940 UnixNanos::default(),
14941 None,
14942 None,
14943 ))
14944 .unwrap();
14945
14946 assert!(
14947 tokio::time::timeout(Duration::from_millis(50), rx.recv())
14948 .await
14949 .is_err(),
14950 "unknown outcome must not emit OrderCancelRejected",
14951 );
14952 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
14953 assert_eq!(
14954 client.dispatch.pending_order_action(&client_order_id),
14955 Some(PendingOrderAction::Cancel),
14956 );
14957 assert_eq!(
14958 client
14959 .dispatch
14960 .nonce_manager
14961 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
14962 Some(TEST_NEXT_NONCE),
14963 );
14964 }
14965
14966 #[tokio::test]
14967 async fn cancel_order_prepare_failure_emits_cancel_denied_without_dispatch() {
14968 let (client, cache, mut rx) = create_execution_client();
14969 let instrument_id = register_test_instrument(&client, &cache);
14970 let mut factory = test_order_factory();
14971 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-NO-VOI");
14972 let client_order_id = order.client_order_id();
14973 cache_pending_cancel_order(&cache, order, VenueOrderId::from("123"), None);
14974
14975 let command = CancelOrder::new(
14976 trader_id(),
14977 Some(client_id()),
14978 strategy_id(),
14979 instrument_id,
14980 client_order_id,
14981 None,
14982 UUID4::new(),
14983 UnixNanos::default(),
14984 None,
14985 None,
14986 );
14987 client.cancel_order(command).unwrap();
14988
14989 let rejected = recv_order_event(&mut rx).await;
14990 match rejected {
14991 OrderEventAny::CancelDenied(event) => {
14992 assert_eq!(event.client_order_id, client_order_id);
14993 assert_eq!(event.instrument_id, instrument_id);
14994 assert_eq!(event.venue_order_id, None);
14995 assert!(event.reason.contains("Lighter cancel_order failed"));
14996 assert!(event.reason.contains("venue order_id not yet known"));
14997 }
14998 event => panic!("expected cancel rejected event, was {event:?}"),
14999 }
15000
15001 assert_nonce_reusable(&client.dispatch);
15002 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15003 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
15004 assert!(
15005 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15006 .await
15007 .is_err(),
15008 "prepare failure must emit exactly one cancel rejection",
15009 );
15010 }
15011
15012 #[tokio::test]
15013 async fn cancel_order_prepare_failure_emits_cancel_denied_when_order_uncached() {
15014 let (client, cache, mut rx) = create_execution_client();
15015 let instrument_id = register_test_instrument(&client, &cache);
15016 let client_order_id = ClientOrderId::from("O-CANCEL-NO-CACHE");
15017 let venue_order_id = VenueOrderId::from("123");
15018
15019 let command = CancelOrder::new(
15020 trader_id(),
15021 Some(client_id()),
15022 strategy_id(),
15023 instrument_id,
15024 client_order_id,
15025 Some(venue_order_id),
15026 UUID4::new(),
15027 UnixNanos::default(),
15028 None,
15029 None,
15030 );
15031 client.cancel_order(command).unwrap();
15032
15033 let rejected = recv_order_event(&mut rx).await;
15034 match rejected {
15035 OrderEventAny::CancelDenied(event) => {
15036 assert_eq!(event.client_order_id, client_order_id);
15037 assert_eq!(event.instrument_id, instrument_id);
15038 assert_eq!(event.venue_order_id, Some(venue_order_id));
15039 assert!(event.reason.contains("Lighter cancel_order failed"));
15040 assert!(event.reason.contains("order not found in cache"));
15041 }
15042 event => panic!("expected cancel rejected event, was {event:?}"),
15043 }
15044
15045 assert_nonce_reusable(&client.dispatch);
15046 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15047 assert!(
15048 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15049 .await
15050 .is_err(),
15051 "prepare failure must emit exactly one cancel rejection",
15052 );
15053 }
15054
15055 #[rstest]
15056 #[case::buy(Some(OrderSide::Buy), false, vec![0, 2])]
15057 #[case::sell(Some(OrderSide::Sell), false, vec![1, 3])]
15058 #[case::unsided(None, false, vec![0, 1, 2, 3])]
15059 #[case::empty_buy(Some(OrderSide::Buy), true, vec![])]
15060 #[case::empty_sell(Some(OrderSide::Sell), true, vec![])]
15061 #[case::empty_unsided(None, true, vec![])]
15062 #[tokio::test]
15063 async fn cancel_all_orders_filters_orders_and_preserves_owners(
15064 #[case] order_side: Option<OrderSide>,
15065 #[case] empty: bool,
15066 #[case] expected_indices: Vec<usize>,
15067 ) {
15068 let (client, cache, mut rx) = create_execution_client();
15069 let instrument_id = register_test_instrument(&client, &cache);
15070 let other_instrument_id = InstrumentId::from("BTC-PERP.LIGHTER");
15071 let selected_account = account_id();
15072 let other_account = AccountId::from("LIGHTER-002");
15073 let mut orders = Vec::new();
15074
15075 for (index, (instrument, owner, side, open, account)) in [
15076 (
15077 instrument_id,
15078 "S-001",
15079 OrderSide::Buy,
15080 true,
15081 selected_account,
15082 ),
15083 (
15084 instrument_id,
15085 "S-001",
15086 OrderSide::Sell,
15087 true,
15088 selected_account,
15089 ),
15090 (
15091 instrument_id,
15092 "S-002",
15093 OrderSide::Buy,
15094 true,
15095 selected_account,
15096 ),
15097 (
15098 instrument_id,
15099 "S-002",
15100 OrderSide::Sell,
15101 true,
15102 selected_account,
15103 ),
15104 (
15105 other_instrument_id,
15106 "S-001",
15107 OrderSide::Buy,
15108 true,
15109 selected_account,
15110 ),
15111 (
15112 other_instrument_id,
15113 "S-002",
15114 OrderSide::Sell,
15115 true,
15116 selected_account,
15117 ),
15118 (
15119 instrument_id,
15120 "S-001",
15121 OrderSide::Buy,
15122 false,
15123 selected_account,
15124 ),
15125 (
15126 instrument_id,
15127 "S-002",
15128 OrderSide::Sell,
15129 false,
15130 selected_account,
15131 ),
15132 (instrument_id, "S-001", OrderSide::Buy, true, other_account),
15133 (instrument_id, "S-002", OrderSide::Sell, true, other_account),
15134 ]
15135 .into_iter()
15136 .enumerate()
15137 {
15138 if empty
15139 && account == selected_account
15140 && instrument == instrument_id
15141 && open
15142 && order_side.is_none_or(|filter| filter == side)
15143 {
15144 continue;
15145 }
15146
15147 let order = OrderTestBuilder::new(OrderType::Limit)
15148 .trader_id(trader_id())
15149 .strategy_id(StrategyId::from(owner))
15150 .instrument_id(instrument)
15151 .client_order_id(ClientOrderId::from(format!("O-CANCEL-ALL-{index}")))
15152 .side(side)
15153 .quantity(Quantity::from("0.1000"))
15154 .price(Price::from("2361.31"))
15155 .build();
15156 let venue_order_id = VenueOrderId::from(format!("{}", 123 + index));
15157 let accepted = TestOrderEventStubs::accepted(&order, account, venue_order_id);
15158 cache_order(&cache, order.clone());
15159 cache.borrow_mut().update_order(&accepted).unwrap();
15160 client
15161 .dispatch
15162 .venue_id_map
15163 .insert(order.client_order_id(), venue_order_id);
15164
15165 let event = if open {
15166 OrderEventAny::PendingCancel(OrderPendingCancel::new(
15167 order.trader_id(),
15168 order.strategy_id(),
15169 instrument,
15170 order.client_order_id(),
15171 Some(account),
15172 UUID4::new(),
15173 UnixNanos::default(),
15174 UnixNanos::default(),
15175 false,
15176 Some(venue_order_id),
15177 ))
15178 } else {
15179 TestOrderEventStubs::canceled(&order, account, Some(venue_order_id))
15180 };
15181
15182 cache.borrow_mut().update_order(&event).unwrap();
15183 orders.push((order, venue_order_id));
15184 }
15185
15186 client
15187 .cancel_all_orders(CancelAllOrders::new(
15188 trader_id(),
15189 Some(client_id()),
15190 strategy_id(),
15191 instrument_id,
15192 order_side,
15193 UUID4::new(),
15194 UnixNanos::default(),
15195 None,
15196 None,
15197 ))
15198 .unwrap();
15199
15200 let mut actual = Vec::new();
15203
15204 for _ in &expected_indices {
15205 match recv_order_event(&mut rx).await {
15206 OrderEventAny::CancelDenied(event) => {
15207 assert!(
15208 event
15209 .reason
15210 .contains("Lighter cancel_order dispatch failed")
15211 );
15212 assert!(event.reason.contains("handler unavailable"));
15213 assert_eq!(event.trader_id, trader_id());
15214 assert_eq!(event.account_id, Some(account_id()));
15215 actual.push((
15216 event.client_order_id,
15217 event.strategy_id,
15218 event.instrument_id,
15219 event.venue_order_id,
15220 ));
15221 }
15222 event => panic!("expected cancel rejected event, was {event:?}"),
15223 }
15224 }
15225
15226 let mut expected: Vec<_> = expected_indices
15227 .iter()
15228 .map(|&index| {
15229 let (order, venue_order_id) = &orders[index];
15230 (
15231 order.client_order_id(),
15232 order.strategy_id(),
15233 order.instrument_id(),
15234 Some(*venue_order_id),
15235 )
15236 })
15237 .collect();
15238
15239 actual.sort();
15240 expected.sort();
15241
15242 assert_eq!(actual, expected);
15243 assert!(
15244 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15245 .await
15246 .is_err()
15247 );
15248 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15249
15250 for (order, _) in orders {
15251 assert_eq!(
15252 client
15253 .dispatch
15254 .pending_order_action(&order.client_order_id()),
15255 None
15256 );
15257 }
15258
15259 if empty {
15260 assert_nonce_reusable(&client.dispatch);
15261 }
15262 }
15263
15264 #[tokio::test]
15265 async fn cancel_all_orders_prepare_failure_denies_open_order() {
15266 let (client, cache, mut rx) = create_execution_client();
15267 let instrument_id = register_test_instrument(&client, &cache);
15268 let mut factory = test_order_factory();
15269 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-ALL-NO-VOI");
15270 let client_order_id = order.client_order_id();
15271 cache_accepted_order(&cache, order, VenueOrderId::from("123"), Some(client_id()));
15272
15273 let command = CancelAllOrders::new(
15274 trader_id(),
15275 Some(client_id()),
15276 strategy_id(),
15277 instrument_id,
15278 Some(OrderSide::Buy),
15279 UUID4::new(),
15280 UnixNanos::default(),
15281 None,
15282 None,
15283 );
15284 client.cancel_all_orders(command).unwrap();
15285
15286 let OrderEventAny::CancelDenied(denied) = recv_order_event(&mut rx).await else {
15287 panic!("Expected cancel denial");
15288 };
15289
15290 assert_eq!(denied.client_order_id, client_order_id);
15291 assert_eq!(denied.instrument_id, instrument_id);
15292 assert_eq!(
15293 cache.borrow().order(&client_order_id).unwrap().status(),
15294 OrderStatus::Accepted
15295 );
15296
15297 assert_nonce_reusable(&client.dispatch);
15298 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15299 assert!(
15300 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15301 .await
15302 .is_err(),
15303 "cancel-all prepare failure must not emit an invalid cancel rejection",
15304 );
15305 }
15306
15307 #[tokio::test]
15308 async fn cancel_order_nonce_prepare_failure_emits_cancel_denied() {
15309 let mut config = test_config();
15310 config.base_url_http = Some(spawn_next_nonce_server(100).await);
15311 let (client, cache, mut rx) = create_execution_client_with_config(config);
15312 let instrument_id = register_test_instrument(&client, &cache);
15313 let mut factory = test_order_factory();
15314 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-NONCE-FAIL");
15315 let client_order_id = order.client_order_id();
15316 let venue_order_id = VenueOrderId::from("123");
15317 cache_pending_cancel_order(&cache, order, venue_order_id, None);
15318
15319 let window = i64::from(client.dispatch.nonce_manager.skip_window());
15320 for _ in 0..window {
15321 client
15322 .dispatch
15323 .nonce_manager
15324 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15325 .unwrap();
15326 }
15327
15328 let command = CancelOrder::new(
15329 trader_id(),
15330 Some(client_id()),
15331 strategy_id(),
15332 instrument_id,
15333 client_order_id,
15334 Some(venue_order_id),
15335 UUID4::new(),
15336 UnixNanos::default(),
15337 None,
15338 None,
15339 );
15340 client.cancel_order(command).unwrap();
15341
15342 let rejected = recv_order_event(&mut rx).await;
15343 match rejected {
15344 OrderEventAny::CancelDenied(event) => {
15345 assert_eq!(event.client_order_id, client_order_id);
15346 assert_eq!(event.venue_order_id, Some(venue_order_id));
15347 assert!(event.reason.contains("failed to allocate Lighter nonce"),);
15348 assert!(event.reason.contains("skip-window exhausted"));
15349 }
15350 event => panic!("expected cancel rejected event, was {event:?}"),
15351 }
15352
15353 wait_for_spawned_tasks(&client).await;
15354 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15355 assert_eq!(
15356 client
15357 .dispatch
15358 .nonce_manager
15359 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15360 .unwrap(),
15361 100,
15362 );
15363 }
15364
15365 struct CancelProbe {
15366 command: CancelOrder,
15367 identity: (u64, i64),
15368 token: CancellationToken,
15369 status: OrderStatus,
15370 }
15371
15372 impl CancelProbe {
15373 fn cache(
15374 client: &LighterExecutionClient,
15375 cache: &Rc<RefCell<Cache>>,
15376 instrument_id: InstrumentId,
15377 marked: bool,
15378 ) -> Self {
15379 let mut factory = test_order_factory();
15380 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-PROBE");
15381 let client_order_id = order.client_order_id();
15382 let venue_order_id = VenueOrderId::from("123");
15383
15384 let command = CancelOrder::new(
15385 trader_id(),
15386 Some(client_id()),
15387 strategy_id(),
15388 instrument_id,
15389 client_order_id,
15390 Some(venue_order_id),
15391 UUID4::new(),
15392 UnixNanos::default(),
15393 None,
15394 None,
15395 );
15396
15397 if marked {
15398 cache_pending_cancel_order(cache, order, venue_order_id, Some(command.command_id));
15399 } else {
15400 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
15401 }
15402
15403 let epoch = client.ws_client.connection_epoch();
15404 let nonce = client
15405 .dispatch
15406 .nonce_manager
15407 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15408 .unwrap();
15409 client
15410 .dispatch
15411 .nonce_manager
15412 .ack_success(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, nonce)
15413 .unwrap();
15414 client
15415 .dispatch
15416 .register_cancel_command(client_order_id, command.command_id, marked);
15417 client.dispatch.promote_pending_cancel(
15418 &client_order_id,
15419 command.command_id,
15420 epoch,
15421 nonce,
15422 marked,
15423 );
15424 let token = client
15425 .dispatch
15426 .pending_cancel_token(&client_order_id, epoch, nonce)
15427 .unwrap();
15428
15429 Self {
15430 command,
15431 identity: (epoch, nonce),
15432 token,
15433 status: if marked {
15434 OrderStatus::PendingCancel
15435 } else {
15436 OrderStatus::Accepted
15437 },
15438 }
15439 }
15440
15441 fn pending_command(&self, cache: &Rc<RefCell<Cache>>) -> CancelOrder {
15442 let mut command = self.command.clone();
15443 command.command_id = UUID4::new();
15444 apply_pending_cancel(
15445 cache,
15446 command.instrument_id,
15447 command.client_order_id,
15448 command.venue_order_id.unwrap(),
15449 Some(command.command_id),
15450 );
15451 command
15452 }
15453
15454 fn assert_preserved(
15455 &self,
15456 client: &LighterExecutionClient,
15457 cache: &Rc<RefCell<Cache>>,
15458 command: &CancelOrder,
15459 event: &OrderEventAny,
15460 ) {
15461 let OrderEventAny::CancelDenied(denied) = event else {
15462 panic!("Expected local cancel denial, was {event:?}");
15463 };
15464
15465 cache.borrow_mut().update_order(event).unwrap();
15466 let id = self.command.client_order_id;
15467 let cached = cache.borrow().try_order_owned(&id).unwrap();
15468 assert_eq!(denied.causation_id, Some(command.command_id));
15469 assert_eq!(denied.trader_id, command.trader_id);
15470 assert_eq!(denied.strategy_id, command.strategy_id);
15471 assert_eq!(denied.instrument_id, command.instrument_id);
15472 assert_eq!(denied.client_order_id, command.client_order_id);
15473 assert_eq!(denied.venue_order_id, command.venue_order_id);
15474 assert_eq!(cached.status(), self.status);
15475 assert_eq!(cached.quantity(), Quantity::from("0.1000"));
15476 assert_eq!(cached.price(), Some(Price::from("2361.31")));
15477 assert_eq!(cached.venue_order_id(), self.command.venue_order_id);
15478 assert_eq!(
15479 cached.has_pending_cancel_command(),
15480 self.status == OrderStatus::PendingCancel
15481 );
15482 assert_eq!(
15483 client.dispatch.pending_order_action(&id),
15484 Some(PendingOrderAction::cancel(
15485 self.status == OrderStatus::PendingCancel
15486 ))
15487 );
15488 assert_eq!(
15489 client.dispatch.pending_cancel_identity(&id),
15490 Some((self.command.command_id, self.identity))
15491 );
15492 assert!(!self.token.is_cancelled());
15493 assert!(
15494 !client
15495 .dispatch
15496 .pending_cancel_token(&id, self.identity.0, self.identity.1)
15497 .unwrap()
15498 .is_cancelled()
15499 );
15500 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15501 }
15502 }
15503
15504 #[rstest]
15505 #[case::marked(true)]
15506 #[case::unmarked(false)]
15507 #[tokio::test]
15508 async fn cancel_order_nonce_failure_preserves_existing_probe(#[case] marked: bool) {
15509 let mut config = test_config();
15510 config.base_url_http = Some(spawn_next_nonce_server(100).await);
15511 let (client, cache, mut rx) = create_execution_client_with_config(config);
15512 let instrument_id = register_test_instrument(&client, &cache);
15513 let probe = CancelProbe::cache(&client, &cache, instrument_id, marked);
15514 let command = probe.pending_command(&cache);
15515 let window = i64::from(client.dispatch.nonce_manager.skip_window());
15516 for _ in 0..window {
15517 client
15518 .dispatch
15519 .nonce_manager
15520 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15521 .unwrap();
15522 }
15523
15524 client.cancel_order(command.clone()).unwrap();
15525 let event = recv_order_event(&mut rx).await;
15526 wait_for_spawned_tasks(&client).await;
15527 probe.assert_preserved(&client, &cache, &command, &event);
15528
15529 let OrderEventAny::CancelDenied(denied) = event else {
15530 unreachable!()
15531 };
15532
15533 assert!(denied.reason.contains("skip-window exhausted"));
15534 assert!(rx.try_recv().is_err());
15535 }
15536
15537 #[rstest]
15538 #[case::marked(true)]
15539 #[case::unmarked(false)]
15540 #[tokio::test]
15541 async fn batch_cancel_validation_failure_preserves_existing_probe(#[case] marked: bool) {
15542 let (client, cache, mut rx) = create_execution_client();
15543 let instrument_id = register_test_instrument(&client, &cache);
15544 let probe = CancelProbe::cache(&client, &cache, instrument_id, marked);
15545 let mut command = probe.pending_command(&cache);
15546 command.venue_order_id = Some(VenueOrderId::from("invalid"));
15547 client
15548 .batch_cancel_orders(BatchCancelOrders::new(
15549 trader_id(),
15550 Some(client_id()),
15551 strategy_id(),
15552 instrument_id,
15553 vec![command.clone()],
15554 UUID4::new(),
15555 UnixNanos::default(),
15556 None,
15557 None,
15558 ))
15559 .unwrap();
15560 let event = recv_order_event(&mut rx).await;
15561 probe.assert_preserved(&client, &cache, &command, &event);
15562
15563 let OrderEventAny::CancelDenied(denied) = event else {
15564 unreachable!()
15565 };
15566
15567 assert!(denied.reason.contains("not an integer index"));
15568 assert!(rx.try_recv().is_err());
15569 }
15570
15571 #[rstest]
15572 #[case::single_marked(false, true)]
15573 #[case::single_unmarked(false, false)]
15574 #[case::batch_marked(true, true)]
15575 #[case::batch_unmarked(true, false)]
15576 #[tokio::test]
15577 async fn cancel_order_send_failure_preserves_existing_probe(
15578 #[case] batch: bool,
15579 #[case] marked: bool,
15580 ) {
15581 let (client, cache, mut rx) = create_execution_client();
15582 let instrument_id = register_test_instrument(&client, &cache);
15583 let probe = CancelProbe::cache(&client, &cache, instrument_id, marked);
15584 let command = probe.pending_command(&cache);
15585
15586 if batch {
15587 client
15588 .batch_cancel_orders(BatchCancelOrders::new(
15589 trader_id(),
15590 Some(client_id()),
15591 strategy_id(),
15592 instrument_id,
15593 vec![command.clone()],
15594 UUID4::new(),
15595 UnixNanos::default(),
15596 None,
15597 None,
15598 ))
15599 .unwrap();
15600 } else {
15601 client.cancel_order(command.clone()).unwrap();
15602 }
15603
15604 let event = recv_order_event(&mut rx).await;
15605 probe.assert_preserved(&client, &cache, &command, &event);
15606
15607 let OrderEventAny::CancelDenied(denied) = event else {
15608 unreachable!()
15609 };
15610
15611 assert!(denied.reason.contains("handler unavailable"));
15612 assert!(rx.try_recv().is_err());
15613 }
15614
15615 #[tokio::test]
15616 async fn batch_cancel_releases_split_reservations_before_waiting_for_ack() {
15617 let (client, cache, _rx) = create_execution_client();
15618 let instrument_id = register_test_instrument(&client, &cache);
15619 client.ws_client.drop_next_send_tx_result_for_test().await;
15620 let context = client
15621 .fanout_dispatch_context(client.credential.as_ref().unwrap())
15622 .unwrap();
15623
15624 let plan = CancelOrderPlan {
15625 command_id: Some(UUID4::new()),
15626 client_order_id: ClientOrderId::from("O-SPLIT-CANCEL"),
15627 strategy_id: strategy_id(),
15628 instrument_id,
15629 venue_order_id: Some(VenueOrderId::from("123")),
15630 market_index: TEST_MARKET_INDEX,
15631 venue_index: 123,
15632 };
15633
15634 let first = context.sign_cancel_order(&plan).unwrap();
15635 let interleaved = context.build_tx_context().unwrap();
15636 let second = context.sign_cancel_order(&plan).unwrap();
15637 let first_nonce = first.nonce;
15638 let second_nonce = second.nonce;
15639 drop(interleaved);
15640
15641 let task = tokio::spawn(async move {
15642 context
15643 .send_batch(vec![
15644 PreparedBatchTx::from((first, false)),
15645 PreparedBatchTx::from((second, false)),
15646 ])
15647 .await;
15648 });
15649
15650 wait_until_async(
15651 || async { client.dispatch.pending_sendtx_len() != 0 },
15652 Duration::from_secs(2),
15653 )
15654 .await;
15655
15656 let released = tokio::time::timeout(
15657 Duration::from_millis(250),
15658 client.nonce_submission_gate.write(),
15659 )
15660 .await
15661 .is_ok();
15662 let pending = client.dispatch.pending_sendtx.lock().clone();
15663 task.abort();
15664 let _ = task.await;
15665
15666 assert!(
15667 released,
15668 "unacknowledged group must not hold later nonce reservations"
15669 );
15670 assert_eq!(second_nonce, first_nonce + 2);
15671 assert_eq!(pending.len(), 1);
15672 assert_eq!(pending[0].nonce, first_nonce);
15673 }
15674
15675 #[rstest]
15676 #[case::not_sent(false)]
15677 #[case::ambiguous(true)]
15678 #[tokio::test]
15679 async fn batch_cancel_orders_send_failure_preserves_delivery_semantics(
15680 #[case] ambiguous: bool,
15681 ) {
15682 let (client, cache, mut rx) = create_execution_client();
15683 let instrument_id = register_test_instrument(&client, &cache);
15684 let mut factory = test_order_factory();
15685 let cancels = ["O-BATCH-CANCEL-A", "O-BATCH-CANCEL-B"]
15686 .iter()
15687 .enumerate()
15688 .map(|(i, id)| {
15689 let order = test_limit_order(&mut factory, instrument_id, id);
15690 let client_order_id = order.client_order_id();
15691 let venue_order_id = VenueOrderId::from(format!("{}", 123 + i).as_str());
15692 cache_pending_cancel_order(&cache, order, venue_order_id, None);
15693
15694 CancelOrder::new(
15695 trader_id(),
15696 Some(client_id()),
15697 strategy_id(),
15698 instrument_id,
15699 client_order_id,
15700 Some(venue_order_id),
15701 UUID4::new(),
15702 UnixNanos::default(),
15703 None,
15704 None,
15705 )
15706 })
15707 .collect::<Vec<_>>();
15708
15709 let command = BatchCancelOrders::new(
15710 trader_id(),
15711 Some(client_id()),
15712 strategy_id(),
15713 instrument_id,
15714 cancels.clone(),
15715 UUID4::new(),
15716 UnixNanos::default(),
15717 None,
15718 None,
15719 );
15720
15721 if ambiguous {
15722 client.ws_client.drop_next_send_tx_result_for_test().await;
15723 }
15724
15725 client.batch_cancel_orders(command).unwrap();
15726
15727 if ambiguous {
15728 assert!(
15729 tokio::time::timeout(Duration::from_millis(100), rx.recv())
15730 .await
15731 .is_err()
15732 );
15733 let pending = client.dispatch.pending_sendtx.lock().clone();
15734 assert_eq!(pending.len(), 2);
15735 assert_eq!(pending[0].batch_id, pending[1].batch_id);
15736 assert!(
15737 pending[0]
15738 .batch_id
15739 .as_ref()
15740 .unwrap()
15741 .starts_with("tx-batch:")
15742 );
15743 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
15744 assert_eq!(pending[1].nonce, TEST_NEXT_NONCE + 1);
15745
15746 for cancel in cancels {
15747 assert_eq!(
15748 client
15749 .dispatch
15750 .pending_order_action(&cancel.client_order_id),
15751 Some(PendingOrderAction::Cancel)
15752 );
15753 }
15754
15755 assert_eq!(
15756 client
15757 .dispatch
15758 .nonce_manager
15759 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
15760 Some(TEST_NEXT_NONCE + 1)
15761 );
15762 return;
15763 }
15764
15765 let first = recv_order_event(&mut rx).await;
15766 let second = recv_order_event(&mut rx).await;
15767
15768 let rejected_ids = [first, second].map(|event| match event {
15769 OrderEventAny::CancelDenied(e) => {
15770 assert!(e.reason.contains("Lighter cancel_order dispatch failed"),);
15771 e.client_order_id
15772 }
15773 other => panic!("expected CancelDenied, was {other:?}"),
15774 });
15775
15776 for cancel in cancels {
15777 assert!(rejected_ids.contains(&cancel.client_order_id));
15778 assert_eq!(
15779 client
15780 .dispatch
15781 .pending_order_action(&cancel.client_order_id),
15782 None,
15783 );
15784 }
15785
15786 assert_nonce_reusable(&client.dispatch);
15787 assert_eq!(
15788 client.dispatch.pending_sendtx_len(),
15789 0,
15790 "local-send-failure must remove batch cancel pending entries",
15791 );
15792 }
15793
15794 #[rstest]
15795 #[case::not_sent(false)]
15796 #[case::ambiguous(true)]
15797 #[tokio::test]
15798 async fn batch_cancel_duplicate_target_resolves_each_command(#[case] ambiguous: bool) {
15799 let (client, cache, mut rx) = create_execution_client();
15800 let instrument_id = register_test_instrument(&client, &cache);
15801 let mut factory = test_order_factory();
15802 let order = test_limit_order(&mut factory, instrument_id, "O-DUPLICATE-CANCEL");
15803 let client_order_id = order.client_order_id();
15804 let venue_order_id = VenueOrderId::from("123");
15805 let first_id = UUID4::new();
15806 let second_id = UUID4::new();
15807 cache_pending_cancel_order(&cache, order, venue_order_id, Some(first_id));
15808
15809 let mut pending = OrderPendingCancel::new(
15810 trader_id(),
15811 strategy_id(),
15812 instrument_id,
15813 client_order_id,
15814 Some(account_id()),
15815 UUID4::new(),
15816 UnixNanos::default(),
15817 UnixNanos::default(),
15818 false,
15819 Some(venue_order_id),
15820 );
15821 pending.causation_id = Some(second_id);
15822 cache
15823 .borrow_mut()
15824 .update_order(&OrderEventAny::PendingCancel(pending))
15825 .unwrap();
15826
15827 let mut cancels = [first_id, second_id].map(|command_id| {
15828 CancelOrder::new(
15829 trader_id(),
15830 Some(client_id()),
15831 strategy_id(),
15832 instrument_id,
15833 client_order_id,
15834 Some(venue_order_id),
15835 command_id,
15836 UnixNanos::default(),
15837 None,
15838 None,
15839 )
15840 });
15841
15842 cancels[0].strategy_id = StrategyId::from("REQUESTER-031");
15843
15844 if ambiguous {
15845 client.ws_client.drop_next_send_tx_result_for_test().await;
15846 }
15847
15848 client
15849 .batch_cancel_orders(BatchCancelOrders::new(
15850 trader_id(),
15851 Some(client_id()),
15852 strategy_id(),
15853 instrument_id,
15854 cancels.to_vec(),
15855 UUID4::new(),
15856 UnixNanos::default(),
15857 None,
15858 None,
15859 ))
15860 .unwrap();
15861 wait_until_async(
15862 || async { client.pending_tasks.is_empty() },
15863 BATCH_ACK_TIMEOUT + Duration::from_secs(2),
15864 )
15865 .await;
15866 let first = recv_order_event(&mut rx).await;
15867
15868 let OrderEventAny::CancelDenied(denied) = &first else {
15869 panic!("expected duplicate command denial, was {first:?}");
15870 };
15871
15872 assert_eq!(denied.causation_id, Some(first_id));
15873 assert_eq!(denied.client_order_id, client_order_id);
15874 assert_eq!(denied.strategy_id, strategy_id());
15875 assert_eq!(denied.instrument_id, instrument_id);
15876 assert_eq!(denied.venue_order_id, Some(venue_order_id));
15877 assert_eq!(denied.reason.as_str(), "Duplicate cancel target in batch");
15878 cache.borrow_mut().update_order(&first).unwrap();
15879 assert_eq!(
15880 cache.borrow().order_ref(&client_order_id).unwrap().status(),
15881 OrderStatus::PendingCancel
15882 );
15883
15884 if ambiguous {
15885 let pending = client.dispatch.pending_sendtx.lock().clone();
15886 assert_eq!(pending.len(), 1);
15887
15888 let PendingSendTxKind::Cancel { command_id, .. } = pending[0].kind else {
15889 panic!("expected one cancel transaction");
15890 };
15891
15892 assert_eq!(command_id, Some(second_id));
15893 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
15894 assert_eq!(
15895 client.dispatch.pending_order_action(&client_order_id),
15896 Some(PendingOrderAction::Cancel)
15897 );
15898 } else {
15899 let second = recv_order_event(&mut rx).await;
15900
15901 let OrderEventAny::CancelDenied(denied) = &second else {
15902 panic!("expected unsent command denial, was {second:?}");
15903 };
15904
15905 assert_eq!(denied.causation_id, Some(second_id));
15906 cache.borrow_mut().update_order(&second).unwrap();
15907 let cached = cache.borrow().try_order_owned(&client_order_id).unwrap();
15908 assert_eq!(cached.status(), OrderStatus::Accepted);
15909 assert_eq!(cached.quantity(), Quantity::from("0.1000"));
15910 assert!(!cached.has_pending_cancel_command());
15911 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15912 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
15913 assert_nonce_reusable(&client.dispatch);
15914 }
15915
15916 assert!(rx.try_recv().is_err());
15917 }
15918
15919 #[rstest]
15920 #[case::spill(16)]
15921 #[case::three(31)]
15922 #[tokio::test]
15923 async fn batch_cancel_orders_chunks_large_commands_on_send_failure(#[case] count: usize) {
15924 let (client, cache, mut rx) = create_execution_client();
15925 let instrument_id = register_test_instrument(&client, &cache);
15926 let mut factory = test_order_factory();
15927
15928 let cancels = (0..count)
15929 .map(|i| {
15930 let order = test_limit_order(
15931 &mut factory,
15932 instrument_id,
15933 &format!("O-BATCH-CANCEL-MAX-{i}"),
15934 );
15935 let id = order.client_order_id();
15936 let venue = VenueOrderId::from(format!("{}", 1_000 + i));
15937 cache_pending_cancel_order(&cache, order, venue, None);
15938
15939 CancelOrder::new(
15940 trader_id(),
15941 Some(client_id()),
15942 strategy_id(),
15943 instrument_id,
15944 id,
15945 Some(venue),
15946 UUID4::new(),
15947 UnixNanos::default(),
15948 None,
15949 None,
15950 )
15951 })
15952 .collect::<Vec<_>>();
15953
15954 let command = BatchCancelOrders::new(
15955 trader_id(),
15956 Some(client_id()),
15957 strategy_id(),
15958 instrument_id,
15959 cancels.clone(),
15960 UUID4::new(),
15961 UnixNanos::default(),
15962 None,
15963 None,
15964 );
15965 client.batch_cancel_orders(command).unwrap();
15966
15967 let mut rejected = Vec::new();
15968
15969 for _ in &cancels {
15970 match recv_order_event(&mut rx).await {
15971 OrderEventAny::CancelDenied(e) => {
15972 assert!(e.reason.contains("Lighter cancel_order dispatch failed"));
15973 rejected.push((e.client_order_id, e.venue_order_id));
15974 }
15975 other => panic!("expected CancelDenied, was {other:?}"),
15976 }
15977 }
15978
15979 let mut expected: Vec<_> = cancels
15980 .iter()
15981 .map(|cancel| (cancel.client_order_id, cancel.venue_order_id))
15982 .collect();
15983 expected.sort();
15984 rejected.sort();
15985 assert_eq!(rejected, expected);
15986
15987 for cancel in &cancels {
15988 assert_eq!(
15989 client
15990 .dispatch
15991 .pending_order_action(&cancel.client_order_id),
15992 None
15993 );
15994 assert_eq!(
15995 client
15996 .dispatch
15997 .pending_cancel_nonce(&cancel.client_order_id),
15998 None
15999 );
16000 }
16001
16002 assert!(
16003 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16004 .await
16005 .is_err(),
16006 );
16007 assert_nonce_reusable(&client.dispatch);
16008 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16009 }
16010
16011 #[rstest]
16012 #[case::success(200, "batch-a", 0, vec!["hash-1", "hash-2"], 2, false)]
16013 #[case::partial_ack(200, "batch-a", 0, vec!["hash-2"], 1, false)]
16014 #[case::missing_hashes(200, "batch-a", 0, vec![], 0, false)]
16015 #[case::wrong_hashes(200, "batch-a", 0, vec!["unknown"], 0, false)]
16016 #[case::wrong_id(200, "unknown", 0, vec!["hash-1", "hash-2"], 0, false)]
16017 #[case::stale(200, "batch-a", 1, vec!["hash-1", "hash-2"], 0, false)]
16018 #[case::invalid_nonce(21104, "batch-a", 0, vec![], 0, true)]
16019 #[case::rejected(21727, "batch-a", 0, vec![], 0, false)]
16020 #[tokio::test]
16021 async fn batch_cancel_response_correlates_members_and_preserves_nonces(
16022 #[case] code: i64,
16023 #[case] id: &str,
16024 #[case] epoch: u64,
16025 #[case] hashes: Vec<&str>,
16026 #[case] acked_count: usize,
16027 #[case] resync_expected: bool,
16028 ) {
16029 let (client, cache, mut rx) = create_execution_client();
16030 let instrument_id = register_test_instrument(&client, &cache);
16031 for i in 0..3 {
16032 let nonce = client
16033 .dispatch
16034 .nonce_manager
16035 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
16036 .unwrap();
16037 let client_order_id = ClientOrderId::from(format!("O-BATCH-{i}"));
16038 let command_id = UUID4::new();
16039 client
16040 .dispatch
16041 .register_cancel_command(client_order_id, command_id, true);
16042 client
16043 .dispatch
16044 .promote_pending_cancel(&client_order_id, command_id, 0, nonce, true);
16045 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
16046 batch_id: Some(if i == 0 { "batch-b" } else { "batch-a" }.to_string()),
16047 connection_epoch: 0,
16048 kind: PendingSendTxKind::Cancel {
16049 command_id: Some(command_id),
16050 emit_cancel_rejected: true,
16051 strategy_id: strategy_id(),
16052 instrument_id,
16053 client_order_id,
16054 venue_order_id: Some(VenueOrderId::from(format!("{}", 100 + i))),
16055 },
16056 submitted_at: UnixNanos::default(),
16057 nonce,
16058 api_key_index: TEST_API_KEY_INDEX,
16059 tx_hash: format!("hash-{i}"),
16060 });
16061 }
16062
16063 assert!(client.dispatch.pop_pending_sendtx_if_only(0).is_none());
16064 let hashes: Vec<_> = hashes.into_iter().map(str::to_string).collect();
16065 let (acked, resync) = handle_send_tx_batch_result(
16066 &client.dispatch,
16067 &client.emitter,
16068 Some(TEST_ACCOUNT_INDEX_I64),
16069 epoch,
16070 UnixNanos::default(),
16071 id,
16072 code,
16073 "batch failure",
16074 &hashes,
16075 );
16076 assert_eq!(resync, resync_expected);
16077 assert_eq!(acked.len(), acked_count);
16078 let rejected = code != 200;
16079 assert_eq!(
16080 client.dispatch.pending_sendtx_len(),
16081 if rejected { 1 } else { 3 - acked_count }
16082 );
16083
16084 if rejected {
16085 for i in [2, 1] {
16086 match recv_order_event(&mut rx).await {
16087 OrderEventAny::CancelRejected(event) => {
16088 assert_eq!(
16089 event.client_order_id,
16090 ClientOrderId::from(format!("O-BATCH-{i}"))
16091 );
16092 assert_eq!(event.strategy_id, strategy_id());
16093 assert_eq!(event.instrument_id, instrument_id);
16094 assert_eq!(
16095 event.venue_order_id,
16096 Some(VenueOrderId::from(format!("{}", 100 + i)))
16097 );
16098 }
16099 event => panic!("expected rejection, was {event:?}"),
16100 }
16101
16102 assert_eq!(
16103 client
16104 .dispatch
16105 .pending_order_action(&ClientOrderId::from(format!("O-BATCH-{i}"))),
16106 None
16107 );
16108 }
16109
16110 assert_eq!(
16111 client
16112 .dispatch
16113 .nonce_manager
16114 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
16115 Some(TEST_NEXT_NONCE)
16116 );
16117 } else {
16118 for pending in &acked {
16119 assert!(hashes.contains(&pending.tx_hash));
16120 }
16121
16122 assert_eq!(
16123 client
16124 .dispatch
16125 .nonce_manager
16126 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
16127 Some(TEST_NEXT_NONCE + 2)
16128 );
16129 }
16130
16131 assert_eq!(
16132 client
16133 .dispatch
16134 .pending_order_action(&ClientOrderId::from("O-BATCH-0")),
16135 Some(PendingOrderAction::Cancel)
16136 );
16137 assert!(
16138 tokio::time::timeout(Duration::from_millis(20), rx.recv())
16139 .await
16140 .is_err()
16141 );
16142 let remaining = client.dispatch.drain_pending_sendtx(0);
16143 assert_eq!(remaining.len(), if rejected { 1 } else { 3 - acked_count });
16144 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16145 }
16146
16147 #[rstest]
16148 fn cancel_order_from_cancel_all_preserves_tracing_ids() {
16149 let instrument_id = InstrumentId::from("ETH-PERP.LIGHTER");
16150 let client_order_id = ClientOrderId::from("O-CANCEL-ALL-CHILD");
16151 let command_id = UUID4::new();
16152 let correlation_id = UUID4::new();
16153 let causation_id = UUID4::new();
16154 let ts_init = UnixNanos::default();
16155 let mut cmd = CancelAllOrders::new(
16156 trader_id(),
16157 Some(client_id()),
16158 strategy_id(),
16159 instrument_id,
16160 Some(OrderSide::Buy),
16161 command_id,
16162 ts_init,
16163 None,
16164 Some(correlation_id),
16165 );
16166 cmd.causation_id = Some(causation_id);
16167
16168 let owner = StrategyId::from("S-002");
16169 let order_cmd = cancel_order_from_cancel_all(&cmd, client_order_id, owner);
16170
16171 assert_eq!(order_cmd.trader_id, trader_id());
16172 assert_eq!(order_cmd.client_id, Some(client_id()));
16173 assert_eq!(order_cmd.strategy_id, owner);
16174 assert_eq!(order_cmd.instrument_id, instrument_id);
16175 assert_eq!(order_cmd.client_order_id, client_order_id);
16176 assert_eq!(order_cmd.venue_order_id, None);
16177 assert_eq!(order_cmd.command_id, command_id);
16178 assert_eq!(order_cmd.ts_init, ts_init);
16179 assert_eq!(order_cmd.params, None);
16180 assert_eq!(order_cmd.correlation_id, Some(correlation_id));
16181 assert_eq!(order_cmd.causation_id, Some(causation_id));
16182 }
16183
16184 #[rstest]
16185 #[case::not_sent(false)]
16186 #[case::ambiguous(true)]
16187 #[tokio::test]
16188 async fn batch_modify_orders_send_failure_preserves_delivery_semantics(
16189 #[case] ambiguous: bool,
16190 ) {
16191 let (client, cache, mut rx) = create_execution_client();
16192 let instrument_id = register_test_instrument(&client, &cache);
16193 let mut factory = test_order_factory();
16194
16195 let modifies: Vec<_> = (0..2)
16196 .map(|i| {
16197 let order =
16198 test_limit_order(&mut factory, instrument_id, &format!("O-BATCH-MODIFY-{i}"));
16199 let id = order.client_order_id();
16200 cache_order(&cache, order);
16201
16202 ModifyOrder::new(
16203 trader_id(),
16204 Some(client_id()),
16205 strategy_id(),
16206 instrument_id,
16207 id,
16208 Some(VenueOrderId::from(format!("{}", 123 + i))),
16209 Some(Quantity::from("0.2000")),
16210 Some(Price::from("2362.00")),
16211 None,
16212 UUID4::new(),
16213 UnixNanos::default(),
16214 None,
16215 None,
16216 )
16217 })
16218 .collect();
16219
16220 if ambiguous {
16221 client.ws_client.drop_next_send_tx_result_for_test().await;
16222 }
16223
16224 client
16225 .batch_modify_orders(BatchModifyOrders::new(
16226 trader_id(),
16227 Some(client_id()),
16228 strategy_id(),
16229 instrument_id,
16230 modifies.clone(),
16231 UUID4::new(),
16232 UnixNanos::default(),
16233 None,
16234 None,
16235 ))
16236 .unwrap();
16237
16238 if ambiguous {
16239 wait_until_async(
16240 || async { client.dispatch.pending_sendtx_len() == 2 },
16241 Duration::from_secs(2),
16242 )
16243 .await;
16244 assert!(
16245 tokio::time::timeout(Duration::from_millis(100), rx.recv())
16246 .await
16247 .is_err()
16248 );
16249 let pending = client.dispatch.pending_sendtx.lock().clone();
16250 assert_eq!(pending.len(), 2);
16251 assert_eq!(pending[0].batch_id, pending[1].batch_id);
16252 assert!(
16253 pending[0]
16254 .batch_id
16255 .as_ref()
16256 .unwrap()
16257 .starts_with(TX_BATCH_ID_PREFIX)
16258 );
16259 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
16260 assert_eq!(pending[1].nonce, TEST_NEXT_NONCE + 1);
16261
16262 for modify in &modifies {
16263 assert_eq!(
16264 client
16265 .dispatch
16266 .pending_order_action(&modify.client_order_id),
16267 Some(PendingOrderAction::Modify)
16268 );
16269 }
16270 } else {
16271 let mut ids = Vec::new();
16272
16273 for _ in 0..2 {
16274 match recv_order_event(&mut rx).await {
16275 OrderEventAny::ModifyDenied(e) => {
16276 assert!(e.reason.contains("Lighter modify_order dispatch failed"));
16277 assert_eq!(e.instrument_id, instrument_id);
16278 assert_eq!(e.strategy_id, strategy_id());
16279 ids.push((e.client_order_id, e.venue_order_id));
16280 }
16281 other => panic!("expected ModifyDenied, was {other:?}"),
16282 }
16283 }
16284
16285 ids.sort();
16286 let mut expected: Vec<_> = modifies
16287 .iter()
16288 .map(|modify| (modify.client_order_id, modify.venue_order_id))
16289 .collect();
16290 expected.sort();
16291 assert_eq!(ids, expected);
16292 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16293
16294 for modify in &modifies {
16295 assert_eq!(
16296 client
16297 .dispatch
16298 .pending_order_action(&modify.client_order_id),
16299 None
16300 );
16301 }
16302
16303 assert_nonce_reusable(&client.dispatch);
16304 }
16305 }
16306
16307 #[tokio::test]
16308 async fn batch_modify_orders_rejects_invalid_child_and_dispatches_supported() {
16309 let (client, cache, mut rx) = create_execution_client();
16310 let instrument_id = register_test_instrument(&client, &cache);
16311 let mut factory = test_order_factory();
16312 let mut modifies = Vec::new();
16313
16314 for (id, venue_id) in [("O-MODIFY-INVALID", "invalid"), ("O-MODIFY-VALID", "456")] {
16315 let order = test_limit_order(&mut factory, instrument_id, id);
16316 let client_order_id = order.client_order_id();
16317 cache_order(&cache, order);
16318 modifies.push(ModifyOrder::new(
16319 trader_id(),
16320 Some(client_id()),
16321 strategy_id(),
16322 instrument_id,
16323 client_order_id,
16324 Some(VenueOrderId::from(venue_id)),
16325 Some(Quantity::from("0.2000")),
16326 Some(Price::from("2362.00")),
16327 None,
16328 UUID4::new(),
16329 UnixNanos::default(),
16330 None,
16331 None,
16332 ));
16333 }
16334
16335 client.ws_client.drop_next_send_tx_result_for_test().await;
16336 client
16337 .batch_modify_orders(BatchModifyOrders::new(
16338 trader_id(),
16339 Some(client_id()),
16340 strategy_id(),
16341 instrument_id,
16342 modifies.clone(),
16343 UUID4::new(),
16344 UnixNanos::default(),
16345 None,
16346 None,
16347 ))
16348 .unwrap();
16349 let event = recv_order_event(&mut rx).await;
16350
16351 let OrderEventAny::ModifyDenied(event) = event else {
16352 panic!("expected invalid-child rejection")
16353 };
16354
16355 assert_eq!(event.client_order_id, modifies[0].client_order_id);
16356 assert_eq!(event.venue_order_id, modifies[0].venue_order_id);
16357 assert!(event.reason.contains("invalid"));
16358 wait_until_async(
16359 || async { client.dispatch.pending_sendtx_len() == 1 },
16360 Duration::from_secs(2),
16361 )
16362 .await;
16363 let pending = client.dispatch.pending_sendtx.lock().clone();
16364 assert_eq!(pending.len(), 1);
16365 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
16366 assert!(
16367 matches!(&pending[0].kind, PendingSendTxKind::Modify { client_order_id, .. } if *client_order_id == modifies[1].client_order_id)
16368 );
16369 assert_eq!(
16370 client
16371 .dispatch
16372 .pending_order_action(&modifies[0].client_order_id),
16373 None
16374 );
16375 assert_eq!(
16376 client
16377 .dispatch
16378 .pending_order_action(&modifies[1].client_order_id),
16379 Some(PendingOrderAction::Modify)
16380 );
16381 assert!(rx.try_recv().is_err());
16382 }
16383
16384 #[rstest]
16385 #[case::empty(0)]
16386 #[case::full(15)]
16387 #[case::spill(16)]
16388 #[case::three(31)]
16389 #[tokio::test]
16390 async fn batch_modify_orders_chunks_large_commands_on_send_failure(#[case] count: usize) {
16391 let (client, cache, mut rx) = create_execution_client();
16392 let instrument_id = register_test_instrument(&client, &cache);
16393 let mut factory = test_order_factory();
16394
16395 let modifies: Vec<_> = (0..count)
16396 .map(|i| {
16397 let order =
16398 test_limit_order(&mut factory, instrument_id, &format!("O-BATCH-SIZE-{i}"));
16399 let id = order.client_order_id();
16400 cache_order(&cache, order);
16401
16402 ModifyOrder::new(
16403 trader_id(),
16404 Some(client_id()),
16405 strategy_id(),
16406 instrument_id,
16407 id,
16408 Some(VenueOrderId::from(format!("{}", 123 + i))),
16409 Some(Quantity::from("0.2000")),
16410 Some(Price::from("2362.00")),
16411 None,
16412 UUID4::new(),
16413 UnixNanos::default(),
16414 None,
16415 None,
16416 )
16417 })
16418 .collect();
16419
16420 client
16421 .batch_modify_orders(BatchModifyOrders::new(
16422 trader_id(),
16423 Some(client_id()),
16424 strategy_id(),
16425 instrument_id,
16426 modifies.clone(),
16427 UUID4::new(),
16428 UnixNanos::default(),
16429 None,
16430 None,
16431 ))
16432 .unwrap();
16433
16434 let mut rejected = Vec::new();
16435
16436 for _ in &modifies {
16437 match recv_order_event(&mut rx).await {
16438 OrderEventAny::ModifyDenied(e) => {
16439 assert!(e.reason.contains("Lighter modify_order dispatch failed"));
16440 rejected.push((e.client_order_id, e.venue_order_id));
16441 }
16442 other => panic!("expected ModifyDenied, was {other:?}"),
16443 }
16444 }
16445
16446 let mut expected: Vec<_> = modifies
16447 .iter()
16448 .map(|modify| (modify.client_order_id, modify.venue_order_id))
16449 .collect();
16450 expected.sort();
16451 rejected.sort();
16452 assert_eq!(rejected, expected);
16453
16454 for modify in &modifies {
16455 assert_eq!(
16456 client
16457 .dispatch
16458 .pending_order_action(&modify.client_order_id),
16459 None
16460 );
16461 assert_eq!(
16462 client
16463 .dispatch
16464 .pending_cancel_nonce(&modify.client_order_id),
16465 None
16466 );
16467 }
16468
16469 assert!(
16470 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16471 .await
16472 .is_err(),
16473 );
16474 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16475 assert_nonce_reusable(&client.dispatch);
16476 }
16477
16478 #[rstest]
16479 #[case::modify(false, false)]
16480 #[case::batch_modify(false, true)]
16481 #[case::cancel(true, false)]
16482 #[case::batch_cancel(true, true)]
16483 #[tokio::test]
16484 async fn command_admission_failure_denies_each_unsent_child(
16485 #[case] cancel: bool,
16486 #[case] batch: bool,
16487 #[values(false, true)] shutdown: bool,
16488 ) {
16489 let (mut client, cache, mut receiver) = create_execution_client();
16490 let instrument = register_test_instrument(&client, &cache);
16491 let mut factory = test_order_factory();
16492
16493 let count = if batch { 2 } else { 1 };
16494 let mut modifies = Vec::new();
16495 let mut cancels = Vec::new();
16496 let mut expected = Vec::new();
16497
16498 for index in 0..count {
16499 let order = test_limit_order(&mut factory, instrument, &format!("O-ADMISSION-{index}"));
16500 let id = order.client_order_id();
16501 let venue = VenueOrderId::from(format!("{}", 123 + index));
16502 cache_accepted_order(&cache, order, venue, Some(client_id()));
16503 let cause = UUID4::new();
16504 expected.push((id, venue, cause));
16505 modifies.push(ModifyOrder::new(
16506 trader_id(),
16507 Some(client_id()),
16508 strategy_id(),
16509 instrument,
16510 id,
16511 Some(venue),
16512 Some(Quantity::from("0.2000")),
16513 Some(Price::from("2362.00")),
16514 None,
16515 cause,
16516 17.into(),
16517 None,
16518 None,
16519 ));
16520 cancels.push(CancelOrder::new(
16521 trader_id(),
16522 Some(client_id()),
16523 strategy_id(),
16524 instrument,
16525 id,
16526 Some(venue),
16527 cause,
16528 19.into(),
16529 None,
16530 None,
16531 ));
16532 }
16533
16534 let before = expected
16535 .iter()
16536 .map(|(id, _, _)| cache.borrow().order(id).unwrap().clone())
16537 .collect::<Vec<_>>();
16538
16539 if shutdown {
16540 client.begin_session_shutdown();
16541 } else {
16542 client.credential = None;
16543 }
16544
16545 let result = match (cancel, batch) {
16546 (false, false) => client.modify_order(modifies.remove(0)),
16547 (false, true) => client.batch_modify_orders(BatchModifyOrders::new(
16548 trader_id(),
16549 Some(client_id()),
16550 strategy_id(),
16551 instrument,
16552 modifies,
16553 UUID4::new(),
16554 23.into(),
16555 None,
16556 None,
16557 )),
16558 (true, false) => client.cancel_order(cancels.remove(0)),
16559 (true, true) => client.batch_cancel_orders(BatchCancelOrders::new(
16560 trader_id(),
16561 Some(client_id()),
16562 strategy_id(),
16563 instrument,
16564 cancels,
16565 UUID4::new(),
16566 29.into(),
16567 None,
16568 None,
16569 )),
16570 };
16571
16572 assert_eq!(result.is_err(), !shutdown || batch);
16573
16574 for ((id, venue, cause), original) in expected.into_iter().zip(before) {
16575 let event = match receiver.try_recv().unwrap() {
16576 ExecutionEvent::Order(event) => event,
16577 other => panic!("expected command denial, was {other:?}"),
16578 };
16579
16580 let (actual_id, actual_instrument, actual_strategy, actual_venue, actual_cause, reason) =
16581 match event {
16582 OrderEventAny::ModifyDenied(event) if !cancel => (
16583 event.client_order_id,
16584 event.instrument_id,
16585 event.strategy_id,
16586 event.venue_order_id,
16587 event.causation_id,
16588 event.reason,
16589 ),
16590 OrderEventAny::CancelDenied(event) if cancel => (
16591 event.client_order_id,
16592 event.instrument_id,
16593 event.strategy_id,
16594 event.venue_order_id,
16595 event.causation_id,
16596 event.reason,
16597 ),
16598 other => panic!("unexpected command outcome {other:?}"),
16599 };
16600
16601 assert_eq!(
16602 (
16603 actual_id,
16604 actual_instrument,
16605 actual_strategy,
16606 actual_venue,
16607 actual_cause
16608 ),
16609 (id, instrument, strategy_id(), Some(venue), Some(cause))
16610 );
16611 assert_eq!(
16612 reason.as_str(),
16613 if shutdown {
16614 "Lighter task admission is closed"
16615 } else if cancel {
16616 "Lighter execution client cannot cancel without credentials"
16617 } else {
16618 "Lighter execution client cannot modify without credentials"
16619 }
16620 );
16621 assert_eq!(
16622 serde_json::to_value(cache.borrow().order(&id).unwrap().clone()).unwrap(),
16623 serde_json::to_value(&original).unwrap()
16624 );
16625 assert_eq!(client.dispatch.pending_order_action(&id), None);
16626 }
16627
16628 assert!(matches!(
16629 receiver.try_recv(),
16630 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
16631 ));
16632 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16633 assert_nonce_reusable(&client.dispatch);
16634 }
16635
16636 #[tokio::test]
16637 async fn modify_order_send_failure_emits_modify_denied_and_rolls_back() {
16638 let (client, cache, mut rx) = create_execution_client();
16639 let instrument_id = register_test_instrument(&client, &cache);
16640 let mut factory = test_order_factory();
16641 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-FAIL");
16642 let client_order_id = order.client_order_id();
16643 let venue_order_id = VenueOrderId::from("123");
16644 cache_order(&cache, order);
16645
16646 let command = ModifyOrder::new(
16647 trader_id(),
16648 Some(client_id()),
16649 strategy_id(),
16650 instrument_id,
16651 client_order_id,
16652 Some(venue_order_id),
16653 Some(Quantity::from("0.2000")),
16654 Some(Price::from("2362.00")),
16655 None,
16656 UUID4::new(),
16657 UnixNanos::default(),
16658 None,
16659 None,
16660 );
16661 client.modify_order(command).unwrap();
16662
16663 let rejected = recv_order_event(&mut rx).await;
16664
16665 match rejected {
16666 OrderEventAny::ModifyDenied(event) => {
16667 assert_eq!(event.client_order_id, client_order_id);
16668 assert_eq!(event.instrument_id, instrument_id);
16669 assert_eq!(event.venue_order_id, Some(venue_order_id));
16670 assert!(
16671 event
16672 .reason
16673 .contains("Lighter modify_order dispatch failed"),
16674 );
16675 assert!(event.reason.contains("handler unavailable"));
16676 }
16677 event => panic!("expected modify rejected event, was {event:?}"),
16678 }
16679
16680 assert_nonce_reusable(&client.dispatch);
16681 assert_eq!(
16682 client.dispatch.pending_sendtx_len(),
16683 0,
16684 "local-send-failure must remove the pending modify entry",
16685 );
16686 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
16687 }
16688
16689 #[tokio::test]
16690 async fn modify_order_unknown_send_outcome_retains_pending_state() {
16691 let (client, cache, mut rx) = create_execution_client();
16692 let instrument_id = register_test_instrument(&client, &cache);
16693 let mut factory = test_order_factory();
16694 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-UNKNOWN");
16695 let client_order_id = order.client_order_id();
16696 let venue_order_id = VenueOrderId::from("123");
16697 cache_order(&cache, order);
16698 client.ws_client.drop_next_send_tx_result_for_test().await;
16699
16700 client
16701 .modify_order(ModifyOrder::new(
16702 trader_id(),
16703 Some(client_id()),
16704 strategy_id(),
16705 instrument_id,
16706 client_order_id,
16707 Some(venue_order_id),
16708 Some(Quantity::from("0.2000")),
16709 Some(Price::from("2362.00")),
16710 None,
16711 UUID4::new(),
16712 UnixNanos::default(),
16713 None,
16714 None,
16715 ))
16716 .unwrap();
16717
16718 assert!(
16719 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16720 .await
16721 .is_err(),
16722 "unknown outcome must not emit OrderModifyRejected",
16723 );
16724 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
16725 assert_eq!(
16726 client.dispatch.pending_order_action(&client_order_id),
16727 Some(PendingOrderAction::Modify),
16728 );
16729 assert_eq!(
16730 client
16731 .dispatch
16732 .nonce_manager
16733 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
16734 Some(TEST_NEXT_NONCE),
16735 );
16736 }
16737
16738 #[tokio::test]
16739 async fn modify_order_prepare_failure_emits_modify_denied_without_dispatch() {
16740 let (client, cache, mut rx) = create_execution_client();
16741 let instrument_id = register_test_instrument(&client, &cache);
16742 let client_order_id = ClientOrderId::from("O-MODIFY-NO-CACHE");
16743 let venue_order_id = VenueOrderId::from("123");
16744
16745 let command = ModifyOrder::new(
16746 trader_id(),
16747 Some(client_id()),
16748 strategy_id(),
16749 instrument_id,
16750 client_order_id,
16751 Some(venue_order_id),
16752 Some(Quantity::from("0.2000")),
16753 Some(Price::from("2362.00")),
16754 None,
16755 UUID4::new(),
16756 UnixNanos::default(),
16757 None,
16758 None,
16759 );
16760 client.modify_order(command).unwrap();
16761
16762 let rejected = recv_order_event(&mut rx).await;
16763 match rejected {
16764 OrderEventAny::ModifyDenied(event) => {
16765 assert_eq!(event.client_order_id, client_order_id);
16766 assert_eq!(event.instrument_id, instrument_id);
16767 assert_eq!(event.venue_order_id, Some(venue_order_id));
16768 assert!(event.reason.contains("Lighter modify_order failed"));
16769 assert!(event.reason.contains("order not found in cache"));
16770 }
16771 event => panic!("expected modify rejected event, was {event:?}"),
16772 }
16773
16774 assert_nonce_reusable(&client.dispatch);
16775 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16776 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
16777 assert!(
16778 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16779 .await
16780 .is_err(),
16781 "prepare failure must emit exactly one modify rejection",
16782 );
16783 }
16784
16785 #[tokio::test]
16786 async fn modify_order_prepare_failure_emits_modify_denied_when_instrument_uncached() {
16787 let (client, cache, mut rx) = create_execution_client();
16788 let instrument_id =
16789 client
16790 .registry
16791 .insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
16792 let mut factory = test_order_factory();
16793 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-NO-INSTRUMENT");
16794 let client_order_id = order.client_order_id();
16795 let venue_order_id = VenueOrderId::from("123");
16796 cache_order(&cache, order);
16797
16798 let command = ModifyOrder::new(
16799 trader_id(),
16800 Some(client_id()),
16801 strategy_id(),
16802 instrument_id,
16803 client_order_id,
16804 Some(venue_order_id),
16805 Some(Quantity::from("0.2000")),
16806 Some(Price::from("2362.00")),
16807 None,
16808 UUID4::new(),
16809 UnixNanos::default(),
16810 None,
16811 None,
16812 );
16813 client.modify_order(command).unwrap();
16814
16815 let rejected = recv_order_event(&mut rx).await;
16816 match rejected {
16817 OrderEventAny::ModifyDenied(event) => {
16818 assert_eq!(event.client_order_id, client_order_id);
16819 assert_eq!(event.instrument_id, instrument_id);
16820 assert_eq!(event.venue_order_id, Some(venue_order_id));
16821 assert!(event.reason.contains("Lighter modify_order failed"));
16822 assert!(event.reason.contains("instrument not found in cache"));
16823 }
16824 event => panic!("expected modify rejected event, was {event:?}"),
16825 }
16826
16827 assert_nonce_reusable(&client.dispatch);
16828 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16829 assert!(
16830 tokio::time::timeout(Duration::from_millis(50), rx.recv())
16831 .await
16832 .is_err(),
16833 "prepare failure must emit exactly one modify rejection",
16834 );
16835 }
16836
16837 #[rstest]
16838 #[case::default(None, "dispatch failed")]
16839 #[case::invalid(Some(serde_json::json!("0")), "market_order_slippage_bps must be an unsigned integer")]
16840 #[case::out_of_range(Some(serde_json::json!(10_000)), "market_order_slippage_bps must be in [0, 10000)")]
16841 #[tokio::test]
16842 async fn modify_stop_market_validates_slippage_and_derives_price_from_trigger(
16843 #[case] slippage: Option<serde_json::Value>,
16844 #[case] expected_reason: &str,
16845 ) {
16846 let (client, cache, mut rx) = create_execution_client();
16847 let instrument_id = register_test_instrument(&client, &cache);
16848 let mut factory = test_order_factory();
16849 let order = factory.stop_market(
16850 instrument_id,
16851 OrderSide::Sell,
16852 Quantity::from("0.1000"),
16853 Price::from("2300.00"), None,
16855 Some(TimeInForce::Gtc),
16856 None,
16857 Some(false),
16858 Some(false),
16859 None,
16860 None,
16861 None,
16862 None,
16863 None,
16864 None,
16865 Some(ClientOrderId::from("O-MODIFY-STOP-MARKET")),
16866 );
16867 let client_order_id = order.client_order_id();
16868 let venue_order_id = VenueOrderId::from("123");
16869 cache_order(&cache, order);
16870
16871 let params = slippage.map(|value| {
16872 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": value})).unwrap()
16873 });
16874
16875 let command = ModifyOrder::new(
16876 trader_id(),
16877 Some(client_id()),
16878 strategy_id(),
16879 instrument_id,
16880 client_order_id,
16881 Some(venue_order_id),
16882 None,
16883 None,
16884 Some(Price::from("2310.00")),
16885 UUID4::new(),
16886 UnixNanos::default(),
16887 params,
16888 None,
16889 );
16890 client.modify_order(command).unwrap();
16891
16892 let rejected = recv_order_event(&mut rx).await;
16893 match rejected {
16894 OrderEventAny::ModifyDenied(event) => {
16895 assert_eq!(event.client_order_id, client_order_id);
16896 assert_eq!(event.instrument_id, instrument_id);
16897 assert_eq!(event.venue_order_id, Some(venue_order_id));
16898 assert!(
16899 event.reason.contains(expected_reason),
16900 "expected {expected_reason}, was: {}",
16901 event.reason,
16902 );
16903 }
16904 event => panic!("expected modify rejected event, was {event:?}"),
16905 }
16906
16907 assert!(rx.try_recv().is_err());
16908 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16909 assert_nonce_reusable(&client.dispatch);
16910 }
16911
16912 #[tokio::test]
16913 async fn modify_order_rejects_sub_tick_quantity_before_nonce_allocation() {
16914 let (client, cache, mut rx) = create_execution_client();
16915 let instrument_id = register_test_instrument(&client, &cache);
16916 let mut factory = test_order_factory();
16917 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-SUB-TICK");
16918 let client_order_id = order.client_order_id();
16919 cache_order(&cache, order);
16920
16921 client
16922 .modify_order(ModifyOrder::new(
16923 trader_id(),
16924 Some(client_id()),
16925 strategy_id(),
16926 instrument_id,
16927 client_order_id,
16928 Some(VenueOrderId::from("123")),
16929 Some(Quantity::from("0.00001")),
16930 Some(Price::from("2362.00")),
16931 None,
16932 UUID4::new(),
16933 UnixNanos::default(),
16934 None,
16935 None,
16936 ))
16937 .unwrap();
16938
16939 assert_modify_denied_reason(&mut rx, "rounds to 0 ticks").await;
16940 assert_nonce_reusable(&client.dispatch);
16941 }
16942
16943 #[tokio::test]
16944 async fn modify_order_rejects_below_min_notional_before_nonce_allocation() {
16945 let (client, cache, mut rx) = create_execution_client();
16946 let instrument_id = register_test_instrument(&client, &cache);
16947 let mut factory = test_order_factory();
16948 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-MIN-NOTIONAL");
16949 let client_order_id = order.client_order_id();
16950 cache_order(&cache, order);
16951
16952 client
16953 .modify_order(ModifyOrder::new(
16954 trader_id(),
16955 Some(client_id()),
16956 strategy_id(),
16957 instrument_id,
16958 client_order_id,
16959 Some(VenueOrderId::from("123")),
16960 Some(Quantity::from("0.0001")),
16961 Some(Price::from("2362.00")),
16962 None,
16963 UUID4::new(),
16964 UnixNanos::default(),
16965 None,
16966 None,
16967 ))
16968 .unwrap();
16969
16970 assert_modify_denied_reason(&mut rx, "below Lighter min_quote_amount").await;
16971 assert_nonce_reusable(&client.dispatch);
16972 }
16973
16974 #[tokio::test]
16975 async fn modify_conditional_order_rejects_sub_tick_trigger_before_nonce_allocation() {
16976 let (client, cache, mut rx) = create_execution_client();
16977 let instrument_id = register_test_instrument(&client, &cache);
16978 let mut factory = test_order_factory();
16979 let order = factory.stop_market(
16980 instrument_id,
16981 OrderSide::Sell,
16982 Quantity::from("0.1000"),
16983 Price::from("2300.00"),
16984 None,
16985 Some(TimeInForce::Gtc),
16986 None,
16987 Some(false),
16988 Some(false),
16989 None,
16990 None,
16991 None,
16992 None,
16993 None,
16994 None,
16995 Some(ClientOrderId::from("O-MODIFY-SUB-TICK-TRIGGER")),
16996 );
16997 let client_order_id = order.client_order_id();
16998 cache_order(&cache, order);
16999
17000 client
17001 .modify_order(ModifyOrder::new(
17002 trader_id(),
17003 Some(client_id()),
17004 strategy_id(),
17005 instrument_id,
17006 client_order_id,
17007 Some(VenueOrderId::from("123")),
17008 None,
17009 None,
17010 Some(Price::from("0.001")),
17011 UUID4::new(),
17012 UnixNanos::default(),
17013 None,
17014 None,
17015 ))
17016 .unwrap();
17017
17018 assert_modify_denied_reason(&mut rx, "rounds to 0 ticks").await;
17019 assert_nonce_reusable(&client.dispatch);
17020 }
17021
17022 #[tokio::test]
17023 async fn modify_order_nonce_prepare_failure_emits_modify_denied() {
17024 let mut config = test_config();
17025 config.base_url_http = Some(spawn_next_nonce_server(101).await);
17026 let (client, cache, mut rx) = create_execution_client_with_config(config);
17027 let instrument_id = register_test_instrument(&client, &cache);
17028 let mut factory = test_order_factory();
17029 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-NONCE-FAIL");
17030 let client_order_id = order.client_order_id();
17031 let venue_order_id = VenueOrderId::from("123");
17032 cache_order(&cache, order);
17033
17034 let window = i64::from(client.dispatch.nonce_manager.skip_window());
17035 for _ in 0..window {
17036 client
17037 .dispatch
17038 .nonce_manager
17039 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
17040 .unwrap();
17041 }
17042
17043 let command = ModifyOrder::new(
17044 trader_id(),
17045 Some(client_id()),
17046 strategy_id(),
17047 instrument_id,
17048 client_order_id,
17049 Some(venue_order_id),
17050 Some(Quantity::from("0.2000")),
17051 Some(Price::from("2362.00")),
17052 None,
17053 UUID4::new(),
17054 UnixNanos::default(),
17055 None,
17056 None,
17057 );
17058 client.modify_order(command).unwrap();
17059
17060 let rejected = recv_order_event(&mut rx).await;
17061 match rejected {
17062 OrderEventAny::ModifyDenied(event) => {
17063 assert_eq!(event.client_order_id, client_order_id);
17064 assert_eq!(event.venue_order_id, Some(venue_order_id));
17065 assert!(event.reason.contains("failed to allocate Lighter nonce"),);
17066 assert!(event.reason.contains("skip-window exhausted"));
17067 }
17068 event => panic!("expected modify rejected event, was {event:?}"),
17069 }
17070
17071 wait_for_spawned_tasks(&client).await;
17072 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
17073 assert_eq!(
17074 client
17075 .dispatch
17076 .nonce_manager
17077 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
17078 .unwrap(),
17079 101,
17080 );
17081 }
17082
17083 #[tokio::test]
17084 async fn update_leverage_requires_credentials() {
17085 let (mut client, _cache, _rx) = create_execution_client();
17086 client.credential = None;
17087 let instrument_id = InstrumentId::from("ETH-PERP.LIGHTER");
17088
17089 let err = client
17090 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
17091 .unwrap_err();
17092
17093 assert!(
17094 err.to_string()
17095 .contains("cannot update leverage without credentials"),
17096 );
17097 }
17098
17099 #[tokio::test]
17100 async fn update_leverage_requires_registered_instrument() {
17101 let (client, _cache, _rx) = create_execution_client();
17102 let unknown = InstrumentId::from("DOGE-PERP.LIGHTER");
17103
17104 let err = client
17105 .update_leverage(unknown, 500, LighterPositionMarginMode::Isolated)
17106 .unwrap_err();
17107
17108 assert!(
17109 err.to_string()
17110 .contains("no Lighter market_index registered")
17111 );
17112 assert_nonce_reusable(&client.dispatch);
17115 }
17116
17117 #[tokio::test]
17118 async fn update_leverage_dispatches_and_rolls_back_on_send_failure() {
17119 let (client, cache, _rx) = create_execution_client();
17120 let instrument_id = register_test_instrument(&client, &cache);
17121
17122 client
17123 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
17124 .unwrap();
17125
17126 wait_for_spawned_tasks(&client).await;
17127 assert_nonce_reusable(&client.dispatch);
17128 }
17129
17130 #[tokio::test]
17131 async fn update_leverage_unknown_send_outcome_retains_pending_nonce() {
17132 let (client, cache, _rx) = create_execution_client();
17133 let instrument_id = register_test_instrument(&client, &cache);
17134 client.ws_client.drop_next_send_tx_result_for_test().await;
17135
17136 client
17137 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
17138 .unwrap();
17139
17140 wait_for_spawned_tasks(&client).await;
17141 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
17142 assert_eq!(
17143 client
17144 .dispatch
17145 .nonce_manager
17146 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
17147 Some(TEST_NEXT_NONCE),
17148 );
17149 }
17150
17151 #[tokio::test]
17152 async fn update_leverage_rejects_zero_margin_fraction() {
17153 let (client, cache, _rx) = create_execution_client();
17154 let instrument_id = register_test_instrument(&client, &cache);
17155
17156 let err = client
17157 .update_leverage(instrument_id, 0, LighterPositionMarginMode::Cross)
17158 .unwrap_err();
17159 assert!(err.to_string().contains("must be in 1..=10_000"));
17160 }
17161
17162 #[tokio::test]
17163 async fn update_leverage_rejects_above_margin_fraction_tick() {
17164 let (client, cache, _rx) = create_execution_client();
17165 let instrument_id = register_test_instrument(&client, &cache);
17166
17167 let err = client
17168 .update_leverage(instrument_id, 10_001, LighterPositionMarginMode::Cross)
17169 .unwrap_err();
17170 assert!(err.to_string().contains("must be in 1..=10_000"));
17171 }
17172
17173 #[tokio::test]
17174 async fn update_leverage_accepts_minimum_margin_fraction() {
17175 let (client, cache, _rx) = create_execution_client();
17178 let instrument_id = register_test_instrument(&client, &cache);
17179
17180 client
17181 .update_leverage(instrument_id, 1, LighterPositionMarginMode::Cross)
17182 .unwrap();
17183
17184 wait_for_spawned_tasks(&client).await;
17185 assert_nonce_reusable(&client.dispatch);
17186 }
17187
17188 #[tokio::test]
17189 async fn update_leverage_accepts_maximum_margin_fraction() {
17190 let (client, cache, _rx) = create_execution_client();
17193 let instrument_id = register_test_instrument(&client, &cache);
17194
17195 client
17196 .update_leverage(instrument_id, 10_000, LighterPositionMarginMode::Isolated)
17197 .unwrap();
17198
17199 wait_for_spawned_tasks(&client).await;
17200 assert_nonce_reusable(&client.dispatch);
17201 }
17202
17203 async fn wait_for_spawned_tasks(client: &LighterExecutionClient) {
17204 wait_until_async(
17205 || async { client.pending_tasks_all_finished() },
17206 Duration::from_secs(2),
17207 )
17208 .await;
17209 }
17210
17211 fn mark_all_streams_ready(client: &LighterExecutionClient) {
17212 let ready = &client.dispatch.account_streams_ready;
17213 ready.mark_orders();
17214 ready.mark_trades();
17215 ready.mark_positions();
17216 ready.mark_assets();
17217 ready.mark_user_stats();
17218 }
17219
17220 #[tokio::test]
17221 async fn await_account_streams_ready_times_out_when_no_frame_arrives() {
17222 let (client, _cache, _rx) = create_execution_client();
17225
17226 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
17227
17228 assert!(
17229 err.to_string().contains("Timeout")
17230 && err.to_string().contains("Lighter account streams"),
17231 "unexpected error message, was {err}",
17232 );
17233 }
17234
17235 #[tokio::test]
17236 async fn await_account_streams_ready_returns_when_all_streams_marked() {
17237 let (client, _cache, _rx) = create_execution_client();
17238 mark_all_streams_ready(&client);
17239
17240 client.await_account_streams_ready(0.05).await.unwrap();
17241 }
17242
17243 #[tokio::test]
17244 async fn await_account_streams_ready_returns_when_streams_arrive_mid_wait() {
17245 let (client, _cache, _rx) = create_execution_client();
17246 let ready = Arc::clone(&client.dispatch.account_streams_ready);
17247 let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
17248 let mut pending_tx = Some(pending_tx);
17249 let wait = client.await_account_streams_ready(30.0);
17250 tokio::pin!(wait);
17251
17252 let producer = tokio::spawn(async move {
17253 pending_rx.await.expect("readiness waiter parked");
17254 ready.mark_orders();
17255 ready.mark_trades();
17256 ready.mark_positions();
17257 ready.mark_assets();
17258 ready.mark_user_stats();
17259 });
17260
17261 tokio::time::timeout(
17263 Duration::from_secs(2),
17264 std::future::poll_fn(|cx| {
17265 let result = wait.as_mut().poll(cx);
17266 if result.is_pending()
17267 && let Some(tx) = pending_tx.take()
17268 {
17269 tx.send(()).expect("readiness producer alive");
17270 }
17271
17272 result
17273 }),
17274 )
17275 .await
17276 .expect("marks must wake the waiter before its deadline")
17277 .unwrap();
17278
17279 assert!(pending_tx.is_none(), "waiter must initially be pending");
17280 producer.await.unwrap();
17281 }
17282
17283 #[tokio::test]
17284 async fn await_account_streams_ready_times_out_with_partial_marks() {
17285 let (client, _cache, _rx) = create_execution_client();
17289 let ready = &client.dispatch.account_streams_ready;
17290 ready.mark_orders();
17291 ready.mark_trades();
17292 ready.mark_positions();
17293
17294 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
17295 assert!(
17296 err.to_string().contains("assets"),
17297 "pending list should call out the missing stream, was {err}",
17298 );
17299 }
17300
17301 #[tokio::test]
17302 async fn await_account_streams_ready_after_reset_requires_new_marks() {
17303 let (client, _cache, _rx) = create_execution_client();
17308 mark_all_streams_ready(&client);
17309 client.await_account_streams_ready(0.05).await.unwrap();
17310
17311 client.dispatch.account_streams_ready.reset();
17312
17313 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
17314 let msg = err.to_string();
17315 assert!(msg.contains("orders"), "pending list missing orders: {msg}");
17316 assert!(msg.contains("trades"), "pending list missing trades: {msg}");
17317 assert!(
17318 msg.contains("positions"),
17319 "pending list missing positions: {msg}",
17320 );
17321 assert!(msg.contains("assets"), "pending list missing assets: {msg}");
17322 assert!(
17323 msg.contains("user_stats"),
17324 "pending list missing user_stats: {msg}",
17325 );
17326 }
17327
17328 fn test_market_order(
17329 factory: &mut OrderFactory,
17330 instrument_id: InstrumentId,
17331 client_order_id: &str,
17332 side: OrderSide,
17333 ) -> OrderAny {
17334 factory.market(
17335 instrument_id,
17336 side,
17337 Quantity::from("0.1000"),
17338 Some(TimeInForce::Ioc),
17339 Some(false),
17340 Some(false),
17341 None,
17342 None,
17343 None,
17344 Some(ClientOrderId::from(client_order_id)),
17345 )
17346 }
17347
17348 fn add_test_quote(
17349 cache: &Rc<RefCell<Cache>>,
17350 instrument_id: InstrumentId,
17351 bid: &str,
17352 ask: &str,
17353 ) {
17354 let quote = QuoteTick::new(
17355 instrument_id,
17356 Price::from(bid),
17357 Price::from(ask),
17358 Quantity::from("1.0000"),
17359 Quantity::from("1.0000"),
17360 UnixNanos::default(),
17361 UnixNanos::default(),
17362 );
17363 cache.borrow_mut().add_quote(quote).unwrap();
17364 }
17365
17366 #[tokio::test]
17367 async fn submit_market_order_without_cached_quote_emits_denied() {
17368 let (client, cache, mut rx) = create_execution_client();
17369 let instrument_id = register_test_instrument(&client, &cache);
17370 let mut factory = test_order_factory();
17371 let order = test_market_order(
17372 &mut factory,
17373 instrument_id,
17374 "O-MARKET-NO-QUOTE",
17375 OrderSide::Buy,
17376 );
17377 cache_order(&cache, order.clone());
17378
17379 let command = SubmitOrder::from_order(
17380 &order,
17381 trader_id(),
17382 Some(client_id()),
17383 None,
17384 UUID4::new(),
17385 UnixNanos::default(),
17386 );
17387 let _ = client.submit_order(command);
17389
17390 let event = recv_order_event(&mut rx).await;
17391 match event {
17392 OrderEventAny::Denied(event) => {
17393 assert!(
17394 event.reason.contains("no cached quote"),
17395 "expected no-cached-quote in reason, was {:?}",
17396 event.reason,
17397 );
17398 }
17399 event => panic!("expected denied event, was {event:?}"),
17400 }
17401
17402 assert_nonce_reusable(&client.dispatch);
17403 }
17404
17405 #[tokio::test]
17406 async fn submit_market_buy_with_quote_uses_ask_widened_by_slippage() {
17407 let (client, cache, mut rx) = create_execution_client();
17408 let instrument_id = register_test_instrument(&client, &cache);
17409 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
17410
17411 let mut factory = test_order_factory();
17412 let order = test_market_order(
17413 &mut factory,
17414 instrument_id,
17415 "O-MARKET-QUOTED-BUY",
17416 OrderSide::Buy,
17417 );
17418 cache_order(&cache, order.clone());
17419
17420 let command = SubmitOrder::from_order(
17421 &order,
17422 trader_id(),
17423 Some(client_id()),
17424 None,
17425 UUID4::new(),
17426 UnixNanos::default(),
17427 );
17428 let _ = client.submit_order(command);
17429
17430 let submitted = recv_order_event(&mut rx).await;
17431 assert!(
17432 matches!(submitted, OrderEventAny::Submitted(_)),
17433 "expected submitted, was {submitted:?}",
17434 );
17435 let rejected = recv_order_event(&mut rx).await;
17436 match rejected {
17437 OrderEventAny::Rejected(event) => {
17438 assert!(
17439 event
17440 .reason
17441 .contains("Lighter submit_order dispatch failed"),
17442 );
17443 }
17444 event => panic!("expected rejected event, was {event:?}"),
17445 }
17446
17447 assert_nonce_reusable(&client.dispatch);
17448 }
17449
17450 #[tokio::test]
17451 async fn submit_order_with_sub_tick_quantity_emits_denied() {
17452 let (client, cache, mut rx) = create_execution_client();
17453 let instrument_id = register_test_instrument(&client, &cache);
17454 let mut factory = test_order_factory();
17455 let order = factory.limit(
17457 instrument_id,
17458 OrderSide::Buy,
17459 Quantity::from("0.00001"),
17460 Price::from("2361.31"),
17461 Some(TimeInForce::Gtc),
17462 None,
17463 Some(false),
17464 Some(false),
17465 None,
17466 None,
17467 None,
17468 None,
17469 None,
17470 None,
17471 None,
17472 Some(ClientOrderId::from("O-SUB-TICK-QTY")),
17473 );
17474 cache_order(&cache, order.clone());
17475
17476 let command = SubmitOrder::from_order(
17477 &order,
17478 trader_id(),
17479 Some(client_id()),
17480 None,
17481 UUID4::new(),
17482 UnixNanos::default(),
17483 );
17484 let _ = client.submit_order(command);
17485
17486 let event = recv_order_event(&mut rx).await;
17487 match event {
17488 OrderEventAny::Denied(event) => {
17489 assert!(
17490 event.reason.contains("rounds to 0 ticks"),
17491 "expected rounds-to-0 in reason, was {:?}",
17492 event.reason,
17493 );
17494 }
17495 event => panic!("expected denied event, was {event:?}"),
17496 }
17497
17498 assert_nonce_reusable(&client.dispatch);
17499 }
17500
17501 #[tokio::test]
17502 async fn submit_order_below_min_notional_emits_denied() {
17503 let (client, cache, mut rx) = create_execution_client();
17504 let instrument_id = register_test_instrument(&client, &cache);
17505 let mut factory = test_order_factory();
17506 let order = factory.limit(
17507 instrument_id,
17508 OrderSide::Buy,
17509 Quantity::from("0.0010"),
17510 Price::from("2361.31"),
17511 Some(TimeInForce::Gtc),
17512 None,
17513 Some(false),
17514 Some(false),
17515 None,
17516 None,
17517 None,
17518 None,
17519 None,
17520 None,
17521 None,
17522 Some(ClientOrderId::from("O-BELOW-MIN-NOTIONAL")),
17523 );
17524 cache_order(&cache, order.clone());
17525
17526 let command = SubmitOrder::from_order(
17527 &order,
17528 trader_id(),
17529 Some(client_id()),
17530 None,
17531 UUID4::new(),
17532 UnixNanos::default(),
17533 );
17534 let _ = client.submit_order(command);
17535
17536 let event = recv_order_event(&mut rx).await;
17537 match event {
17538 OrderEventAny::Denied(event) => {
17539 assert!(
17540 event.reason.contains("min_quote_amount"),
17541 "expected min_quote_amount in reason, was {:?}",
17542 event.reason,
17543 );
17544 }
17545 event => panic!("expected denied event, was {event:?}"),
17546 }
17547
17548 assert_nonce_reusable(&client.dispatch);
17549 }
17550
17551 #[tokio::test]
17552 async fn submit_stop_market_with_sub_tick_trigger_emits_denied() {
17553 let (client, cache, mut rx) = create_execution_client();
17554 let instrument_id = register_test_instrument(&client, &cache);
17555 let mut factory = test_order_factory();
17556 let order = factory.stop_market(
17558 instrument_id,
17559 OrderSide::Buy,
17560 Quantity::from("0.1000"),
17561 Price::from("0.001"),
17562 None,
17563 Some(TimeInForce::Gtc),
17564 None,
17565 Some(false),
17566 Some(false),
17567 None,
17568 None,
17569 None,
17570 None,
17571 None,
17572 None,
17573 Some(ClientOrderId::from("O-STOP-SUB-TICK")),
17574 );
17575 cache_order(&cache, order.clone());
17576
17577 let command = SubmitOrder::from_order(
17578 &order,
17579 trader_id(),
17580 Some(client_id()),
17581 None,
17582 UUID4::new(),
17583 UnixNanos::default(),
17584 );
17585 let _ = client.submit_order(command);
17586
17587 let event = recv_order_event(&mut rx).await;
17588 match event {
17589 OrderEventAny::Denied(event) => {
17590 assert!(
17591 event.reason.contains("rounds to 0 ticks"),
17592 "expected rounds-to-0 in reason, was {:?}",
17593 event.reason,
17594 );
17595 }
17596 event => panic!("expected denied event, was {event:?}"),
17597 }
17598
17599 assert_nonce_reusable(&client.dispatch);
17600 }
17601
17602 #[tokio::test]
17603 async fn submit_stop_market_dispatches_using_trigger_widened_by_slippage() {
17604 let (client, cache, mut rx) = create_execution_client();
17605 let instrument_id = register_test_instrument(&client, &cache);
17606 let mut factory = test_order_factory();
17607 let order = factory.stop_market(
17608 instrument_id,
17609 OrderSide::Sell,
17610 Quantity::from("0.1000"),
17611 Price::from("2300.00"), None,
17613 Some(TimeInForce::Gtc),
17614 None,
17615 Some(false),
17616 Some(false),
17617 None,
17618 None,
17619 None,
17620 None,
17621 None,
17622 None,
17623 Some(ClientOrderId::from("O-STOP-MARKET")),
17624 );
17625 cache_order(&cache, order.clone());
17626
17627 let command = SubmitOrder::from_order(
17628 &order,
17629 trader_id(),
17630 Some(client_id()),
17631 None,
17632 UUID4::new(),
17633 UnixNanos::default(),
17634 );
17635 let _ = client.submit_order(command);
17636
17637 let submitted = recv_order_event(&mut rx).await;
17638 assert!(matches!(submitted, OrderEventAny::Submitted(_)));
17639 let rejected = recv_order_event(&mut rx).await;
17640 assert!(matches!(rejected, OrderEventAny::Rejected(_)));
17641 assert_nonce_reusable(&client.dispatch);
17642 }
17643
17644 #[tokio::test]
17645 async fn submit_market_order_respects_per_order_slippage_override() {
17646 let (client, cache, mut rx) = create_execution_client();
17649 let instrument_id = register_test_instrument(&client, &cache);
17650 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
17651
17652 let mut factory = test_order_factory();
17653 let order = test_market_order(
17654 &mut factory,
17655 instrument_id,
17656 "O-MARKET-ZERO-SLIP",
17657 OrderSide::Buy,
17658 );
17659 cache_order(&cache, order.clone());
17660
17661 let params: Params =
17662 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 0})).unwrap();
17663 let mut command = SubmitOrder::from_order(
17664 &order,
17665 trader_id(),
17666 Some(client_id()),
17667 None,
17668 UUID4::new(),
17669 UnixNanos::default(),
17670 );
17671 command.params = Some(params);
17672 let _ = client.submit_order(command);
17673
17674 let submitted = recv_order_event(&mut rx).await;
17675 assert!(matches!(submitted, OrderEventAny::Submitted(_)));
17676 let rejected = recv_order_event(&mut rx).await;
17677 assert!(matches!(rejected, OrderEventAny::Rejected(_)));
17678 assert_nonce_reusable(&client.dispatch);
17679 }
17680
17681 #[tokio::test]
17682 async fn resolve_slippage_bps_prefers_params_over_config_default() {
17683 let config = LighterExecutionClientConfig {
17684 market_order_slippage_bps: 73,
17685 ..test_config()
17686 };
17687
17688 let (client, _cache, _rx) = create_execution_client_with_config(config);
17689 assert_eq!(client.resolve_slippage_bps(None).unwrap(), 73);
17690
17691 let override_params: Params =
17692 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 100})).unwrap();
17693 assert_eq!(
17694 client.resolve_slippage_bps(Some(&override_params)).unwrap(),
17695 100
17696 );
17697
17698 let unrelated_params: Params =
17699 serde_json::from_value(serde_json::json!({"other_key": 999})).unwrap();
17700 assert_eq!(
17701 client
17702 .resolve_slippage_bps(Some(&unrelated_params))
17703 .unwrap(),
17704 73
17705 );
17706 }
17707
17708 #[rstest]
17709 #[case::zero(serde_json::json!(0), Ok(0))]
17710 #[case::integer(serde_json::json!(37), Ok(37))]
17711 #[case::maximum(serde_json::json!(9_999), Ok(9_999))]
17712 #[case::out_of_range(serde_json::json!(10_000), Err("must be in [0, 10000)"))]
17713 #[case::maximum_u32(serde_json::json!(u32::MAX), Err("must be in [0, 10000)"))]
17714 #[case::float(serde_json::json!(0.0), Err("must be an unsigned integer"))]
17715 #[case::negative(serde_json::json!(-1), Err("must be an unsigned integer"))]
17716 #[case::string(serde_json::json!("0"), Err("must be an unsigned integer"))]
17717 #[case::null(serde_json::Value::Null, Err("must be an unsigned integer"))]
17718 #[case::boolean(serde_json::json!(false), Err("must be an unsigned integer"))]
17719 #[case::overflow(serde_json::json!(u64::from(u32::MAX) + 1), Err("exceeds u32 range"))]
17720 #[case::maximum_u64(serde_json::json!(u64::MAX), Err("exceeds u32 range"))]
17721 fn resolve_slippage_bps_validates_present_override(
17722 #[case] value: serde_json::Value,
17723 #[case] expected: Result<u32, &str>,
17724 ) {
17725 let (client, _cache, _rx) = create_execution_client();
17726 let params = serde_json::from_value(serde_json::json!({
17727 "market_order_slippage_bps": value,
17728 }))
17729 .unwrap();
17730
17731 let result = client.resolve_slippage_bps(Some(¶ms));
17732
17733 assert_eq!(
17734 result.map_err(|e| e.to_string()),
17735 expected.map_err(|reason| format!("Lighter market_order_slippage_bps {reason}")),
17736 );
17737 }
17738
17739 #[tokio::test]
17740 async fn submit_order_denies_invalid_slippage_before_dispatch() {
17741 let (client, cache, mut rx) = create_execution_client();
17742 let instrument_id = register_test_instrument(&client, &cache);
17743 let mut factory = test_order_factory();
17744 let order = test_limit_order(&mut factory, instrument_id, "O-SLIPPAGE-INVALID");
17745 cache_order(&cache, order.clone());
17746 let mut command = SubmitOrder::from_order(
17747 &order,
17748 trader_id(),
17749 Some(client_id()),
17750 None,
17751 UUID4::new(),
17752 UnixNanos::default(),
17753 );
17754 command.params = Some(
17755 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 0.0})).unwrap(),
17756 );
17757
17758 client.submit_order(command).unwrap();
17759
17760 match recv_order_event(&mut rx).await {
17761 OrderEventAny::Denied(event) => {
17762 assert_eq!(event.client_order_id, order.client_order_id());
17763 assert_eq!(
17764 event.reason,
17765 "VALIDATION_FAILED: Lighter submit_order failed: Lighter market_order_slippage_bps must be an unsigned integer",
17766 );
17767 }
17768 event => panic!("expected denied event, was {event:?}"),
17769 }
17770
17771 assert!(rx.try_recv().is_err());
17772 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
17773 assert_nonce_reusable(&client.dispatch);
17774 }
17775
17776 #[rstest]
17777 #[case::out_of_range(10_000, "market_order_slippage_bps must be in [0, 10000)")]
17778 #[case::overflow(u64::from(u32::MAX) + 1, "market_order_slippage_bps exceeds u32 range")]
17779 #[tokio::test]
17780 async fn submit_order_list_denies_invalid_slippage_before_dispatch(
17781 #[values(false, true)] grouped: bool,
17782 #[case] value: u64,
17783 #[case] expected_reason: &str,
17784 ) {
17785 let (client, cache, mut rx) = create_execution_client();
17786 let instrument_id = register_test_instrument(&client, &cache);
17787 let mut factory = test_order_factory();
17788
17789 let orders = if grouped {
17790 test_native_group(instrument_id, LighterGroupingType::OneCancelsTheOther)
17791 } else {
17792 vec![
17793 test_limit_order(&mut factory, instrument_id, "O-SLIPPAGE-LIST-A"),
17794 test_limit_order(&mut factory, instrument_id, "O-SLIPPAGE-LIST-B"),
17795 ]
17796 };
17797
17798 for order in &orders {
17799 cache_order(&cache, order.clone());
17800 }
17801
17802 let mut command = submit_order_list_command(&orders, "OL-SLIPPAGE-INVALID");
17803 command.params = Some(
17804 serde_json::from_value(serde_json::json!({
17805 "market_order_slippage_bps": value,
17806 }))
17807 .unwrap(),
17808 );
17809
17810 client.submit_order_list(command).unwrap();
17811
17812 for order in &orders {
17813 match recv_order_event(&mut rx).await {
17814 OrderEventAny::Denied(event) => {
17815 assert_eq!(event.client_order_id, order.client_order_id());
17816 assert!(event.reason.contains(expected_reason));
17817 }
17818 event => panic!("expected denied event, was {event:?}"),
17819 }
17820 }
17821
17822 assert!(rx.try_recv().is_err());
17823 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
17824 assert_nonce_reusable(&client.dispatch);
17825 }
17826
17827 #[tokio::test]
17828 async fn submit_market_sell_with_quote_uses_bid_widened_by_slippage() {
17829 let (client, cache, mut rx) = create_execution_client();
17830 let instrument_id = register_test_instrument(&client, &cache);
17831 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
17832
17833 let mut factory = test_order_factory();
17834 let order = test_market_order(
17835 &mut factory,
17836 instrument_id,
17837 "O-MARKET-QUOTED-SELL",
17838 OrderSide::Sell,
17839 );
17840 cache_order(&cache, order.clone());
17841
17842 let command = SubmitOrder::from_order(
17843 &order,
17844 trader_id(),
17845 Some(client_id()),
17846 None,
17847 UUID4::new(),
17848 UnixNanos::default(),
17849 );
17850 let _ = client.submit_order(command);
17851
17852 let submitted = recv_order_event(&mut rx).await;
17853 assert!(
17854 matches!(submitted, OrderEventAny::Submitted(_)),
17855 "expected submitted, was {submitted:?}",
17856 );
17857 let rejected = recv_order_event(&mut rx).await;
17858 match rejected {
17859 OrderEventAny::Rejected(event) => {
17860 assert!(
17861 event
17862 .reason
17863 .contains("Lighter submit_order dispatch failed"),
17864 );
17865 }
17866 event => panic!("expected rejected event, was {event:?}"),
17867 }
17868
17869 assert_nonce_reusable(&client.dispatch);
17870 }
17871
17872 #[rstest]
17873 fn integrator_attributes_tag_mainnet_orders() {
17874 assert_eq!(
17875 integrator_attributes(Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)),
17876 L2TxAttributes {
17877 integrator_account_index: LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
17878 ..Default::default()
17879 },
17880 );
17881 }
17882
17883 #[rstest]
17884 #[case("expire_maker", "account_index", 0, 0)]
17885 #[case("expire_maker", "master_account_index", 0, 1)]
17886 #[case("expire_taker", "account_index", 1, 0)]
17887 #[case("EXPIRE_TAKER", "MASTER_ACCOUNT_INDEX", 1, 1)]
17888 #[case("cancel_both", "account_index", 2, 0)]
17889 #[case("cancel_both", "master_account_index", 2, 1)]
17890 #[case("reduce", "account_index", 3, 0)]
17891 fn resolve_self_trade_params(
17892 #[case] behavior: &str,
17893 #[case] equality: &str,
17894 #[case] expected_behavior: u8,
17895 #[case] expected_equality: u8,
17896 ) {
17897 let (client, _cache, _rx) = create_execution_client();
17898 let params: Params = serde_json::from_value(serde_json::json!({
17899 "self_trade_behavior_mode": behavior,
17900 "self_trade_equality_mode": equality,
17901 }))
17902 .unwrap();
17903
17904 let attrs = client.resolve_create_attributes(Some(¶ms)).unwrap();
17905
17906 assert_eq!(
17907 attrs,
17908 L2TxAttributes {
17909 self_trade_behavior_mode: Some(
17910 serde_json::from_value(serde_json::json!(expected_behavior)).unwrap()
17911 ),
17912 self_trade_equality_mode: Some(
17913 serde_json::from_value(serde_json::json!(expected_equality)).unwrap()
17914 ),
17915 ..Default::default()
17916 }
17917 );
17918 }
17919
17920 #[rstest]
17921 fn resolve_self_trade_defaults_are_explicit() {
17922 let (client, _cache, _rx) = create_execution_client();
17923 let attrs = client.resolve_create_attributes(None).unwrap();
17924
17925 assert_eq!(
17926 attrs,
17927 L2TxAttributes {
17928 self_trade_behavior_mode: Some(LighterSelfTradeBehaviorMode::ExpireMaker),
17929 self_trade_equality_mode: Some(LighterSelfTradeEqualityMode::AccountIndex),
17930 ..Default::default()
17931 }
17932 );
17933 }
17934
17935 #[rstest]
17936 #[case(serde_json::json!({"self_trade_behavior_mode": "unknown"}), "invalid Lighter self_trade_behavior_mode")]
17937 #[case(serde_json::json!({"self_trade_equality_mode": "unknown"}), "invalid Lighter self_trade_equality_mode")]
17938 #[case(serde_json::json!({"self_trade_behavior_mode": 1}), "Lighter self_trade_behavior_mode must be a string")]
17939 #[case(serde_json::json!({"self_trade_equality_mode": null}), "Lighter self_trade_equality_mode must be a string")]
17940 #[case(serde_json::json!({"self_trade_behavior_mode": "reduce", "self_trade_equality_mode": "master_account_index"}), "Lighter reduce self-trade behavior cannot combine with master-account equality")]
17941 fn resolve_self_trade_rejects_invalid_params(
17942 #[case] value: serde_json::Value,
17943 #[case] expected: &str,
17944 ) {
17945 let (client, _cache, _rx) = create_execution_client();
17946 let params: Params = serde_json::from_value(value).unwrap();
17947
17948 let error = client.resolve_create_attributes(Some(¶ms)).unwrap_err();
17949
17950 assert_eq!(error.to_string(), expected);
17951 assert_nonce_reusable(&client.dispatch);
17952 }
17953
17954 #[rstest]
17955 #[case::single(false, false)]
17956 #[case::list(true, false)]
17957 #[case::group(true, true)]
17958 #[tokio::test]
17959 async fn submit_self_trade_invalid_combination_denies_before_dispatch(
17960 #[case] list: bool,
17961 #[case] grouped: bool,
17962 ) {
17963 let (client, cache, mut rx) = create_execution_client();
17964 let instrument = register_test_instrument(&client, &cache);
17965 let mut factory = test_order_factory();
17966
17967 let orders = if grouped {
17968 test_native_group(
17969 instrument,
17970 LighterGroupingType::OneTriggersOneCancelsTheOther,
17971 )
17972 } else {
17973 vec![test_limit_order(
17974 &mut factory,
17975 instrument,
17976 "O-SELF-TRADE-INVALID",
17977 )]
17978 };
17979
17980 for order in &orders {
17981 cache_order(&cache, order.clone());
17982 }
17983
17984 let params: Params = serde_json::from_value(serde_json::json!({
17985 "self_trade_behavior_mode": "reduce",
17986 "self_trade_equality_mode": "master_account_index",
17987 }))
17988 .unwrap();
17989
17990 if list {
17991 let mut command = submit_order_list_command(&orders, "OL-SELF-TRADE-INVALID");
17992 command.params = Some(params);
17993 client.submit_order_list(command).unwrap();
17994 } else {
17995 let mut command = SubmitOrder::from_order(
17996 &orders[0],
17997 trader_id(),
17998 Some(client_id()),
17999 None,
18000 UUID4::new(),
18001 UnixNanos::default(),
18002 );
18003 command.params = Some(params);
18004 client.submit_order(command).unwrap();
18005 }
18006
18007 for order in &orders {
18008 match recv_order_event(&mut rx).await {
18009 OrderEventAny::Denied(event) => {
18010 assert_eq!(event.client_order_id, order.client_order_id());
18011 assert!(event.reason.contains("Lighter reduce self-trade behavior cannot combine with master-account equality"));
18012 }
18013 event => panic!("expected OrderDenied, was {event:?}"),
18014 }
18015 }
18016
18017 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
18018 assert_nonce_reusable(&client.dispatch);
18019 }
18020
18021 #[rstest]
18022 fn integrator_attributes_leave_testnet_orders_unattributed() {
18023 assert_eq!(integrator_attributes(None), L2TxAttributes::default());
18024 }
18025
18026 #[rstest]
18027 #[case::lighter_mainnet(
18028 LighterDeployment::Lighter,
18029 LighterEnvironment::Mainnet,
18030 Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)
18031 )]
18032 #[case::lighter_testnet(LighterDeployment::Lighter, LighterEnvironment::Testnet, None)]
18033 #[case::robinhood_mainnet(
18034 LighterDeployment::Robinhood,
18035 LighterEnvironment::Mainnet,
18036 Some(LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)
18037 )]
18038 #[case::robinhood_testnet(LighterDeployment::Robinhood, LighterEnvironment::Testnet, None)]
18039 fn integrator_account_index_follows_deployment(
18040 #[case] deployment: LighterDeployment,
18041 #[case] environment: LighterEnvironment,
18042 #[case] expected: Option<u64>,
18043 ) {
18044 let mut config = test_config();
18045 config.deployment = deployment;
18046 config.environment = environment;
18047 config.venue = Some(*LIGHTER_VENUE);
18048 let (client, _cache, _rx) = create_execution_client_with_config(config);
18049
18050 assert_eq!(client.integrator_account_index(), expected);
18051 }
18052
18053 #[rstest]
18054 fn robinhood_orders_carry_integrator_attributes() {
18055 let mut config = test_config();
18056 config.deployment = LighterDeployment::Robinhood;
18057 config.environment = LighterEnvironment::Mainnet;
18058
18059 assert_eq!(
18060 integrator_attributes(deployment::integrator_account_index(
18061 config.deployment,
18062 config.environment,
18063 )),
18064 L2TxAttributes {
18065 integrator_account_index: LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
18066 ..Default::default()
18067 },
18068 );
18069 }
18070
18071 use std::str::FromStr;
18072
18073 use nautilus_live::ExecutionEventEmitter;
18074 use rust_decimal::Decimal;
18075
18076 use crate::{
18077 common::enums::{
18078 LighterOrderKind, LighterOrderSide, LighterOrderStatus, LighterOrderTimeInForce,
18079 LighterTradeType, LighterTriggerStatus,
18080 },
18081 http::models::{LighterOrder, LighterTrade},
18082 };
18083
18084 fn dispatcher_emitter() -> (
18085 ExecutionEventEmitter,
18086 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
18087 ) {
18088 let mut emitter = ExecutionEventEmitter::new(
18089 get_atomic_clock_realtime(),
18090 trader_id(),
18091 account_id(),
18092 AccountType::Margin,
18093 None,
18094 );
18095 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
18096 emitter.set_sender(sender);
18097 (emitter, receiver)
18098 }
18099
18100 struct DispatcherRig {
18104 dispatch: WsDispatchState,
18105 registry: Arc<MarketRegistry>,
18106 emitter: ExecutionEventEmitter,
18107 rx: tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
18108 instrument_id: InstrumentId,
18109 cloid: ClientOrderId,
18110 }
18111
18112 fn dispatcher_rig(cloid_suffix: &str) -> DispatcherRig {
18113 let registry = Arc::new(MarketRegistry::new());
18114 let instrument_id = registry.insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
18119 let instrument = InstrumentAny::CryptoPerpetual(
18120 CryptoPerpetual::builder()
18121 .instrument_id(instrument_id)
18122 .raw_symbol(Symbol::new("ETH-PERP"))
18123 .base_currency(Currency::from("ETH"))
18124 .quote_currency(Currency::from("USDC"))
18125 .settlement_currency(Currency::from("USDC"))
18126 .is_inverse(false)
18127 .price_precision(2)
18128 .size_precision(4)
18129 .price_increment(Price::from("0.01"))
18130 .size_increment(Quantity::from("0.0001"))
18131 .min_notional(Money::from("10.000000 USDC"))
18132 .ts_event(UnixNanos::default())
18133 .ts_init(UnixNanos::default())
18134 .build()
18135 .unwrap(),
18136 );
18137 LIGHTER_INSTRUMENT_CACHE.insert(instrument_id, instrument);
18138 let (emitter, rx) = dispatcher_emitter();
18139
18140 DispatcherRig {
18141 dispatch: WsDispatchState::new(),
18142 registry,
18143 emitter,
18144 rx,
18145 instrument_id,
18146 cloid: ClientOrderId::new(format!("CLOID-{cloid_suffix}")),
18147 }
18148 }
18149
18150 fn register_identity(rig: &DispatcherRig) {
18151 register_identity_for(rig, OrderType::Limit);
18152 }
18153
18154 fn register_identity_for(rig: &DispatcherRig, order_type: OrderType) {
18155 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
18156 let client_order_index = rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
18157 rig.dispatch.register_order_identity(
18158 rig.cloid,
18159 OrderIdentity::new(
18160 rig.instrument_id,
18161 strategy_id(),
18162 OrderSide::Buy,
18163 order_type,
18164 client_order_index,
18165 ),
18166 );
18167 rig.dispatch
18168 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
18169 }
18170
18171 fn register_unowned_identity(rig: &DispatcherRig) {
18172 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
18173 let client_order_index = rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
18174 rig.dispatch.register_order_identity(
18175 rig.cloid,
18176 OrderIdentity::new(
18177 rig.instrument_id,
18178 strategy_id(),
18179 OrderSide::Buy,
18180 OrderType::Limit,
18181 client_order_index,
18182 ),
18183 );
18184 }
18185
18186 fn dispatcher_test_order(rig: &DispatcherRig, status: LighterOrderStatus) -> LighterOrder {
18187 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
18188 rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
18189
18190 LighterOrder {
18191 order_index: 281_476_929_510_110,
18192 client_order_index: derived,
18193 order_id: "281476929510110".to_string(),
18194 client_order_id: derived.to_string(),
18195 market_index: TEST_MARKET_INDEX,
18196 owner_account_index: TEST_ACCOUNT_INDEX_I64,
18197 initial_base_amount: Decimal::from_str("0.0050").unwrap(),
18198 price: Decimal::from_str("2352.74").unwrap(),
18199 nonce: TEST_ORDER_NONCE,
18200 remaining_base_amount: Decimal::from_str("0.0050").unwrap(),
18201 is_ask: false,
18202 base_size: 50,
18203 base_price: 235_274,
18204 filled_base_amount: Decimal::ZERO,
18205 filled_quote_amount: Decimal::ZERO,
18206 side: Some(LighterOrderSide::Buy),
18207 order_type: LighterOrderKind::Limit,
18208 time_in_force: LighterOrderTimeInForce::GoodTillTime,
18209 reduce_only: false,
18210 trigger_price: Decimal::ZERO,
18211 order_expiry: 1_780_360_584_479,
18212 status,
18213 trigger_status: LighterTriggerStatus::Na,
18214 trigger_time: 0,
18215 parent_order_index: 0,
18216 parent_order_id: "0".to_string(),
18217 to_trigger_order_id_0: "0".to_string(),
18218 to_trigger_order_id_1: "0".to_string(),
18219 to_cancel_order_id_0: "0".to_string(),
18220 integrator_fee_collector_index: "0".to_string(),
18221 integrator_taker_fee: Decimal::ZERO,
18222 integrator_maker_fee: Decimal::ZERO,
18223 block_height: 227_535_532,
18224 timestamp: 1_777_941_383_576,
18225 created_at: 1_777_941_383_576,
18226 updated_at: 1_777_941_383_900,
18227 transaction_time: 1_777_941_383_576_735,
18228 }
18229 }
18230
18231 fn dispatcher_test_trade(rig: &DispatcherRig, user_is_bidder: bool) -> LighterTrade {
18232 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
18233 rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
18234
18235 LighterTrade {
18236 trade_id: 19_209_006_902,
18237 trade_id_str: Some("19209006902".to_string()),
18238 tx_hash: "000000128b1ee814".to_string(),
18239 trade_type: LighterTradeType::Trade,
18240 market_id: TEST_MARKET_INDEX,
18241 size: Decimal::from_str("0.1336").unwrap(),
18242 price: Decimal::from_str("2352.73").unwrap(),
18243 usd_amount: Decimal::from_str("314.324728").unwrap(),
18244 ask_id: 281_476_929_510_102,
18245 ask_id_str: Some("281476929510102".to_string()),
18246 bid_id: 562_947_905_631_053,
18247 bid_id_str: Some("562947905631053".to_string()),
18248 ask_client_id: if user_is_bidder { 0 } else { derived },
18249 ask_client_id_str: Some(if user_is_bidder {
18250 "0".to_string()
18251 } else {
18252 derived.to_string()
18253 }),
18254 bid_client_id: if user_is_bidder { derived } else { 0 },
18255 bid_client_id_str: Some(if user_is_bidder {
18256 derived.to_string()
18257 } else {
18258 "0".to_string()
18259 }),
18260 ask_account_id: if user_is_bidder {
18261 91_249
18262 } else {
18263 TEST_ACCOUNT_INDEX_I64
18264 },
18265 bid_account_id: if user_is_bidder {
18266 TEST_ACCOUNT_INDEX_I64
18267 } else {
18268 91_249
18269 },
18270 is_maker_ask: false,
18271 block_height: 227_535_535,
18272 timestamp: 1_777_941_384_181,
18273 taker_fee: Some(196),
18274 taker_position_size_before: None,
18275 taker_entry_quote_before: None,
18276 taker_initial_margin_fraction_before: None,
18277 taker_position_sign_changed: None,
18278 maker_fee: Some(28),
18279 maker_position_size_before: None,
18280 maker_entry_quote_before: None,
18281 maker_initial_margin_fraction_before: None,
18282 maker_position_sign_changed: None,
18283 transaction_time: 1_777_941_384_181_586,
18284 ask_account_pnl: None,
18285 bid_account_pnl: None,
18286 }
18287 }
18288
18289 fn drain_events(
18293 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
18294 ) -> Vec<ExecutionEvent> {
18295 let mut events = Vec::new();
18296 while let Ok(event) = rx.try_recv() {
18297 events.push(event);
18298 }
18299
18300 events
18301 }
18302
18303 #[rstest]
18304 fn dispatch_lighter_order_tracked_emits_accepted_then_silent_repeat() {
18305 let mut rig = dispatcher_rig("1");
18306 register_identity(&rig);
18307 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18308
18309 dispatch_lighter_order(
18310 &order,
18311 &rig.dispatch,
18312 &rig.emitter,
18313 &rig.registry,
18314 account_id(),
18315 trader_id(),
18316 UnixNanos::from(1),
18317 );
18318 dispatch_lighter_order(
18319 &order,
18320 &rig.dispatch,
18321 &rig.emitter,
18322 &rig.registry,
18323 account_id(),
18324 trader_id(),
18325 UnixNanos::from(2),
18326 );
18327
18328 let events = drain_events(&mut rig.rx);
18329 assert_eq!(
18330 events.len(),
18331 1,
18332 "exactly one event expected, was {events:?}",
18333 );
18334
18335 match &events[0] {
18336 ExecutionEvent::Order(OrderEventAny::Accepted(e)) => {
18337 assert_eq!(e.client_order_id, rig.cloid);
18338 assert_eq!(e.venue_order_id.to_string(), "281476929510110");
18339 }
18340 other => panic!("expected Accepted, was {other:?}"),
18341 }
18342
18343 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
18344 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_some());
18345 }
18346
18347 #[rstest]
18348 #[case::tracked(true)]
18349 #[case::untracked(false)]
18350 fn dispatch_lighter_order_skips_unknown_status_and_keeps_fill(#[case] tracked: bool) {
18351 let mut rig = dispatcher_rig(if tracked { "UNK-T" } else { "UNK-U" });
18352 if tracked {
18353 register_identity(&rig);
18354 }
18355
18356 let order = dispatcher_test_order(&rig, LighterOrderStatus::Unknown);
18357
18358 dispatch_lighter_order(
18359 &order,
18360 &rig.dispatch,
18361 &rig.emitter,
18362 &rig.registry,
18363 account_id(),
18364 trader_id(),
18365 UnixNanos::from(1),
18366 );
18367
18368 assert!(drain_events(&mut rig.rx).is_empty());
18369 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
18370 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_none());
18371 assert!(rig.dispatch.order_events.lock().is_empty());
18372 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
18373
18374 let trade = dispatcher_test_trade(&rig, true);
18375 dispatch_lighter_trade(
18376 &trade,
18377 &rig.dispatch,
18378 &rig.emitter,
18379 &rig.registry,
18380 account_id(),
18381 trader_id(),
18382 Some(TEST_ACCOUNT_INDEX_I64),
18383 UnixNanos::from(2),
18384 );
18385
18386 let events = drain_events(&mut rig.rx);
18387
18388 if tracked {
18389 assert_eq!(
18390 events.len(),
18391 2,
18392 "expected Accepted then Filled, was {events:?}"
18393 );
18394
18395 let ExecutionEvent::Order(OrderEventAny::Filled(filled)) = &events[1] else {
18396 panic!("expected Filled, was {:?}", events[1]);
18397 };
18398
18399 assert_eq!(filled.client_order_id, rig.cloid);
18400 assert_eq!(filled.last_qty, Quantity::from("0.1336"));
18401 assert_eq!(filled.last_px, Price::from("2352.73"));
18402 } else {
18403 assert_eq!(events.len(), 1, "expected one fill report, was {events:?}");
18404
18405 let ExecutionEvent::Report(EngineExecutionReport::Fill(report)) = &events[0] else {
18406 panic!("expected fill report, was {:?}", events[0]);
18407 };
18408
18409 assert_eq!(report.last_qty, Quantity::from("0.1336"));
18410 assert_eq!(report.last_px, Price::from("2352.73"));
18411 }
18412 }
18413
18414 #[rstest]
18415 fn dispatch_lighter_order_tracked_emits_updated_on_shape_change() {
18416 let mut rig = dispatcher_rig("2");
18417 register_identity(&rig);
18418 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18419
18420 dispatch_lighter_order(
18421 &order,
18422 &rig.dispatch,
18423 &rig.emitter,
18424 &rig.registry,
18425 account_id(),
18426 trader_id(),
18427 UnixNanos::from(1),
18428 );
18429 assert_eq!(drain_events(&mut rig.rx).len(), 1);
18430
18431 let mut modified = order;
18432 modified.price = Decimal::from_str("2400.00").unwrap();
18433
18434 dispatch_lighter_order(
18435 &modified,
18436 &rig.dispatch,
18437 &rig.emitter,
18438 &rig.registry,
18439 account_id(),
18440 trader_id(),
18441 UnixNanos::from(2),
18442 );
18443
18444 let events = drain_events(&mut rig.rx);
18445 assert_eq!(
18446 events.len(),
18447 1,
18448 "expected one Updated event, was {events:?}",
18449 );
18450
18451 match &events[0] {
18452 ExecutionEvent::Order(OrderEventAny::Updated(e)) => {
18453 assert_eq!(e.client_order_id, rig.cloid);
18454 assert_eq!(e.price, Some(Price::from("2400.00")));
18455 }
18456 other => panic!("expected Updated, was {other:?}"),
18457 }
18458
18459 let snapshot = rig.dispatch.snapshot_for(&rig.cloid).expect("snapshot");
18460 assert_eq!(snapshot.price, Some(Price::from("2400.00")));
18461 }
18462
18463 #[rstest]
18464 #[case(LighterOrderStatus::Pending)]
18465 #[case(LighterOrderStatus::Open)]
18466 fn dispatch_lighter_order_defers_live_frame_while_cancel_pending(
18467 #[case] status: LighterOrderStatus,
18468 ) {
18469 let mut rig = dispatcher_rig(match status {
18470 LighterOrderStatus::Pending => "24-PENDING",
18471 LighterOrderStatus::Open => "24-OPEN",
18472 other => panic!("unexpected case {other:?}"),
18473 });
18474
18475 register_identity(&rig);
18476 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18477
18478 dispatch_lighter_order(
18479 &order,
18480 &rig.dispatch,
18481 &rig.emitter,
18482 &rig.registry,
18483 account_id(),
18484 trader_id(),
18485 UnixNanos::from(1),
18486 );
18487 assert_eq!(drain_events(&mut rig.rx).len(), 1);
18488
18489 rig.dispatch
18490 .set_pending_order_action(rig.cloid, PendingOrderAction::Cancel);
18491 let mut late = order;
18492 late.status = status;
18493 late.initial_base_amount = Decimal::from_str("0.0075").unwrap();
18494 late.price = Decimal::from_str("2400.00").unwrap();
18495
18496 dispatch_lighter_order(
18497 &late,
18498 &rig.dispatch,
18499 &rig.emitter,
18500 &rig.registry,
18501 account_id(),
18502 trader_id(),
18503 UnixNanos::from(2),
18504 );
18505
18506 let events = drain_events(&mut rig.rx);
18507 assert_eq!(events.len(), 0, "late frame must be silent: {events:?}");
18508 assert_eq!(
18509 rig.dispatch.pending_order_action(&rig.cloid),
18510 Some(PendingOrderAction::Cancel),
18511 );
18512 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
18513 assert!(!rig.dispatch.triggered_was_emitted(&rig.cloid));
18514 assert_eq!(
18515 rig.dispatch.lookup_venue_order_id(&rig.cloid),
18516 Some(VenueOrderId::new("281476929510110")),
18517 );
18518 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
18519 assert_eq!(
18520 rig.dispatch.snapshot_for(&rig.cloid),
18521 Some(crate::websocket::dispatch::OrderShapeSnapshot {
18522 quantity: Quantity::from("0.0050"),
18523 price: Some(Price::from("2352.74")),
18524 trigger_price: None,
18525 }),
18526 );
18527 }
18528
18529 #[rstest]
18530 #[case::standalone(false)]
18531 #[case::native_child(true)]
18532 fn dispatch_lighter_order_emits_updated_then_triggered_for_one_frame(
18533 #[case] native_child: bool,
18534 ) {
18535 let mut rig = dispatcher_rig("25");
18536
18537 if native_child {
18538 let child = OrderTestBuilder::new(OrderType::StopLimit)
18539 .trader_id(trader_id())
18540 .strategy_id(strategy_id())
18541 .instrument_id(rig.instrument_id)
18542 .client_order_id(rig.cloid)
18543 .side(OrderSide::Buy)
18544 .quantity(Quantity::from("0.0050"))
18545 .price(Price::from("2352.74"))
18546 .trigger_price(Price::from("2300.00"))
18547 .parent_order_id(ClientOrderId::new("TRIGGER-PARENT"))
18548 .build();
18549 rig.dispatch.register_create_identity(&child).unwrap();
18550 rig.dispatch
18551 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
18552 } else {
18553 register_identity_for(&rig, OrderType::StopLimit);
18554 }
18555
18556 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
18557 order.order_type = LighterOrderKind::StopLossLimit;
18558 order.trigger_price = Decimal::from_str("2300.00").unwrap();
18559
18560 dispatch_lighter_order(
18561 &order,
18562 &rig.dispatch,
18563 &rig.emitter,
18564 &rig.registry,
18565 account_id(),
18566 trader_id(),
18567 UnixNanos::from(1),
18568 );
18569 let accepted = drain_events(&mut rig.rx);
18570 assert_eq!(accepted.len(), 1, "expected one acceptance: {accepted:?}");
18571 assert!(matches!(
18572 accepted[0],
18573 ExecutionEvent::Order(OrderEventAny::Accepted(_)),
18574 ));
18575
18576 let command = ModifyOrder::new(
18577 trader_id(),
18578 Some(client_id()),
18579 strategy_id(),
18580 rig.instrument_id,
18581 rig.cloid,
18582 Some(VenueOrderId::new("281476929510110")),
18583 Some(Quantity::from("0.0075")),
18584 Some(Price::from("2400.00")),
18585 Some(Price::from("2390.00")),
18586 UUID4::new(),
18587 UnixNanos::from(2),
18588 None,
18589 None,
18590 );
18591 rig.dispatch.set_pending_modify_action(&command);
18592 rig.dispatch.set_pending_modify_shape(
18593 &command,
18594 OrderShapeSnapshot {
18595 quantity: command.quantity.unwrap(),
18596 price: command.price,
18597 trigger_price: command.trigger_price,
18598 },
18599 );
18600
18601 rig.dispatch
18602 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 29);
18603 let mut triggered = order;
18604 triggered.status = LighterOrderStatus::Open;
18605 triggered.trigger_status = LighterTriggerStatus::Ready;
18606 triggered.initial_base_amount = Decimal::from_str("0.0075").unwrap();
18607 triggered.remaining_base_amount = triggered.initial_base_amount;
18608 triggered.price = Decimal::from_str("2400.00").unwrap();
18609 triggered.trigger_price = Decimal::from_str("2390.00").unwrap();
18610
18611 dispatch_lighter_order(
18612 &triggered,
18613 &rig.dispatch,
18614 &rig.emitter,
18615 &rig.registry,
18616 account_id(),
18617 trader_id(),
18618 UnixNanos::from(2),
18619 );
18620
18621 assert!(drain_events(&mut rig.rx).is_empty());
18622 let (client, _, _) = create_execution_client();
18623 let mut context = acked_probe_context(&client);
18624 context.dispatch = rig.dispatch.clone();
18625 context.registry = rig.registry.clone();
18626 context.emitter = rig.emitter.clone();
18627
18628 let pending = PendingSendTx {
18629 batch_id: None,
18630 connection_epoch: 0,
18631 kind: PendingSendTxKind::Modify {
18632 command_id: UUID4::default(),
18633 strategy_id: strategy_id(),
18634 instrument_id: rig.instrument_id,
18635 client_order_id: rig.cloid,
18636 venue_order_id: command.venue_order_id,
18637 },
18638 submitted_at: UnixNanos::from(2),
18639 nonce: 29,
18640 api_key_index: TEST_API_KEY_INDEX,
18641 tx_hash: "confirmed-trigger".to_string(),
18642 };
18643
18644 let mut unknown = triggered.clone();
18645 unknown.status = LighterOrderStatus::Unknown;
18646 let err = confirm_modify_order(&pending, &context, &unknown).unwrap_err();
18647 assert_eq!(
18648 err.to_string(),
18649 "Lighter modified order has an unknown status"
18650 );
18651 assert!(drain_events(&mut rig.rx).is_empty());
18652
18653 confirm_modify_order(&pending, &context, &triggered).unwrap();
18654 let events = drain_events(&mut rig.rx);
18655 assert_eq!(
18656 events.len(),
18657 2,
18658 "expected Updated then Triggered: {events:?}",
18659 );
18660
18661 match &events[0] {
18662 ExecutionEvent::Order(OrderEventAny::Updated(event)) => {
18663 assert_eq!(event.trader_id, trader_id());
18664 assert_eq!(event.strategy_id, strategy_id());
18665 assert_eq!(event.instrument_id, rig.instrument_id);
18666 assert_eq!(event.client_order_id, rig.cloid);
18667 assert_eq!(
18668 event.venue_order_id,
18669 Some(VenueOrderId::new("281476929510110")),
18670 );
18671 assert_eq!(event.account_id, Some(account_id()));
18672 assert_eq!(event.quantity, Quantity::from("0.0075"));
18673 assert_eq!(event.price, Some(Price::from("2400.00")));
18674 assert_eq!(event.trigger_price, Some(Price::from("2390.00")));
18675 }
18676 other => panic!("first event should be Updated, was {other:?}"),
18677 }
18678
18679 match &events[1] {
18680 ExecutionEvent::Order(OrderEventAny::Triggered(event)) => {
18681 assert_eq!(event.trader_id, trader_id());
18682 assert_eq!(event.strategy_id, strategy_id());
18683 assert_eq!(event.instrument_id, rig.instrument_id);
18684 assert_eq!(event.client_order_id, rig.cloid);
18685 assert_eq!(
18686 event.venue_order_id,
18687 Some(VenueOrderId::new("281476929510110")),
18688 );
18689 assert_eq!(event.account_id, Some(account_id()));
18690 }
18691 other => panic!("second event should be Triggered, was {other:?}"),
18692 }
18693
18694 assert_eq!(rig.dispatch.pending_order_action(&rig.cloid), None);
18695 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
18696 assert!(rig.dispatch.triggered_was_emitted(&rig.cloid));
18697 assert_eq!(
18698 rig.dispatch.snapshot_for(&rig.cloid),
18699 Some(crate::websocket::dispatch::OrderShapeSnapshot {
18700 quantity: Quantity::from("0.0075"),
18701 price: Some(Price::from("2400.00")),
18702 trigger_price: Some(Price::from("2390.00")),
18703 }),
18704 );
18705
18706 dispatch_lighter_order(
18707 &triggered,
18708 &rig.dispatch,
18709 &rig.emitter,
18710 &rig.registry,
18711 account_id(),
18712 trader_id(),
18713 UnixNanos::from(3),
18714 );
18715 assert!(
18716 drain_events(&mut rig.rx).is_empty(),
18717 "replayed frame must be silent",
18718 );
18719 }
18720
18721 #[rstest]
18722 #[case(LighterTriggerStatus::Ready)]
18723 #[case(LighterTriggerStatus::Na)]
18724 fn dispatch_lighter_order_defers_trigger_until_modify_resolves(
18725 #[case] trigger_status: LighterTriggerStatus,
18726 ) {
18727 let mut rig = dispatcher_rig("26");
18728 register_identity_for(&rig, OrderType::StopLimit);
18729 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
18730 order.order_type = LighterOrderKind::StopLossLimit;
18731 order.trigger_price = Decimal::from_str("2300.00").unwrap();
18732
18733 dispatch_lighter_order(
18734 &order,
18735 &rig.dispatch,
18736 &rig.emitter,
18737 &rig.registry,
18738 account_id(),
18739 trader_id(),
18740 UnixNanos::from(1),
18741 );
18742 assert_eq!(drain_events(&mut rig.rx).len(), 1);
18743
18744 let command = ModifyOrder::new(
18745 trader_id(),
18746 Some(client_id()),
18747 strategy_id(),
18748 rig.instrument_id,
18749 rig.cloid,
18750 Some(VenueOrderId::new("281476929510110")),
18751 Some(Quantity::from("0.0075")),
18752 Some(Price::from("2400.00")),
18753 Some(Price::from("2390.00")),
18754 UUID4::new(),
18755 UnixNanos::from(2),
18756 None,
18757 None,
18758 );
18759 rig.dispatch.set_pending_modify_action(&command);
18760 rig.dispatch.set_pending_modify_shape(
18761 &command,
18762 OrderShapeSnapshot {
18763 quantity: command.quantity.unwrap(),
18764 price: command.price,
18765 trigger_price: command.trigger_price,
18766 },
18767 );
18768
18769 order.status = LighterOrderStatus::Open;
18770 order.trigger_status = trigger_status;
18771 dispatch_lighter_order(
18772 &order,
18773 &rig.dispatch,
18774 &rig.emitter,
18775 &rig.registry,
18776 account_id(),
18777 trader_id(),
18778 UnixNanos::from(2),
18779 );
18780
18781 assert!(drain_events(&mut rig.rx).is_empty());
18782 assert_eq!(
18783 rig.dispatch.pending_order_action(&rig.cloid),
18784 Some(PendingOrderAction::Modify),
18785 );
18786 assert!(!rig.dispatch.triggered_was_emitted(&rig.cloid));
18787
18788 assert!(
18789 rig.dispatch
18790 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
18791 );
18792 dispatch_lighter_order(
18793 &order,
18794 &rig.dispatch,
18795 &rig.emitter,
18796 &rig.registry,
18797 account_id(),
18798 trader_id(),
18799 UnixNanos::from(3),
18800 );
18801
18802 let events = drain_events(&mut rig.rx);
18803 assert_eq!(events.len(), 1, "expected deferred trigger: {events:?}");
18804
18805 match &events[0] {
18806 ExecutionEvent::Order(OrderEventAny::Triggered(event)) => {
18807 assert_eq!(event.client_order_id, rig.cloid);
18808 assert_eq!(
18809 event.venue_order_id,
18810 Some(VenueOrderId::new("281476929510110")),
18811 );
18812 assert_eq!(event.account_id, Some(account_id()));
18813 }
18814 other => panic!("expected Triggered, was {other:?}"),
18815 }
18816
18817 assert!(rig.dispatch.triggered_was_emitted(&rig.cloid));
18818 }
18819
18820 #[rstest]
18821 fn dispatch_lighter_order_pending_emits_accepted_then_updated() {
18822 let mut rig = dispatcher_rig("23");
18823 register_identity(&rig);
18824 let order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
18825
18826 dispatch_lighter_order(
18827 &order,
18828 &rig.dispatch,
18829 &rig.emitter,
18830 &rig.registry,
18831 account_id(),
18832 trader_id(),
18833 UnixNanos::from(1),
18834 );
18835 let accepted = drain_events(&mut rig.rx);
18836
18837 let mut modified = order;
18838 modified.price = Decimal::from_str("2400.00").unwrap();
18839 dispatch_lighter_order(
18840 &modified,
18841 &rig.dispatch,
18842 &rig.emitter,
18843 &rig.registry,
18844 account_id(),
18845 trader_id(),
18846 UnixNanos::from(2),
18847 );
18848 let updated = drain_events(&mut rig.rx);
18849
18850 assert_eq!(
18851 accepted.len(),
18852 1,
18853 "expected one Accepted event: {accepted:?}"
18854 );
18855 assert!(matches!(
18856 accepted[0],
18857 ExecutionEvent::Order(OrderEventAny::Accepted(_))
18858 ));
18859 assert_eq!(updated.len(), 1, "expected one Updated event: {updated:?}");
18860 assert!(matches!(
18861 updated[0],
18862 ExecutionEvent::Order(OrderEventAny::Updated(_))
18863 ));
18864 assert_eq!(
18865 rig.dispatch
18866 .snapshot_for(&rig.cloid)
18867 .expect("snapshot")
18868 .price,
18869 Some(Price::from("2400.00")),
18870 );
18871 }
18872
18873 #[rstest]
18874 fn dispatch_lighter_order_untracked_emits_report() {
18875 let mut rig = dispatcher_rig("3");
18876 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18878 order.client_order_id = "external-1".to_string();
18879 order.client_order_index = 0;
18880
18881 dispatch_lighter_order(
18882 &order,
18883 &rig.dispatch,
18884 &rig.emitter,
18885 &rig.registry,
18886 account_id(),
18887 trader_id(),
18888 UnixNanos::from(1),
18889 );
18890
18891 let events = drain_events(&mut rig.rx);
18892 assert_eq!(events.len(), 1);
18893
18894 match &events[0] {
18895 ExecutionEvent::Report(report) => match report {
18896 EngineExecutionReport::Order(r) => {
18897 assert_eq!(r.venue_order_id.to_string(), "281476929510110");
18898 }
18899 other => panic!("expected order report, was {other:?}"),
18900 },
18901 other => panic!("expected report, was {other:?}"),
18902 }
18903
18904 assert!(
18905 !rig.dispatch
18906 .accepted_was_emitted(&ClientOrderId::new("external-1"))
18907 );
18908 }
18909
18910 #[rstest]
18911 fn same_index_external_order_waits_for_create_submission_ownership() {
18912 let mut rig = dispatcher_rig("EPOCH-ORDER");
18913 register_unowned_identity(&rig);
18914 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18915 let venue_order_id = VenueOrderId::new(order.order_id.as_str());
18916
18917 dispatch_lighter_order(
18918 &order,
18919 &rig.dispatch,
18920 &rig.emitter,
18921 &rig.registry,
18922 account_id(),
18923 trader_id(),
18924 UnixNanos::from(1),
18925 );
18926
18927 let external = drain_events(&mut rig.rx);
18928 assert_eq!(external.len(), 1);
18929
18930 match &external[0] {
18931 ExecutionEvent::Report(EngineExecutionReport::Order(report)) => {
18932 assert_eq!(report.venue_order_id, venue_order_id);
18933 assert_eq!(
18934 report.client_order_id,
18935 Some(ClientOrderId::new(venue_order_id.as_str())),
18936 );
18937 }
18938 other => panic!("expected one external order report, was {other:?}"),
18939 }
18940
18941 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
18942 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
18943
18944 rig.dispatch
18945 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
18946 dispatch_lighter_order(
18947 &order,
18948 &rig.dispatch,
18949 &rig.emitter,
18950 &rig.registry,
18951 account_id(),
18952 trader_id(),
18953 UnixNanos::from(2),
18954 );
18955
18956 let replayed_external = drain_events(&mut rig.rx);
18957 assert_eq!(replayed_external.len(), 1);
18958
18959 match &replayed_external[0] {
18960 ExecutionEvent::Report(EngineExecutionReport::Order(report)) => {
18961 assert_eq!(report.venue_order_id, venue_order_id);
18962 assert_eq!(
18963 report.client_order_id,
18964 Some(ClientOrderId::new(venue_order_id.as_str())),
18965 );
18966 }
18967 other => panic!("expected replayed external order report, was {other:?}"),
18968 }
18969
18970 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
18971 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
18972
18973 let mut local_order = order.clone();
18974 local_order.order_id = "281476929510111".to_string();
18975 let local_venue_order_id = VenueOrderId::new(local_order.order_id.as_str());
18976 dispatch_lighter_order(
18977 &local_order,
18978 &rig.dispatch,
18979 &rig.emitter,
18980 &rig.registry,
18981 account_id(),
18982 trader_id(),
18983 UnixNanos::from(3),
18984 );
18985
18986 let local = drain_events(&mut rig.rx);
18987 assert_eq!(local.len(), 1);
18988
18989 match &local[0] {
18990 ExecutionEvent::Order(OrderEventAny::Accepted(event)) => {
18991 assert_eq!(event.client_order_id, rig.cloid);
18992 assert_eq!(event.venue_order_id, local_venue_order_id);
18993 assert_eq!(event.account_id, account_id());
18994 }
18995 other => panic!("expected one local OrderAccepted, was {other:?}"),
18996 }
18997
18998 assert_eq!(
18999 rig.dispatch
19000 .venue_id_map
19001 .get(&rig.cloid)
19002 .map(|entry| *entry),
19003 Some(local_venue_order_id),
19004 );
19005 assert_eq!(
19006 rig.dispatch
19007 .resolve_live_order_cloid(order.client_order_id.as_str(), local_venue_order_id,),
19008 Some(rig.cloid),
19009 "a bound identity remains stable across replay epochs",
19010 );
19011 }
19012
19013 #[rstest]
19014 #[case::order_first(false)]
19015 #[case::trade_first(true)]
19016 fn quantity_amend_immediate_fill_orders_update_before_fill(
19017 #[case] trade_first: bool,
19018 #[values(false, true)] reduced: bool,
19019 ) {
19020 let mut rig = dispatcher_rig("AMEND-FILL");
19021 let mut cached = OrderTestBuilder::new(OrderType::Limit)
19022 .trader_id(trader_id())
19023 .strategy_id(strategy_id())
19024 .instrument_id(rig.instrument_id)
19025 .client_order_id(rig.cloid)
19026 .side(OrderSide::Buy)
19027 .quantity(Quantity::from("0.0050"))
19028 .price(Price::from("2352.74"))
19029 .build();
19030 rig.dispatch.register_create_identity(&cached).unwrap();
19031 rig.dispatch
19032 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
19033 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
19034 dispatch_lighter_order(
19035 &frame,
19036 &rig.dispatch,
19037 &rig.emitter,
19038 &rig.registry,
19039 account_id(),
19040 trader_id(),
19041 UnixNanos::from(1),
19042 );
19043
19044 for event in drain_events(&mut rig.rx) {
19045 let ExecutionEvent::Order(event) = event else {
19046 panic!("expected order event");
19047 };
19048
19049 cached.apply(event).unwrap();
19050 }
19051
19052 let command = ModifyOrder::new(
19053 trader_id(),
19054 Some(client_id()),
19055 strategy_id(),
19056 rig.instrument_id,
19057 rig.cloid,
19058 cached.venue_order_id(),
19059 Some(Quantity::from("0.0100")),
19060 Some(Price::from("2400.00")),
19061 None,
19062 UUID4::new(),
19063 UnixNanos::from(2),
19064 None,
19065 None,
19066 );
19067 rig.dispatch.set_pending_modify_action(&command);
19068 rig.dispatch.set_pending_modify_shape(
19069 &command,
19070 OrderShapeSnapshot {
19071 quantity: Quantity::from("0.0100"),
19072 price: Some(Price::from("2400.00")),
19073 trigger_price: None,
19074 },
19075 );
19076
19077 rig.dispatch
19078 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 19);
19079
19080 let pending = PendingSendTx {
19081 batch_id: None,
19082 connection_epoch: 0,
19083 kind: PendingSendTxKind::Modify {
19084 command_id: UUID4::default(),
19085 strategy_id: strategy_id(),
19086 instrument_id: rig.instrument_id,
19087 client_order_id: rig.cloid,
19088 venue_order_id: cached.venue_order_id(),
19089 },
19090 submitted_at: UnixNanos::from(2),
19091 nonce: 19,
19092 api_key_index: TEST_API_KEY_INDEX,
19093 tx_hash: "confirmed-amend".to_string(),
19094 };
19095
19096 let mut trade = dispatcher_test_trade(&rig, true);
19097 trade.bid_id = frame.order_index;
19098 trade.bid_id_str = Some(frame.order_id.clone());
19099 let expected_qty = Quantity::from(if reduced { "0.0040" } else { "0.0100" });
19100 trade.size = expected_qty.as_decimal();
19101 trade.usd_amount = trade.size * trade.price;
19102
19103 if trade_first {
19104 dispatch_lighter_trade(
19105 &trade,
19106 &rig.dispatch,
19107 &rig.emitter,
19108 &rig.registry,
19109 account_id(),
19110 trader_id(),
19111 Some(TEST_ACCOUNT_INDEX_I64),
19112 UnixNanos::from(3),
19113 );
19114 assert!(
19115 drain_events(&mut rig.rx).is_empty(),
19116 "unconfirmed amend fill must wait"
19117 );
19118 assert!(
19119 !rig.dispatch
19120 .seen_trade_ids
19121 .contains(&parse_lighter_trade_id(&trade).unwrap())
19122 );
19123 }
19124
19125 frame.status = LighterOrderStatus::Filled;
19126 frame.initial_base_amount = Decimal::from_str("0.0100").unwrap();
19127 frame.remaining_base_amount = Decimal::ZERO;
19128 frame.filled_base_amount = trade.size;
19129 frame.price = Decimal::from_str("2400.00").unwrap();
19130 dispatch_lighter_order(
19131 &frame,
19132 &rig.dispatch,
19133 &rig.emitter,
19134 &rig.registry,
19135 account_id(),
19136 trader_id(),
19137 UnixNanos::from(4),
19138 );
19139
19140 if !trade_first {
19141 dispatch_lighter_trade(
19142 &trade,
19143 &rig.dispatch,
19144 &rig.emitter,
19145 &rig.registry,
19146 account_id(),
19147 trader_id(),
19148 Some(TEST_ACCOUNT_INDEX_I64),
19149 UnixNanos::from(5),
19150 );
19151 }
19152
19153 assert!(
19154 drain_events(&mut rig.rx).is_empty(),
19155 "order echo must wait for transaction confirmation"
19156 );
19157 let (client, _, _) = create_execution_client();
19158 let mut context = acked_probe_context(&client);
19159 context.dispatch = rig.dispatch.clone();
19160 context.registry = rig.registry.clone();
19161 context.emitter = rig.emitter.clone();
19162 confirm_modify_order(&pending, &context, &frame).unwrap();
19163 let events = drain_events(&mut rig.rx);
19164 assert_eq!(events.len(), 2, "expected update then fill, was {events:?}");
19165
19166 let ExecutionEvent::Order(OrderEventAny::Updated(update)) = &events[0] else {
19167 panic!("expected update before fill, was {:?}", events[0]);
19168 };
19169
19170 assert_eq!(update.client_order_id, rig.cloid);
19171 assert_eq!(update.quantity, expected_qty);
19172 assert_eq!(update.price, Some(Price::from("2400.00")));
19173 assert_eq!(update.trigger_price, None);
19174
19175 let ExecutionEvent::Order(OrderEventAny::Filled(fill)) = &events[1] else {
19176 panic!("expected real fill, was {:?}", events[1]);
19177 };
19178
19179 assert_eq!(fill.trade_id, parse_lighter_trade_id(&trade).unwrap());
19180 assert_eq!(fill.last_qty, expected_qty);
19181 assert_eq!(fill.last_px, Price::from("2352.73"));
19182
19183 for event in events {
19184 let ExecutionEvent::Order(event) = event else {
19185 panic!("expected order event");
19186 };
19187
19188 cached.apply(event).unwrap();
19189 }
19190
19191 assert_eq!(cached.quantity(), expected_qty);
19192 assert_eq!(cached.filled_qty(), expected_qty);
19193 assert_eq!(cached.price(), Some(Price::from("2400.00")));
19194 assert_eq!(cached.status(), OrderStatus::Filled);
19195 dispatch_lighter_order(
19196 &frame,
19197 &rig.dispatch,
19198 &rig.emitter,
19199 &rig.registry,
19200 account_id(),
19201 trader_id(),
19202 UnixNanos::from(6),
19203 );
19204 dispatch_lighter_trade(
19205 &trade,
19206 &rig.dispatch,
19207 &rig.emitter,
19208 &rig.registry,
19209 account_id(),
19210 trader_id(),
19211 Some(TEST_ACCOUNT_INDEX_I64),
19212 UnixNanos::from(7),
19213 );
19214 assert!(
19215 drain_events(&mut rig.rx)
19216 .iter()
19217 .all(|event| matches!(event, ExecutionEvent::Report(_))),
19218 "terminal replay must not emit duplicate order events"
19219 );
19220 }
19221
19222 #[rstest]
19223 #[case(OrderType::StopMarket)]
19224 #[case(OrderType::StopLimit)]
19225 #[case(OrderType::MarketIfTouched)]
19226 #[case(OrderType::LimitIfTouched)]
19227 fn conditional_fill_synthesizes_trigger_once_across_retirement(#[case] kind: OrderType) {
19228 let mut rig = dispatcher_rig("CONDITIONAL-FILL");
19229 register_identity_for(&rig, kind);
19230 let trade = dispatcher_test_trade(&rig, true);
19231 dispatch_lighter_trade(
19232 &trade,
19233 &rig.dispatch,
19234 &rig.emitter,
19235 &rig.registry,
19236 account_id(),
19237 trader_id(),
19238 Some(TEST_ACCOUNT_INDEX_I64),
19239 UnixNanos::from(1),
19240 );
19241 let first = drain_events(&mut rig.rx);
19242 let triggerable = TRIGGERABLE_ORDER_TYPES.contains(&kind);
19243 let mut builder = OrderTestBuilder::new(kind);
19244 builder
19245 .trader_id(trader_id())
19246 .strategy_id(strategy_id())
19247 .instrument_id(rig.instrument_id)
19248 .client_order_id(rig.cloid)
19249 .side(OrderSide::Buy)
19250 .quantity(Quantity::from("0.2672"))
19251 .trigger_price(Price::from("2300.00"));
19252
19253 if LIMIT_ORDER_TYPES.contains(&kind) {
19254 builder.price(Price::from("2352.74"));
19255 }
19256
19257 let mut order = builder.build();
19258
19259 for event in &first {
19260 let ExecutionEvent::Order(event) = event else {
19261 panic!("expected order event");
19262 };
19263
19264 order.apply(event.clone()).unwrap();
19265 }
19266
19267 rig.dispatch.retire_order_identity(&rig.cloid);
19268 let mut subsequent = trade;
19269 subsequent.trade_id += 1;
19270 subsequent.trade_id_str = Some(subsequent.trade_id.to_string());
19271 dispatch_lighter_trade(
19272 &subsequent,
19273 &rig.dispatch,
19274 &rig.emitter,
19275 &rig.registry,
19276 account_id(),
19277 trader_id(),
19278 Some(TEST_ACCOUNT_INDEX_I64),
19279 UnixNanos::from(2),
19280 );
19281 let next = drain_events(&mut rig.rx);
19282 assert_eq!(first.len(), 2 + usize::from(triggerable));
19283 assert!(
19284 matches!(&first[0], ExecutionEvent::Order(OrderEventAny::Accepted(event)) if event.client_order_id == rig.cloid)
19285 );
19286
19287 if triggerable {
19288 assert!(
19289 matches!(&first[1], ExecutionEvent::Order(OrderEventAny::Triggered(event)) if event.client_order_id == rig.cloid)
19290 );
19291 }
19292
19293 assert!(
19294 matches!(first.last().unwrap(), ExecutionEvent::Order(OrderEventAny::Filled(event)) if event.client_order_id == rig.cloid && event.order_type == kind && event.last_qty == Quantity::from("0.1336"))
19295 );
19296 assert_eq!(next.len(), 1);
19297 assert!(
19298 matches!(&next[0], ExecutionEvent::Order(OrderEventAny::Filled(event)) if event.client_order_id == rig.cloid && event.trade_id.as_str() == subsequent.trade_id.to_string())
19299 );
19300
19301 let ExecutionEvent::Order(event) = &next[0] else {
19302 panic!("expected trailing fill");
19303 };
19304
19305 order.apply(event.clone()).unwrap();
19306 assert_eq!(order.status(), OrderStatus::Filled);
19307 assert_eq!(order.filled_qty(), Quantity::from("0.2672"));
19308 assert_eq!(order.is_triggered(), Some(triggerable));
19309 assert_eq!(rig.dispatch.triggered_was_emitted(&rig.cloid), triggerable);
19310 }
19311
19312 #[rstest]
19313 #[case(OrderType::Limit)]
19314 #[case(OrderType::StopLimit)]
19315 #[case(OrderType::StopMarket)]
19316 fn cached_order_restores_before_first_fill(
19317 #[case] kind: OrderType,
19318 #[values(false, true)] retired: bool,
19319 ) {
19320 let mut rig = dispatcher_rig("CACHED-FIRST-FILL");
19321 let trade = dispatcher_test_trade(&rig, true);
19322 rig.dispatch.forget_cloid(trade.bid_client_id);
19323 let venue = VenueOrderId::new(trade.bid_id_str.as_deref().unwrap());
19324 let mut builder = OrderTestBuilder::new(kind);
19325 builder
19326 .trader_id(trader_id())
19327 .strategy_id(strategy_id())
19328 .instrument_id(rig.instrument_id)
19329 .client_order_id(rig.cloid)
19330 .side(OrderSide::Buy)
19331 .quantity(Quantity::from("0.1336"));
19332
19333 if LIMIT_ORDER_TYPES.contains(&kind) {
19334 builder.price(Price::from("2352.74"));
19335 }
19336
19337 if STOP_ORDER_TYPES.contains(&kind) {
19338 builder.trigger_price(Price::from("2300.00"));
19339 }
19340
19341 let mut order = builder.build();
19342 order
19343 .apply(OrderEventAny::Accepted(OrderAccepted::new(
19344 trader_id(),
19345 strategy_id(),
19346 rig.instrument_id,
19347 rig.cloid,
19348 venue,
19349 account_id(),
19350 UUID4::new(),
19351 UnixNanos::from(1),
19352 UnixNanos::from(1),
19353 false,
19354 )))
19355 .unwrap();
19356 rig.dispatch.seed_order_restorations(vec![order.clone()]);
19357 if retired {
19358 rig.dispatch
19359 .restore_reconciled_order(&order, trade.bid_client_id, venue, true)
19360 .unwrap();
19361 }
19362
19363 dispatch_lighter_trade(
19364 &trade,
19365 &rig.dispatch,
19366 &rig.emitter,
19367 &rig.registry,
19368 account_id(),
19369 trader_id(),
19370 Some(TEST_ACCOUNT_INDEX_I64),
19371 UnixNanos::from(2),
19372 );
19373 let events = drain_events(&mut rig.rx);
19374 let triggerable = TRIGGERABLE_ORDER_TYPES.contains(&kind);
19375 assert_eq!(events.len(), 1 + usize::from(triggerable));
19376
19377 for event in events {
19378 let ExecutionEvent::Order(event) = event else {
19379 panic!("cached fill must remain tracked");
19380 };
19381
19382 assert_eq!(event.client_order_id(), rig.cloid);
19383 order.apply(event).unwrap();
19384 }
19385
19386 assert_eq!(order.status(), OrderStatus::Filled);
19387 assert_eq!(order.filled_qty(), Quantity::from("0.1336"));
19388 assert_eq!(order.venue_order_id(), Some(venue));
19389 assert_eq!(
19390 rig.dispatch.client_order_index(&rig.cloid),
19391 Some(trade.bid_client_id)
19392 );
19393 }
19394
19395 #[rstest]
19396 fn dispatch_lighter_trade_tracked_synthesizes_accepted_before_filled() {
19397 let mut rig = dispatcher_rig("4");
19401 register_identity(&rig);
19402
19403 let trade = dispatcher_test_trade(&rig, true);
19404
19405 dispatch_lighter_trade(
19406 &trade,
19407 &rig.dispatch,
19408 &rig.emitter,
19409 &rig.registry,
19410 account_id(),
19411 trader_id(),
19412 Some(TEST_ACCOUNT_INDEX_I64),
19413 UnixNanos::from(1),
19414 );
19415
19416 let events = drain_events(&mut rig.rx);
19417 assert_eq!(
19418 events.len(),
19419 2,
19420 "expected Accepted then Filled, was {events:?}",
19421 );
19422
19423 match &events[0] {
19424 ExecutionEvent::Order(OrderEventAny::Accepted(_)) => {}
19425 other => panic!("first event should be Accepted, was {other:?}"),
19426 }
19427
19428 match &events[1] {
19429 ExecutionEvent::Order(OrderEventAny::Filled(e)) => {
19430 assert_eq!(e.client_order_id, rig.cloid);
19431 assert_eq!(e.last_qty, Quantity::from("0.1336"));
19432 assert_eq!(e.last_px, Price::from("2352.73"));
19433 }
19434 other => panic!("second event should be Filled, was {other:?}"),
19435 }
19436
19437 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
19438 }
19439
19440 #[rstest]
19441 fn dispatch_lighter_trade_dedupes_repeated_trade_ids() {
19442 let mut rig = dispatcher_rig("5");
19443 register_identity(&rig);
19444 let trade = dispatcher_test_trade(&rig, true);
19445
19446 for _ in 0..3 {
19447 dispatch_lighter_trade(
19448 &trade,
19449 &rig.dispatch,
19450 &rig.emitter,
19451 &rig.registry,
19452 account_id(),
19453 trader_id(),
19454 Some(TEST_ACCOUNT_INDEX_I64),
19455 UnixNanos::from(1),
19456 );
19457 }
19458
19459 let events = drain_events(&mut rig.rx);
19460 assert_eq!(
19462 events.len(),
19463 2,
19464 "expected dedup after first dispatch, was {events:?}"
19465 );
19466 }
19467
19468 #[rstest]
19469 #[case::commission_overflow(true)]
19470 #[case::invalid_timestamp(false)]
19471 fn invalid_tracked_fill_preserves_unaccepted_order_for_replay(#[case] overflow: bool) {
19472 let mut rig = dispatcher_rig("INVALID-FILL-REPLAY");
19473 register_identity(&rig);
19474 let trade = dispatcher_test_trade(&rig, true);
19475 let mut invalid = trade.clone();
19476 if overflow {
19477 invalid.price = Decimal::from(1_000_000_000);
19478 invalid.size = Decimal::from(1_000_000_000);
19479 } else {
19480 invalid.timestamp = -1;
19481 }
19482
19483 let trade_id = parse_lighter_trade_id(&trade).unwrap();
19484
19485 if overflow {
19486 let error = parse_lighter_order_filled(
19487 &invalid,
19488 &LIGHTER_INSTRUMENT_CACHE.get(&rig.instrument_id).unwrap(),
19489 &rig.dispatch.order_identity(&rig.cloid).unwrap(),
19490 rig.cloid,
19491 account_id(),
19492 trader_id(),
19493 TEST_ACCOUNT_INDEX_I64,
19494 UnixNanos::from(1),
19495 )
19496 .unwrap_err();
19497 assert!(is_commission_error(&error));
19498 }
19499
19500 dispatch_lighter_trade(
19501 &invalid,
19502 &rig.dispatch,
19503 &rig.emitter,
19504 &rig.registry,
19505 account_id(),
19506 trader_id(),
19507 Some(TEST_ACCOUNT_INDEX_I64),
19508 UnixNanos::from(1),
19509 );
19510
19511 assert!(drain_events(&mut rig.rx).is_empty());
19512 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
19513 assert!(!rig.dispatch.seen_trade_ids.contains(&trade_id));
19514 assert!(
19515 !rig.dispatch
19516 .order_identity(&rig.cloid)
19517 .unwrap()
19518 .has_publication()
19519 );
19520 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
19521
19522 dispatch_lighter_trade(
19523 &trade,
19524 &rig.dispatch,
19525 &rig.emitter,
19526 &rig.registry,
19527 account_id(),
19528 trader_id(),
19529 Some(TEST_ACCOUNT_INDEX_I64),
19530 UnixNanos::from(2),
19531 );
19532
19533 let events = drain_events(&mut rig.rx);
19534 assert_eq!(events.len(), 2);
19535 assert!(
19536 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Accepted(event)) if event.client_order_id == rig.cloid)
19537 );
19538
19539 let ExecutionEvent::Order(OrderEventAny::Filled(actual)) = &events[1] else {
19540 panic!("expected typed fill");
19541 };
19542
19543 assert_eq!(actual.trade_id, trade_id);
19544 assert_eq!(actual.trader_id, trader_id());
19545 assert_eq!(actual.strategy_id, strategy_id());
19546 assert_eq!(actual.instrument_id, rig.instrument_id);
19547 assert_eq!(actual.client_order_id, rig.cloid);
19548 assert_eq!(actual.venue_order_id, VenueOrderId::from("562947905631053"));
19549 assert_eq!(actual.account_id, account_id());
19550 assert_eq!(actual.order_side, OrderSide::Buy);
19551 assert_eq!(actual.order_type, OrderType::Limit);
19552 assert_eq!(actual.last_qty, Quantity::from("0.1336"));
19553 assert_eq!(actual.last_px, Price::from("2352.73"));
19554 assert_eq!(actual.currency, Currency::USDC());
19555 assert_eq!(actual.liquidity_side, LiquiditySide::Maker);
19556 assert_eq!(actual.ts_event, UnixNanos::from(1_777_941_384_181_000_000));
19557 assert_eq!(actual.ts_init, UnixNanos::from(2));
19558 assert!(!actual.reconciliation);
19559 assert_eq!(actual.position_id, None);
19560 assert_eq!(actual.commission, Some(Money::from("0.00880109 USDC")));
19561 assert_eq!(actual.info, None);
19562 assert_eq!(actual.causation_id, None);
19563 assert!(rig.dispatch.seen_trade_ids.contains(&trade_id));
19564 }
19565
19566 #[rstest]
19567 #[case::tracked(true)]
19568 #[case::external(false)]
19569 fn malformed_duplicate_preserves_delivered_fill(#[case] tracked: bool) {
19570 let mut rig = dispatcher_rig("DUPLICATE-MALFORMED");
19571 if tracked {
19572 register_identity(&rig);
19573 }
19574
19575 let original = dispatcher_test_trade(&rig, true);
19576 let mut malformed = original.clone();
19577 malformed.timestamp = -1;
19578 for trade in [&original, &malformed, &original] {
19579 dispatch_lighter_trade(
19580 trade,
19581 &rig.dispatch,
19582 &rig.emitter,
19583 &rig.registry,
19584 account_id(),
19585 trader_id(),
19586 Some(TEST_ACCOUNT_INDEX_I64),
19587 UnixNanos::from(1),
19588 );
19589 }
19590
19591 let events = drain_events(&mut rig.rx);
19592 assert_eq!(events.len(), if tracked { 2 } else { 1 });
19593
19594 let fills = events
19595 .iter()
19596 .filter(|event| {
19597 matches!(
19598 event,
19599 ExecutionEvent::Order(OrderEventAny::Filled(_)) | ExecutionEvent::Report(_)
19600 )
19601 })
19602 .count();
19603
19604 assert_eq!(fills, 1);
19605 assert!(
19606 rig.dispatch
19607 .seen_trade_ids
19608 .contains(&parse_lighter_trade_id(&original).unwrap())
19609 );
19610 }
19611
19612 #[rstest]
19613 #[case::confirmed(true, false, false)]
19614 #[case::confirmed_stale_http(true, true, false)]
19615 #[case::rejected(false, false, false)]
19616 #[case::confirmed_stream(true, false, true)]
19617 #[case::rejected_stream(false, false, true)]
19618 fn cancel_overlap_preserves_amendment_and_real_fill(
19619 #[case] confirmed: bool,
19620 #[case] stale_http: bool,
19621 #[case] via_stream: bool,
19622 ) {
19623 let mut rig = dispatcher_rig("AMEND-CANCEL-FILL");
19624 let mut cached = OrderTestBuilder::new(OrderType::Limit)
19625 .trader_id(trader_id())
19626 .strategy_id(strategy_id())
19627 .instrument_id(rig.instrument_id)
19628 .client_order_id(rig.cloid)
19629 .side(OrderSide::Buy)
19630 .quantity(Quantity::from("0.0050"))
19631 .price(Price::from("2352.74"))
19632 .build();
19633 rig.dispatch.register_create_identity(&cached).unwrap();
19634 rig.dispatch
19635 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
19636 let mut raw = dispatcher_test_order(&rig, LighterOrderStatus::Open);
19637 dispatch_lighter_order(
19638 &raw,
19639 &rig.dispatch,
19640 &rig.emitter,
19641 &rig.registry,
19642 account_id(),
19643 trader_id(),
19644 UnixNanos::from(1),
19645 );
19646
19647 for event in drain_events(&mut rig.rx) {
19648 let ExecutionEvent::Order(event) = event else {
19649 panic!("expected acceptance");
19650 };
19651
19652 cached.apply(event).unwrap();
19653 }
19654
19655 let command = ModifyOrder::new(
19656 trader_id(),
19657 Some(client_id()),
19658 strategy_id(),
19659 rig.instrument_id,
19660 rig.cloid,
19661 cached.venue_order_id(),
19662 Some(Quantity::from("0.0100")),
19663 Some(Price::from("2400.00")),
19664 None,
19665 UUID4::new(),
19666 UnixNanos::from(2),
19667 None,
19668 None,
19669 );
19670 assert!(rig.dispatch.set_pending_modify_action(&command));
19671 rig.dispatch.set_pending_modify_shape(
19672 &command,
19673 OrderShapeSnapshot {
19674 quantity: command.quantity.unwrap(),
19675 price: command.price,
19676 trigger_price: None,
19677 },
19678 );
19679
19680 rig.dispatch
19681 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 19);
19682 rig.dispatch
19683 .set_pending_order_action(rig.cloid, PendingOrderAction::Cancel);
19684 rig.dispatch
19685 .set_pending_cancel_nonce(&rig.cloid, 0, 20, true);
19686 assert_eq!(rig.dispatch.pending_modify_nonce(&rig.cloid), Some((0, 19)));
19687 assert_eq!(rig.dispatch.pending_cancel_nonce(&rig.cloid), Some((0, 20)));
19688 let mut trade = dispatcher_test_trade(&rig, true);
19689 trade.bid_id = raw.order_index;
19690 trade.bid_id_str = Some(raw.order_id.clone());
19691 trade.size = Decimal::from_str("0.0030").unwrap();
19692 trade.usd_amount = trade.size * trade.price;
19693 dispatch_lighter_trade(
19694 &trade,
19695 &rig.dispatch,
19696 &rig.emitter,
19697 &rig.registry,
19698 account_id(),
19699 trader_id(),
19700 Some(TEST_ACCOUNT_INDEX_I64),
19701 UnixNanos::from(3),
19702 );
19703 raw.status = LighterOrderStatus::Canceled;
19704 raw.filled_base_amount = trade.size;
19705 if confirmed {
19706 raw.initial_base_amount = Decimal::from_str("0.0100").unwrap();
19707 raw.price = Decimal::from_str("2400.00").unwrap();
19708 }
19709
19710 raw.remaining_base_amount = raw.initial_base_amount - trade.size;
19711 dispatch_lighter_order(
19712 &raw,
19713 &rig.dispatch,
19714 &rig.emitter,
19715 &rig.registry,
19716 account_id(),
19717 trader_id(),
19718 UnixNanos::from(4),
19719 );
19720 assert!(drain_events(&mut rig.rx).is_empty());
19721 assert!(
19722 !rig.dispatch
19723 .seen_trade_ids
19724 .contains(&parse_lighter_trade_id(&trade).unwrap())
19725 );
19726
19727 let pending = PendingSendTx {
19728 batch_id: None,
19729 connection_epoch: 0,
19730 kind: PendingSendTxKind::Modify {
19731 command_id: command.command_id,
19732 strategy_id: strategy_id(),
19733 instrument_id: rig.instrument_id,
19734 client_order_id: rig.cloid,
19735 venue_order_id: cached.venue_order_id(),
19736 },
19737 submitted_at: UnixNanos::from(2),
19738 nonce: 19,
19739 api_key_index: TEST_API_KEY_INDEX,
19740 tx_hash: "amend-cancel".to_string(),
19741 };
19742
19743 let (client, _, _) = create_execution_client();
19744 let mut context = acked_probe_context(&client);
19745 context.dispatch = rig.dispatch.clone();
19746 context.registry = rig.registry.clone();
19747 context.emitter = rig.emitter.clone();
19748
19749 match (via_stream, confirmed) {
19750 (true, _) => {
19751 rig.dispatch.enqueue_pending_sendtx(pending.clone());
19752 handle_account_tx(
19753 0,
19754 &[LighterWsTx {
19755 hash: pending.tx_hash,
19756 status: if confirmed {
19757 LighterTxStatus::Executed
19758 } else {
19759 LighterTxStatus::Failed
19760 },
19761 event_info: String::new(),
19762 }],
19763 &context,
19764 );
19765 }
19766 (false, true) => {
19767 let mut http = raw;
19768 if stale_http {
19769 http.status = LighterOrderStatus::Open;
19770 }
19771
19772 confirm_modify_order(&pending, &context, &http).unwrap();
19773 }
19774 (false, false) => {
19775 reject_modify_order(
19776 &rig.dispatch,
19777 &rig.emitter,
19778 &pending,
19779 &CommandFailure::venue_rejected("amendment failed"),
19780 UnixNanos::from(5),
19781 false,
19782 );
19783 }
19784 }
19785
19786 let events = drain_events(&mut rig.rx);
19787 assert_eq!(events.len(), 3);
19788
19789 match &events[0] {
19790 ExecutionEvent::Order(OrderEventAny::Updated(update)) if confirmed => {
19791 assert_eq!(update.client_order_id, rig.cloid);
19792 assert_eq!(update.quantity, Quantity::from("0.0100"));
19793 assert_eq!(update.price, Some(Price::from("2400.00")));
19794 }
19795 ExecutionEvent::Order(OrderEventAny::ModifyRejected(rejected)) if !confirmed => {
19796 assert_eq!(rejected.client_order_id, rig.cloid);
19797 assert_eq!(
19798 rejected.reason.as_str(),
19799 if via_stream {
19800 "Lighter modify transaction failed: transaction failed without an application error"
19801 } else {
19802 "amendment failed"
19803 }
19804 );
19805 }
19806 other => panic!("expected amendment resolution before fill, was {other:?}"),
19807 }
19808
19809 let ExecutionEvent::Order(OrderEventAny::Filled(fill)) = &events[1] else {
19810 panic!("expected held real fill");
19811 };
19812
19813 assert_eq!(fill.client_order_id, rig.cloid);
19814 assert_eq!(fill.trade_id, parse_lighter_trade_id(&trade).unwrap());
19815 assert_eq!(fill.last_qty, Quantity::from("0.0030"));
19816 assert_eq!(fill.last_px, Price::from("2352.73"));
19817
19818 let ExecutionEvent::Order(OrderEventAny::Canceled(cancel)) = &events[2] else {
19819 panic!("expected cancellation after fill");
19820 };
19821
19822 assert_eq!(cancel.client_order_id, rig.cloid);
19823 assert_eq!(cancel.venue_order_id, cached.venue_order_id());
19824
19825 for event in events {
19826 let ExecutionEvent::Order(event) = event else {
19827 panic!("expected order event");
19828 };
19829
19830 cached.apply(event).unwrap();
19831 }
19832
19833 assert_eq!(cached.status(), OrderStatus::Canceled);
19834 assert_eq!(
19835 cached.quantity(),
19836 Quantity::from(if confirmed { "0.0100" } else { "0.0050" })
19837 );
19838 assert_eq!(cached.filled_qty(), Quantity::from("0.0030"));
19839 assert_eq!(
19840 cached.leaves_qty(),
19841 Quantity::from(if confirmed { "0.0070" } else { "0.0020" })
19842 );
19843 assert_eq!(rig.dispatch.pending_modify_nonce(&rig.cloid), None);
19844 assert_eq!(rig.dispatch.pending_cancel_nonce(&rig.cloid), None);
19845 assert!(
19846 rig.dispatch
19847 .seen_trade_ids
19848 .contains(&parse_lighter_trade_id(&trade).unwrap())
19849 );
19850 }
19851
19852 #[rstest]
19853 fn overlapping_modify_rejection_preserves_first_submission() {
19854 let (client, cache, mut rx) = create_execution_client();
19855 let instrument_id = register_test_instrument(&client, &cache);
19856 let cloid = ClientOrderId::from("OVERLAPPING-MODIFY");
19857 let venue = VenueOrderId::from("789");
19858 enqueue_modify(&client, instrument_id, cloid, venue, 23);
19859
19860 let command = ModifyOrder::new(
19861 trader_id(),
19862 Some(client_id()),
19863 strategy_id(),
19864 instrument_id,
19865 cloid,
19866 Some(venue),
19867 Some(Quantity::from("0.3000")),
19868 Some(Price::from("2363.00")),
19869 None,
19870 UUID4::new(),
19871 UnixNanos::from(2),
19872 None,
19873 None,
19874 );
19875 assert!(!client.admit_modify_order(&command));
19876 let events = drain_events(&mut rx);
19877 assert_eq!(events.len(), 1);
19878
19879 let ExecutionEvent::Order(OrderEventAny::ModifyDenied(event)) = &events[0] else {
19880 panic!("expected overlap rejection");
19881 };
19882
19883 assert_eq!(event.trader_id, trader_id());
19884 assert_eq!(event.strategy_id, strategy_id());
19885 assert_eq!(event.instrument_id, instrument_id);
19886 assert_eq!(event.client_order_id, cloid);
19887 assert_eq!(event.venue_order_id, Some(venue));
19888 assert_eq!(event.account_id, Some(account_id()));
19889 assert_eq!(
19890 event.reason.as_str(),
19891 "Lighter permits one unresolved modification per order; wait for its update or rejection"
19892 );
19893 assert_eq!(event.causation_id, Some(command.command_id));
19894 assert_eq!(client.dispatch.pending_modify_nonce(&cloid), Some((0, 23)));
19895 assert_eq!(
19896 client.dispatch.pending_modify_shape(&cloid),
19897 Some(OrderShapeSnapshot {
19898 quantity: Quantity::from("0.2000"),
19899 price: Some(Price::from("2362.00")),
19900 trigger_price: None
19901 })
19902 );
19903 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
19904 }
19905
19906 #[rstest]
19907 fn recovery_waits_for_confirmed_update_in_cache() {
19908 let (client, cache, mut rx) = create_execution_client();
19909 let instrument_id = register_test_instrument(&client, &cache);
19910 let rig = dispatcher_rig("HTTP-AMEND");
19911 let cloid = rig.cloid;
19912 let mut raw = dispatcher_test_order(&rig, LighterOrderStatus::Filled);
19913 let venue = VenueOrderId::new(&raw.order_id);
19914 let order = OrderTestBuilder::new(OrderType::Limit)
19915 .trader_id(trader_id())
19916 .strategy_id(strategy_id())
19917 .instrument_id(instrument_id)
19918 .client_order_id(cloid)
19919 .side(OrderSide::Buy)
19920 .quantity(Quantity::from("0.0050"))
19921 .price(Price::from("2352.74"))
19922 .build();
19923 cache_accepted_order(&cache, order, venue, Some(client_id()));
19924 let cached = cache.borrow().order_owned(&cloid).unwrap();
19925 client
19926 .dispatch
19927 .restore_reconciled_order(&cached, raw.client_order_index, venue, false)
19928 .unwrap();
19929 enqueue_modify(&client, instrument_id, cloid, venue, 23);
19930 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
19931 raw.initial_base_amount = Decimal::from_str("0.2000").unwrap();
19932 raw.filled_base_amount = raw.initial_base_amount;
19933 raw.remaining_base_amount = Decimal::ZERO;
19934 raw.price = Decimal::from_str("2362.00").unwrap();
19935 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&instrument_id).unwrap();
19936 let report =
19937 parse_ws_order_status_report(&raw, &instrument, account_id(), UnixNanos::from(3))
19938 .unwrap()
19939 .with_client_order_id(cloid);
19940 drop(instrument);
19941
19942 let command = GenerateOrderStatusReports::new(
19943 UUID4::new(),
19944 UnixNanos::default(),
19945 false,
19946 Some(instrument_id),
19947 None,
19948 None,
19949 None,
19950 None,
19951 );
19952
19953 let result = finish_order_reports(
19954 &client.core,
19955 &client.dispatch,
19956 OrderReportSweep {
19957 orders: vec![(raw.client_order_index, report)],
19958 ..Default::default()
19959 },
19960 &command,
19961 None,
19962 );
19963
19964 assert_eq!(
19965 result.unwrap_err().to_string(),
19966 "incomplete Lighter order reports: Lighter execution confirmation is pending"
19967 );
19968 assert_eq!(client.dispatch.pending_modify_nonce(&cloid), Some((0, 23)));
19969 assert!(client.dispatch.order_identities.contains_key(&cloid));
19970 let fill = reconciliation_fill_report(
19971 instrument_id,
19972 raw.client_order_index,
19973 venue,
19974 "HTTP-AMEND-TRADE",
19975 );
19976
19977 let withheld = finish_fill_reports(
19978 &client.core,
19979 &client.dispatch,
19980 BTreeSet::new(),
19981 Ok(FillSweep {
19982 reports: vec![fill.clone()],
19983 covers_window: true,
19984 withheld_instruments: AHashSet::new(),
19985 }),
19986 )
19987 .unwrap();
19988
19989 assert!(withheld.reports.is_empty());
19990 assert!(!withheld.covers_window);
19991 assert!(!client.dispatch.seen_trade_ids.contains(&fill.trade_id));
19992 confirm_modify_order(&pending, &acked_probe_context(&client), &raw).unwrap();
19993 let events = drain_events(&mut rx);
19994 assert_eq!(events.len(), 2);
19995 let mut events = events.into_iter();
19996
19997 let ExecutionEvent::Order(update @ OrderEventAny::Updated(_)) = events.next().unwrap()
19998 else {
19999 panic!("expected confirmed update before real fill");
20000 };
20001
20002 let ExecutionEvent::Order(delivered_fill @ OrderEventAny::Filled(_)) =
20003 events.next().unwrap()
20004 else {
20005 panic!("expected buffered real fill");
20006 };
20007
20008 let OrderEventAny::Filled(actual) = &delivered_fill else {
20009 unreachable!()
20010 };
20011
20012 assert_eq!(actual.trade_id, fill.trade_id);
20013 assert_eq!(actual.client_order_id, cloid);
20014 assert_eq!(actual.venue_order_id, fill.venue_order_id);
20015 assert_eq!(actual.last_qty, fill.last_qty);
20016 assert_eq!(actual.last_px, fill.last_px);
20017 assert_eq!(actual.commission, Some(fill.commission));
20018 assert_eq!(actual.liquidity_side, fill.liquidity_side);
20019 assert_eq!(actual.ts_event, fill.ts_event);
20020 assert!(!actual.reconciliation);
20021
20022 let still_withheld = finish_fill_reports(
20023 &client.core,
20024 &client.dispatch,
20025 BTreeSet::new(),
20026 Ok(FillSweep {
20027 reports: vec![fill.clone()],
20028 covers_window: true,
20029 withheld_instruments: AHashSet::new(),
20030 }),
20031 )
20032 .unwrap();
20033
20034 assert!(still_withheld.reports.is_empty());
20035 assert!(!still_withheld.covers_window);
20036 assert!(client.dispatch.seen_trade_ids.contains(&fill.trade_id));
20037 cache.borrow_mut().update_order(&update).unwrap();
20038 assert!(order_shape_pending(&client.core, &client.dispatch, &cloid));
20039 cache.borrow_mut().update_order(&delivered_fill).unwrap();
20040
20041 let delivered = finish_fill_reports(
20042 &client.core,
20043 &client.dispatch,
20044 BTreeSet::new(),
20045 Ok(FillSweep {
20046 reports: vec![fill.clone()],
20047 covers_window: true,
20048 withheld_instruments: AHashSet::new(),
20049 }),
20050 )
20051 .unwrap();
20052
20053 assert!(delivered.reports.is_empty());
20054 assert!(delivered.covers_window);
20055 assert_eq!(
20056 client.dispatch.mark_trade_reconciled(fill.trade_id),
20057 Some(TradeDedupSource::Live)
20058 );
20059 }
20060
20061 #[rstest]
20062 fn dispatch_lighter_trade_parse_failure_rolls_back_dedup() {
20063 let mut rig = dispatcher_rig("21");
20065 register_identity(&rig);
20066 let mut trade = dispatcher_test_trade(&rig, true);
20067 trade.timestamp = -1;
20068 let trade_id = parse_lighter_trade_id(&trade).expect("trade id parses");
20069
20070 dispatch_lighter_trade(
20071 &trade,
20072 &rig.dispatch,
20073 &rig.emitter,
20074 &rig.registry,
20075 account_id(),
20076 trader_id(),
20077 Some(TEST_ACCOUNT_INDEX_I64),
20078 UnixNanos::from(1),
20079 );
20080
20081 let events = drain_events(&mut rig.rx);
20082 assert!(
20083 !events
20084 .iter()
20085 .any(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Filled(_)))),
20086 "malformed trade must not emit a fill, was {events:?}",
20087 );
20088 assert!(
20089 rig.dispatch.mark_trade_seen(trade_id),
20090 "parse-failed trade must be re-markable (dedup rolled back)",
20091 );
20092 }
20093
20094 #[rstest]
20095 fn dispatch_lighter_trade_untracked_parse_failure_rolls_back_dedup() {
20096 let mut rig = dispatcher_rig("22");
20098 let mut trade = dispatcher_test_trade(&rig, true);
20099 trade.timestamp = -1;
20100 let trade_id = parse_lighter_trade_id(&trade).expect("trade id parses");
20101
20102 dispatch_lighter_trade(
20103 &trade,
20104 &rig.dispatch,
20105 &rig.emitter,
20106 &rig.registry,
20107 account_id(),
20108 trader_id(),
20109 Some(TEST_ACCOUNT_INDEX_I64),
20110 UnixNanos::from(1),
20111 );
20112
20113 let events = drain_events(&mut rig.rx);
20114 assert!(
20115 events.is_empty(),
20116 "untracked malformed trade must emit nothing, was {events:?}",
20117 );
20118 assert!(
20119 rig.dispatch.mark_trade_seen(trade_id),
20120 "untracked parse-failed trade must be re-markable (dedup rolled back)",
20121 );
20122 }
20123
20124 #[rstest]
20125 fn dispatch_tracked_order_event_terminal_cancel_retires_identity_and_removes_snapshot() {
20126 let mut rig = dispatcher_rig("6");
20127 register_identity(&rig);
20128 rig.dispatch.mark_accepted_emitted(rig.cloid);
20129 rig.dispatch.store_snapshot(
20130 rig.cloid,
20131 crate::websocket::dispatch::OrderShapeSnapshot {
20132 quantity: Quantity::from("0.0050"),
20133 price: Some(Price::from("2352.74")),
20134 trigger_price: None,
20135 },
20136 );
20137 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
20138 order.filled_base_amount = Decimal::ZERO;
20139 let mut trade = dispatcher_test_trade(&rig, true);
20140 trade.bid_id = order.order_index;
20141 trade.bid_id_str = Some(order.order_id.clone());
20142 dispatch_lighter_order(
20143 &order,
20144 &rig.dispatch,
20145 &rig.emitter,
20146 &rig.registry,
20147 account_id(),
20148 trader_id(),
20149 UnixNanos::from(1),
20150 );
20151
20152 let events = drain_events(&mut rig.rx);
20153 let canceled = events
20154 .iter()
20155 .find(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Canceled(_))))
20156 .expect("expected a Canceled event");
20157 if let ExecutionEvent::Order(OrderEventAny::Canceled(e)) = canceled {
20158 assert_eq!(e.client_order_id, rig.cloid);
20159 }
20160 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
20161 assert!(rig.dispatch.order_identity(&rig.cloid).is_some());
20162 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_none());
20163 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
20164
20165 dispatch_lighter_trade(
20166 &trade,
20167 &rig.dispatch,
20168 &rig.emitter,
20169 &rig.registry,
20170 account_id(),
20171 trader_id(),
20172 Some(TEST_ACCOUNT_INDEX_I64),
20173 UnixNanos::from(2),
20174 );
20175 let trailing = drain_events(&mut rig.rx);
20176 assert_eq!(
20177 trailing.len(),
20178 1,
20179 "expected one trailing fill: {trailing:?}"
20180 );
20181
20182 match &trailing[0] {
20183 ExecutionEvent::Order(OrderEventAny::Filled(event)) => {
20184 assert_eq!(event.client_order_id, rig.cloid);
20185 }
20186 other => panic!("expected typed trailing OrderFilled, was {other:?}"),
20187 }
20188 }
20189
20190 #[rstest]
20191 fn dispatch_tracked_cancel_after_report_seed_skips_synthesized_accept() {
20192 let mut rig = dispatcher_rig("10");
20193 register_identity(&rig);
20194
20195 let report_order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
20196 let instrument = LIGHTER_INSTRUMENT_CACHE
20197 .get(&rig.instrument_id)
20198 .expect("instrument cached");
20199 let report = parse_ws_order_status_report(
20200 &report_order,
20201 instrument.value(),
20202 account_id(),
20203 UnixNanos::from(1),
20204 )
20205 .map(|report| report.with_client_order_id(rig.cloid))
20206 .expect("report parses");
20207
20208 assert_eq!(report.order_status, OrderStatus::Accepted);
20209 rig.dispatch.seed_accepted_from_report(&report);
20210
20211 let mut cancel_order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
20212 cancel_order.filled_base_amount = Decimal::ZERO;
20213 dispatch_lighter_order(
20214 &cancel_order,
20215 &rig.dispatch,
20216 &rig.emitter,
20217 &rig.registry,
20218 account_id(),
20219 trader_id(),
20220 UnixNanos::from(2),
20221 );
20222
20223 let events = drain_events(&mut rig.rx);
20224 assert_eq!(
20225 events.len(),
20226 1,
20227 "report-seeded cancel should emit only Canceled, was {events:?}",
20228 );
20229 assert!(
20230 !events
20231 .iter()
20232 .any(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Accepted(_)))),
20233 "typed cancel must not synthesize a second Accepted",
20234 );
20235
20236 match &events[0] {
20237 ExecutionEvent::Order(OrderEventAny::Canceled(e)) => {
20238 assert_eq!(e.client_order_id, rig.cloid);
20239 assert_eq!(e.venue_order_id, Some(VenueOrderId::new("281476929510110")));
20240 }
20241 other => panic!("expected Canceled, was {other:?}"),
20242 }
20243 }
20244
20245 #[rstest]
20246 fn dispatch_tracked_cancel_after_submitted_report_seed_skips_synthesized_accept() {
20247 let mut rig = dispatcher_rig("11");
20248 register_identity(&rig);
20249
20250 let report_order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
20251 let instrument = LIGHTER_INSTRUMENT_CACHE
20252 .get(&rig.instrument_id)
20253 .expect("instrument cached");
20254 let mut report = parse_ws_order_status_report(
20255 &report_order,
20256 instrument.value(),
20257 account_id(),
20258 UnixNanos::from(1),
20259 )
20260 .map(|report| report.with_client_order_id(rig.cloid))
20261 .expect("report parses");
20262 report.order_status = OrderStatus::Submitted;
20263
20264 rig.dispatch.seed_accepted_from_report(&report);
20265 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
20266
20267 let mut cancel_order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
20268 cancel_order.filled_base_amount = Decimal::ZERO;
20269
20270 dispatch_lighter_order(
20271 &cancel_order,
20272 &rig.dispatch,
20273 &rig.emitter,
20274 &rig.registry,
20275 account_id(),
20276 trader_id(),
20277 UnixNanos::from(2),
20278 );
20279
20280 let events = drain_events(&mut rig.rx);
20281 assert_eq!(
20282 events.len(),
20283 1,
20284 "Cancel after Submitted report should emit only Canceled, was {events:?}",
20285 );
20286
20287 match &events[0] {
20288 ExecutionEvent::Order(OrderEventAny::Canceled(e)) => {
20289 assert_eq!(e.client_order_id, rig.cloid);
20290 assert_eq!(e.venue_order_id, Some(VenueOrderId::new("281476929510110")));
20291 }
20292 other => panic!("expected Canceled, was {other:?}"),
20293 }
20294 }
20295
20296 #[rstest]
20297 fn dispatch_tracked_order_event_accept_dedup_is_idempotent() {
20298 let mut rig = dispatcher_rig("7");
20299 register_identity(&rig);
20300 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
20301
20302 dispatch_lighter_order(
20305 &order,
20306 &rig.dispatch,
20307 &rig.emitter,
20308 &rig.registry,
20309 account_id(),
20310 trader_id(),
20311 UnixNanos::from(1),
20312 );
20313 dispatch_lighter_order(
20314 &order,
20315 &rig.dispatch,
20316 &rig.emitter,
20317 &rig.registry,
20318 account_id(),
20319 trader_id(),
20320 UnixNanos::from(2),
20321 );
20322
20323 let events = drain_events(&mut rig.rx);
20324 let accepted_count = events
20325 .iter()
20326 .filter(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Accepted(_))))
20327 .count();
20328 assert_eq!(accepted_count, 1, "Accepted must be emitted exactly once");
20329 }
20330
20331 #[rstest]
20332 fn dispatch_lighter_order_drops_when_instrument_uncached() {
20333 let registry = Arc::new(MarketRegistry::new());
20335 let (emitter, mut rx) = dispatcher_emitter();
20336 let dispatch = WsDispatchState::new();
20337 let cloid = ClientOrderId::new("CLOID-MISSING");
20338 dispatch.register_order_identity(
20339 cloid,
20340 OrderIdentity::new(
20341 InstrumentId::from("MISSING-PERP.LIGHTER"),
20342 strategy_id(),
20343 OrderSide::Buy,
20344 OrderType::Limit,
20345 1,
20346 ),
20347 );
20348 let mut order = LighterOrder {
20349 order_index: 1,
20350 client_order_index: 1,
20351 order_id: "1".to_string(),
20352 client_order_id: "1".to_string(),
20353 market_index: 999, owner_account_index: TEST_ACCOUNT_INDEX_I64,
20355 initial_base_amount: Decimal::ZERO,
20356 price: Decimal::ZERO,
20357 nonce: 0,
20358 remaining_base_amount: Decimal::ZERO,
20359 is_ask: false,
20360 base_size: 0,
20361 base_price: 0,
20362 filled_base_amount: Decimal::ZERO,
20363 filled_quote_amount: Decimal::ZERO,
20364 side: Some(LighterOrderSide::Buy),
20365 order_type: LighterOrderKind::Limit,
20366 time_in_force: LighterOrderTimeInForce::GoodTillTime,
20367 reduce_only: false,
20368 trigger_price: Decimal::ZERO,
20369 order_expiry: 0,
20370 status: LighterOrderStatus::Open,
20371 trigger_status: LighterTriggerStatus::Na,
20372 trigger_time: 0,
20373 parent_order_index: 0,
20374 parent_order_id: "0".to_string(),
20375 to_trigger_order_id_0: "0".to_string(),
20376 to_trigger_order_id_1: "0".to_string(),
20377 to_cancel_order_id_0: "0".to_string(),
20378 integrator_fee_collector_index: "0".to_string(),
20379 integrator_taker_fee: Decimal::ZERO,
20380 integrator_maker_fee: Decimal::ZERO,
20381 block_height: 0,
20382 timestamp: 0,
20383 created_at: 0,
20384 updated_at: 0,
20385 transaction_time: 0,
20386 };
20387 order.client_order_id = "1".to_string();
20388
20389 dispatch_lighter_order(
20390 &order,
20391 &dispatch,
20392 &emitter,
20393 ®istry,
20394 account_id(),
20395 trader_id(),
20396 UnixNanos::from(1),
20397 );
20398
20399 let events = drain_events(&mut rx);
20400 assert!(
20401 events.is_empty(),
20402 "no event for uncached instrument, was {events:?}"
20403 );
20404 assert!(dispatch.order_identities.contains_key(&cloid));
20405 }
20406
20407 #[rstest]
20408 fn dispatch_lighter_trade_filters_non_account_trades_defensively() {
20409 let mut rig = dispatcher_rig("8");
20410 let mut trade = dispatcher_test_trade(&rig, true);
20415 trade.bid_account_id = 91_249;
20416 trade.ask_account_id = 91_250;
20417
20418 dispatch_lighter_trade(
20419 &trade,
20420 &rig.dispatch,
20421 &rig.emitter,
20422 &rig.registry,
20423 account_id(),
20424 trader_id(),
20425 Some(TEST_ACCOUNT_INDEX_I64),
20426 UnixNanos::from(1),
20427 );
20428
20429 let events = drain_events(&mut rig.rx);
20430 assert!(
20431 events.is_empty(),
20432 "foreign trade must produce no event, was {events:?}"
20433 );
20434 assert!(
20435 !rig.dispatch
20436 .seen_trade_ids
20437 .contains(&TradeId::new("19209006902"),)
20438 );
20439 }
20440
20441 #[rstest]
20442 fn register_cloid_in_submit_path_uses_probed_index_on_collision() {
20443 let dispatch = WsDispatchState::new();
20448 let cloid = ClientOrderId::new("PROBE-CLOID");
20449 let derived = dispatch.derive_client_order_index(&cloid);
20450
20451 let intruder = ClientOrderId::new("INTRUDER");
20452 dispatch.cloid_map.insert(derived, intruder);
20453
20454 let chosen = dispatch.register_cloid(derived, cloid).unwrap();
20455
20456 assert_ne!(chosen, derived);
20457 assert_eq!(
20458 dispatch.cloid_map.get(&derived).map(|e| *e.value()),
20459 Some(intruder),
20460 );
20461 assert_eq!(
20462 dispatch.cloid_map.get(&chosen).map(|e| *e.value()),
20463 Some(cloid),
20464 );
20465 }
20466
20467 #[rstest]
20468 fn reconciliation_restores_active_collision_probed_identity() {
20469 let (client, cache, _rx) = create_execution_client();
20470 let instrument_id = register_test_instrument(&client, &cache);
20471 let mut factory = test_order_factory();
20472 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-ACTIVE");
20473 let cloid = order.client_order_id();
20474 let venue_order_id = VenueOrderId::from("281476929510110");
20475 cache_accepted_order(&cache, order, venue_order_id, None);
20476 let (base_index, client_order_index) = forced_probed_index(cloid);
20477 let raw =
20478 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
20479
20480 restore_reconciled_order(
20481 &client.core,
20482 &client.dispatch,
20483 VenueOrderId::new(raw.order_id.as_str()),
20484 raw.client_order_index,
20485 false,
20486 );
20487 let report = parse_http_order_to_report(
20488 &raw,
20489 &client.registry,
20490 account_id(),
20491 UnixNanos::from(1),
20492 &client.dispatch,
20493 )
20494 .unwrap();
20495 let translated = client.dispatch.translate_order_cloid(report);
20496 let replacement = ClientOrderId::from("O-RECON-ACTIVE-REPLACEMENT");
20497 let replacement_index = client
20498 .dispatch
20499 .register_cloid(client_order_index, replacement)
20500 .unwrap();
20501 let wrong_venue_order_id = VenueOrderId::from("281476929510111");
20502 let wrong_raw = reconciliation_raw_order(
20503 client_order_index,
20504 wrong_venue_order_id,
20505 LighterOrderStatus::Open,
20506 );
20507 let wrong_report = parse_http_order_to_report(
20508 &wrong_raw,
20509 &client.registry,
20510 account_id(),
20511 UnixNanos::from(2),
20512 &client.dispatch,
20513 )
20514 .unwrap();
20515 let wrong_translated = client.dispatch.translate_order_cloid(wrong_report);
20516
20517 assert_ne!(client_order_index, base_index);
20518 assert_eq!(
20519 client.dispatch.client_order_index(&cloid),
20520 Some(client_order_index),
20521 );
20522 assert_eq!(
20523 client
20524 .dispatch
20525 .resolve_client_order_index(client_order_index),
20526 Some(cloid),
20527 );
20528 assert_eq!(
20529 client.dispatch.lookup_venue_order_id(&cloid),
20530 Some(venue_order_id),
20531 );
20532 assert!(client.dispatch.order_identities.contains_key(&cloid));
20533 assert!(client.dispatch.accepted_was_emitted(&cloid));
20534 assert_eq!(translated.client_order_id, Some(cloid));
20535 assert_eq!(translated.venue_order_id, venue_order_id);
20536 assert_ne!(replacement_index, client_order_index);
20537 assert_eq!(
20538 client
20539 .dispatch
20540 .resolve_client_order_index(replacement_index),
20541 Some(replacement),
20542 );
20543 assert_eq!(
20544 wrong_translated.client_order_id,
20545 Some(ClientOrderId::new(wrong_venue_order_id.as_str())),
20546 );
20547 assert_eq!(wrong_translated.venue_order_id, wrong_venue_order_id);
20548 }
20549
20550 #[rstest]
20551 fn reconciliation_retires_stale_active_cache_from_terminal_report() {
20552 let (client, cache, _rx) = create_execution_client();
20553 let instrument_id = register_test_instrument(&client, &cache);
20554 let mut factory = test_order_factory();
20555 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-FILLED");
20556 let cloid = order.client_order_id();
20557 let venue_order_id = VenueOrderId::from("281476929510119");
20558 cache_accepted_order(&cache, order, venue_order_id, None);
20559 let (_, client_order_index) = forced_probed_index(cloid);
20560 let raw = reconciliation_raw_order(
20561 client_order_index,
20562 venue_order_id,
20563 LighterOrderStatus::Filled,
20564 );
20565
20566 restore_reconciled_order(
20567 &client.core,
20568 &client.dispatch,
20569 VenueOrderId::new(raw.order_id.as_str()),
20570 raw.client_order_index,
20571 true,
20572 );
20573 let fill = client
20574 .dispatch
20575 .translate_fill_cloid(reconciliation_fill_report(
20576 instrument_id,
20577 client_order_index,
20578 venue_order_id,
20579 "19209006919",
20580 ));
20581
20582 assert!(!client.dispatch.cloid_map.contains_key(&client_order_index));
20583 assert!(!client.dispatch.order_identities.contains_key(&cloid));
20584 assert_eq!(
20585 client.dispatch.client_order_index(&cloid),
20586 Some(client_order_index),
20587 );
20588 assert_eq!(fill.client_order_id, Some(cloid));
20589 assert_eq!(fill.venue_order_id, venue_order_id);
20590 }
20591
20592 #[rstest]
20593 fn reconciliation_restores_reused_retired_index_for_fill_translation() {
20594 let (client, cache, _rx) = create_execution_client();
20595 let instrument_id = register_test_instrument(&client, &cache);
20596 let mut factory = test_order_factory();
20597 let first_order = test_limit_order(&mut factory, instrument_id, "O-RECON-RETIRED-A");
20598 let second_order = test_limit_order(&mut factory, instrument_id, "O-RECON-RETIRED-B");
20599 let first_cloid = first_order.client_order_id();
20600 let second_cloid = second_order.client_order_id();
20601 let first_venue_order_id = VenueOrderId::from("281476929510120");
20602 let second_venue_order_id = VenueOrderId::from("281476929510121");
20603 cache_canceled_order(&cache, first_order, first_venue_order_id);
20604 cache_canceled_order(&cache, second_order, second_venue_order_id);
20605 let (_, client_order_index) = forced_probed_index(first_cloid);
20606 let first_raw = reconciliation_raw_order(
20607 client_order_index,
20608 first_venue_order_id,
20609 LighterOrderStatus::Canceled,
20610 );
20611 let second_raw = reconciliation_raw_order(
20612 client_order_index,
20613 second_venue_order_id,
20614 LighterOrderStatus::Canceled,
20615 );
20616
20617 restore_reconciled_order(
20618 &client.core,
20619 &client.dispatch,
20620 VenueOrderId::new(first_raw.order_id.as_str()),
20621 first_raw.client_order_index,
20622 true,
20623 );
20624 restore_reconciled_order(
20625 &client.core,
20626 &client.dispatch,
20627 VenueOrderId::new(second_raw.order_id.as_str()),
20628 second_raw.client_order_index,
20629 true,
20630 );
20631 let first_fill = client
20632 .dispatch
20633 .translate_fill_cloid(reconciliation_fill_report(
20634 instrument_id,
20635 client_order_index,
20636 first_venue_order_id,
20637 "19209006920",
20638 ));
20639 let second_fill = client
20640 .dispatch
20641 .translate_fill_cloid(reconciliation_fill_report(
20642 instrument_id,
20643 client_order_index,
20644 second_venue_order_id,
20645 "19209006921",
20646 ));
20647 let replacement = ClientOrderId::from("O-RECON-RETIRED-REPLACEMENT");
20648 let replacement_index = client
20649 .dispatch
20650 .register_cloid(client_order_index, replacement)
20651 .unwrap();
20652
20653 assert!(!client.dispatch.cloid_map.contains_key(&client_order_index));
20654 assert!(!client.dispatch.order_identities.contains_key(&first_cloid));
20655 assert!(!client.dispatch.order_identities.contains_key(&second_cloid));
20656 assert_eq!(
20657 client.dispatch.client_order_index(&first_cloid),
20658 Some(client_order_index),
20659 );
20660 assert_eq!(
20661 client.dispatch.client_order_index(&second_cloid),
20662 Some(client_order_index),
20663 );
20664 assert_eq!(
20665 client
20666 .dispatch
20667 .resolve_client_order_index(client_order_index),
20668 None,
20669 );
20670 assert_eq!(first_fill.client_order_id, Some(first_cloid));
20671 assert_eq!(first_fill.venue_order_id, first_venue_order_id);
20672 assert_eq!(second_fill.client_order_id, Some(second_cloid));
20673 assert_eq!(second_fill.venue_order_id, second_venue_order_id);
20674 assert_ne!(replacement_index, client_order_index);
20675 assert_eq!(
20676 client
20677 .dispatch
20678 .resolve_client_order_index(replacement_index),
20679 Some(replacement),
20680 );
20681 }
20682
20683 #[rstest]
20684 fn reconciliation_restores_triggered_marker() {
20685 let (client, cache, _rx) = create_execution_client();
20686 let instrument_id = register_test_instrument(&client, &cache);
20687 let mut factory = test_order_factory();
20688 let order = factory.stop_limit(
20689 instrument_id,
20690 OrderSide::Sell,
20691 Quantity::from("0.1000"),
20692 Price::from("2290.00"),
20693 Price::from("2300.00"),
20694 None,
20695 Some(TimeInForce::Gtc),
20696 None,
20697 Some(false),
20698 Some(false),
20699 Some(false),
20700 None,
20701 None,
20702 None,
20703 None,
20704 None,
20705 None,
20706 Some(ClientOrderId::from("O-RECON-TRIGGERED")),
20707 );
20708 let cloid = order.client_order_id();
20709 let venue_order_id = VenueOrderId::from("281476929510124");
20710 cache_accepted_order(&cache, order, venue_order_id, None);
20711 let triggered = OrderEventAny::Triggered(OrderTriggered::new(
20712 trader_id(),
20713 strategy_id(),
20714 instrument_id,
20715 cloid,
20716 UUID4::new(),
20717 UnixNanos::from(1),
20718 UnixNanos::from(2),
20719 false,
20720 Some(venue_order_id),
20721 Some(account_id()),
20722 ));
20723 cache.borrow_mut().update_order(&triggered).unwrap();
20724 let (_, client_order_index) = forced_probed_index(cloid);
20725 let raw =
20726 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
20727
20728 restore_reconciled_order(
20729 &client.core,
20730 &client.dispatch,
20731 VenueOrderId::new(raw.order_id.as_str()),
20732 raw.client_order_index,
20733 false,
20734 );
20735
20736 assert!(client.dispatch.order_identities.contains_key(&cloid));
20737 assert!(client.dispatch.accepted_was_emitted(&cloid));
20738 assert!(client.dispatch.triggered_was_emitted(&cloid));
20739 }
20740
20741 #[rstest]
20742 fn reconciliation_routes_reused_index_by_exact_venue_order_id() {
20743 let (client, cache, _rx) = create_execution_client();
20744 let instrument_id = register_test_instrument(&client, &cache);
20745 let mut factory = test_order_factory();
20746 let active_order = test_limit_order(&mut factory, instrument_id, "O-RECON-LIVE-ACTIVE");
20747 let retired_order = test_limit_order(&mut factory, instrument_id, "O-RECON-LIVE-RETIRED");
20748 let active_cloid = active_order.client_order_id();
20749 let retired_cloid = retired_order.client_order_id();
20750 let active_venue_order_id = VenueOrderId::from("281476929510125");
20751 let retired_venue_order_id = VenueOrderId::from("281476929510126");
20752 cache_accepted_order(
20753 &cache,
20754 active_order,
20755 active_venue_order_id,
20756 Some(client_id()),
20757 );
20758 cache_canceled_order(&cache, retired_order, retired_venue_order_id);
20759 let (_, client_order_index) = forced_probed_index(active_cloid);
20760 let active_raw = reconciliation_raw_order(
20761 client_order_index,
20762 active_venue_order_id,
20763 LighterOrderStatus::Open,
20764 );
20765 let retired_raw = reconciliation_raw_order(
20766 client_order_index,
20767 retired_venue_order_id,
20768 LighterOrderStatus::Canceled,
20769 );
20770 let raw_client_id = client_order_index.to_string();
20771
20772 restore_reconciled_order(
20773 &client.core,
20774 &client.dispatch,
20775 VenueOrderId::new(active_raw.order_id.as_str()),
20776 active_raw.client_order_index,
20777 false,
20778 );
20779 restore_reconciled_order(
20780 &client.core,
20781 &client.dispatch,
20782 VenueOrderId::new(retired_raw.order_id.as_str()),
20783 retired_raw.client_order_index,
20784 true,
20785 );
20786
20787 assert_eq!(
20788 client
20789 .dispatch
20790 .resolve_live_order_cloid(&raw_client_id, active_venue_order_id,),
20791 Some(active_cloid),
20792 );
20793 assert_eq!(
20794 client
20795 .dispatch
20796 .resolve_live_order_cloid(&raw_client_id, retired_venue_order_id,),
20797 Some(retired_cloid),
20798 );
20799 assert_eq!(
20800 client
20801 .dispatch
20802 .resolve_live_trade_cloid(&raw_client_id, active_venue_order_id),
20803 Some(active_cloid),
20804 );
20805 assert_eq!(
20806 client
20807 .dispatch
20808 .resolve_live_trade_cloid(&raw_client_id, retired_venue_order_id),
20809 Some(retired_cloid),
20810 );
20811 }
20812
20813 #[rstest]
20814 fn reconciliation_does_not_restore_from_numeric_client_index_alone() {
20815 let (client, cache, _rx) = create_execution_client();
20816 let instrument_id = register_test_instrument(&client, &cache);
20817 let mut factory = test_order_factory();
20818 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-NUMERIC-ONLY");
20819 let cloid = order.client_order_id();
20820 let client_order_index = client.dispatch.derive_client_order_index(&cloid);
20821 let venue_order_id = VenueOrderId::from("281476929510130");
20822 cache_order(&cache, order);
20823 let raw =
20824 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
20825
20826 restore_reconciled_order(
20827 &client.core,
20828 &client.dispatch,
20829 VenueOrderId::new(raw.order_id.as_str()),
20830 raw.client_order_index,
20831 false,
20832 );
20833 let report = parse_http_order_to_report(
20834 &raw,
20835 &client.registry,
20836 account_id(),
20837 UnixNanos::from(1),
20838 &client.dispatch,
20839 )
20840 .unwrap();
20841 let translated = client.dispatch.translate_order_cloid(report);
20842
20843 assert_eq!(client.dispatch.client_order_index(&cloid), None);
20844 assert_eq!(
20845 client
20846 .dispatch
20847 .resolve_client_order_index(client_order_index),
20848 None,
20849 );
20850 assert!(!client.dispatch.order_identities.contains_key(&cloid));
20851 assert_eq!(
20852 translated.client_order_id,
20853 Some(ClientOrderId::new(venue_order_id.as_str())),
20854 );
20855 assert_eq!(translated.venue_order_id, venue_order_id);
20856 }
20857
20858 #[rstest]
20859 #[case::known_partner(true)]
20860 #[case::external_partner(false)]
20861 fn lookup_after_restart_preserves_collision_partner(#[case] partner_cached: bool) {
20862 let (client, cache, _rx) = create_execution_client();
20863 let instrument_id = register_test_instrument(&client, &cache);
20864 let mut factory = test_order_factory();
20865 let order = test_limit_order(&mut factory, instrument_id, "O-UNACKNOWLEDGED-PROBED");
20866 let cloid = order.client_order_id();
20867 let partner = test_limit_order(&mut factory, instrument_id, "O-COLLISION-PARTNER");
20868 let partner_cloid = partner.client_order_id();
20869 let venue_order_id = VenueOrderId::from("281476929510140");
20870 let source = WsDispatchState::new();
20871 let base_index = source.derive_client_order_index(&cloid);
20872 source.register_cloid(base_index, partner_cloid).unwrap();
20873 let selected_index = source.register_cloid(base_index, cloid).unwrap();
20874 cache_order(&cache, order);
20875
20876 let submitted = OrderSubmitted::new(
20877 trader_id(),
20878 strategy_id(),
20879 instrument_id,
20880 cloid,
20881 account_id(),
20882 UUID4::new(),
20883 UnixNanos::from(1),
20884 UnixNanos::from(2),
20885 );
20886 cache
20887 .borrow_mut()
20888 .update_order(&OrderEventAny::Submitted(submitted))
20889 .unwrap();
20890
20891 if partner_cached {
20892 cache_accepted_order(&cache, partner, venue_order_id, Some(client_id()));
20893 restore_reconciled_order(
20894 &client.core,
20895 &client.dispatch,
20896 venue_order_id,
20897 base_index,
20898 false,
20899 );
20900 }
20901
20902 let raw = reconciliation_raw_order(base_index, venue_order_id, LighterOrderStatus::Open);
20903 let lookup = OrderReportLookup::new(
20904 &client.registry,
20905 &client.dispatch,
20906 Some(instrument_id),
20907 Some(&cloid),
20908 None,
20909 )
20910 .unwrap();
20911 let parsed = lookup
20912 .parse(
20913 &raw,
20914 &client.registry,
20915 account_id(),
20916 UnixNanos::from(3),
20917 &client.dispatch,
20918 )
20919 .unwrap();
20920
20921 let expected_cloid = if partner_cached {
20922 partner_cloid
20923 } else {
20924 ClientOrderId::new(venue_order_id.as_str())
20925 };
20926
20927 let expected = parsed.0.clone().with_client_order_id(expected_cloid);
20928 let report = lookup.finish(&client.dispatch, parsed);
20929 let cached = cache.borrow().order_owned(&cloid).unwrap();
20930
20931 assert_eq!(selected_index, base_index + 1);
20932 assert_eq!(
20933 serde_json::to_value(&report).unwrap(),
20934 serde_json::to_value(&expected).unwrap()
20935 );
20936 assert_eq!(cached.status(), OrderStatus::Submitted);
20937 assert_eq!(cached.venue_order_id(), None);
20938 assert_eq!(client.dispatch.client_order_index(&cloid), None);
20939 }
20940
20941 fn forced_probed_index(cloid: ClientOrderId) -> (i64, i64) {
20942 let source = WsDispatchState::new();
20943 let base_index = source.derive_client_order_index(&cloid);
20944 source
20945 .cloid_map
20946 .insert(base_index, ClientOrderId::from("O-FORCED-COLLISION"));
20947 let client_order_index = source.register_cloid(base_index, cloid).unwrap();
20948 (base_index, client_order_index)
20949 }
20950
20951 fn reconciliation_raw_order(
20952 client_order_index: i64,
20953 venue_order_id: VenueOrderId,
20954 status: LighterOrderStatus,
20955 ) -> LighterOrder {
20956 let rig = dispatcher_rig("RECONCILIATION-RAW");
20957 let mut raw = dispatcher_test_order(&rig, status);
20958 raw.order_index = venue_order_id.as_str().parse().unwrap();
20959 raw.order_id = venue_order_id.to_string();
20960 raw.client_order_index = client_order_index;
20961 raw.client_order_id = client_order_index.to_string();
20962 raw
20963 }
20964
20965 fn cache_canceled_order(
20966 cache: &Rc<RefCell<Cache>>,
20967 order: OrderAny,
20968 venue_order_id: VenueOrderId,
20969 ) {
20970 let (instrument_id, client_order_id) =
20971 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
20972 let canceled = OrderEventAny::Canceled(OrderCanceled::new(
20973 trader_id(),
20974 strategy_id(),
20975 instrument_id,
20976 client_order_id,
20977 UUID4::new(),
20978 UnixNanos::from(1),
20979 UnixNanos::from(2),
20980 false,
20981 Some(venue_order_id),
20982 Some(account_id()),
20983 None,
20984 ));
20985 cache.borrow_mut().update_order(&canceled).unwrap();
20986 }
20987
20988 fn reconciliation_fill_report(
20989 instrument_id: InstrumentId,
20990 client_order_index: i64,
20991 venue_order_id: VenueOrderId,
20992 trade_id: &str,
20993 ) -> FillReport {
20994 FillReport::new(
20995 account_id(),
20996 instrument_id,
20997 venue_order_id,
20998 TradeId::new(trade_id),
20999 OrderSide::Buy,
21000 Quantity::from("0.1336"),
21001 Price::from("2352.73"),
21002 Money::from("0.000196 USDC"),
21003 LiquiditySide::Taker,
21004 Some(ClientOrderId::new(client_order_index.to_string())),
21005 None,
21006 UnixNanos::from(1),
21007 UnixNanos::from(2),
21008 Some(UUID4::new()),
21009 )
21010 }
21011
21012 #[rstest]
21013 fn dispatch_lighter_order_seeds_snapshot_after_synthesized_accept() {
21014 let mut rig = dispatcher_rig("9");
21019 register_identity(&rig);
21020
21021 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
21022 let mut trade = dispatcher_test_trade(&rig, true);
21023 trade.bid_id = order.order_index;
21024 trade.bid_id_str = Some(order.order_id.clone());
21025 dispatch_lighter_trade(
21026 &trade,
21027 &rig.dispatch,
21028 &rig.emitter,
21029 &rig.registry,
21030 account_id(),
21031 trader_id(),
21032 Some(TEST_ACCOUNT_INDEX_I64),
21033 UnixNanos::from(1),
21034 );
21035 assert_eq!(drain_events(&mut rig.rx).len(), 2);
21036 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
21037 assert!(
21038 rig.dispatch.snapshot_for(&rig.cloid).is_none(),
21039 "synthesized Accept has no snapshot until the Open frame seeds one",
21040 );
21041
21042 dispatch_lighter_order(
21046 &order,
21047 &rig.dispatch,
21048 &rig.emitter,
21049 &rig.registry,
21050 account_id(),
21051 trader_id(),
21052 UnixNanos::from(2),
21053 );
21054 assert!(
21055 rig.dispatch.snapshot_for(&rig.cloid).is_some(),
21056 "Open frame after synthesized accept must seed the snapshot",
21057 );
21058
21059 let mut modified = order;
21061 modified.price = Decimal::from_str("2400.00").unwrap();
21062 dispatch_lighter_order(
21063 &modified,
21064 &rig.dispatch,
21065 &rig.emitter,
21066 &rig.registry,
21067 account_id(),
21068 trader_id(),
21069 UnixNanos::from(3),
21070 );
21071 let events = drain_events(&mut rig.rx);
21072 let updated = events
21073 .iter()
21074 .find(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Updated(_))));
21075 assert!(
21076 updated.is_some(),
21077 "real modify must produce Updated, events={events:?}"
21078 );
21079 }
21080
21081 fn enqueue_create(client: &LighterExecutionClient, order: &OrderAny, nonce: i64) -> i64 {
21082 let client_order_index = client
21083 .dispatch
21084 .derive_client_order_index(&order.client_order_id());
21085 client
21086 .dispatch
21087 .register_cloid(client_order_index, order.client_order_id())
21088 .unwrap();
21089 client.dispatch.register_order_identity(
21090 order.client_order_id(),
21091 OrderIdentity::new(
21092 order.instrument_id(),
21093 order.strategy_id(),
21094 order.order_side(),
21095 order.order_type(),
21096 client_order_index,
21097 ),
21098 );
21099 let now = UnixNanos::from(1_000_000_000);
21100 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
21101 batch_id: None,
21102 connection_epoch: 0,
21103 kind: PendingSendTxKind::Create {
21104 order: Box::new(order.clone()),
21105 client_order_index,
21106 },
21107 submitted_at: now,
21108 nonce,
21109 api_key_index: TEST_API_KEY_INDEX,
21110 tx_hash: format!("hash{nonce:02x}"),
21111 });
21112 client.dispatch.nonce_manager.refresh(
21113 TEST_ACCOUNT_INDEX_I64,
21114 TEST_API_KEY_INDEX,
21115 nonce + 1,
21116 );
21117 let _ = client
21118 .dispatch
21119 .nonce_manager
21120 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX);
21121 client_order_index
21122 }
21123
21124 fn enqueue_other(client: &LighterExecutionClient, nonce: i64) {
21125 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
21126 batch_id: None,
21127 connection_epoch: 0,
21128 kind: PendingSendTxKind::Other,
21129 submitted_at: UnixNanos::from(1_000_000_000),
21130 nonce,
21131 api_key_index: TEST_API_KEY_INDEX,
21132 tx_hash: format!("hash{nonce:02x}"),
21133 });
21134 }
21135
21136 #[rstest]
21137 #[case(serde_json::json!({"self_trade_behavior_mode": "reduce"}), LighterSelfTradeBehaviorMode::Reduce, LighterSelfTradeEqualityMode::AccountIndex)]
21138 #[case(serde_json::json!({"self_trade_equality_mode": "master_account_index"}), LighterSelfTradeBehaviorMode::ExpireMaker, LighterSelfTradeEqualityMode::MasterAccountIndex)]
21139 #[case(serde_json::json!({"market_order_slippage_bps": 37}), LighterSelfTradeBehaviorMode::ExpireMaker, LighterSelfTradeEqualityMode::AccountIndex)]
21140 fn resolve_self_trade_partial_params(
21141 #[case] value: serde_json::Value,
21142 #[case] behavior: LighterSelfTradeBehaviorMode,
21143 #[case] equality: LighterSelfTradeEqualityMode,
21144 ) {
21145 let (client, _cache, _rx) = create_execution_client();
21146 let params: Params = serde_json::from_value(value).unwrap();
21147
21148 let attrs = client.resolve_create_attributes(Some(¶ms)).unwrap();
21149
21150 assert_eq!(
21151 attrs,
21152 L2TxAttributes {
21153 self_trade_behavior_mode: Some(behavior),
21154 self_trade_equality_mode: Some(equality),
21155 ..Default::default()
21156 }
21157 );
21158 }
21159
21160 fn enqueue_cancel(
21161 client: &LighterExecutionClient,
21162 instrument_id: InstrumentId,
21163 client_order_id: ClientOrderId,
21164 venue_order_id: VenueOrderId,
21165 nonce: i64,
21166 ) -> UUID4 {
21167 let command_id = UUID4::new();
21168 let pending = enqueue_cancel_pending(
21169 client,
21170 instrument_id,
21171 client_order_id,
21172 venue_order_id,
21173 nonce,
21174 command_id,
21175 );
21176 promote_cancel_submission(&client.dispatch, &pending);
21177 command_id
21178 }
21179
21180 fn enqueue_cancel_pending(
21181 client: &LighterExecutionClient,
21182 instrument_id: InstrumentId,
21183 client_order_id: ClientOrderId,
21184 venue_order_id: VenueOrderId,
21185 nonce: i64,
21186 command_id: UUID4,
21187 ) -> PendingSendTx {
21188 client
21189 .dispatch
21190 .register_cancel_command(client_order_id, command_id, true);
21191
21192 let pending = PendingSendTx {
21193 batch_id: None,
21194 connection_epoch: 0,
21195 kind: PendingSendTxKind::Cancel {
21196 command_id: Some(command_id),
21197 emit_cancel_rejected: true,
21198 strategy_id: strategy_id(),
21199 instrument_id,
21200 client_order_id,
21201 venue_order_id: Some(venue_order_id),
21202 },
21203 submitted_at: UnixNanos::from(1_000_000_000),
21204 nonce,
21205 api_key_index: TEST_API_KEY_INDEX,
21206 tx_hash: format!("hash{nonce:02x}"),
21207 };
21208
21209 client.dispatch.enqueue_pending_sendtx(pending.clone());
21210
21211 pending
21212 }
21213
21214 fn enqueue_modify(
21215 client: &LighterExecutionClient,
21216 instrument_id: InstrumentId,
21217 client_order_id: ClientOrderId,
21218 venue_order_id: VenueOrderId,
21219 nonce: i64,
21220 ) -> UUID4 {
21221 let command = ModifyOrder::new(
21222 trader_id(),
21223 Some(client_id()),
21224 strategy_id(),
21225 instrument_id,
21226 client_order_id,
21227 Some(venue_order_id),
21228 Some(Quantity::from("0.2000")),
21229 Some(Price::from("2362.00")),
21230 None,
21231 UUID4::new(),
21232 UnixNanos::default(),
21233 None,
21234 None,
21235 );
21236 client.dispatch.set_pending_modify_action(&command);
21237 client.dispatch.set_pending_modify_shape(
21238 &command,
21239 OrderShapeSnapshot {
21240 quantity: command.quantity.unwrap(),
21241 price: command.price,
21242 trigger_price: command.trigger_price,
21243 },
21244 );
21245
21246 client
21247 .dispatch
21248 .set_pending_modify_nonce(&client_order_id, command.command_id, 0, nonce);
21249 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
21250 batch_id: None,
21251 connection_epoch: 0,
21252 kind: PendingSendTxKind::Modify {
21253 command_id: command.command_id,
21254 strategy_id: strategy_id(),
21255 instrument_id,
21256 client_order_id,
21257 venue_order_id: Some(venue_order_id),
21258 },
21259 submitted_at: UnixNanos::from(1_000_000_000),
21260 nonce,
21261 api_key_index: TEST_API_KEY_INDEX,
21262 tx_hash: format!("hash{nonce:02x}"),
21263 });
21264
21265 command.command_id
21266 }
21267
21268 fn acked_probe_context(client: &LighterExecutionClient) -> AckedOrderProbeContext {
21269 AckedOrderProbeContext {
21270 http_client: client.http_client.clone(),
21271 registry: Arc::clone(&client.registry),
21272 credential: test_credential(),
21273 dispatch: client.dispatch.clone(),
21274 account_id: client.core.account_id,
21275 clock: client.clock,
21276 emitter: client.emitter.clone(),
21277 connection_epoch: client.ws_client.connection_epoch_atomic(),
21278 cancellation_token: client.cancellation_token.clone(),
21279 pending_tasks: client.pending_tasks.spawner().unwrap(),
21280 }
21281 }
21282
21283 fn failed_modify_tx(pending: &PendingSendTx) -> LighterTx {
21284 LighterTx {
21285 code: 200,
21286 message: None,
21287 hash: pending.tx_hash.clone(),
21288 tx_type: LighterTxType::ModifyOrder as u8,
21289 info: String::new(),
21290 event_info: serde_json::json!({"ae":"invalid order price"}).to_string(),
21291 status: LighterTxStatus::Failed,
21292 account_index: TEST_ACCOUNT_INDEX_I64,
21293 nonce: pending.nonce,
21294 api_key_index: pending.api_key_index,
21295 }
21296 }
21297
21298 #[rstest]
21299 #[case::hash("hash")]
21300 #[case::type_mismatch("type")]
21301 #[case::account("account")]
21302 #[case::key("key")]
21303 #[case::nonce("nonce")]
21304 #[tokio::test]
21305 async fn acknowledged_modify_requires_exact_transaction_identity(#[case] mismatch: &str) {
21306 let (client, cache, mut rx) = create_execution_client();
21307 let instrument = register_test_instrument(&client, &cache);
21308 let id = ClientOrderId::from("MODIFY-IDENTITY");
21309 enqueue_modify(&client, instrument, id, VenueOrderId::from("789"), 23);
21310 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21311 let mut tx = failed_modify_tx(&pending);
21312 match mismatch {
21313 "hash" => tx.hash = "different".to_string(),
21314 "type" => tx.tx_type = LighterTxType::CancelOrder as u8,
21315 "account" => tx.account_index += 1,
21316 "key" => tx.api_key_index += 1,
21317 "nonce" => tx.nonce += 1,
21318 _ => unreachable!(),
21319 }
21320
21321 let result = resolve_acked_action(&pending, &acked_probe_context(&client), tx).await;
21322 assert_eq!(
21323 result.unwrap_err().to_string(),
21324 "Lighter transaction lookup did not match acknowledged action identity"
21325 );
21326 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 23)));
21327 assert_eq!(
21328 client.dispatch.pending_order_action(&id),
21329 Some(PendingOrderAction::Modify)
21330 );
21331 assert!(rx.try_recv().is_err());
21332 }
21333
21334 #[rstest]
21335 #[case::failed_status(LighterTxStatus::Failed, "", true)]
21336 #[case::malformed_failed_status(LighterTxStatus::Failed, "invalid json", true)]
21337 #[case::application_error(LighterTxStatus::Executed, "application", true)]
21338 #[case::pending(LighterTxStatus::Pending, "", false)]
21339 #[case::pending_final(LighterTxStatus::PendingFinal, "", false)]
21340 #[tokio::test]
21341 async fn acknowledged_modify_outcomes_consume_only_matching_action(
21342 #[case] status: LighterTxStatus,
21343 #[case] event_info: &str,
21344 #[case] failed: bool,
21345 ) {
21346 let (client, cache, mut rx) = create_execution_client();
21347 let instrument = register_test_instrument(&client, &cache);
21348 let id = ClientOrderId::from("MODIFY-OUTCOME");
21349 let venue = VenueOrderId::from("789");
21350 let nonce = client
21351 .dispatch
21352 .nonce_manager
21353 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21354 .unwrap();
21355 enqueue_modify(&client, instrument, id, venue, nonce);
21356 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21357 let context = acked_probe_context(&client);
21358 let mut tx = failed_modify_tx(&pending);
21359 tx.status = status;
21360
21361 tx.event_info = if event_info == "application" {
21362 serde_json::json!({"ae":"invalid order price"}).to_string()
21363 } else {
21364 event_info.to_string()
21365 };
21366
21367 let resolved = resolve_acked_action(&pending, &context, tx.clone())
21368 .await
21369 .unwrap();
21370 assert_eq!(resolved, failed);
21371
21372 if failed {
21373 let OrderEventAny::ModifyRejected(event) = recv_order_event(&mut rx).await else {
21374 panic!("expected ModifyRejected")
21375 };
21376
21377 assert_eq!(event.client_order_id, id);
21378 assert_eq!(event.venue_order_id, Some(venue));
21379 assert_eq!(event.instrument_id, instrument);
21380 assert_eq!(event.strategy_id, strategy_id());
21381 assert_eq!(
21382 event.reason.as_str(),
21383 if event_info == "application" {
21384 "Lighter modify transaction failed: invalid order price"
21385 } else {
21386 "Lighter modify transaction failed: transaction failed without an application error"
21387 }
21388 );
21389 assert_eq!(client.dispatch.pending_modify_nonce(&id), None);
21390 assert_eq!(client.dispatch.pending_order_action(&id), None);
21391 assert!(resolve_acked_action(&pending, &context, tx).await.unwrap());
21392 } else {
21393 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, nonce)));
21394 }
21395
21396 assert!(rx.try_recv().is_err());
21397 assert_eq!(
21398 client
21399 .dispatch
21400 .nonce_manager
21401 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21402 .unwrap(),
21403 nonce + 1
21404 );
21405 }
21406
21407 #[rstest]
21408 #[case::empty_error("")]
21409 #[case::malformed_error("invalid json")]
21410 #[tokio::test]
21411 async fn acknowledged_cancel_failure_consumes_action_once(
21412 #[case] event_info: &str,
21413 #[values(false, true)] emit_rejected: bool,
21414 ) {
21415 let (client, cache, mut rx) = create_execution_client();
21416 let instrument = register_test_instrument(&client, &cache);
21417 let id = ClientOrderId::from("CANCEL-FAILED");
21418 let venue = VenueOrderId::from("789");
21419 let nonce = client
21420 .dispatch
21421 .nonce_manager
21422 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21423 .unwrap();
21424 enqueue_cancel(&client, instrument, id, venue, nonce);
21425 let mut pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21426
21427 if let PendingSendTxKind::Cancel {
21428 emit_cancel_rejected,
21429 ..
21430 } = &mut pending.kind
21431 {
21432 *emit_cancel_rejected = emit_rejected;
21433 }
21434
21435 let PendingSendTxKind::Cancel {
21436 command_id: Some(command_id),
21437 ..
21438 } = pending.kind
21439 else {
21440 unreachable!()
21441 };
21442
21443 client.dispatch.clear_pending_order_action(&id);
21444 client
21445 .dispatch
21446 .register_cancel_command(id, command_id, emit_rejected);
21447 client
21448 .dispatch
21449 .promote_pending_cancel(&id, command_id, 0, nonce, emit_rejected);
21450 let mut tx = failed_modify_tx(&pending);
21451 tx.tx_type = LighterTxType::CancelOrder as u8;
21452 tx.event_info = event_info.to_string();
21453 let context = acked_probe_context(&client);
21454 let resolved = resolve_acked_action(&pending, &context, tx.clone())
21455 .await
21456 .unwrap();
21457
21458 {
21459 let OrderEventAny::CancelRejected(event) = recv_order_event(&mut rx).await else {
21460 panic!("expected CancelRejected")
21461 };
21462
21463 let PendingSendTxKind::Cancel { command_id, .. } = pending.kind else {
21464 unreachable!()
21465 };
21466
21467 assert_eq!(event.causation_id, command_id);
21468 assert_eq!(event.client_order_id, id);
21469 assert_eq!(event.venue_order_id, Some(venue));
21470 assert_eq!(event.instrument_id, instrument);
21471 assert_eq!(event.strategy_id, strategy_id());
21472 assert_eq!(
21473 event.reason.as_str(),
21474 "Lighter cancel transaction failed: transaction failed without an application error"
21475 );
21476 }
21477
21478 let duplicate_resolved = resolve_acked_action(&pending, &context, tx).await.unwrap();
21479 assert!(resolved);
21480 assert!(duplicate_resolved);
21481 assert_eq!(client.dispatch.pending_cancel_nonce(&id), None);
21482 assert_eq!(client.dispatch.pending_order_action(&id), None);
21483 assert!(rx.try_recv().is_err());
21484 assert_eq!(
21485 client
21486 .dispatch
21487 .nonce_manager
21488 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX,)
21489 .unwrap(),
21490 nonce + 1
21491 );
21492 }
21493
21494 #[rstest]
21495 #[case::unknown_hash("unmatched", 0, LighterTxStatus::Failed, "")]
21496 #[case::old_connection("hash17", 1, LighterTxStatus::Failed, "")]
21497 #[case::pending("hash17", 0, LighterTxStatus::Pending, "")]
21498 #[case::pending_final("hash17", 0, LighterTxStatus::PendingFinal, "")]
21499 #[case::malformed_success("hash17", 0, LighterTxStatus::Executed, "invalid json")]
21500 fn account_tx_preserves_unconfirmed_modification(
21501 #[case] hash: &str,
21502 #[case] connection_epoch: u64,
21503 #[case] status: LighterTxStatus,
21504 #[case] event_info: &str,
21505 ) {
21506 let (client, cache, mut rx) = create_execution_client();
21507 let instrument = register_test_instrument(&client, &cache);
21508 let id = ClientOrderId::from("STREAM-UNCONFIRMED");
21509 enqueue_modify(&client, instrument, id, VenueOrderId::from("789"), 23);
21510 handle_account_tx(
21511 connection_epoch,
21512 &[LighterWsTx {
21513 hash: hash.to_owned(),
21514 status,
21515 event_info: event_info.to_owned(),
21516 }],
21517 &acked_probe_context(&client),
21518 );
21519
21520 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 23)));
21521 assert!(client.dispatch.pending_modify_executed(&id).is_none());
21522 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
21523 assert!(rx.try_recv().is_err());
21524 }
21525
21526 #[rstest]
21527 #[case::executed(LighterTxStatus::Executed)]
21528 #[case::failed(LighterTxStatus::Failed)]
21529 fn account_tx_preserves_ack_attribution(#[case] status: LighterTxStatus) {
21530 let (client, cache, mut rx) = create_execution_client();
21531 let instrument = register_test_instrument(&client, &cache);
21532 let id_a = ClientOrderId::from("STREAM-ACK-A");
21533 let id_b = ClientOrderId::from("STREAM-ACK-B");
21534 let nonce_a = client
21535 .dispatch
21536 .nonce_manager
21537 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21538 .unwrap();
21539 enqueue_modify(
21540 &client,
21541 instrument,
21542 id_a,
21543 VenueOrderId::from("789"),
21544 nonce_a,
21545 );
21546 let nonce_b = client
21547 .dispatch
21548 .nonce_manager
21549 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21550 .unwrap();
21551 let command_b = enqueue_modify(
21552 &client,
21553 instrument,
21554 id_b,
21555 VenueOrderId::from("790"),
21556 nonce_b,
21557 );
21558 handle_account_tx(
21559 0,
21560 &[LighterWsTx {
21561 hash: format!("hash{nonce_a:02x}"),
21562 status,
21563 event_info: String::new(),
21564 }],
21565 &acked_probe_context(&client),
21566 );
21567
21568 let stream_events = drain_events(&mut rx);
21569 assert_eq!(
21570 stream_events.len(),
21571 usize::from(status == LighterTxStatus::Failed)
21572 );
21573 assert!(
21574 handle_send_tx_ack_for_connection(
21575 &client.dispatch,
21576 Some(TEST_ACCOUNT_INDEX_I64),
21577 0,
21578 200,
21579 None,
21580 )
21581 .is_none()
21582 );
21583 handle_send_tx_rejection(
21584 &client.dispatch,
21585 &client.emitter,
21586 Some(TEST_ACCOUNT_INDEX_I64),
21587 UnixNanos::from(5),
21588 SendTxRejectionSource::Ack,
21589 Some(21702),
21590 "B rejected",
21591 Some(&format!("hash{nonce_b:02x}")),
21592 );
21593
21594 let ExecutionEvent::Order(OrderEventAny::ModifyRejected(rejected)) = rx.try_recv().unwrap()
21595 else {
21596 panic!("expected rejection of the second command");
21597 };
21598
21599 assert_eq!(rejected.client_order_id, id_b);
21600 assert_eq!(rejected.trader_id, trader_id());
21601 assert_eq!(rejected.strategy_id, strategy_id());
21602 assert_eq!(rejected.instrument_id, instrument);
21603 assert_eq!(rejected.account_id, Some(account_id()));
21604 assert_eq!(rejected.venue_order_id, Some(VenueOrderId::from("790")));
21605 assert_eq!(rejected.causation_id, Some(command_b));
21606 assert_eq!(rejected.reason.as_str(), "LIGHTER_21702: B rejected");
21607 assert_eq!(client.dispatch.pending_modify_nonce(&id_b), None);
21608 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
21609 assert_eq!(
21610 client
21611 .dispatch
21612 .nonce_manager
21613 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21614 .unwrap(),
21615 nonce_b
21616 );
21617 assert!(rx.try_recv().is_err());
21618 }
21619
21620 #[rstest]
21621 fn account_tx_rejection_cannot_publish_after_modification_resolves() {
21622 let (client, cache, mut rx) = create_execution_client();
21623 let instrument = register_test_instrument(&client, &cache);
21624 let id = ClientOrderId::from("STREAM-STALE-REJECTION");
21625 let venue = VenueOrderId::from("789");
21626 enqueue_modify(&client, instrument, id, venue, 23);
21627 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21628 assert!(client.dispatch.clear_pending_modify_if_nonce(&id, 0, 23));
21629 enqueue_modify(&client, instrument, id, venue, 24);
21630 reject_modify_order(
21631 &client.dispatch,
21632 &client.emitter,
21633 &pending,
21634 &CommandFailure::venue_rejected("stale execution failure"),
21635 UnixNanos::from(5),
21636 true,
21637 );
21638 client.dispatch.set_pending_modify_executed(&id, &pending);
21639 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 24)));
21640 assert!(client.dispatch.pending_modify_executed(&id).is_none());
21641 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
21642 assert!(rx.try_recv().is_err());
21643 }
21644
21645 #[rstest]
21646 #[case::newer_nonce(false)]
21647 #[case::reconnected(true)]
21648 #[tokio::test]
21649 async fn acknowledged_modify_does_not_consume_newer_submission(#[case] reconnected: bool) {
21650 let (client, cache, mut rx) = create_execution_client();
21651 let instrument = register_test_instrument(&client, &cache);
21652 let id = ClientOrderId::from("MODIFY-NEWER");
21653 let venue = VenueOrderId::from("789");
21654 enqueue_modify(&client, instrument, id, venue, 23);
21655 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
21656 let context = acked_probe_context(&client);
21657 client.dispatch.clear_pending_modify_if_nonce(&id, 0, 23);
21658
21659 if reconnected {
21660 context.connection_epoch.store(1, Ordering::Release);
21661 enqueue_modify(&client, instrument, id, venue, 24);
21662 } else {
21663 enqueue_modify(&client, instrument, id, venue, 24);
21664 }
21665
21666 assert!(
21667 resolve_acked_action(&pending, &context, failed_modify_tx(&pending))
21668 .await
21669 .unwrap()
21670 );
21671 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 24)));
21672 assert!(rx.try_recv().is_err());
21673 }
21674
21675 #[rstest]
21676 fn queued_modify_cannot_bind_a_newer_command() {
21677 let (client, cache, _rx) = create_execution_client();
21678 let instrument = register_test_instrument(&client, &cache);
21679 let id = ClientOrderId::from("MODIFY-QUEUED");
21680
21681 let mut cmd = ModifyOrder::new(
21682 trader_id(),
21683 Some(client_id()),
21684 strategy_id(),
21685 instrument,
21686 id,
21687 Some(VenueOrderId::from("789")),
21688 Some(Quantity::from("0.2000")),
21689 Some(Price::from("2362.00")),
21690 None,
21691 UUID4::new(),
21692 UnixNanos::default(),
21693 None,
21694 None,
21695 );
21696 let old_command = cmd.command_id;
21697 let old_cmd = cmd.clone();
21698 client.dispatch.set_pending_modify_action(&cmd);
21699 assert!(
21700 client
21701 .dispatch
21702 .clear_pending_modify_if_command(&id, old_command)
21703 );
21704 cmd.command_id = UUID4::new();
21705 cmd.price = Some(Price::from("2363.00"));
21706 client.dispatch.set_pending_modify_action(&cmd);
21707 client.dispatch.set_pending_modify_shape(
21708 &cmd,
21709 OrderShapeSnapshot {
21710 quantity: cmd.quantity.unwrap(),
21711 price: cmd.price,
21712 trigger_price: cmd.trigger_price,
21713 },
21714 );
21715
21716 client.dispatch.set_pending_modify_shape(
21717 &old_cmd,
21718 OrderShapeSnapshot {
21719 quantity: old_cmd.quantity.unwrap(),
21720 price: old_cmd.price,
21721 trigger_price: old_cmd.trigger_price,
21722 },
21723 );
21724
21725 client
21726 .dispatch
21727 .set_pending_modify_nonce(&id, old_command, 0, 23);
21728 assert_eq!(client.dispatch.pending_modify_nonce(&id), None);
21729 assert!(
21730 !client
21731 .dispatch
21732 .clear_pending_modify_if_command(&id, old_command)
21733 );
21734 assert_eq!(
21735 client.dispatch.pending_order_action(&id),
21736 Some(PendingOrderAction::Modify)
21737 );
21738 client
21739 .dispatch
21740 .set_pending_modify_nonce(&id, cmd.command_id, 0, 24);
21741 client.dispatch.clear_pending_modify_if_update(
21742 &id,
21743 Quantity::from("0.2000"),
21744 Some(Price::from("2362.00")),
21745 None,
21746 );
21747 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 24)));
21748 client.dispatch.clear_pending_modify_if_update(
21749 &id,
21750 Quantity::from("0.2000"),
21751 Some(Price::from("2363.00")),
21752 None,
21753 );
21754 assert_eq!(client.dispatch.pending_order_action(&id), None);
21755 }
21756
21757 #[tokio::test]
21758 async fn handle_send_tx_ack_removes_hash_matched_entry() {
21759 let (client, cache, mut rx) = create_execution_client();
21760 let instrument_id = register_test_instrument(&client, &cache);
21761 let mut factory = test_order_factory();
21762 let order_a = test_limit_order(&mut factory, instrument_id, "ACK-A");
21763 let order_b = test_limit_order(&mut factory, instrument_id, "ACK-B");
21764 enqueue_create(&client, &order_a, 10);
21765 enqueue_create(&client, &order_b, 11);
21766
21767 let acked = handle_send_tx_ack(
21769 &client.dispatch,
21770 Some(TEST_ACCOUNT_INDEX_I64),
21771 200,
21772 Some("hash0b"),
21773 );
21774
21775 let acked = acked.expect("create ack");
21776 assert!(matches!(acked.kind, PendingSendTxKind::Create { .. }));
21777 assert!(matches!(
21778 AckedOrderProbe::from_pending(&acked),
21779 Some(AckedOrderProbe {
21780 nonce: 11,
21781 connection_epoch: 0,
21782 ..
21783 }),
21784 ));
21785 assert_eq!(client.dispatch.pending_sendtx_len(), 1, "only B pops");
21786 let head = client.dispatch.pop_pending_sendtx_head().unwrap();
21787 match head.kind {
21788 PendingSendTxKind::Create { order, .. } => {
21789 assert_eq!(order.client_order_id(), order_a.client_order_id());
21790 }
21791 _ => panic!("expected Create kind"),
21792 }
21793
21794 assert!(
21795 tokio::time::timeout(Duration::from_millis(50), rx.recv())
21796 .await
21797 .is_err(),
21798 "ack must not emit an event",
21799 );
21800 }
21801
21802 #[rstest]
21803 #[case::executed(LighterTxStatus::Executed, false)]
21804 #[case::pending_final(LighterTxStatus::PendingFinal, true)]
21805 #[tokio::test]
21806 async fn acknowledged_create_probe_retains_unqueryable_order(
21807 #[case] status: LighterTxStatus,
21808 #[case] pending: bool,
21809 ) {
21810 let (client, cache, mut rx) = create_execution_client();
21811 let instrument_id = register_test_instrument(&client, &cache);
21812 let mut factory = test_order_factory();
21813 let order = test_limit_order(&mut factory, instrument_id, "PROBE-UNQUERYABLE");
21814 let index = enqueue_create(&client, &order, TEST_SUBMISSION_NONCE);
21815 let submission = client.dispatch.pop_pending_sendtx_head().unwrap();
21816
21817 let tx = LighterTx {
21818 code: 200,
21819 message: None,
21820 hash: submission.tx_hash.clone(),
21821 tx_type: LighterTxType::CreateOrder as u8,
21822 info: serde_json::json!({"ClientOrderIndex": index}).to_string(),
21823 event_info: serde_json::json!({"ae": ""}).to_string(),
21824 status,
21825 account_index: TEST_ACCOUNT_INDEX_I64,
21826 nonce: TEST_SUBMISSION_NONCE,
21827 api_key_index: TEST_API_KEY_INDEX,
21828 };
21829
21830 let (http_client, server) = spawn_create_probe_server(&tx).await;
21831 let mut context = acked_probe_context(&client);
21832 context.http_client = http_client;
21833 let cloid = order.client_order_id();
21834 assert!(
21835 client
21836 .dispatch
21837 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE)
21838 );
21839
21840 probe_acked_create(
21841 &AckedOrderProbe::from_pending(&submission).unwrap(),
21842 &context,
21843 )
21844 .await
21845 .unwrap();
21846 server.abort();
21847 let _ = server.await;
21848
21849 let identity = client.dispatch.order_identity(&cloid).unwrap();
21850 assert_eq!(
21851 client
21852 .dispatch
21853 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE),
21854 pending
21855 );
21856 assert_eq!(identity.instrument_id, instrument_id);
21857 assert_eq!(identity.strategy_id, strategy_id());
21858 assert_eq!(identity.order_side, order.order_side());
21859 assert_eq!(identity.order_type, order.order_type());
21860 assert_eq!(identity.client_order_index, index);
21861 assert_eq!(identity.venue_order_id(), None);
21862 assert!(!client.dispatch.accepted_was_emitted(&cloid));
21863 assert!(!client.dispatch.venue_id_map.contains_key(&cloid));
21864 assert_eq!(
21865 client.dispatch.cloid_map.get(&index).map(|id| *id),
21866 Some(cloid)
21867 );
21868 assert!(drain_events(&mut rx).is_empty());
21869 }
21870
21871 #[rstest]
21872 #[case::hash("hash")]
21873 #[case::type_mismatch("type")]
21874 #[case::account("account")]
21875 #[case::key("key")]
21876 #[case::nonce("nonce")]
21877 #[case::index("index")]
21878 #[tokio::test]
21879 async fn acknowledged_create_probe_requires_exact_http_transaction_identity(
21880 #[case] mismatch: &str,
21881 ) {
21882 let (client, cache, mut rx) = create_execution_client();
21883 let instrument_id = register_test_instrument(&client, &cache);
21884 let mut factory = test_order_factory();
21885 let order = test_limit_order(&mut factory, instrument_id, "PROBE-IDENTITY");
21886 let index = enqueue_create(&client, &order, TEST_SUBMISSION_NONCE);
21887 let submission = client.dispatch.pop_pending_sendtx_head().unwrap();
21888
21889 let mut tx = LighterTx {
21890 code: 200,
21891 message: None,
21892 hash: submission.tx_hash.clone(),
21893 tx_type: LighterTxType::CreateOrder as u8,
21894 info: serde_json::json!({"ClientOrderIndex": index}).to_string(),
21895 event_info: serde_json::json!({"ae": "foreign transaction failed"}).to_string(),
21896 status: LighterTxStatus::Failed,
21897 account_index: TEST_ACCOUNT_INDEX_I64,
21898 nonce: TEST_SUBMISSION_NONCE,
21899 api_key_index: TEST_API_KEY_INDEX,
21900 };
21901
21902 match mismatch {
21903 "hash" => tx.hash = "foreign".to_string(),
21904 "type" => tx.tx_type = LighterTxType::CancelOrder as u8,
21905 "account" => tx.account_index += 1,
21906 "key" => tx.api_key_index += 1,
21907 "nonce" => tx.nonce += 1,
21908 "index" => tx.info = serde_json::json!({"ClientOrderIndex": index + 1}).to_string(),
21909 _ => unreachable!(),
21910 }
21911
21912 let (http_client, server) = spawn_create_probe_server(&tx).await;
21913 let mut context = acked_probe_context(&client);
21914 context.http_client = http_client;
21915 let error = probe_acked_create(
21916 &AckedOrderProbe::from_pending(&submission).unwrap(),
21917 &context,
21918 )
21919 .await
21920 .unwrap_err();
21921 server.abort();
21922 let _ = server.await;
21923
21924 let expected = if mismatch == "index" {
21925 format!(
21926 "Lighter transaction lookup returned client_order_index {} for acknowledged create {index}",
21927 index + 1
21928 )
21929 } else {
21930 "Lighter transaction lookup did not match acknowledged create identity".to_string()
21931 };
21932
21933 let cloid = order.client_order_id();
21934 assert_eq!(error.to_string(), expected);
21935 assert!(
21936 client
21937 .dispatch
21938 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE)
21939 );
21940 assert_eq!(
21941 client
21942 .dispatch
21943 .order_identity(&cloid)
21944 .unwrap()
21945 .client_order_index,
21946 index
21947 );
21948 assert_eq!(
21949 client.dispatch.cloid_map.get(&index).map(|id| *id),
21950 Some(cloid)
21951 );
21952 assert!(!client.dispatch.accepted_was_emitted(&cloid));
21953 assert!(!client.dispatch.venue_id_map.contains_key(&cloid));
21954 assert!(drain_events(&mut rx).is_empty());
21955 }
21956
21957 async fn spawn_create_probe_server(
21958 tx: &LighterTx,
21959 ) -> (LighterHttpClient, tokio::task::JoinHandle<()>) {
21960 let body = serde_json::to_string(tx).unwrap();
21961
21962 let app = Router::new()
21963 .route(
21964 "/api/v1/accountActiveOrders",
21965 get(|| async { r#"{"code":200,"orders":[]}"# }),
21966 )
21967 .route(
21968 "/api/v1/tx",
21969 get(move || {
21970 let body = body.clone();
21971 async move { body }
21972 }),
21973 );
21974
21975 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
21976 let addr = listener.local_addr().unwrap();
21977 let server = tokio::spawn(async move {
21978 axum::serve(listener, app).await.unwrap();
21979 });
21980
21981 let client = LighterHttpClient::new(
21982 LighterEnvironment::Testnet,
21983 Some(format!("http://{addr}")),
21984 1,
21985 None,
21986 )
21987 .unwrap();
21988 (client, server)
21989 }
21990
21991 #[rstest]
21992 #[case::pending(LighterTxStatus::Pending, ControlFlow::Continue(false))]
21993 #[case::executed(LighterTxStatus::Executed, ControlFlow::Continue(true))]
21994 #[case::failed(LighterTxStatus::Failed, ControlFlow::Break(()))]
21995 #[tokio::test]
21996 async fn acknowledged_create_probe_handles_empty_returned_signatures(
21997 #[case] status: LighterTxStatus,
21998 #[case] expected: ControlFlow<(), bool>,
21999 ) {
22000 let (client, cache, mut rx) = create_execution_client();
22001 let instrument_id = register_test_instrument(&client, &cache);
22002 let mut factory = test_order_factory();
22003 let order = test_limit_order(&mut factory, instrument_id, "PROBE-EMPTY-SIG");
22004 let index = enqueue_create(&client, &order, TEST_SUBMISSION_NONCE);
22005 let submission = client.dispatch.pop_pending_sendtx_head().unwrap();
22006
22007 let tx = LighterTx {
22008 code: 200,
22009 message: None,
22010 hash: submission.tx_hash.clone(),
22011 tx_type: LighterTxType::CreateOrder as u8,
22012 info: serde_json::json!({
22013 "ClientOrderIndex": index,
22014 "Sig": "",
22015 "L1Sig": "",
22016 })
22017 .to_string(),
22018 event_info: serde_json::json!({"ae": ""}).to_string(),
22019 status,
22020 account_index: TEST_ACCOUNT_INDEX_I64,
22021 nonce: TEST_SUBMISSION_NONCE,
22022 api_key_index: TEST_API_KEY_INDEX,
22023 };
22024
22025 let (http_client, server) = spawn_create_probe_server(&tx).await;
22026 let mut context = acked_probe_context(&client);
22027 context.http_client = http_client;
22028 let probe = AckedOrderProbe::from_pending(&submission).unwrap();
22029
22030 let result = probe_acked_create_tx(&probe, &context, 1).await.unwrap();
22031 server.abort();
22032 let _ = server.await;
22033
22034 let cloid = order.client_order_id();
22035 let events = drain_events(&mut rx);
22036 assert_eq!(result, expected);
22037 assert_eq!(
22038 client
22039 .dispatch
22040 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE),
22041 status != LighterTxStatus::Failed,
22042 );
22043 assert_eq!(events.len(), usize::from(status == LighterTxStatus::Failed));
22044
22045 if status == LighterTxStatus::Failed {
22046 let ExecutionEvent::Order(OrderEventAny::Rejected(rejected)) = &events[0] else {
22047 panic!("expected Rejected, was {:?}", events[0]);
22048 };
22049
22050 assert_eq!(rejected.client_order_id, cloid);
22051 assert_eq!(
22052 rejected.reason.as_str(),
22053 format!(
22054 "Lighter sequencer rejected acknowledged create transaction {}: transaction failed without an application error",
22055 submission.tx_hash,
22056 ),
22057 );
22058 }
22059 }
22060
22061 #[rstest]
22062 #[case::returned("returned-signature")]
22063 #[case::empty("")]
22064 fn acknowledged_create_tx_validation_requires_exact_identity(#[case] signature: &str) {
22065 let tx = LighterTx {
22066 code: 200,
22067 message: None,
22068 hash: "ABCDEF".to_string(),
22069 tx_type: LighterTxType::CreateOrder as u8,
22070 info: serde_json::json!({
22071 "ClientOrderIndex": 42,
22072 "Sig": signature,
22073 "L1Sig": signature,
22074 })
22075 .to_string(),
22076 event_info: serde_json::json!({"ae": ""}).to_string(),
22077 status: LighterTxStatus::Failed,
22078 account_index: TEST_ACCOUNT_INDEX_I64,
22079 nonce: 10,
22080 api_key_index: TEST_API_KEY_INDEX,
22081 };
22082 assert!(
22083 validate_acked_create_tx(
22084 &tx,
22085 TEST_ACCOUNT_INDEX_I64,
22086 TEST_API_KEY_INDEX,
22087 42,
22088 10,
22089 "0xabcdef",
22090 )
22091 .is_ok(),
22092 );
22093
22094 let mismatches = [
22095 LighterTx {
22096 hash: "different".to_string(),
22097 ..tx.clone()
22098 },
22099 LighterTx {
22100 tx_type: LighterTxType::CancelOrder as u8,
22101 ..tx.clone()
22102 },
22103 LighterTx {
22104 account_index: TEST_ACCOUNT_INDEX_I64 + 1,
22105 ..tx.clone()
22106 },
22107 LighterTx {
22108 nonce: 11,
22109 ..tx.clone()
22110 },
22111 LighterTx {
22112 api_key_index: TEST_API_KEY_INDEX + 1,
22113 ..tx.clone()
22114 },
22115 ];
22116
22117 for mismatch in mismatches {
22118 let error = validate_acked_create_tx(
22119 &mismatch,
22120 TEST_ACCOUNT_INDEX_I64,
22121 TEST_API_KEY_INDEX,
22122 42,
22123 10,
22124 "abcdef",
22125 )
22126 .expect_err("mismatched transaction identity must fail");
22127 assert_eq!(
22128 error.to_string(),
22129 "Lighter transaction lookup did not match acknowledged create identity",
22130 );
22131 }
22132
22133 let wrong_index = LighterTx {
22134 info: serde_json::json!({"ClientOrderIndex": 43}).to_string(),
22135 ..tx
22136 };
22137
22138 let error = validate_acked_create_tx(
22139 &wrong_index,
22140 TEST_ACCOUNT_INDEX_I64,
22141 TEST_API_KEY_INDEX,
22142 42,
22143 10,
22144 "abcdef",
22145 )
22146 .expect_err("mismatched client order index must fail");
22147 assert_eq!(
22148 error.to_string(),
22149 "Lighter transaction lookup returned client_order_index 43 for acknowledged create 42",
22150 );
22151 }
22152
22153 #[rstest]
22154 #[case(41, true)]
22155 #[case(42, true)]
22156 #[case(43, true)]
22157 #[case(44, false)]
22158 fn acknowledged_group_tx_validation_requires_membership(
22159 #[case] client_order_index: i64,
22160 #[case] expected: bool,
22161 #[values("returned-signature", "")] signature: &str,
22162 ) {
22163 let tx = LighterTx {
22164 code: 200,
22165 message: None,
22166 hash: "ABCDEF".to_string(),
22167 tx_type: LighterTxType::CreateGroupedOrders as u8,
22168 info: serde_json::json!({
22169 "Orders": [
22170 {"ClientOrderIndex": 41}, {"ClientOrderIndex": 42}, {"ClientOrderIndex": 43}
22171 ],
22172 "Sig": signature,
22173 "L1Sig": signature,
22174 })
22175 .to_string(),
22176 event_info: serde_json::json!({"ae": ""}).to_string(),
22177 status: LighterTxStatus::Executed,
22178 account_index: TEST_ACCOUNT_INDEX_I64,
22179 nonce: 10,
22180 api_key_index: TEST_API_KEY_INDEX,
22181 };
22182
22183 let result = validate_acked_create_tx(
22184 &tx,
22185 TEST_ACCOUNT_INDEX_I64,
22186 TEST_API_KEY_INDEX,
22187 client_order_index,
22188 10,
22189 "0xabcdef",
22190 );
22191 assert_eq!(result.is_ok(), expected);
22192
22193 if !expected {
22194 assert_eq!(
22195 result.unwrap_err().to_string(),
22196 "Lighter transaction lookup did not include acknowledged client_order_index 44"
22197 );
22198 }
22199 }
22200
22201 #[tokio::test]
22202 async fn send_tx_responses_only_attribute_within_their_connection_epoch() {
22203 let (client, _cache, mut rx) = create_execution_client();
22204 for (connection_epoch, nonce) in [(4, 20), (5, 21)] {
22205 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
22206 batch_id: None,
22207 connection_epoch,
22208 kind: PendingSendTxKind::Other,
22209 submitted_at: UnixNanos::from(1_000_000_000 + nonce as u64),
22210 nonce,
22211 api_key_index: TEST_API_KEY_INDEX,
22212 tx_hash: "same-hash".to_string(),
22213 });
22214 }
22215
22216 let acked =
22217 handle_send_tx_ack_for_connection(&client.dispatch, None, 5, 200, Some("same-hash"))
22218 .expect("replacement epoch response should match");
22219 assert_eq!(acked.connection_epoch, 5);
22220 assert_eq!(acked.nonce, 21);
22221 {
22222 let queue = client.dispatch.pending_sendtx.lock();
22223 assert_eq!(queue.len(), 1);
22224 assert_eq!(queue[0].connection_epoch, 4);
22225 assert_eq!(queue[0].nonce, 20);
22226 }
22227
22228 assert!(!handle_send_tx_rejection_for_connection(
22229 &client.dispatch,
22230 &client.emitter,
22231 None,
22232 4,
22233 UnixNanos::from(1_000_000_100),
22234 SendTxRejectionSource::Ack,
22235 Some(21702),
22236 "invalid price",
22237 Some("same-hash"),
22238 ));
22239 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
22240 assert!(
22241 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22242 .await
22243 .is_err(),
22244 "non-order responses must not emit order events",
22245 );
22246 }
22247
22248 #[tokio::test]
22249 async fn handle_send_tx_ack_unmatched_hash_pops_nothing() {
22250 let (client, cache, mut rx) = create_execution_client();
22251 let instrument_id = register_test_instrument(&client, &cache);
22252 let mut factory = test_order_factory();
22253 let order = test_limit_order(&mut factory, instrument_id, "ACK-UNMATCHED");
22254 enqueue_create(&client, &order, 10);
22255
22256 let acked = handle_send_tx_ack(
22257 &client.dispatch,
22258 Some(TEST_ACCOUNT_INDEX_I64),
22259 200,
22260 Some("0xabc"),
22261 );
22262
22263 assert!(acked.is_none());
22264 assert_eq!(
22265 client.dispatch.pending_sendtx_len(),
22266 1,
22267 "an echoed hash with no matching entry must not pop the head",
22268 );
22269 assert!(
22270 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22271 .await
22272 .is_err(),
22273 "unmatched ack must not emit an event",
22274 );
22275 }
22276
22277 #[tokio::test]
22278 async fn handle_send_tx_ack_hashless_with_multiple_pending_pops_nothing() {
22279 let (client, _cache, _rx) = create_execution_client();
22280 enqueue_other(&client, 10);
22281 enqueue_other(&client, 11);
22282
22283 let acked = handle_send_tx_ack(&client.dispatch, Some(TEST_ACCOUNT_INDEX_I64), 200, None);
22284
22285 assert!(acked.is_none());
22286 assert_eq!(
22287 client.dispatch.pending_sendtx_len(),
22288 2,
22289 "a hashless ack is ambiguous when multiple transactions are pending",
22290 );
22291 }
22292
22293 #[tokio::test]
22294 async fn prepared_create_tx_hash_round_trips_through_ack() {
22295 let (client, cache, _rx) = create_execution_client();
22298 let instrument_id = register_test_instrument(&client, &cache);
22299 let mut factory = test_order_factory();
22300 let order = test_limit_order(&mut factory, instrument_id, "ACK-REAL-HASH");
22301
22302 let credential = test_credential();
22303 let plan = client
22304 .prepare_create_order_plan(&order, None)
22305 .expect("prepare must validate");
22306 let prepared = client
22307 .fanout_dispatch_context(&credential)
22308 .expect("task admission")
22309 .sign_create_order(&plan)
22310 .expect("prepare must sign");
22311
22312 assert_eq!(prepared.tx_hash.len(), TX_HASH_BYTES * 2);
22313 assert!(
22314 prepared
22315 .tx_hash
22316 .chars()
22317 .all(|c| matches!(c, '0'..='9' | 'a'..='f')),
22318 "tx_hash must be lowercase hex, was `{}`",
22319 prepared.tx_hash,
22320 );
22321
22322 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
22323 batch_id: None,
22324 connection_epoch: 0,
22325 kind: PendingSendTxKind::Create {
22326 order: Box::new(order),
22327 client_order_index: prepared.client_order_index,
22328 },
22329 submitted_at: UnixNanos::from(1_000_000_000),
22330 nonce: prepared.nonce,
22331 api_key_index: prepared.api_key_index,
22332 tx_hash: prepared.tx_hash.clone(),
22333 });
22334
22335 let acked = handle_send_tx_ack(
22336 &client.dispatch,
22337 Some(TEST_ACCOUNT_INDEX_I64),
22338 200,
22339 Some(&prepared.tx_hash),
22340 );
22341
22342 assert!(acked.is_some());
22343 assert_eq!(
22344 client.dispatch.pending_sendtx_len(),
22345 0,
22346 "venue echo of the signed hash must match the enqueued entry",
22347 );
22348 }
22349
22350 #[tokio::test]
22351 async fn handle_send_tx_ack_retains_cancel_for_transaction_probe() {
22352 let (client, cache, mut rx) = create_execution_client();
22353 let instrument_id = register_test_instrument(&client, &cache);
22354 let client_order_id = ClientOrderId::from("ACK-CANCEL-TX");
22355 let venue_order_id = VenueOrderId::from("123");
22356 let expected_command_id =
22357 enqueue_cancel(&client, instrument_id, client_order_id, venue_order_id, 12);
22358
22359 let acked = handle_send_tx_ack(
22360 &client.dispatch,
22361 Some(TEST_ACCOUNT_INDEX_I64),
22362 200,
22363 Some("hash0c"),
22364 )
22365 .expect("acked cancel pending entry");
22366 assert!(AckedOrderProbe::from_pending(&acked).is_none());
22367
22368 let PendingSendTxKind::Cancel {
22369 instrument_id: actual_instrument_id,
22370 client_order_id: actual_client_order_id,
22371 venue_order_id: actual_venue_order_id,
22372 emit_cancel_rejected,
22373 strategy_id: actual_strategy_id,
22374 command_id,
22375 } = acked.kind
22376 else {
22377 panic!("expected acknowledged cancel")
22378 };
22379
22380 assert_eq!(command_id, Some(expected_command_id));
22381 assert_eq!(actual_instrument_id, instrument_id);
22382 assert_eq!(actual_client_order_id, client_order_id);
22383 assert_eq!(actual_venue_order_id, Some(venue_order_id));
22384 assert_eq!(actual_strategy_id, strategy_id());
22385 assert!(emit_cancel_rejected);
22386 assert_eq!(acked.connection_epoch, 0);
22387 assert_eq!(acked.nonce, 12);
22388 assert_eq!(acked.api_key_index, TEST_API_KEY_INDEX);
22389 assert_eq!(acked.tx_hash, "hash0c");
22390 assert_eq!(
22391 client.dispatch.pending_cancel_nonce(&client_order_id),
22392 Some((0, 12))
22393 );
22394 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
22395 assert!(
22396 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22397 .await
22398 .is_err(),
22399 "ack itself must not emit before the transaction probe runs",
22400 );
22401 }
22402
22403 #[rstest]
22404 #[tokio::test]
22405 async fn acknowledged_order_probe_is_retained_until_session_shutdown() {
22406 let (mut client, cache, _rx) = create_execution_client();
22407 let instrument_id = register_test_instrument(&client, &cache);
22408 let pending = PendingSendTx {
22409 batch_id: None,
22410 connection_epoch: 0,
22411 kind: PendingSendTxKind::Cancel {
22412 command_id: Some(UUID4::new()),
22413 emit_cancel_rejected: true,
22414 strategy_id: strategy_id(),
22415 instrument_id,
22416 client_order_id: ClientOrderId::from("ACK-RETAINED"),
22417 venue_order_id: Some(VenueOrderId::from("123")),
22418 },
22419 submitted_at: UnixNanos::from(1_000_000_000),
22420 nonce: 12,
22421 api_key_index: TEST_API_KEY_INDEX,
22422 tx_hash: "hash0c".to_string(),
22423 };
22424 let context = AckedOrderProbeContext {
22425 http_client: client.http_client.clone(),
22426 registry: Arc::clone(&client.registry),
22427 credential: test_credential(),
22428 dispatch: client.dispatch.clone(),
22429 account_id: client.core.account_id,
22430 clock: client.clock,
22431 emitter: client.emitter.clone(),
22432 connection_epoch: client.ws_client.connection_epoch_atomic(),
22433 cancellation_token: client.cancellation_token.clone(),
22434 pending_tasks: client.pending_tasks.spawner().expect("task spawner"),
22435 };
22436
22437 spawn_acked_order_probe(&pending, context);
22438
22439 assert_eq!(client.pending_tasks.len(), 1);
22440
22441 client.begin_session_shutdown();
22442 client
22443 .finish_session_shutdown()
22444 .await
22445 .expect("session shutdown");
22446
22447 assert!(client.pending_tasks.is_empty());
22448 }
22449
22450 #[tokio::test]
22451 async fn handle_send_tx_ack_retains_modify_for_transaction_probe() {
22452 let (client, cache, mut rx) = create_execution_client();
22453 let instrument_id = register_test_instrument(&client, &cache);
22454 let client_order_id = ClientOrderId::from("ACK-MODIFY-TX");
22455 let venue_order_id = VenueOrderId::from("456");
22456 let expected_command_id =
22457 enqueue_modify(&client, instrument_id, client_order_id, venue_order_id, 13);
22458
22459 let acked = handle_send_tx_ack(
22460 &client.dispatch,
22461 Some(TEST_ACCOUNT_INDEX_I64),
22462 200,
22463 Some("hash0d"),
22464 )
22465 .expect("acked modify pending entry");
22466 assert!(AckedOrderProbe::from_pending(&acked).is_none());
22467
22468 match acked.kind {
22469 PendingSendTxKind::Modify {
22470 instrument_id: actual_instrument_id,
22471 client_order_id: actual_client_order_id,
22472 venue_order_id: actual_venue_order_id,
22473 command_id,
22474 ..
22475 } => {
22476 assert_eq!(command_id, expected_command_id);
22477 assert_eq!(actual_instrument_id, instrument_id);
22478 assert_eq!(actual_client_order_id, client_order_id);
22479 assert_eq!(actual_venue_order_id, Some(venue_order_id));
22480 }
22481 other => panic!("expected acknowledged modify, was {other:?}"),
22482 }
22483
22484 assert_eq!(acked.connection_epoch, 0);
22485 assert_eq!(acked.nonce, 13);
22486 assert_eq!(acked.api_key_index, TEST_API_KEY_INDEX);
22487 assert_eq!(acked.tx_hash, "hash0d");
22488 assert_eq!(
22489 client.dispatch.pending_modify_nonce(&client_order_id),
22490 Some((0, 13))
22491 );
22492 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
22493 assert!(
22494 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22495 .await
22496 .is_err(),
22497 "ack itself must not emit before the transaction probe runs",
22498 );
22499 }
22500
22501 #[rstest]
22502 #[case::missing(false)]
22503 #[case::working(true)]
22504 #[tokio::test]
22505 async fn acknowledged_cancel_unresolved_status_retains_pending(#[case] found: bool) {
22506 let (client, cache, mut rx) = create_execution_client();
22507 let instrument = register_test_instrument(&client, &cache);
22508 let id = ClientOrderId::from("ACK-CANCEL-UNRESOLVED");
22509 let venue = VenueOrderId::from("123");
22510 let mut factory = test_order_factory();
22511 let order = test_limit_order(&mut factory, instrument, id.as_str());
22512 let index = client.dispatch.register_create_identity(&order).unwrap();
22513 enqueue_cancel(&client, instrument, id, venue, 12);
22514 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
22515 let order = reconciliation_raw_order(index, venue, LighterOrderStatus::Open);
22516
22517 let orders = if found { vec![order] } else { vec![] };
22518 let app = Router::new()
22519 .route(
22520 "/api/v1/accountActiveOrders",
22521 get(move || {
22522 let orders = orders.clone();
22523 async move { axum::Json(serde_json::json!({"code":200,"orders":orders})) }
22524 }),
22525 )
22526 .route(
22527 "/api/v1/accountInactiveOrders",
22528 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
22529 );
22530
22531 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22532 let addr = listener.local_addr().unwrap();
22533 let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
22534 let mut context = acked_probe_context(&client);
22535 context.http_client = LighterHttpClient::new(
22536 LighterEnvironment::Testnet,
22537 Some(format!("http://{addr}")),
22538 1,
22539 None,
22540 )
22541 .unwrap();
22542 let resolved = recover_acked_action_report(&pending, &context)
22543 .await
22544 .unwrap();
22545 server.abort();
22546 let _ = server.await;
22547 assert!(!resolved);
22548 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((0, 12)));
22549 assert_eq!(
22550 client.dispatch.pending_order_action(&id),
22551 Some(PendingOrderAction::Cancel)
22552 );
22553 let events = drain_events(&mut rx);
22554 if found {
22555 let [ExecutionEvent::Report(ExecutionReport::Order(report))] = events.as_slice() else {
22556 panic!("expected pending cancellation report, was {events:?}")
22557 };
22558
22559 assert_eq!(report.client_order_id, Some(id));
22560 assert_eq!(report.venue_order_id, venue);
22561 assert_eq!(report.order_status, OrderStatus::PendingCancel);
22562 } else {
22563 assert!(events.is_empty());
22564 }
22565 }
22566
22567 #[tokio::test]
22568 async fn send_tx_acks_recover_skip_window_past_window_size() {
22569 let (client, _cache, _rx) = create_execution_client();
22570 let window = i64::from(client.dispatch.nonce_manager.skip_window());
22571 for _ in 0..window {
22572 let nonce = client
22573 .dispatch
22574 .nonce_manager
22575 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22576 .unwrap();
22577 enqueue_other(&client, nonce);
22578 }
22579
22580 assert!(
22581 client
22582 .dispatch
22583 .nonce_manager
22584 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22585 .is_err(),
22586 "window must trip after {window} unacked txs",
22587 );
22588
22589 for i in 0..window {
22592 let acked = handle_send_tx_ack(
22593 &client.dispatch,
22594 Some(TEST_ACCOUNT_INDEX_I64),
22595 200,
22596 Some(&format!("hash{:02x}", TEST_NEXT_NONCE + i)),
22597 );
22598 assert!(matches!(
22599 acked.map(|pending| pending.kind),
22600 Some(PendingSendTxKind::Other),
22601 ));
22602 let nonce = client
22603 .dispatch
22604 .nonce_manager
22605 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22606 .unwrap();
22607 assert_eq!(nonce, TEST_NEXT_NONCE + window + i);
22608 enqueue_other(&client, nonce);
22609 }
22610 }
22611
22612 #[tokio::test]
22613 async fn handle_send_tx_rejection_rolls_back_latest_nonce() {
22614 let (client, cache, mut rx) = create_execution_client();
22615 let instrument_id = register_test_instrument(&client, &cache);
22616 let mut factory = test_order_factory();
22617 let order = test_limit_order(&mut factory, instrument_id, "REJECT-LATEST");
22618
22619 let nonce = client
22620 .dispatch
22621 .nonce_manager
22622 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22623 .unwrap();
22624 let client_order_index = client.dispatch.register_create_identity(&order).unwrap();
22625 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
22626 batch_id: None,
22627 connection_epoch: 0,
22628 kind: PendingSendTxKind::Create {
22629 order: Box::new(order.clone()),
22630 client_order_index,
22631 },
22632 submitted_at: UnixNanos::from(1_000_000_000),
22633 nonce,
22634 api_key_index: TEST_API_KEY_INDEX,
22635 tx_hash: format!("hash{nonce:02x}"),
22636 });
22637
22638 handle_send_tx_rejection(
22639 &client.dispatch,
22640 &client.emitter,
22641 Some(TEST_ACCOUNT_INDEX_I64),
22642 UnixNanos::from(1_000_000_000),
22643 SendTxRejectionSource::Ack,
22644 Some(21702),
22645 "invalid price",
22646 None,
22647 );
22648
22649 let event = recv_order_event(&mut rx).await;
22650 assert!(matches!(event, OrderEventAny::Rejected(_)));
22651 assert_nonce_reusable(&client.dispatch);
22652 }
22653
22654 #[tokio::test]
22655 async fn handle_send_tx_rejection_hashless_with_multiple_pending_is_unattributed() {
22656 let (client, cache, mut rx) = create_execution_client();
22657 let instrument_id = register_test_instrument(&client, &cache);
22658 let mut factory = test_order_factory();
22659 let order_a = test_limit_order(&mut factory, instrument_id, "REJECT-AMBIGUOUS-A");
22660 let order_b = test_limit_order(&mut factory, instrument_id, "REJECT-AMBIGUOUS-B");
22661 enqueue_create(&client, &order_a, 10);
22662 enqueue_create(&client, &order_b, 11);
22663
22664 handle_send_tx_rejection(
22665 &client.dispatch,
22666 &client.emitter,
22667 Some(TEST_ACCOUNT_INDEX_I64),
22668 UnixNanos::from(1_000_000_000),
22669 SendTxRejectionSource::Ack,
22670 Some(21702),
22671 "invalid price",
22672 None,
22673 );
22674
22675 assert_eq!(client.dispatch.pending_sendtx_len(), 2);
22676 assert!(
22677 tokio::time::timeout(Duration::from_millis(50), rx.recv())
22678 .await
22679 .is_err(),
22680 "an ambiguous hashless rejection must not reject either order",
22681 );
22682 }
22683
22684 #[tokio::test]
22685 async fn handle_send_tx_rejection_with_newer_issuance_skips_rollback() {
22686 let (client, cache, mut rx) = create_execution_client();
22687 let instrument_id = register_test_instrument(&client, &cache);
22688 let mut factory = test_order_factory();
22689 let order = test_limit_order(&mut factory, instrument_id, "REJECT-STALE");
22690
22691 let rejected_nonce = client
22692 .dispatch
22693 .nonce_manager
22694 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22695 .unwrap();
22696 let newer_nonce = client
22697 .dispatch
22698 .nonce_manager
22699 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22700 .unwrap();
22701 let client_order_index = client.dispatch.register_create_identity(&order).unwrap();
22702 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
22703 batch_id: None,
22704 connection_epoch: 0,
22705 kind: PendingSendTxKind::Create {
22706 order: Box::new(order.clone()),
22707 client_order_index,
22708 },
22709 submitted_at: UnixNanos::from(1_000_000_000),
22710 nonce: rejected_nonce,
22711 api_key_index: TEST_API_KEY_INDEX,
22712 tx_hash: format!("hash{rejected_nonce:02x}"),
22713 });
22714
22715 handle_send_tx_rejection(
22716 &client.dispatch,
22717 &client.emitter,
22718 Some(TEST_ACCOUNT_INDEX_I64),
22719 UnixNanos::from(1_000_000_000),
22720 SendTxRejectionSource::Ack,
22721 Some(21702),
22722 "invalid price",
22723 None,
22724 );
22725
22726 let event = recv_order_event(&mut rx).await;
22727 assert!(matches!(event, OrderEventAny::Rejected(_)));
22728 assert_eq!(
22731 client
22732 .dispatch
22733 .nonce_manager
22734 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22735 Some(newer_nonce),
22736 );
22737 assert_eq!(
22738 client
22739 .dispatch
22740 .nonce_manager
22741 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22742 .unwrap(),
22743 newer_nonce + 1,
22744 );
22745 }
22746
22747 async fn spawn_next_nonce_server(nonce: i64) -> String {
22748 let body = serde_json::to_string(&LighterNextNonce {
22749 code: 200,
22750 message: None,
22751 nonce,
22752 })
22753 .unwrap();
22754
22755 let app = Router::new().route(
22756 "/api/v1/nextNonce",
22757 get(move || {
22758 let body = body.clone();
22759 async move { body }
22760 }),
22761 );
22762
22763 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22764 let addr = listener.local_addr().unwrap();
22765 tokio::spawn(async move {
22766 axum::serve(listener, app).await.unwrap();
22767 });
22768
22769 format!("http://{addr}")
22770 }
22771
22772 async fn spawn_counting_next_nonce_server(nonce: i64) -> (String, Arc<AtomicUsize>) {
22773 let requests = Arc::new(AtomicUsize::new(0));
22774 let requests_for_route = Arc::clone(&requests);
22775
22776 let body = serde_json::to_string(&LighterNextNonce {
22777 code: 200,
22778 message: None,
22779 nonce,
22780 })
22781 .unwrap();
22782
22783 let app = Router::new().route(
22784 "/api/v1/nextNonce",
22785 get(move || {
22786 requests_for_route.fetch_add(1, Ordering::SeqCst);
22787 let body = body.clone();
22788 async move { body }
22789 }),
22790 );
22791
22792 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22793 let addr = listener.local_addr().unwrap();
22794 tokio::spawn(async move {
22795 axum::serve(listener, app).await.unwrap();
22796 });
22797
22798 (format!("http://{addr}"), requests)
22799 }
22800
22801 #[tokio::test]
22802 async fn hard_nonce_refresh_waits_for_submission_and_replaces_its_baseline() {
22803 let (base_url_http, requests) = spawn_counting_next_nonce_server(100).await;
22804 let mut config = test_config();
22805 config.base_url_http = Some(base_url_http);
22806 let (client, _cache, _rx) = create_execution_client_with_config(config);
22807 let credential = test_credential();
22808 client
22809 .dispatch
22810 .nonce_manager
22811 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, 40);
22812
22813 let reserved = client.build_tx_context(&credential).unwrap();
22814 assert_eq!(reserved.context.nonce, 40);
22815 let mut refresh = Box::pin(client.refresh_nonce());
22816 assert!(
22817 tokio::time::timeout(Duration::from_millis(50), refresh.as_mut())
22818 .await
22819 .is_err(),
22820 "hard refresh must wait while a submission retains its nonce",
22821 );
22822 assert_eq!(requests.load(Ordering::SeqCst), 0);
22823 assert_eq!(
22824 client.nonce_ready_connection_epoch.load(Ordering::Acquire),
22825 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
22826 );
22827 assert_eq!(
22828 client
22829 .dispatch
22830 .nonce_manager
22831 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22832 Some(40),
22833 );
22834
22835 drop(reserved);
22836 refresh.await.unwrap();
22837
22838 assert_eq!(requests.load(Ordering::SeqCst), 1);
22839 assert_eq!(
22840 client.nonce_ready_connection_epoch.load(Ordering::Acquire),
22841 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
22842 );
22843 client
22844 .nonce_ready_connection_epoch
22845 .store(client.ws_client.connection_epoch(), Ordering::Release);
22846 let next = client.build_tx_context(&credential).unwrap();
22847 assert_eq!(next.context.nonce, 100);
22848 assert_eq!(
22849 client
22850 .dispatch
22851 .nonce_manager
22852 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22853 Some(100),
22854 );
22855 }
22856
22857 #[tokio::test]
22858 async fn nonce_refresh_retry_reopens_submission_with_replacement_baseline() {
22859 let (base_url_http, requests) = spawn_counting_next_nonce_server(100).await;
22860 let mut config = test_config();
22861 config.base_url_http = Some(base_url_http);
22862 let (client, _cache, _rx) = create_execution_client_with_config(config);
22863 let credential = test_credential();
22864 let connection_epoch = client.ws_client.connection_epoch();
22865 client
22866 .dispatch
22867 .nonce_manager
22868 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, 40);
22869 client
22870 .nonce_ready_connection_epoch
22871 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
22872
22873 NonceRefreshRetry {
22874 http_client: client.http_client.clone(),
22875 dispatch: client.dispatch.clone(),
22876 credential: Some(credential.clone()),
22877 submission_gate: Arc::clone(&client.nonce_submission_gate),
22878 ready_connection_epoch: Arc::clone(&client.nonce_ready_connection_epoch),
22879 ws_client: client.ws_client.clone(),
22880 cancellation_token: client.cancellation_token.clone(),
22881 pending_tasks: client.pending_tasks.spawner().expect("task spawner"),
22882 }
22883 .spawn(connection_epoch);
22884
22885 assert_eq!(client.pending_tasks.len(), 1);
22886
22887 let err = client
22888 .build_tx_context(&credential)
22889 .expect_err("submission must stay closed until retry succeeds");
22890 assert_eq!(
22891 err.to_string(),
22892 format!("Lighter nonce refresh is pending for connection epoch {connection_epoch}"),
22893 );
22894 wait_until_async(
22895 || async {
22896 client.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
22897 },
22898 Duration::from_secs(3),
22899 )
22900 .await;
22901
22902 let reserved = client.build_tx_context(&credential).unwrap();
22903 assert_eq!(requests.load(Ordering::SeqCst), 1);
22904 assert_eq!(reserved.context.nonce, 100);
22905 assert_eq!(
22906 client
22907 .dispatch
22908 .nonce_manager
22909 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22910 Some(99),
22911 );
22912 assert_eq!(
22913 client
22914 .dispatch
22915 .nonce_manager
22916 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22917 Some(100),
22918 );
22919 assert!(client.pending_tasks.all_finished());
22920 }
22921
22922 async fn spawn_integrator_approval_rejection_server() -> String {
22923 let app = Router::new()
22924 .route(
22925 "/api/v1/getMakerOnlyApiKeys",
22926 get(|| async { r#"{"code":200,"api_key_indexes":[]}"# }),
22927 )
22928 .route(
22929 "/api/v1/sendTx",
22930 post(|| async { r#"{"code":21702,"message":"invalid approval"}"# }),
22931 );
22932 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22933 let addr = listener.local_addr().unwrap();
22934 tokio::spawn(async move {
22935 axum::serve(listener, app).await.unwrap();
22936 });
22937
22938 format!("http://{addr}")
22939 }
22940
22941 #[tokio::test]
22942 async fn integrator_approval_api_rejection_releases_reservation_and_nonce() {
22943 let mut config = test_config();
22944 config.environment = LighterEnvironment::Mainnet;
22945 config.base_url_http = Some(spawn_integrator_approval_rejection_server().await);
22946 let (client, _cache, _rx) = create_execution_client_with_config(config);
22947
22948 let err = client.submit_integrator_auto_approval().await.unwrap_err();
22949
22950 assert!(format!("{err:#}").contains("invalid approval"));
22951 assert_nonce_reusable(&client.dispatch);
22952 }
22953
22954 #[tokio::test]
22955 async fn skip_window_exhaustion_resyncs_baseline_from_venue() {
22956 let venue_next_nonce = 100;
22957 let mut config = test_config();
22958 config.base_url_http = Some(spawn_next_nonce_server(venue_next_nonce).await);
22959 let (client, _cache, _rx) = create_execution_client_with_config(config);
22960 let credential = test_credential();
22961
22962 let window = i64::from(client.dispatch.nonce_manager.skip_window());
22963 for _ in 0..window {
22964 client
22965 .dispatch
22966 .nonce_manager
22967 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
22968 .unwrap();
22969 }
22970
22971 let err = client
22972 .build_tx_context(&credential)
22973 .expect_err("window must trip");
22974 assert!(
22975 err.to_string().contains("skip-window exhausted"),
22976 "unexpected error, was {err}",
22977 );
22978
22979 wait_for_spawned_tasks(&client).await;
22980
22981 assert_eq!(
22982 client
22983 .dispatch
22984 .nonce_manager
22985 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
22986 Some(venue_next_nonce - 1),
22987 "venue resync must advance the baseline",
22988 );
22989 assert!(
22990 !client.nonce_recovery_inflight.load(Ordering::Acquire),
22991 "recovery latch must release for the next exhaustion",
22992 );
22993 let context = client.build_tx_context(&credential).unwrap().context;
22994 assert_eq!(
22995 context.nonce, venue_next_nonce,
22996 "allocation must resume at the venue nonce",
22997 );
22998 }
22999
23000 #[tokio::test]
23001 async fn skip_window_recovery_fetch_failure_releases_latch() {
23002 let app = Router::new();
23004 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
23005 let addr = listener.local_addr().unwrap();
23006 tokio::spawn(async move {
23007 axum::serve(listener, app).await.unwrap();
23008 });
23009
23010 let mut config = test_config();
23011 config.base_url_http = Some(format!("http://{addr}"));
23012 let (client, _cache, _rx) = create_execution_client_with_config(config);
23013 let credential = test_credential();
23014
23015 let window = i64::from(client.dispatch.nonce_manager.skip_window());
23016 for _ in 0..window {
23017 client
23018 .dispatch
23019 .nonce_manager
23020 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23021 .unwrap();
23022 }
23023
23024 client
23025 .build_tx_context(&credential)
23026 .expect_err("window must trip");
23027
23028 wait_for_spawned_tasks(&client).await;
23029
23030 assert_eq!(
23031 client
23032 .dispatch
23033 .nonce_manager
23034 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
23035 Some(TEST_NEXT_NONCE - 1),
23036 "failed venue fetch must leave the baseline unchanged",
23037 );
23038 assert!(
23039 !client.nonce_recovery_inflight.load(Ordering::Acquire),
23040 "failed venue fetch must release the recovery latch",
23041 );
23042 }
23043
23044 #[tokio::test]
23045 async fn skip_window_recovery_dedupes_concurrent_fetches() {
23046 let hits = Arc::new(AtomicUsize::new(0));
23047 let server_hits = Arc::clone(&hits);
23048
23049 let body = serde_json::to_string(&LighterNextNonce {
23050 code: 200,
23051 message: None,
23052 nonce: 100,
23053 })
23054 .unwrap();
23055
23056 let app = Router::new().route(
23057 "/api/v1/nextNonce",
23058 get(move || {
23059 server_hits.fetch_add(1, Ordering::AcqRel);
23060 let body = body.clone();
23061 async move { body }
23062 }),
23063 );
23064 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
23065 let addr = listener.local_addr().unwrap();
23066 tokio::spawn(async move {
23067 axum::serve(listener, app).await.unwrap();
23068 });
23069
23070 let mut config = test_config();
23071 config.base_url_http = Some(format!("http://{addr}"));
23072 let (client, _cache, _rx) = create_execution_client_with_config(config);
23073 let credential = test_credential();
23074
23075 let window = i64::from(client.dispatch.nonce_manager.skip_window());
23076 for _ in 0..window {
23077 client
23078 .dispatch
23079 .nonce_manager
23080 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23081 .unwrap();
23082 }
23083
23084 client
23087 .build_tx_context(&credential)
23088 .expect_err("window must trip");
23089 client
23090 .build_tx_context(&credential)
23091 .expect_err("window must still be exhausted");
23092
23093 wait_for_spawned_tasks(&client).await;
23094
23095 assert_eq!(
23096 hits.load(Ordering::Acquire),
23097 1,
23098 "burst exhaustion must trigger a single venue fetch",
23099 );
23100 assert!(!client.nonce_recovery_inflight.load(Ordering::Acquire));
23101 }
23102
23103 #[tokio::test]
23104 async fn refresh_nonce_releases_recovery_latch() {
23105 let venue_next_nonce = 77;
23106 let mut config = test_config();
23107 config.base_url_http = Some(spawn_next_nonce_server(venue_next_nonce).await);
23108 let (client, _cache, _rx) = create_execution_client_with_config(config);
23109
23110 client
23112 .nonce_recovery_inflight
23113 .store(true, Ordering::Release);
23114
23115 client.refresh_nonce().await.unwrap();
23116
23117 assert!(
23118 !client.nonce_recovery_inflight.load(Ordering::Acquire),
23119 "connect-time refresh must release a stuck recovery latch",
23120 );
23121 assert_eq!(
23122 client
23123 .dispatch
23124 .nonce_manager
23125 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23126 .unwrap(),
23127 venue_next_nonce,
23128 "refresh must hard-reset allocation to the venue nonce",
23129 );
23130 }
23131
23132 #[tokio::test]
23133 async fn handle_send_tx_rejection_ack_create_emits_order_rejected() {
23134 let (client, cache, mut rx) = create_execution_client();
23135 let instrument_id = register_test_instrument(&client, &cache);
23136 let mut factory = test_order_factory();
23137 let order = test_limit_order(&mut factory, instrument_id, "REJECT-CREATE");
23138 let client_order_index = enqueue_create(&client, &order, 42);
23139
23140 handle_send_tx_rejection(
23141 &client.dispatch,
23142 &client.emitter,
23143 Some(TEST_ACCOUNT_INDEX_I64),
23144 UnixNanos::from(1_000_000_000),
23145 SendTxRejectionSource::Ack,
23146 Some(21702),
23147 "invalid price",
23148 None,
23149 );
23150
23151 let event = recv_order_event(&mut rx).await;
23152 match event {
23153 OrderEventAny::Rejected(e) => {
23154 assert_eq!(e.client_order_id, order.client_order_id());
23155 assert_eq!(e.reason, "LIGHTER_21702: invalid price");
23156 assert!(!e.due_post_only);
23157 }
23158 other => panic!("expected Rejected, was {other:?}"),
23159 }
23160
23161 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23162 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
23163 }
23164
23165 #[tokio::test]
23166 async fn handle_send_tx_rejection_ack_create_sets_due_post_only() {
23167 let (client, cache, mut rx) = create_execution_client();
23168 let instrument_id = register_test_instrument(&client, &cache);
23169 let mut factory = test_order_factory();
23170 let order = test_limit_order(&mut factory, instrument_id, "REJECT-POST-ONLY");
23171 enqueue_create(&client, &order, 43);
23172
23173 handle_send_tx_rejection(
23174 &client.dispatch,
23175 &client.emitter,
23176 Some(TEST_ACCOUNT_INDEX_I64),
23177 UnixNanos::from(1_000_000_000),
23178 SendTxRejectionSource::Ack,
23179 Some(21700),
23180 "post-only order would execute",
23181 None,
23182 );
23183
23184 let event = recv_order_event(&mut rx).await;
23185 match event {
23186 OrderEventAny::Rejected(e) => {
23187 assert_eq!(e.client_order_id, order.client_order_id());
23188 assert!(e.due_post_only);
23189 }
23190 other => panic!("expected Rejected, was {other:?}"),
23191 }
23192
23193 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23194 }
23195
23196 #[tokio::test]
23197 async fn cancel_ack_promotes_before_send_continuation() {
23198 let (client, cache, mut rx) = create_execution_client();
23199 let instrument_id = register_test_instrument(&client, &cache);
23200 let id = ClientOrderId::from("CANCEL-ACK-RACE");
23201 let venue = VenueOrderId::from("123");
23202 let first = enqueue_cancel(&client, instrument_id, id, venue, 41);
23203 let old = client.dispatch.pending_cancel_token(&id, 0, 41).unwrap();
23204 let second = UUID4::new();
23205 let pending = enqueue_cancel_pending(&client, instrument_id, id, venue, 42, second);
23206
23207 assert_eq!(
23208 client.dispatch.pending_cancel_identity(&id),
23209 Some((first, (0, 41)))
23210 );
23211 assert!(!old.is_cancelled());
23212
23213 let acked = handle_send_tx_ack_for_connection(
23214 &client.dispatch,
23215 None,
23216 0,
23217 200,
23218 Some(&pending.tx_hash),
23219 )
23220 .unwrap();
23221 let current = client.dispatch.pending_cancel_token(&id, 0, 42).unwrap();
23222 promote_cancel_submission(&client.dispatch, &pending);
23223 handle_send_tx_ack_for_connection(&client.dispatch, None, 0, 200, Some("hash29")).unwrap();
23224
23225 assert_eq!(acked.nonce, 42);
23226 assert_eq!(
23227 client.dispatch.pending_cancel_identity(&id),
23228 Some((second, (0, 42)))
23229 );
23230 assert!(old.is_cancelled());
23231 assert!(!current.is_cancelled());
23232 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23233 assert!(rx.try_recv().is_err());
23234 }
23235
23236 #[tokio::test]
23237 async fn cancel_rejection_before_send_continuation_preserves_previous_probe() {
23238 let (client, cache, mut rx) = create_execution_client();
23239 let instrument_id = register_test_instrument(&client, &cache);
23240 let id = ClientOrderId::from("CANCEL-REJECTION-RACE");
23241 let venue = VenueOrderId::from("123");
23242 let first = enqueue_cancel(&client, instrument_id, id, venue, 41);
23243 let token = client.dispatch.pending_cancel_token(&id, 0, 41).unwrap();
23244 let second = UUID4::new();
23245 let pending = enqueue_cancel_pending(&client, instrument_id, id, venue, 42, second);
23246 handle_send_tx_rejection_for_connection(
23247 &client.dispatch,
23248 &client.emitter,
23249 None,
23250 0,
23251 UnixNanos::default(),
23252 SendTxRejectionSource::Ack,
23253 Some(21727),
23254 "later cancel failed",
23255 Some(&pending.tx_hash),
23256 );
23257 promote_cancel_submission(&client.dispatch, &pending);
23258
23259 let OrderEventAny::CancelRejected(event) = recv_order_event(&mut rx).await else {
23260 panic!("Expected venue rejection");
23261 };
23262
23263 assert_eq!(event.causation_id, Some(second));
23264 assert_eq!(event.client_order_id, id);
23265 assert_eq!(event.venue_order_id, Some(venue));
23266 assert_eq!(event.reason, "LIGHTER_21727: later cancel failed");
23267 assert_eq!(
23268 client.dispatch.pending_cancel_identity(&id),
23269 Some((first, (0, 41)))
23270 );
23271 assert!(!token.is_cancelled());
23272 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
23273 assert!(rx.try_recv().is_err());
23274 }
23275
23276 #[rstest]
23277 #[case::explicit_enqueued(true)]
23278 #[case::explicit_waiting(false)]
23279 #[tokio::test]
23280 async fn implicit_cancel_enqueue_preserves_explicit_rejection(
23281 #[case] explicit_enqueued: bool,
23282 #[values(false, true)] implicit_marked: bool,
23283 ) {
23284 let (client, cache, mut rx) = create_execution_client();
23285 let instrument_id = register_test_instrument(&client, &cache);
23286 let mut factory = test_order_factory();
23287 let order = test_limit_order(&mut factory, instrument_id, "CANCEL-ELIGIBILITY");
23288 let id = order.client_order_id();
23289 let venue_id = VenueOrderId::from("123");
23290 let cause = UUID4::new();
23291 cache_pending_cancel_order(&cache, order, venue_id, Some(cause));
23292 let context = client
23293 .fanout_dispatch_context(client.credential.as_ref().unwrap())
23294 .unwrap();
23295
23296 let plan = CancelOrderPlan {
23297 command_id: Some(cause),
23298 client_order_id: id,
23299 strategy_id: strategy_id(),
23300 instrument_id,
23301 venue_order_id: Some(venue_id),
23302 market_index: TEST_MARKET_INDEX,
23303 venue_index: 123,
23304 };
23305
23306 let first = context.sign_cancel_order(&plan).unwrap();
23307 let second = context.sign_cancel_order(&plan).unwrap();
23308 let (explicit, batch) = if explicit_enqueued {
23309 (first, second)
23310 } else {
23311 (second, first)
23312 };
23313
23314 client.dispatch.register_cancel_command(id, cause, true);
23315
23316 if explicit_enqueued {
23317 context.enqueue_cancel_order(&explicit, true);
23318 client
23319 .dispatch
23320 .promote_pending_cancel(&id, cause, 0, explicit.nonce, true);
23321 }
23322
23323 let mut batch = PreparedBatchTx::from((batch, implicit_marked));
23324 if let PendingSendTxKind::Cancel { command_id, .. } = &mut batch.pending.kind {
23325 *command_id = None;
23326 }
23327
23328 batch.pending.batch_id = Some("batch-unmarked".to_string());
23329 context.enqueue_batch_tx(&batch);
23330 assert_eq!(
23331 client.dispatch.pending_cancel_nonce(&id),
23332 explicit_enqueued.then_some((0, explicit.nonce))
23333 );
23334
23335 if !explicit_enqueued {
23336 context.enqueue_cancel_order(&explicit, true);
23337 client
23338 .dispatch
23339 .promote_pending_cancel(&id, cause, 0, explicit.nonce, true);
23340 }
23341
23342 handle_send_tx_rejection_for_connection(
23343 &client.dispatch,
23344 &client.emitter,
23345 Some(TEST_ACCOUNT_INDEX_I64),
23346 0,
23347 UnixNanos::default(),
23348 SendTxRejectionSource::Ack,
23349 Some(21727),
23350 "unmarked batch failed",
23351 Some(&batch.pending.tx_hash),
23352 );
23353 assert_eq!(
23354 client.dispatch.pending_cancel_nonce(&id),
23355 Some((0, explicit.nonce))
23356 );
23357 assert!(rx.try_recv().is_err());
23358 handle_send_tx_rejection_for_connection(
23359 &client.dispatch,
23360 &client.emitter,
23361 Some(TEST_ACCOUNT_INDEX_I64),
23362 0,
23363 UnixNanos::default(),
23364 SendTxRejectionSource::Ack,
23365 Some(21727),
23366 "explicit cancel failed",
23367 Some(&explicit.tx_hash),
23368 );
23369 let event = recv_order_event(&mut rx).await;
23370 cache.borrow_mut().update_order(&event).unwrap();
23371 match event {
23372 OrderEventAny::CancelRejected(event) => {
23373 assert_eq!(event.causation_id, Some(cause));
23374 assert_eq!(event.client_order_id, id);
23375 assert_eq!(event.venue_order_id, Some(venue_id));
23376 assert_eq!(event.instrument_id, instrument_id);
23377 assert_eq!(event.strategy_id, strategy_id());
23378 assert_eq!(event.reason, "LIGHTER_21727: explicit cancel failed");
23379 }
23380 other => panic!("expected explicit cancel rejection, was {other:?}"),
23381 }
23382
23383 assert_eq!(
23384 cache.borrow().order(&id).unwrap().status(),
23385 OrderStatus::Accepted
23386 );
23387 assert_eq!(client.dispatch.pending_cancel_nonce(&id), None);
23388 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23389 }
23390
23391 #[rstest]
23392 #[case::pending(PendingOrderAction::Cancel, true)]
23393 #[case::unmarked(PendingOrderAction::CancelUnmarked, false)]
23394 #[tokio::test]
23395 async fn cancel_rejection_preserves_newer_nonce(
23396 #[case] action: PendingOrderAction,
23397 #[case] emit_cancel_rejected: bool,
23398 ) {
23399 let (client, cache, mut rx) = create_execution_client();
23400 let instrument_id = register_test_instrument(&client, &cache);
23401 let id = ClientOrderId::from("CANCEL-NEWER");
23402
23403 let cause = emit_cancel_rejected.then(UUID4::new);
23404
23405 let pending = PendingSendTx {
23406 batch_id: Some("batch-old".to_string()),
23407 connection_epoch: 3,
23408 kind: PendingSendTxKind::Cancel {
23409 command_id: cause,
23410 emit_cancel_rejected,
23411 strategy_id: strategy_id(),
23412 instrument_id,
23413 client_order_id: id,
23414 venue_order_id: Some(VenueOrderId::from("123")),
23415 },
23416 submitted_at: UnixNanos::default(),
23417 nonce: 41,
23418 api_key_index: TEST_API_KEY_INDEX,
23419 tx_hash: "old-hash".to_string(),
23420 };
23421
23422 client.dispatch.set_pending_order_action(id, action);
23423 client
23424 .dispatch
23425 .set_pending_cancel_nonce(&id, 3, 42, emit_cancel_rejected);
23426 reject_cancel_order(
23427 &client.dispatch,
23428 &client.emitter,
23429 &pending,
23430 &CommandFailure::venue_rejected("old failure"),
23431 UnixNanos::default(),
23432 );
23433 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((3, 42)));
23434 assert_eq!(client.dispatch.pending_order_action(&id), Some(action));
23435
23436 if let Some(cause) = cause {
23437 let event = recv_order_event(&mut rx).await;
23438
23439 let OrderEventAny::CancelRejected(event) = event else {
23440 panic!("expected cancel rejection");
23441 };
23442
23443 assert_eq!(event.causation_id, Some(cause));
23444 assert_eq!(event.client_order_id, id);
23445 assert_eq!(event.reason, "old failure");
23446 }
23447
23448 assert!(rx.try_recv().is_err());
23449 }
23450
23451 #[tokio::test]
23452 async fn handle_send_tx_rejection_cancel_emits_cancel_rejected() {
23453 let (client, cache, mut rx) = create_execution_client();
23454 let instrument_id = register_test_instrument(&client, &cache);
23455 let client_order_id = ClientOrderId::from("REJECT-CANCEL");
23456 let venue_order_id = VenueOrderId::from("123");
23457 let nonce = client
23458 .dispatch
23459 .nonce_manager
23460 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23461 .unwrap();
23462
23463 let cause = enqueue_cancel(
23464 &client,
23465 instrument_id,
23466 client_order_id,
23467 venue_order_id,
23468 nonce,
23469 );
23470
23471 handle_send_tx_rejection(
23472 &client.dispatch,
23473 &client.emitter,
23474 Some(TEST_ACCOUNT_INDEX_I64),
23475 UnixNanos::from(1_000_000_000),
23476 SendTxRejectionSource::Ack,
23477 Some(21727),
23478 "order is not cancelable",
23479 None,
23480 );
23481
23482 let event = recv_order_event(&mut rx).await;
23483 match event {
23484 OrderEventAny::CancelRejected(e) => {
23485 assert_eq!(e.causation_id, Some(cause));
23486 assert_eq!(e.client_order_id, client_order_id);
23487 assert_eq!(e.instrument_id, instrument_id);
23488 assert_eq!(e.venue_order_id, Some(venue_order_id));
23489 assert_eq!(e.reason, "LIGHTER_21727: order is not cancelable",);
23490 }
23491 other => panic!("expected CancelRejected, was {other:?}"),
23492 }
23493
23494 assert_eq!(
23495 client.dispatch.pending_cancel_identity(&client_order_id),
23496 None
23497 );
23498 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23499 assert_nonce_reusable(&client.dispatch);
23500 }
23501
23502 #[tokio::test]
23503 async fn handle_send_tx_rejection_modify_emits_modify_rejected() {
23504 let (client, cache, mut rx) = create_execution_client();
23505 let instrument_id = register_test_instrument(&client, &cache);
23506 let client_order_id = ClientOrderId::from("REJECT-MODIFY");
23507 let venue_order_id = VenueOrderId::from("456");
23508 let nonce = client
23509 .dispatch
23510 .nonce_manager
23511 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23512 .unwrap();
23513
23514 enqueue_modify(
23515 &client,
23516 instrument_id,
23517 client_order_id,
23518 venue_order_id,
23519 nonce,
23520 );
23521
23522 handle_send_tx_rejection(
23523 &client.dispatch,
23524 &client.emitter,
23525 Some(TEST_ACCOUNT_INDEX_I64),
23526 UnixNanos::from(1_000_000_000),
23527 SendTxRejectionSource::Ack,
23528 Some(21702),
23529 "modify rejected by venue",
23530 None,
23531 );
23532
23533 let event = recv_order_event(&mut rx).await;
23534 match event {
23535 OrderEventAny::ModifyRejected(e) => {
23536 assert_eq!(e.client_order_id, client_order_id);
23537 assert_eq!(e.instrument_id, instrument_id);
23538 assert_eq!(e.venue_order_id, Some(venue_order_id));
23539 assert_eq!(e.reason, "LIGHTER_21702: modify rejected by venue",);
23540 }
23541 other => panic!("expected ModifyRejected, was {other:?}"),
23542 }
23543
23544 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23545 assert_nonce_reusable(&client.dispatch);
23546 }
23547
23548 #[rstest]
23549 #[case::newer_modify(PendingOrderAction::Modify)]
23550 #[case::newer_cancel(PendingOrderAction::Cancel)]
23551 fn handle_send_tx_rejection_preserves_superseding_modify_action(
23552 #[case] action: PendingOrderAction,
23553 ) {
23554 let (client, cache, mut rx) = create_execution_client();
23555 let instrument = register_test_instrument(&client, &cache);
23556 let id = ClientOrderId::from("REJECT-MODIFY-SUPERSEDED");
23557 let venue = VenueOrderId::from("456");
23558 let nonce = client
23559 .dispatch
23560 .nonce_manager
23561 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23562 .unwrap();
23563 enqueue_modify(&client, instrument, id, venue, nonce);
23564 let current_nonce = client
23565 .dispatch
23566 .nonce_manager
23567 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23568 .unwrap();
23569
23570 let pending = client
23571 .dispatch
23572 .pending_sendtx
23573 .lock()
23574 .front()
23575 .unwrap()
23576 .clone();
23577
23578 let PendingSendTxKind::Modify {
23579 command_id: cause, ..
23580 } = pending.kind
23581 else {
23582 unreachable!()
23583 };
23584
23585 assert!(client.dispatch.clear_pending_modify_if_nonce(&id, 0, nonce));
23586
23587 if action == PendingOrderAction::Modify {
23588 enqueue_modify(&client, instrument, id, venue, current_nonce);
23589 } else {
23590 client.dispatch.set_pending_order_action(id, action);
23591 client
23592 .dispatch
23593 .set_pending_cancel_nonce(&id, 0, current_nonce, true);
23594 }
23595
23596 handle_send_tx_rejection(
23597 &client.dispatch,
23598 &client.emitter,
23599 Some(TEST_ACCOUNT_INDEX_I64),
23600 UnixNanos::from(1_000_000_000),
23601 SendTxRejectionSource::Ack,
23602 Some(21702),
23603 "older modify rejected by venue",
23604 Some(&format!("hash{nonce:02x}")),
23605 );
23606
23607 let ExecutionEvent::Order(OrderEventAny::ModifyRejected(event)) = rx.try_recv().unwrap()
23608 else {
23609 panic!("expected older command rejection");
23610 };
23611
23612 assert_eq!(event.client_order_id, id);
23613 assert_eq!(
23614 event.reason,
23615 "LIGHTER_21702: older modify rejected by venue"
23616 );
23617 assert_eq!(event.causation_id, Some(cause));
23618 assert!(rx.try_recv().is_err());
23619 assert_eq!(client.dispatch.pending_order_action(&id), Some(action));
23620 assert_eq!(
23621 client.dispatch.pending_modify_nonce(&id),
23622 (action == PendingOrderAction::Modify).then_some((0, current_nonce))
23623 );
23624 assert_eq!(
23625 client.dispatch.pending_cancel_nonce(&id),
23626 (action == PendingOrderAction::Cancel).then_some((0, current_nonce))
23627 );
23628 assert_eq!(
23629 client.dispatch.pending_sendtx_len(),
23630 usize::from(action == PendingOrderAction::Modify)
23631 );
23632 assert_eq!(
23633 client
23634 .dispatch
23635 .nonce_manager
23636 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23637 .unwrap(),
23638 current_nonce + 1
23639 );
23640 }
23641
23642 #[tokio::test]
23643 async fn handle_send_tx_rejection_hash_match_attributes_past_the_head() {
23644 let (client, cache, mut rx) = create_execution_client();
23645 let instrument_id = register_test_instrument(&client, &cache);
23646 let mut factory = test_order_factory();
23647 let order_a = test_limit_order(&mut factory, instrument_id, "REJECT-HASH-A");
23648 let order_b = test_limit_order(&mut factory, instrument_id, "REJECT-HASH-B");
23649 enqueue_create(&client, &order_a, 10);
23650 enqueue_create(&client, &order_b, 11);
23651
23652 handle_send_tx_rejection(
23654 &client.dispatch,
23655 &client.emitter,
23656 Some(TEST_ACCOUNT_INDEX_I64),
23657 UnixNanos::from(1_000_000_000),
23658 SendTxRejectionSource::Ack,
23659 Some(21702),
23660 "invalid price",
23661 Some("hash0b"),
23662 );
23663
23664 let event = recv_order_event(&mut rx).await;
23665 match event {
23666 OrderEventAny::Rejected(e) => {
23667 assert_eq!(e.client_order_id, order_b.client_order_id());
23668 }
23669 other => panic!("expected Rejected, was {other:?}"),
23670 }
23671
23672 assert_eq!(client.dispatch.pending_sendtx_len(), 1, "A must survive");
23673 let head = client.dispatch.pop_pending_sendtx_head().unwrap();
23674 match head.kind {
23675 PendingSendTxKind::Create { order, .. } => {
23676 assert_eq!(order.client_order_id(), order_a.client_order_id());
23677 }
23678 _ => panic!("expected Create kind"),
23679 }
23680 }
23681
23682 #[tokio::test]
23683 async fn handle_send_tx_rejection_unmatched_hash_pops_nothing() {
23684 let (client, cache, mut rx) = create_execution_client();
23685 let instrument_id = register_test_instrument(&client, &cache);
23686 let mut factory = test_order_factory();
23687 let order = test_limit_order(&mut factory, instrument_id, "REJECT-UNMATCHED");
23688 enqueue_create(&client, &order, 10);
23689
23690 handle_send_tx_rejection(
23691 &client.dispatch,
23692 &client.emitter,
23693 Some(TEST_ACCOUNT_INDEX_I64),
23694 UnixNanos::from(1_000_000_000),
23695 SendTxRejectionSource::Ack,
23696 Some(21702),
23697 "invalid price",
23698 Some("0xbeef"),
23699 );
23700
23701 assert_eq!(
23702 client.dispatch.pending_sendtx_len(),
23703 1,
23704 "an echoed hash with no matching entry must not pop the head",
23705 );
23706 assert!(
23707 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23708 .await
23709 .is_err(),
23710 "unmatched rejection must not emit an event",
23711 );
23712 }
23713
23714 #[tokio::test]
23715 async fn handle_send_tx_rejection_bare_error_within_window_attributes() {
23716 let (client, cache, mut rx) = create_execution_client();
23717 let instrument_id = register_test_instrument(&client, &cache);
23718 let mut factory = test_order_factory();
23719 let order = test_limit_order(&mut factory, instrument_id, "BARE-IN");
23720 enqueue_create(&client, &order, 50);
23721
23722 let within_window = UnixNanos::from(1_000_000_000 + 500 * 1_000_000);
23723 handle_send_tx_rejection(
23724 &client.dispatch,
23725 &client.emitter,
23726 Some(TEST_ACCOUNT_INDEX_I64),
23727 within_window,
23728 SendTxRejectionSource::BareError,
23729 Some(21149),
23730 "integrator is not approved",
23731 None,
23732 );
23733
23734 let event = recv_order_event(&mut rx).await;
23735 assert!(matches!(event, OrderEventAny::Rejected(_)));
23736 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23737 }
23738
23739 #[tokio::test]
23740 async fn handle_send_tx_rejection_bare_error_outside_window_skips() {
23741 let (client, cache, mut rx) = create_execution_client();
23742 let instrument_id = register_test_instrument(&client, &cache);
23743 let mut factory = test_order_factory();
23744 let order = test_limit_order(&mut factory, instrument_id, "BARE-OUT");
23745 enqueue_create(&client, &order, 60);
23746
23747 let outside_window = UnixNanos::from(1_000_000_000 + 2_000 * 1_000_000);
23748 handle_send_tx_rejection(
23749 &client.dispatch,
23750 &client.emitter,
23751 Some(TEST_ACCOUNT_INDEX_I64),
23752 outside_window,
23753 SendTxRejectionSource::BareError,
23754 Some(99),
23755 "late error",
23756 None,
23757 );
23758
23759 assert_eq!(
23760 client.dispatch.pending_sendtx_len(),
23761 1,
23762 "head must remain queued past the 1s attribution window",
23763 );
23764 assert!(
23765 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23766 .await
23767 .is_err(),
23768 "no event must be emitted outside the window",
23769 );
23770 }
23771
23772 #[tokio::test]
23773 async fn handle_send_tx_rejection_bare_error_with_multiple_pending_skips() {
23774 let (client, cache, mut rx) = create_execution_client();
23775 let instrument_id = register_test_instrument(&client, &cache);
23776 let mut factory = test_order_factory();
23777 let order_a = test_limit_order(&mut factory, instrument_id, "BARE-MULTI-A");
23778 let order_b = test_limit_order(&mut factory, instrument_id, "BARE-MULTI-B");
23779 enqueue_create(&client, &order_a, 70);
23780 enqueue_create(&client, &order_b, 71);
23781
23782 handle_send_tx_rejection(
23783 &client.dispatch,
23784 &client.emitter,
23785 Some(TEST_ACCOUNT_INDEX_I64),
23786 UnixNanos::from(1_000_000_000 + 500 * 1_000_000),
23787 SendTxRejectionSource::BareError,
23788 Some(21149),
23789 "integrator is not approved",
23790 None,
23791 );
23792
23793 assert_eq!(client.dispatch.pending_sendtx_len(), 2);
23794 assert!(
23795 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23796 .await
23797 .is_err(),
23798 "a bare error cannot be attributed with multiple transactions pending",
23799 );
23800 }
23801
23802 #[tokio::test]
23803 async fn handle_send_tx_rejection_other_kind_rolls_back_latest_nonce() {
23804 let (client, _cache, _rx) = create_execution_client();
23805 let nonce = client
23806 .dispatch
23807 .nonce_manager
23808 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23809 .unwrap();
23810 enqueue_other(&client, nonce);
23811
23812 let needs_resync = handle_send_tx_rejection(
23813 &client.dispatch,
23814 &client.emitter,
23815 Some(TEST_ACCOUNT_INDEX_I64),
23816 UnixNanos::from(1_000_000_000),
23817 SendTxRejectionSource::Ack,
23818 Some(23000),
23819 "Too Many Requests",
23820 None,
23821 );
23822
23823 assert!(!needs_resync, "rate-limit rejection must not force resync");
23824 assert_nonce_reusable(&client.dispatch);
23825 }
23826
23827 #[tokio::test]
23828 async fn handle_send_tx_rejection_other_kind_skips_rollback_with_newer_issuance() {
23829 let (client, _cache, _rx) = create_execution_client();
23830 let rejected_nonce = client
23831 .dispatch
23832 .nonce_manager
23833 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23834 .unwrap();
23835 let newer_nonce = client
23836 .dispatch
23837 .nonce_manager
23838 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
23839 .unwrap();
23840 enqueue_other(&client, rejected_nonce);
23841
23842 handle_send_tx_rejection(
23843 &client.dispatch,
23844 &client.emitter,
23845 Some(TEST_ACCOUNT_INDEX_I64),
23846 UnixNanos::from(1_000_000_000),
23847 SendTxRejectionSource::Ack,
23848 Some(23000),
23849 "Too Many Requests",
23850 None,
23851 );
23852
23853 assert_eq!(
23854 client
23855 .dispatch
23856 .nonce_manager
23857 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
23858 Some(newer_nonce),
23859 "non-latest rejection must leave last_issued alone",
23860 );
23861 }
23862
23863 #[tokio::test]
23864 async fn handle_send_tx_rejection_invalid_nonce_signals_resync() {
23865 let (client, _cache, _rx) = create_execution_client();
23866
23867 enqueue_other(&client, 70);
23868 let attributed = handle_send_tx_rejection(
23869 &client.dispatch,
23870 &client.emitter,
23871 Some(TEST_ACCOUNT_INDEX_I64),
23872 UnixNanos::from(1_000_000_000),
23873 SendTxRejectionSource::Ack,
23874 Some(LIGHTER_ERROR_CODE_INVALID_NONCE),
23875 "invalid nonce",
23876 None,
23877 );
23878 assert!(attributed, "attributed invalid nonce must signal resync");
23879
23880 let unattributed = handle_send_tx_rejection(
23881 &client.dispatch,
23882 &client.emitter,
23883 Some(TEST_ACCOUNT_INDEX_I64),
23884 UnixNanos::from(1_000_000_000),
23885 SendTxRejectionSource::Ack,
23886 Some(LIGHTER_ERROR_CODE_INVALID_NONCE),
23887 "invalid nonce",
23888 None,
23889 );
23890 assert!(
23891 unattributed,
23892 "unattributed invalid nonce must still signal resync",
23893 );
23894
23895 enqueue_other(&client, 71);
23896 let other_code = handle_send_tx_rejection(
23897 &client.dispatch,
23898 &client.emitter,
23899 Some(TEST_ACCOUNT_INDEX_I64),
23900 UnixNanos::from(1_000_000_000),
23901 SendTxRejectionSource::Ack,
23902 Some(21702),
23903 "invalid price",
23904 None,
23905 );
23906 assert!(!other_code, "other rejection codes must not force resync");
23907 }
23908
23909 #[tokio::test]
23910 async fn handle_send_tx_rejection_other_kind_logs_and_skips_emit() {
23911 let (client, _cache, mut rx) = create_execution_client();
23912 enqueue_other(&client, 70);
23913
23914 handle_send_tx_rejection(
23915 &client.dispatch,
23916 &client.emitter,
23917 Some(TEST_ACCOUNT_INDEX_I64),
23918 UnixNanos::from(1_000_000_000),
23919 SendTxRejectionSource::Ack,
23920 Some(21727),
23921 "invalid client order index",
23922 None,
23923 );
23924
23925 assert_eq!(client.dispatch.pending_sendtx_len(), 0, "Other head pops");
23926 assert!(
23927 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23928 .await
23929 .is_err(),
23930 "Other-kind rejection must not emit OrderRejected",
23931 );
23932 }
23933
23934 #[tokio::test]
23935 async fn handle_send_tx_rejection_empty_queue_logs_warn() {
23936 let (client, _cache, mut rx) = create_execution_client();
23937
23938 handle_send_tx_rejection(
23939 &client.dispatch,
23940 &client.emitter,
23941 Some(TEST_ACCOUNT_INDEX_I64),
23942 UnixNanos::from(1_000_000_000),
23943 SendTxRejectionSource::Ack,
23944 Some(1),
23945 "no pending",
23946 None,
23947 );
23948
23949 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
23950 assert!(
23951 tokio::time::timeout(Duration::from_millis(50), rx.recv())
23952 .await
23953 .is_err(),
23954 );
23955 }
23956
23957 #[rstest]
23958 #[case::standard_no_override(LighterAccountTier::Standard, None, 60, None)]
23959 #[case::standard_zero_is_default(LighterAccountTier::Standard, Some(0), 60, None)]
23960 #[case::standard_override_above_tier(
23961 LighterAccountTier::Standard,
23962 Some(24_000),
23963 24_000,
23964 Some(TierCrossCheck::AboveTier { documented: 60 })
23965 )]
23966 #[case::premium_raise_hint(
23967 LighterAccountTier::Premium,
23968 None,
23969 60,
23970 Some(TierCrossCheck::RaiseHint { documented: 24_000 })
23971 )]
23972 #[case::premium_configured_no_advisory(LighterAccountTier::Premium, Some(24_000), 24_000, None)]
23973 #[case::unknown_no_advisory(LighterAccountTier::Unknown(7), None, 60, None)]
23974 fn test_tier_quota_report(
23975 #[case] tier: LighterAccountTier,
23976 #[case] configured: Option<u32>,
23977 #[case] expected_active: u32,
23978 #[case] expected_cross_check: Option<TierCrossCheck>,
23979 ) {
23980 assert_eq!(
23981 tier_quota_report(tier, configured, 60),
23982 (expected_active, expected_cross_check),
23983 );
23984 }
23985
23986 #[rstest]
23987 fn cold_first_frame_updates_cached_shape() {
23988 let mut rig = dispatcher_rig("COLD-SHAPE");
23989 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
23990 rig.dispatch.forget_cloid(frame.client_order_index);
23991 let order = OrderTestBuilder::new(OrderType::Limit)
23992 .trader_id(trader_id())
23993 .strategy_id(strategy_id())
23994 .instrument_id(rig.instrument_id)
23995 .client_order_id(rig.cloid)
23996 .side(OrderSide::Buy)
23997 .quantity(Quantity::from("0.1000"))
23998 .price(Price::from("2352.74"))
23999 .build();
24000 let accepted =
24001 TestOrderEventStubs::accepted(&order, account_id(), VenueOrderId::new(&frame.order_id));
24002 let mut order = order;
24003 order.apply(accepted).unwrap();
24004 rig.dispatch.seed_order_restorations(vec![order]);
24005 frame.initial_base_amount = Decimal::from_str("0.0400").unwrap();
24006 frame.remaining_base_amount = frame.initial_base_amount;
24007 dispatch_lighter_order(
24008 &frame,
24009 &rig.dispatch,
24010 &rig.emitter,
24011 &rig.registry,
24012 account_id(),
24013 trader_id(),
24014 UnixNanos::from(2),
24015 );
24016 let events = drain_events(&mut rig.rx);
24017 assert_eq!(events.len(), 1);
24018
24019 let ExecutionEvent::Order(OrderEventAny::Updated(update)) = &events[0] else {
24020 panic!("expected restored shape update");
24021 };
24022
24023 assert_eq!(update.quantity, Quantity::from("0.0400"));
24024 assert_eq!(update.price, Some(Price::from("2352.74")));
24025 assert_eq!(update.client_order_id, rig.cloid);
24026 assert_eq!(
24027 update.venue_order_id,
24028 Some(VenueOrderId::new(&frame.order_id))
24029 );
24030 }
24031
24032 #[rstest]
24033 #[case(false)]
24034 #[case(true)]
24035 fn native_group_child_sizing_preserves_real_fill(#[case] trade_first: bool) {
24036 let mut rig = dispatcher_rig("CHILD-SIZING");
24037 let order = OrderTestBuilder::new(OrderType::StopLimit)
24038 .trader_id(trader_id())
24039 .strategy_id(strategy_id())
24040 .instrument_id(rig.instrument_id)
24041 .client_order_id(rig.cloid)
24042 .side(OrderSide::Buy)
24043 .quantity(Quantity::from("0.1000"))
24044 .price(Price::from("2352.74"))
24045 .trigger_price(Price::from("2300.00"))
24046 .parent_order_id(ClientOrderId::new("PARENT-CHILD-SIZING"))
24047 .reduce_only(true)
24048 .build();
24049 rig.dispatch.register_create_identity(&order).unwrap();
24050 rig.dispatch
24051 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
24052 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
24053 frame.initial_base_amount = Decimal::ZERO;
24054 frame.remaining_base_amount = Decimal::ZERO;
24055 frame.order_type = LighterOrderKind::StopLossLimit;
24056 frame.trigger_price = Decimal::from_str("2300.00").unwrap();
24057 frame.reduce_only = true;
24058 let cache = Rc::new(RefCell::new(Cache::default()));
24059 cache
24060 .borrow_mut()
24061 .add_instrument(
24062 LIGHTER_INSTRUMENT_CACHE
24063 .get(&rig.instrument_id)
24064 .unwrap()
24065 .value()
24066 .clone(),
24067 )
24068 .unwrap();
24069 cache
24070 .borrow_mut()
24071 .add_order(order, None, Some(client_id()), false)
24072 .unwrap();
24073 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
24074 let mut engine = ExecutionEngine::new(clock, cache.clone(), None);
24075 dispatch_lighter_order(
24076 &frame,
24077 &rig.dispatch,
24078 &rig.emitter,
24079 &rig.registry,
24080 account_id(),
24081 trader_id(),
24082 UnixNanos::from(1),
24083 );
24084
24085 for event in drain_events(&mut rig.rx) {
24086 let ExecutionEvent::Order(event) = event else {
24087 panic!("expected acceptance");
24088 };
24089
24090 engine.process(&event);
24091 }
24092
24093 let mut trade = dispatcher_test_trade(&rig, true);
24094 trade.bid_id = frame.order_index;
24095 trade.bid_id_str = Some(frame.order_id.clone());
24096 trade.size = Decimal::from_str("0.0400").unwrap();
24097 trade.usd_amount = trade.size * trade.price;
24098 frame.status = LighterOrderStatus::Filled;
24099 frame.initial_base_amount = trade.size;
24100 frame.filled_base_amount = trade.size;
24101 frame.filled_quote_amount = trade.usd_amount;
24102 frame.order_type = LighterOrderKind::Limit;
24103 frame.trigger_price = Decimal::ZERO;
24104
24105 if trade_first {
24106 dispatch_lighter_trade(
24107 &trade,
24108 &rig.dispatch,
24109 &rig.emitter,
24110 &rig.registry,
24111 account_id(),
24112 trader_id(),
24113 Some(TEST_ACCOUNT_INDEX_I64),
24114 UnixNanos::from(2),
24115 );
24116 }
24117
24118 dispatch_lighter_order(
24119 &frame,
24120 &rig.dispatch,
24121 &rig.emitter,
24122 &rig.registry,
24123 account_id(),
24124 trader_id(),
24125 UnixNanos::from(3),
24126 );
24127
24128 if !trade_first {
24129 dispatch_lighter_trade(
24130 &trade,
24131 &rig.dispatch,
24132 &rig.emitter,
24133 &rig.registry,
24134 account_id(),
24135 trader_id(),
24136 Some(TEST_ACCOUNT_INDEX_I64),
24137 UnixNanos::from(4),
24138 );
24139 }
24140
24141 let events = drain_events(&mut rig.rx);
24142 for event in events {
24143 match event {
24144 ExecutionEvent::Order(event) => engine.process(&event),
24145 ExecutionEvent::Report(report) => engine.reconcile_execution_report(&report),
24146 _ => panic!("unexpected child event"),
24147 }
24148 }
24149
24150 let cache = cache.borrow();
24151 let order = cache.order(&rig.cloid).unwrap();
24152
24153 let fills = order
24154 .events()
24155 .into_iter()
24156 .filter_map(|event| match event {
24157 OrderEventAny::Filled(fill) => Some(fill),
24158 _ => None,
24159 })
24160 .collect::<Vec<_>>();
24161
24162 assert_eq!(order.status(), OrderStatus::Filled);
24163 assert_eq!(order.quantity(), Quantity::from("0.0400"));
24164 assert_eq!(order.filled_qty(), Quantity::from("0.0400"));
24165 assert_eq!(order.leaves_qty(), Quantity::from("0.0000"));
24166 assert_eq!(order.is_triggered(), Some(true));
24167 assert_eq!(fills.len(), 1);
24168 assert_eq!(fills[0].trade_id, parse_lighter_trade_id(&trade).unwrap());
24169 assert_eq!(fills[0].last_qty, Quantity::from("0.0400"));
24170 assert_eq!(fills[0].last_px, Price::from("2352.73"));
24171 assert_eq!(fills[0].commission, Some(Money::from("0.00263506 USDC")));
24172 }
24173
24174 #[rstest]
24175 #[case::distinct_timestamps(false)]
24176 #[case::same_timestamp(true)]
24177 fn native_group_preserves_position_fill_order(#[case] same_timestamp: bool) {
24178 let mut rig = dispatcher_rig("POSITION-PARENT");
24179 let parent_id = rig.cloid;
24180 let child_id = ClientOrderId::new("POSITION-CHILD");
24181 let extra_id = ClientOrderId::new("POSITION-EXTRA");
24182 let parent = OrderTestBuilder::new(OrderType::Limit)
24183 .trader_id(trader_id())
24184 .strategy_id(strategy_id())
24185 .instrument_id(rig.instrument_id)
24186 .client_order_id(parent_id)
24187 .side(OrderSide::Buy)
24188 .quantity(Quantity::from("0.1000"))
24189 .price(Price::from("100.00"))
24190 .build();
24191 rig.dispatch.register_create_identity(&parent).unwrap();
24192 rig.dispatch
24193 .mark_order_submission(&parent_id, TEST_SUBMISSION_NONCE);
24194 let mut parent_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24195 parent_frame.order_index = 562_947_905_631_053;
24196 parent_frame.order_id = parent_frame.order_index.to_string();
24197 parent_frame.initial_base_amount = Decimal::from_str("0.1000").unwrap();
24198 parent_frame.remaining_base_amount = parent_frame.initial_base_amount;
24199 parent_frame.price = Decimal::from_str("100.00").unwrap();
24200 dispatch_lighter_order(
24201 &parent_frame,
24202 &rig.dispatch,
24203 &rig.emitter,
24204 &rig.registry,
24205 account_id(),
24206 trader_id(),
24207 UnixNanos::from(1),
24208 );
24209 let mut opening = dispatcher_test_trade(&rig, true);
24210 opening.bid_id = parent_frame.order_index;
24211 opening.bid_id_str = Some(parent_frame.order_id);
24212 opening.trade_id_str = Some("900001".to_string());
24213 opening.size = Decimal::from_str("0.1000").unwrap();
24214 opening.price = Decimal::from_str("100.00").unwrap();
24215 opening.usd_amount = opening.size * opening.price;
24216 opening.maker_fee = Some(0);
24217 opening.taker_fee = Some(0);
24218 dispatch_lighter_trade(
24219 &opening,
24220 &rig.dispatch,
24221 &rig.emitter,
24222 &rig.registry,
24223 account_id(),
24224 trader_id(),
24225 Some(TEST_ACCOUNT_INDEX_I64),
24226 UnixNanos::from(2),
24227 );
24228 rig.cloid = child_id;
24229 let child = OrderTestBuilder::new(OrderType::StopLimit)
24230 .trader_id(trader_id())
24231 .strategy_id(strategy_id())
24232 .instrument_id(rig.instrument_id)
24233 .client_order_id(child_id)
24234 .side(OrderSide::Sell)
24235 .quantity(Quantity::from("0.1000"))
24236 .price(Price::from("200.00"))
24237 .trigger_price(Price::from("150.00"))
24238 .parent_order_id(parent_id)
24239 .reduce_only(true)
24240 .build();
24241 rig.dispatch.register_create_identity(&child).unwrap();
24242 rig.dispatch
24243 .mark_order_submission(&child_id, TEST_SUBMISSION_NONCE);
24244 let mut child_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24245 child_frame.initial_base_amount = Decimal::from_str("0.0400").unwrap();
24246 child_frame.remaining_base_amount = child_frame.initial_base_amount;
24247 child_frame.price = Decimal::from_str("200.00").unwrap();
24248 child_frame.side = Some(LighterOrderSide::Sell);
24249 child_frame.is_ask = true;
24250 child_frame.reduce_only = true;
24251 dispatch_lighter_order(
24252 &child_frame,
24253 &rig.dispatch,
24254 &rig.emitter,
24255 &rig.registry,
24256 account_id(),
24257 trader_id(),
24258 UnixNanos::from(3),
24259 );
24260 let mut first = dispatcher_test_trade(&rig, false);
24261 first.ask_id = child_frame.order_index;
24262 first.ask_id_str = Some(child_frame.order_id.clone());
24263 first.trade_id_str = Some("900002".to_string());
24264 first.timestamp = opening.timestamp + i64::from(!same_timestamp);
24265 first.size = Decimal::from_str("0.0200").unwrap();
24266 first.price = Decimal::from_str("200.00").unwrap();
24267 first.usd_amount = first.size * first.price;
24268 first.maker_fee = Some(0);
24269 first.taker_fee = Some(0);
24270 dispatch_lighter_trade(
24271 &first,
24272 &rig.dispatch,
24273 &rig.emitter,
24274 &rig.registry,
24275 account_id(),
24276 trader_id(),
24277 Some(TEST_ACCOUNT_INDEX_I64),
24278 UnixNanos::from(4),
24279 );
24280 rig.cloid = extra_id;
24281 let extra = OrderTestBuilder::new(OrderType::Limit)
24282 .trader_id(trader_id())
24283 .strategy_id(strategy_id())
24284 .instrument_id(rig.instrument_id)
24285 .client_order_id(extra_id)
24286 .side(OrderSide::Buy)
24287 .quantity(Quantity::from("0.1000"))
24288 .price(Price::from("100.00"))
24289 .time_in_force(TimeInForce::Ioc)
24290 .build();
24291 rig.dispatch.register_create_identity(&extra).unwrap();
24292 rig.dispatch
24293 .mark_order_submission(&extra_id, TEST_SUBMISSION_NONCE + 1);
24294 let mut extra_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24295 extra_frame.order_index += 5;
24296 extra_frame.order_id = extra_frame.order_index.to_string();
24297 extra_frame.initial_base_amount = Decimal::from_str("0.1000").unwrap();
24298 extra_frame.remaining_base_amount = extra_frame.initial_base_amount;
24299 extra_frame.price = Decimal::from_str("100.00").unwrap();
24300 dispatch_lighter_order(
24301 &extra_frame,
24302 &rig.dispatch,
24303 &rig.emitter,
24304 &rig.registry,
24305 account_id(),
24306 trader_id(),
24307 UnixNanos::from(5),
24308 );
24309 let mut addition = dispatcher_test_trade(&rig, true);
24310 addition.bid_id = extra_frame.order_index;
24311 addition.bid_id_str = Some(extra_frame.order_id);
24312 addition.trade_id_str = Some("900003".to_string());
24313 addition.timestamp = opening.timestamp + 2 * i64::from(!same_timestamp);
24314 addition.size = Decimal::from_str("0.0600").unwrap();
24315 addition.price = Decimal::from_str("50.00").unwrap();
24316 addition.usd_amount = addition.size * addition.price;
24317 addition.maker_fee = Some(0);
24318 addition.taker_fee = Some(0);
24319 dispatch_lighter_trade(
24320 &addition,
24321 &rig.dispatch,
24322 &rig.emitter,
24323 &rig.registry,
24324 account_id(),
24325 trader_id(),
24326 Some(TEST_ACCOUNT_INDEX_I64),
24327 UnixNanos::from(6),
24328 );
24329 rig.cloid = child_id;
24330 let mut last = first;
24331 last.trade_id_str = Some("900004".to_string());
24332 last.timestamp = opening.timestamp + 3 * i64::from(!same_timestamp);
24333 child_frame.status = LighterOrderStatus::Filled;
24334 child_frame.remaining_base_amount = Decimal::ZERO;
24335 child_frame.filled_base_amount = Decimal::from_str("0.0400").unwrap();
24336 child_frame.filled_quote_amount = Decimal::from(8);
24337 dispatch_lighter_order(
24338 &child_frame,
24339 &rig.dispatch,
24340 &rig.emitter,
24341 &rig.registry,
24342 account_id(),
24343 trader_id(),
24344 UnixNanos::from(7),
24345 );
24346 dispatch_lighter_trade(
24347 &last,
24348 &rig.dispatch,
24349 &rig.emitter,
24350 &rig.registry,
24351 account_id(),
24352 trader_id(),
24353 Some(TEST_ACCOUNT_INDEX_I64),
24354 UnixNanos::from(8),
24355 );
24356 let mut fills = Vec::new();
24357
24358 for event in drain_events(&mut rig.rx) {
24359 match event {
24360 ExecutionEvent::Order(OrderEventAny::Filled(fill)) => fills.push(fill),
24361 ExecutionEvent::Report(EngineExecutionReport::OrderWithFills(_, reports)) => {
24362 for report in reports {
24363 fills.push(OrderFilled::new(
24364 trader_id(),
24365 strategy_id(),
24366 rig.instrument_id,
24367 report.client_order_id.unwrap(),
24368 report.venue_order_id,
24369 account_id(),
24370 report.trade_id,
24371 report.order_side,
24372 OrderType::StopLimit,
24373 report.last_qty,
24374 report.last_px,
24375 Currency::from("USDC"),
24376 report.liquidity_side,
24377 UUID4::new(),
24378 report.ts_event,
24379 report.ts_init,
24380 true,
24381 None,
24382 Some(report.commission),
24383 None,
24384 ));
24385 }
24386 }
24387 _ => {}
24388 }
24389 }
24390
24391 let ids = fills.iter().map(|fill| fill.trade_id).collect::<Vec<_>>();
24392 let position_id = PositionId::new("POSITION-ORDER");
24393 for fill in &mut fills {
24394 fill.position_id = Some(position_id);
24395 }
24396
24397 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&rig.instrument_id).unwrap();
24398 let mut position = Position::new(&instrument, fills[0].clone());
24399 for fill in &fills[1..] {
24400 position.apply(fill);
24401 }
24402
24403 let expected_pnl = Money::from_decimal_native(
24405 Decimal::from(31) / Decimal::from(7),
24406 Currency::from("USDC"),
24407 )
24408 .unwrap();
24409
24410 assert_eq!(
24411 ids,
24412 ["900001", "900002", "900003", "900004"].map(TradeId::new)
24413 );
24414 assert_eq!(position.quantity, Quantity::from("0.1200"));
24415 assert_eq!(position.realized_pnl, Some(expected_pnl));
24416 }
24417
24418 #[rstest]
24419 #[case::filled(LighterOrderStatus::Filled)]
24420 #[case::partial_then_canceled(LighterOrderStatus::Canceled)]
24421 fn terminal_after_fill_retires_identity_with_publication(#[case] status: LighterOrderStatus) {
24422 let mut rig = dispatcher_rig("TERMINAL-PUBLICATION");
24423 register_identity(&rig);
24424 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24425 dispatch_lighter_order(
24426 &frame,
24427 &rig.dispatch,
24428 &rig.emitter,
24429 &rig.registry,
24430 account_id(),
24431 trader_id(),
24432 UnixNanos::from(1),
24433 );
24434 let mut trade = dispatcher_test_trade(&rig, true);
24435 trade.bid_id = frame.order_index;
24436 trade.bid_id_str = Some(frame.order_id.clone());
24437 trade.size = if status == LighterOrderStatus::Filled {
24438 frame.initial_base_amount
24439 } else {
24440 frame.initial_base_amount / Decimal::from(2)
24441 };
24442
24443 trade.usd_amount = trade.size * trade.price;
24444 dispatch_lighter_trade(
24445 &trade,
24446 &rig.dispatch,
24447 &rig.emitter,
24448 &rig.registry,
24449 account_id(),
24450 trader_id(),
24451 Some(TEST_ACCOUNT_INDEX_I64),
24452 UnixNanos::from(2),
24453 );
24454 frame.status = status;
24455 frame.filled_base_amount = trade.size;
24456 frame.remaining_base_amount = Decimal::ZERO;
24457 dispatch_lighter_order(
24458 &frame,
24459 &rig.dispatch,
24460 &rig.emitter,
24461 &rig.registry,
24462 account_id(),
24463 trader_id(),
24464 UnixNanos::from(3),
24465 );
24466 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
24467 let events = drain_events(&mut rig.rx);
24468 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
24469 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
24470 assert!(
24471 !rig.dispatch
24472 .cloid_map
24473 .contains_key(&frame.client_order_index)
24474 );
24475 assert!(!rig.dispatch.order_snapshots.contains_key(&rig.cloid));
24476 assert!(identity.has_publication());
24477 assert!(
24478 rig.dispatch
24479 .any_order_identity(|id, other, _| *id == rig.cloid && other.has_publication())
24480 );
24481 assert_eq!(
24482 events
24483 .iter()
24484 .filter(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Filled(_))))
24485 .count(),
24486 1
24487 );
24488 }
24489
24490 #[rstest]
24491 #[case::modify("modify", false)]
24492 #[case::terminal("terminal", false)]
24493 #[case::fill("fill", false)]
24494 #[case::update("update", false)]
24495 #[case::retired_fill("fill", true)]
24496 #[case::retired_update("update", true)]
24497 fn position_reports_wait_for_execution(#[case] hold: &str, #[case] retired: bool) {
24498 let (client, cache, _) = create_execution_client();
24499 let instrument_id = register_test_instrument(&client, &cache);
24500 let mut factory = test_order_factory();
24501 let order = test_limit_order(&mut factory, instrument_id, "POSITION-HOLD");
24502 let cloid = order.client_order_id();
24503 client.dispatch.register_create_identity(&order).unwrap();
24504 let identity = client.dispatch.order_identity(&cloid).unwrap();
24505
24506 match hold {
24507 "modify" => {
24508 enqueue_modify(&client, instrument_id, cloid, VenueOrderId::new("700"), 23);
24509 }
24510 "terminal" => {
24511 let rig = dispatcher_rig("POSITION-TERMINAL");
24512 client
24513 .dispatch
24514 .order_events
24515 .lock()
24516 .entry(cloid)
24517 .or_default()
24518 .order = Some(dispatcher_test_order(&rig, LighterOrderStatus::Canceled));
24519 }
24520 "update" => identity.publish_update(UUID4::new()),
24521 "fill" => {
24522 let instrument = cache.borrow().instrument(&instrument_id).unwrap().clone();
24523
24524 let OrderEventAny::Filled(fill) = TestOrderEventStubs::filled(
24525 &order,
24526 &instrument,
24527 Some(TradeId::new("HELD-FILL")),
24528 None,
24529 Some(Price::from("2360.00")),
24530 Some(Quantity::from("0.0050")),
24531 None,
24532 Some(Money::from("0.01000000 USDC")),
24533 None,
24534 Some(account_id()),
24535 ) else {
24536 unreachable!()
24537 };
24538
24539 identity.publish_fill(&fill);
24540 }
24541 _ => unreachable!(),
24542 }
24543
24544 if retired {
24545 client.dispatch.retire_order_identity(&cloid);
24546 }
24547
24548 let report = PositionStatusReport::new(
24549 account_id(),
24550 instrument_id,
24551 PositionSide::Long,
24552 Quantity::from("0.0050"),
24553 UnixNanos::from(4),
24554 UnixNanos::from(5),
24555 None,
24556 None,
24557 Some(Decimal::from_str("2360.00").unwrap()),
24558 );
24559 client
24560 .dispatch
24561 .replace_position_snapshot(std::slice::from_ref(&report), &[], &[]);
24562
24563 let cmd = GeneratePositionStatusReports::new(
24564 UUID4::new(),
24565 UnixNanos::from(6),
24566 Some(instrument_id),
24567 None,
24568 None,
24569 None,
24570 None,
24571 );
24572 let (reports, complete, coverage) = client.cached_position_reports(&cmd).unwrap();
24573 assert!(reports.is_empty());
24574 assert!(!complete);
24575 assert_eq!(coverage, Some(AHashSet::from_iter([TEST_MARKET_INDEX])));
24576 let other_id = client.registry.insert(1, "BTC", LighterProductType::Perp);
24577 let mut other_cmd = cmd;
24578 other_cmd.instrument_id = Some(other_id);
24579 let (reports, complete, _) = client.cached_position_reports(&other_cmd).unwrap();
24580 assert!(reports.is_empty());
24581 assert!(complete);
24582 }
24583
24584 #[rstest]
24585 #[case::explicit_flat(true)]
24586 #[case::empty_snapshot(false)]
24587 #[tokio::test]
24588 async fn restart_held_child_fill_does_not_synthesize_position(#[case] explicit_flat: bool) {
24589 let (mut client, cache, mut rx) = create_execution_client();
24590 let core = client.core.clone();
24591 let instrument_id = register_test_instrument(&client, &cache);
24592 let mut rig = dispatcher_rig("RESTART-CHILD");
24593 rig.dispatch = client.dispatch.clone();
24594 rig.registry = Arc::clone(&client.registry);
24595
24596 let account = MarginAccount::new(
24597 AccountState::new(
24598 account_id(),
24599 AccountType::Margin,
24600 vec![AccountBalance::new(
24601 Money::from("10000.00000000 USDC"),
24602 Money::from("0.00000000 USDC"),
24603 Money::from("10000.00000000 USDC"),
24604 )],
24605 Vec::new(),
24606 true,
24607 UUID4::new(),
24608 UnixNanos::from(1),
24609 UnixNanos::from(1),
24610 Some(Currency::from("USDC")),
24611 ),
24612 true,
24613 );
24614 cache
24615 .borrow_mut()
24616 .add_account(AccountAny::Margin(account))
24617 .unwrap();
24618 let parent_id = ClientOrderId::new("RESTART-PARENT");
24619 let position_id = PositionId::new("RESTART-POSITION");
24620 let parent = OrderTestBuilder::new(OrderType::Limit)
24621 .trader_id(trader_id())
24622 .strategy_id(strategy_id())
24623 .instrument_id(instrument_id)
24624 .client_order_id(parent_id)
24625 .side(OrderSide::Buy)
24626 .quantity(Quantity::from("0.1000"))
24627 .price(Price::from("2352.73"))
24628 .build();
24629 let instrument = LIGHTER_INSTRUMENT_CACHE
24630 .get(&instrument_id)
24631 .unwrap()
24632 .value()
24633 .clone();
24634 let opening = TestOrderEventStubs::filled(
24635 &parent,
24636 &instrument,
24637 Some(TradeId::new("RESTART-OPENING")),
24638 Some(position_id),
24639 Some(Price::from("2352.73")),
24640 Some(Quantity::from("0.1000")),
24641 Some(LiquiditySide::Taker),
24642 Some(Money::from("0.00000000 USDC")),
24643 Some(UnixNanos::from(1)),
24644 Some(account_id()),
24645 );
24646 let mut parent = parent;
24647 parent
24648 .apply(TestOrderEventStubs::accepted(
24649 &parent,
24650 account_id(),
24651 VenueOrderId::new("600"),
24652 ))
24653 .unwrap();
24654 parent.apply(opening.clone()).unwrap();
24655 cache
24656 .borrow_mut()
24657 .add_order(parent, Some(position_id), Some(client_id()), false)
24658 .unwrap();
24659
24660 let OrderEventAny::Filled(opening) = opening else {
24661 unreachable!()
24662 };
24663
24664 let position = Position::new(&instrument, opening);
24665 cache
24666 .borrow_mut()
24667 .add_position(&position, OmsType::Netting)
24668 .unwrap();
24669 let child = OrderTestBuilder::new(OrderType::StopLimit)
24670 .trader_id(trader_id())
24671 .strategy_id(strategy_id())
24672 .instrument_id(instrument_id)
24673 .client_order_id(rig.cloid)
24674 .side(OrderSide::Sell)
24675 .quantity(Quantity::from("0.1000"))
24676 .price(Price::from("2352.74"))
24677 .trigger_price(Price::from("2300.00"))
24678 .parent_order_id(parent_id)
24679 .reduce_only(true)
24680 .build();
24681 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
24682 frame.side = Some(LighterOrderSide::Sell);
24683 frame.is_ask = true;
24684 frame.order_type = LighterOrderKind::Limit;
24685 frame.reduce_only = true;
24686 let venue_id = VenueOrderId::new(&frame.order_id);
24687 cache_accepted_order(&cache, child, venue_id, Some(client_id()));
24688 cache
24689 .borrow_mut()
24690 .add_position_id(
24691 &position_id,
24692 &instrument_id.venue,
24693 &rig.cloid,
24694 &strategy_id(),
24695 )
24696 .unwrap();
24697 let child = cache.borrow().order_owned(&rig.cloid).unwrap();
24698 client
24699 .dispatch
24700 .restore_reconciled_order(&child, frame.client_order_index, venue_id, false)
24701 .unwrap();
24702 let mut trade = dispatcher_test_trade(&rig, false);
24703 trade.ask_id = frame.order_index;
24704 trade.ask_id_str = Some(frame.order_id.clone());
24705 trade.size = Decimal::from_str("0.1000").unwrap();
24706 trade.usd_amount = trade.size * trade.price;
24707 trade.maker_fee = Some(0);
24708 trade.taker_fee = Some(0);
24709 dispatch_lighter_trade(
24710 &trade,
24711 &client.dispatch,
24712 &client.emitter,
24713 &client.registry,
24714 account_id(),
24715 trader_id(),
24716 Some(TEST_ACCOUNT_INDEX_I64),
24717 UnixNanos::from(2),
24718 );
24719
24720 let flat = PositionStatusReport::new(
24721 account_id(),
24722 instrument_id,
24723 PositionSide::Flat,
24724 Quantity::from("0.0000"),
24725 UnixNanos::from(3),
24726 UnixNanos::from(4),
24727 None,
24728 None,
24729 None,
24730 );
24731
24732 let positions = if explicit_flat {
24733 vec![flat]
24734 } else {
24735 Vec::new()
24736 };
24737
24738 client
24739 .dispatch
24740 .replace_position_snapshot(&positions, &[], &[]);
24741 let mut external = frame.clone();
24742 external.order_index += 1;
24743 external.order_id = external.order_index.to_string();
24744 external.client_order_index = 0;
24745 external.client_order_id = "0".to_string();
24746 external.reduce_only = false;
24747 let active = serde_json::json!({"code":200,"orders":[external]});
24748 let app = Router::new()
24749 .route(
24750 "/api/v1/accountActiveOrders",
24751 get(move || {
24752 let active = active.clone();
24753 async move { axum::Json(active) }
24754 }),
24755 )
24756 .route(
24757 "/api/v1/accountInactiveOrders",
24758 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
24759 )
24760 .route(
24761 "/api/v1/trades",
24762 get(|| async { axum::Json(serde_json::json!({"code":200,"trades":[]})) }),
24763 );
24764
24765 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
24766 let addr = listener.local_addr().unwrap();
24767 let server = tokio::spawn(async move {
24768 axum::serve(listener, app).await.unwrap();
24769 });
24770
24771 client.http_client = LighterHttpClient::new(
24772 LighterEnvironment::Testnet,
24773 Some(format!("http://{addr}")),
24774 1,
24775 None,
24776 )
24777 .unwrap();
24778 let mass = client
24779 .generate_mass_status(Some(60))
24780 .await
24781 .unwrap()
24782 .unwrap();
24783 server.abort();
24784 let _ = server.await;
24785
24786 let command = GeneratePositionStatusReports::new(
24787 UUID4::new(),
24788 UnixNanos::from(5),
24789 None,
24790 None,
24791 None,
24792 None,
24793 None,
24794 );
24795 let direct = client.generate_position_status_reports(&command).await;
24796 let worker = client
24797 .generate_position_status_reports_task(&command)
24798 .unwrap();
24799 worker.collection.await;
24800 let worker = (worker.finish)();
24801 assert!(mass.position_reports().is_empty());
24802 assert!(!mass.reports_complete());
24803 assert!(direct.is_err());
24804 assert!(worker.is_err());
24805 assert!(drain_events(&mut rx).is_empty());
24806 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
24807
24808 let engine = Rc::new(RefCell::new(ExecutionEngine::new(
24809 clock.clone(),
24810 cache.clone(),
24811 None,
24812 )));
24813 let mut manager =
24814 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default()).unwrap();
24815 let result = manager.reconcile_execution_mass_status(&mass, &engine);
24816 assert!(
24817 result
24818 .events
24819 .iter()
24820 .all(|event| !matches!(event, OrderEventAny::Filled(_)))
24821 );
24822 assert_eq!(
24823 cache.borrow().position(&position_id).unwrap().quantity,
24824 Quantity::from("0.1000")
24825 );
24826 frame.status = LighterOrderStatus::Filled;
24827 frame.initial_base_amount = trade.size;
24828 frame.filled_base_amount = trade.size;
24829 frame.remaining_base_amount = Decimal::ZERO;
24830 frame.filled_quote_amount = trade.usd_amount;
24831 dispatch_lighter_order(
24832 &frame,
24833 &client.dispatch,
24834 &client.emitter,
24835 &client.registry,
24836 account_id(),
24837 trader_id(),
24838 UnixNanos::from(6),
24839 );
24840 engine
24841 .borrow_mut()
24842 .register_client(Box::new(client))
24843 .unwrap();
24844 let released = drain_events(&mut rx);
24845 assert!(
24846 released
24847 .iter()
24848 .any(|event| matches!(event, ExecutionEvent::Order(OrderEventAny::Filled(_)))),
24849 "released child events: {released:?}"
24850 );
24851
24852 for event in released {
24853 match event {
24854 ExecutionEvent::Order(event) => engine.borrow_mut().process(&event),
24855 ExecutionEvent::Report(report) => {
24856 engine.borrow_mut().reconcile_execution_report(&report);
24857 }
24858 _ => panic!("unexpected restart event"),
24859 }
24860 }
24861
24862 let cache_ref = cache.borrow();
24863 let position = cache_ref.position(&position_id).unwrap();
24864 let child = cache_ref.order(&rig.cloid).unwrap();
24865 assert_eq!(position.quantity, Quantity::from("0.0000"));
24866 assert_eq!(position.realized_pnl, Some(Money::from("0.00000000 USDC")));
24867 assert_eq!(
24868 child.trade_ids(),
24869 vec![&parse_lighter_trade_id(&trade).unwrap()]
24870 );
24871 assert_eq!(child.filled_qty(), Quantity::from("0.1000"));
24872 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
24873 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
24874
24875 let command = GeneratePositionStatusReports::new(
24876 UUID4::new(),
24877 UnixNanos::from(7),
24878 None,
24879 None,
24880 None,
24881 None,
24882 None,
24883 );
24884 let (reports, complete, coverage) =
24885 cached_position_reports(&core, &rig.dispatch, &rig.registry, &command).unwrap();
24886 assert!(complete);
24887 assert_eq!(coverage, Some(AHashSet::new()));
24888 assert_eq!(reports.len(), usize::from(explicit_flat));
24889 }
24890
24891 #[rstest]
24892 fn resting_child_does_not_withhold_position_reports() {
24893 let (client, cache, _) = create_execution_client();
24894 let instrument_id = register_test_instrument(&client, &cache);
24895 let rig = dispatcher_rig("RESTING-POSITION");
24896 let order = OrderTestBuilder::new(OrderType::StopLimit)
24897 .trader_id(trader_id())
24898 .strategy_id(strategy_id())
24899 .instrument_id(instrument_id)
24900 .client_order_id(rig.cloid)
24901 .side(OrderSide::Sell)
24902 .quantity(Quantity::from("0.1000"))
24903 .price(Price::from("2300.00"))
24904 .trigger_price(Price::from("2301.00"))
24905 .parent_order_id(ClientOrderId::new("RESTING-PARENT"))
24906 .reduce_only(true)
24907 .build();
24908 client.dispatch.register_create_identity(&order).unwrap();
24909 client
24910 .dispatch
24911 .order_events
24912 .lock()
24913 .entry(rig.cloid)
24914 .or_default()
24915 .order = Some(dispatcher_test_order(&rig, LighterOrderStatus::Pending));
24916
24917 let report = PositionStatusReport::new(
24918 account_id(),
24919 instrument_id,
24920 PositionSide::Long,
24921 Quantity::from("0.1000"),
24922 UnixNanos::from(4),
24923 UnixNanos::from(5),
24924 None,
24925 None,
24926 Some(Decimal::from_str("2360.00").unwrap()),
24927 );
24928 client
24929 .dispatch
24930 .replace_position_snapshot(std::slice::from_ref(&report), &[], &[]);
24931
24932 let cmd = GeneratePositionStatusReports::new(
24933 UUID4::new(),
24934 UnixNanos::from(6),
24935 None,
24936 None,
24937 None,
24938 None,
24939 None,
24940 );
24941 let (reports, complete, coverage) = client.cached_position_reports(&cmd).unwrap();
24942 assert_eq!(reports, vec![report]);
24943 assert!(complete);
24944 assert_eq!(coverage, Some(AHashSet::new()));
24945 }
24946
24947 #[rstest]
24948 #[case::cached_fill("2352.73", false)]
24949 #[case::price_restatement("2352.74", true)]
24950 fn cached_parent_fill_is_not_deferred_during_child_recovery(
24951 #[case] price: &str,
24952 #[case] deferred: bool,
24953 ) {
24954 let (client, cache, mut rx) = create_execution_client();
24955 let instrument_id = register_test_instrument(&client, &cache);
24956 let instrument = cache.borrow().instrument(&instrument_id).unwrap().clone();
24957 LIGHTER_INSTRUMENT_CACHE.insert(instrument_id, instrument.clone());
24958 let parent_id = ClientOrderId::new("HTTP-CACHED-PARENT");
24959 let parent_venue = VenueOrderId::new("710001");
24960 let child_id = ClientOrderId::new("HTTP-RECOVERING-CHILD");
24961 let child_venue = VenueOrderId::new("710002");
24962 let parent = OrderTestBuilder::new(OrderType::Limit)
24963 .trader_id(trader_id())
24964 .strategy_id(strategy_id())
24965 .instrument_id(instrument_id)
24966 .client_order_id(parent_id)
24967 .side(OrderSide::Buy)
24968 .quantity(Quantity::from("0.1336"))
24969 .price(Price::from("2352.73"))
24970 .build();
24971 cache_accepted_order(&cache, parent, parent_venue, Some(client_id()));
24972 let parent = cache.borrow().order_owned(&parent_id).unwrap();
24973 let mut report =
24974 reconciliation_fill_report(instrument_id, 101, parent_venue, "CACHED-PARENT-FILL");
24975 let opening = TestOrderEventStubs::filled(
24976 &parent,
24977 &instrument,
24978 Some(report.trade_id),
24979 None,
24980 Some(report.last_px),
24981 Some(report.last_qty),
24982 Some(report.liquidity_side),
24983 Some(report.commission),
24984 Some(report.ts_event),
24985 Some(account_id()),
24986 );
24987 cache.borrow_mut().update_order(&opening).unwrap();
24988 let child = OrderTestBuilder::new(OrderType::StopLimit)
24989 .trader_id(trader_id())
24990 .strategy_id(strategy_id())
24991 .instrument_id(instrument_id)
24992 .client_order_id(child_id)
24993 .side(OrderSide::Sell)
24994 .quantity(Quantity::from("0.1336"))
24995 .price(Price::from("2301.00"))
24996 .trigger_price(Price::from("2302.00"))
24997 .parent_order_id(parent_id)
24998 .reduce_only(true)
24999 .build();
25000 cache_accepted_order(&cache, child, child_venue, Some(client_id()));
25001 let child = cache.borrow().order_owned(&child_id).unwrap();
25002 client
25003 .dispatch
25004 .restore_reconciled_order(&child, 102, child_venue, false)
25005 .unwrap();
25006 let identity = client.dispatch.order_identity(&child_id).unwrap();
25007 let mut child_report =
25008 reconciliation_fill_report(instrument_id, 102, child_venue, "NEW-CHILD-FILL");
25009 child_report.order_side = OrderSide::Sell;
25010 child_report.last_qty = Quantity::from("0.0444");
25011 child_report.last_px = Price::from("2301.23");
25012 let child_fill = child_fill_from_report(&child_report, &identity, trader_id()).unwrap();
25013 client.dispatch.defer_fill(
25014 client
25015 .dispatch
25016 .order_events
25017 .lock()
25018 .entry(child_id)
25019 .or_default(),
25020 child_fill.clone(),
25021 );
25022 report.last_px = Price::from(price);
25023
25024 let sweep = finish_fill_reports(
25025 &client.core,
25026 &client.dispatch,
25027 BTreeSet::new(),
25028 Ok(FillSweep {
25029 reports: vec![report.clone()],
25030 covers_window: true,
25031 withheld_instruments: AHashSet::new(),
25032 }),
25033 )
25034 .unwrap();
25035
25036 let events = client.dispatch.order_events.lock();
25037 assert!(sweep.reports.is_empty());
25038 assert!(!sweep.covers_window);
25039 assert_eq!(
25040 sweep.withheld_instruments,
25041 AHashSet::from_iter([instrument_id])
25042 );
25043 assert_eq!(events.get(&parent_id).is_some(), deferred);
25044 assert_eq!(events[&child_id].fills, vec![child_fill]);
25045
25046 if deferred {
25047 let fills = &events[&parent_id].fills;
25048 assert_eq!(fills.len(), 1);
25049 assert_eq!(fills[0].trade_id, report.trade_id);
25050 assert_eq!(fills[0].client_order_id, parent_id);
25051 assert_eq!(fills[0].venue_order_id, parent_venue);
25052 assert_eq!(fills[0].last_qty, report.last_qty);
25053 assert_eq!(fills[0].last_px, report.last_px);
25054 assert_eq!(fills[0].commission, Some(report.commission));
25055 }
25056
25057 assert_eq!(
25058 cache.borrow().order(&parent_id).unwrap().trade_ids(),
25059 vec![&report.trade_id]
25060 );
25061 assert!(drain_events(&mut rx).is_empty());
25062 }
25063
25064 #[rstest]
25065 #[case::immediate(false, false, 1)]
25066 #[case::deferred(false, true, 1)]
25067 #[case::repeated(false, false, 16)]
25068 #[case::deferred_repeated(false, true, 16)]
25069 #[case::first_frame(true, false, 1)]
25070 fn self_trade_reduce_updates_cache_and_reports(
25071 #[case] first_frame: bool,
25072 #[case] deferred: bool,
25073 #[case] repeats: usize,
25074 ) {
25075 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
25076 let mut events = Vec::new();
25077
25078 if !first_frame {
25079 dispatch_reduce_frame(&rig, &frame, false);
25080 if !deferred {
25081 events.extend(consume_reduce_events(&mut rig, &mut engine));
25082 }
25083 }
25084
25085 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
25086 frame.updated_at += 1_000;
25087 for _ in 0..repeats {
25088 dispatch_reduce_frame(&rig, &frame, false);
25089 if !deferred {
25090 events.extend(consume_reduce_events(&mut rig, &mut engine));
25091 }
25092 }
25093
25094 events.extend(consume_reduce_events(&mut rig, &mut engine));
25095 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
25096 let report = rig.dispatch.translate_order_cloid(
25097 parse_http_order_to_report(
25098 &frame,
25099 &rig.registry,
25100 account_id(),
25101 UnixNanos::from(5),
25102 &rig.dispatch,
25103 )
25104 .unwrap(),
25105 );
25106 engine.reconcile_order_status_report(&report);
25107 engine.reconcile_order_status_report(&report);
25108 let reconciled = cache.borrow().order(&rig.cloid).unwrap().clone();
25109 rig.dispatch = WsDispatchState::new();
25110 rig.dispatch
25111 .seed_order_restorations(vec![reconciled.clone()]);
25112 dispatch_reduce_frame(&rig, &frame, false);
25113 let replayed = consume_reduce_events(&mut rig, &mut engine);
25114
25115 assert_eq!(events.len(), 2);
25116 assert!(matches!(events[0], OrderEventAny::Accepted(_)));
25117
25118 let OrderEventAny::Updated(update) = &events[1] else {
25119 panic!("expected quantity update");
25120 };
25121
25122 assert_eq!(update.quantity, Quantity::from("0.0001"));
25123 assert!(update.reconciliation);
25124 assert_eq!(order.status(), OrderStatus::Accepted);
25125 assert_eq!(order.quantity(), Quantity::from("0.0001"));
25126 assert_eq!(order.leaves_qty(), Quantity::from("0.0001"));
25127 assert_eq!(order.filled_qty(), Quantity::from("0.0000"));
25128 assert_eq!(report.quantity, Quantity::from("0.0001"));
25129 assert_eq!(report.filled_qty, Quantity::from("0.0000"));
25130 assert_eq!(reconciled, order);
25131 assert!(replayed.is_empty());
25132 }
25133
25134 #[rstest]
25135 #[case::frame_first(false, false, false)]
25136 #[case::trade_first(true, false, false)]
25137 #[case::reduced_then_frame(false, true, false)]
25138 #[case::reduced_then_trade(true, true, false)]
25139 #[case::deferred_frame(false, false, true)]
25140 #[case::deferred_trade(true, false, true)]
25141 fn self_trade_reduce_final_fill_ordering(
25142 #[case] trade_first: bool,
25143 #[case] reduced_open: bool,
25144 #[case] deferred: bool,
25145 #[values(false, true)] first_frame: bool,
25146 ) {
25147 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
25148 if !first_frame {
25149 dispatch_reduce_frame(&rig, &frame, false);
25150 consume_reduce_events(&mut rig, &mut engine);
25151 }
25152
25153 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
25154 frame.updated_at += 1_000;
25155 if reduced_open {
25156 dispatch_reduce_frame(&rig, &frame, false);
25157 consume_reduce_events(&mut rig, &mut engine);
25158 }
25159
25160 let mut trade = dispatcher_test_trade(&rig, true);
25161 trade.bid_id = frame.order_index;
25162 trade.bid_id_str = Some(frame.order_id.clone());
25163 trade.size = Decimal::from_str("0.0001").unwrap();
25164 trade.usd_amount = trade.size * trade.price;
25165 frame.status = LighterOrderStatus::Filled;
25166 frame.remaining_base_amount = Decimal::ZERO;
25167 frame.filled_base_amount = trade.size;
25168 frame.filled_quote_amount = trade.usd_amount;
25169 frame.updated_at += 1_000;
25170
25171 if trade_first {
25172 dispatch_lighter_trade(
25173 &trade,
25174 &rig.dispatch,
25175 &rig.emitter,
25176 &rig.registry,
25177 account_id(),
25178 trader_id(),
25179 Some(TEST_ACCOUNT_INDEX_I64),
25180 UnixNanos::from(3),
25181 );
25182 consume_reduce_events(&mut rig, &mut engine);
25183 }
25184
25185 dispatch_reduce_frame(&rig, &frame, deferred);
25186 consume_reduce_events(&mut rig, &mut engine);
25187
25188 if !trade_first {
25189 dispatch_lighter_trade(
25190 &trade,
25191 &rig.dispatch,
25192 &rig.emitter,
25193 &rig.registry,
25194 account_id(),
25195 trader_id(),
25196 Some(TEST_ACCOUNT_INDEX_I64),
25197 UnixNanos::from(4),
25198 );
25199 consume_reduce_events(&mut rig, &mut engine);
25200 }
25201
25202 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
25203
25204 assert_eq!(order.quantity(), Quantity::from("0.0001"));
25205 assert_eq!(order.filled_qty(), Quantity::from("0.0001"));
25206 assert_eq!(order.leaves_qty(), Quantity::from("0.0000"));
25207 assert_eq!(order.status(), OrderStatus::Filled);
25208 assert_eq!(
25209 order.trade_ids(),
25210 vec![&parse_lighter_trade_id(&trade).unwrap()]
25211 );
25212 }
25213
25214 #[rstest]
25215 #[case::stream(false)]
25216 #[case::report(true)]
25217 fn self_trade_reduce_cancellation_remains_canceled(#[case] report: bool) {
25218 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
25219 dispatch_reduce_frame(&rig, &frame, false);
25220 consume_reduce_events(&mut rig, &mut engine);
25221 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
25222 frame.updated_at += 1_000;
25223 dispatch_reduce_frame(&rig, &frame, false);
25224 consume_reduce_events(&mut rig, &mut engine);
25225 frame.status = LighterOrderStatus::Canceled;
25226 frame.remaining_base_amount = Decimal::ZERO;
25227 frame.updated_at += 1_000;
25228
25229 if report {
25230 let report = rig.dispatch.translate_order_cloid(
25231 parse_http_order_to_report(
25232 &frame,
25233 &rig.registry,
25234 account_id(),
25235 UnixNanos::from(5),
25236 &rig.dispatch,
25237 )
25238 .unwrap(),
25239 );
25240 engine.reconcile_order_status_report(&report);
25241 } else {
25242 dispatch_reduce_frame(&rig, &frame, false);
25243 consume_reduce_events(&mut rig, &mut engine);
25244 }
25245
25246 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
25247
25248 assert_eq!(order.status(), OrderStatus::Canceled);
25249 assert_eq!(order.filled_qty(), Quantity::from("0.0000"));
25250 assert!(order.trade_ids().is_empty());
25251 assert_eq!(
25252 order.quantity(),
25253 Quantity::from(if report { "0.0027" } else { "0.0001" })
25254 );
25255 }
25256
25257 fn reduce_order_replay() -> (
25258 DispatcherRig,
25259 Rc<RefCell<Cache>>,
25260 ExecutionEngine,
25261 LighterOrder,
25262 ) {
25263 let rig = dispatcher_rig("REDUCE-REPLAY");
25264 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
25265 frame.initial_base_amount = Decimal::from_str("0.0027").unwrap();
25266 frame.remaining_base_amount = frame.initial_base_amount;
25267 let order = OrderTestBuilder::new(OrderType::Limit)
25268 .trader_id(trader_id())
25269 .strategy_id(strategy_id())
25270 .instrument_id(rig.instrument_id)
25271 .client_order_id(rig.cloid)
25272 .side(OrderSide::Buy)
25273 .quantity(Quantity::from("0.0027"))
25274 .price(Price::from("2352.74"))
25275 .build();
25276 rig.dispatch.register_create_identity(&order).unwrap();
25277 rig.dispatch
25278 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
25279 let cache = Rc::new(RefCell::new(Cache::default()));
25280 cache
25281 .borrow_mut()
25282 .add_instrument(
25283 LIGHTER_INSTRUMENT_CACHE
25284 .get(&rig.instrument_id)
25285 .unwrap()
25286 .value()
25287 .clone(),
25288 )
25289 .unwrap();
25290 cache
25291 .borrow_mut()
25292 .add_order(order, None, Some(client_id()), false)
25293 .unwrap();
25294 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
25295 let mut engine = ExecutionEngine::new(clock, cache.clone(), None);
25296 engine.process(&OrderEventAny::Submitted(OrderSubmitted::new(
25297 trader_id(),
25298 strategy_id(),
25299 rig.instrument_id,
25300 rig.cloid,
25301 account_id(),
25302 UUID4::new(),
25303 UnixNanos::from(1),
25304 UnixNanos::from(1),
25305 )));
25306 (rig, cache, engine, frame)
25307 }
25308
25309 fn dispatch_reduce_frame(rig: &DispatcherRig, frame: &LighterOrder, deferred: bool) {
25310 if deferred {
25311 dispatch_deferred_order(rig.cloid, frame, &rig.dispatch, &rig.emitter);
25312 } else {
25313 dispatch_lighter_order(
25314 frame,
25315 &rig.dispatch,
25316 &rig.emitter,
25317 &rig.registry,
25318 account_id(),
25319 trader_id(),
25320 UnixNanos::from(3),
25321 );
25322 }
25323 }
25324
25325 fn consume_reduce_events(
25326 rig: &mut DispatcherRig,
25327 engine: &mut ExecutionEngine,
25328 ) -> Vec<OrderEventAny> {
25329 let mut orders = Vec::new();
25330
25331 for event in drain_events(&mut rig.rx) {
25332 match event {
25333 ExecutionEvent::Order(event) => {
25334 engine.process(&event);
25335 orders.push(event);
25336 }
25337 ExecutionEvent::Report(report) => engine.reconcile_execution_report(&report),
25338 other => panic!("unexpected event: {other:?}"),
25339 }
25340 }
25341
25342 orders
25343 }
25344
25345 #[rstest]
25346 #[case::ioc(LighterOrderStatus::CanceledNotEnoughLiquidity, false)]
25347 #[case::expiry(LighterOrderStatus::CanceledExpired, false)]
25348 #[case::post_only(LighterOrderStatus::CanceledPostOnly, false)]
25349 #[case::resting(LighterOrderStatus::Canceled, true)]
25350 fn terminal_waits_for_all_distinct_fills(
25351 #[case] status: LighterOrderStatus,
25352 #[case] trade_first: bool,
25353 ) {
25354 let mut rig = dispatcher_rig("TERMINAL-FILLS");
25355 register_identity(&rig);
25356 rig.dispatch.mark_accepted_emitted(rig.cloid);
25357 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
25358 let mut order = dispatcher_test_order(&rig, status);
25359 order.filled_base_amount = Decimal::from_str("0.0030").unwrap();
25360 let mut first = dispatcher_test_trade(&rig, true);
25361 first.bid_id = order.order_index;
25362 first.bid_id_str = Some(order.order_id.clone());
25363 first.size = Decimal::from_str("0.0010").unwrap();
25364 let mut second = first.clone();
25365 second.trade_id += 1;
25366 second.trade_id_str = Some(second.trade_id.to_string());
25367 second.size = Decimal::from_str("0.0020").unwrap();
25368
25369 let send_trade = |trade: &LighterTrade| {
25370 dispatch_lighter_trade(
25371 trade,
25372 &rig.dispatch,
25373 &rig.emitter,
25374 &rig.registry,
25375 account_id(),
25376 trader_id(),
25377 Some(TEST_ACCOUNT_INDEX_I64),
25378 UnixNanos::from(2),
25379 );
25380 };
25381
25382 if trade_first {
25383 send_trade(&first);
25384 }
25385
25386 dispatch_lighter_order(
25387 &order,
25388 &rig.dispatch,
25389 &rig.emitter,
25390 &rig.registry,
25391 account_id(),
25392 trader_id(),
25393 UnixNanos::from(1),
25394 );
25395 let deadline = rig.dispatch.terminals_pending.lock()[&rig.cloid].retry_at;
25396 dispatch_lighter_order(
25397 &order,
25398 &rig.dispatch,
25399 &rig.emitter,
25400 &rig.registry,
25401 account_id(),
25402 trader_id(),
25403 UnixNanos::from(3),
25404 );
25405 assert_eq!(
25406 rig.dispatch.terminals_pending.lock()[&rig.cloid].retry_at,
25407 deadline
25408 );
25409 send_trade(&first);
25410 send_trade(&first);
25411 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
25412 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
25413 let mut events = drain_events(&mut rig.rx);
25414 assert_eq!(events.len(), 1);
25415 assert!(
25416 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
25417 if fill.trade_id.as_str() == first.trade_id_str.as_deref().unwrap()
25418 && fill.last_qty == Quantity::from("0.0010"))
25419 );
25420 dispatch_lighter_trade(
25421 &second,
25422 &rig.dispatch,
25423 &rig.emitter,
25424 &rig.registry,
25425 account_id(),
25426 trader_id(),
25427 Some(TEST_ACCOUNT_INDEX_I64),
25428 UnixNanos::from(4),
25429 );
25430 events.extend(drain_events(&mut rig.rx));
25431 assert_eq!(events.len(), 3);
25432 assert!(
25433 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Filled(fill))
25434 if fill.last_qty == Quantity::from("0.0020"))
25435 );
25436
25437 let terminal_ts = match &events[2] {
25438 ExecutionEvent::Order(OrderEventAny::Canceled(event)) => {
25439 assert_ne!(status, LighterOrderStatus::CanceledExpired);
25440 event.ts_event
25441 }
25442 ExecutionEvent::Order(OrderEventAny::Expired(event)) => {
25443 assert_eq!(status, LighterOrderStatus::CanceledExpired);
25444 event.ts_event
25445 }
25446 event => panic!("expected terminal after fills, was {event:?}"),
25447 };
25448
25449 assert_eq!(terminal_ts, UnixNanos::from(1_777_941_383_900_000_000));
25450 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0030").unwrap());
25451 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25452 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
25453 }
25454
25455 #[rstest]
25456 fn terminal_after_fills_has_no_delay() {
25457 let mut rig = dispatcher_rig("TERMINAL-COVERED");
25458 register_identity(&rig);
25459 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
25460 let mut trade = dispatcher_test_trade(&rig, true);
25461 trade.bid_id = order.order_index;
25462 trade.bid_id_str = Some(order.order_id.clone());
25463 order.filled_base_amount = trade.size;
25464 dispatch_lighter_trade(
25465 &trade,
25466 &rig.dispatch,
25467 &rig.emitter,
25468 &rig.registry,
25469 account_id(),
25470 trader_id(),
25471 Some(TEST_ACCOUNT_INDEX_I64),
25472 UnixNanos::from(1),
25473 );
25474 dispatch_lighter_order(
25475 &order,
25476 &rig.dispatch,
25477 &rig.emitter,
25478 &rig.registry,
25479 account_id(),
25480 trader_id(),
25481 UnixNanos::from(2),
25482 );
25483 let events = drain_events(&mut rig.rx);
25484 assert_eq!(events.len(), 3);
25485 assert!(matches!(
25486 &events[0],
25487 ExecutionEvent::Order(OrderEventAny::Accepted(_))
25488 ));
25489 assert!(matches!(
25490 &events[1],
25491 ExecutionEvent::Order(OrderEventAny::Filled(_))
25492 ));
25493 assert!(matches!(
25494 &events[2],
25495 ExecutionEvent::Order(OrderEventAny::Canceled(_))
25496 ));
25497 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25498 }
25499
25500 #[rstest]
25501 #[case::modify_hold(true)]
25502 #[case::fill_hold(false)]
25503 #[tokio::test]
25504 async fn terminal_preserves_later_holds_and_emits_once(#[case] modify_pending: bool) {
25505 let mut rig = dispatcher_rig("TERMINAL-MODIFY");
25506 register_identity(&rig);
25507 rig.dispatch.mark_accepted_emitted(rig.cloid);
25508 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
25509 let mut trade = dispatcher_test_trade(&rig, true);
25510 trade.bid_id = order.order_index;
25511 trade.bid_id_str = Some(order.order_id.clone());
25512 order.filled_base_amount = trade.size;
25513 dispatch_lighter_order(
25514 &order,
25515 &rig.dispatch,
25516 &rig.emitter,
25517 &rig.registry,
25518 account_id(),
25519 trader_id(),
25520 UnixNanos::from(1),
25521 );
25522
25523 let command = ModifyOrder::new(
25524 trader_id(),
25525 Some(client_id()),
25526 strategy_id(),
25527 rig.instrument_id,
25528 rig.cloid,
25529 Some(VenueOrderId::new(&order.order_id)),
25530 None,
25531 Some(Price::from("2350.00")),
25532 None,
25533 UUID4::new(),
25534 UnixNanos::from(2),
25535 None,
25536 None,
25537 );
25538 assert!(rig.dispatch.set_pending_modify_action(&command));
25539 dispatch_lighter_order(
25540 &order,
25541 &rig.dispatch,
25542 &rig.emitter,
25543 &rig.registry,
25544 account_id(),
25545 trader_id(),
25546 UnixNanos::from(3),
25547 );
25548 dispatch_lighter_trade(
25549 &trade,
25550 &rig.dispatch,
25551 &rig.emitter,
25552 &rig.registry,
25553 account_id(),
25554 trader_id(),
25555 Some(TEST_ACCOUNT_INDEX_I64),
25556 UnixNanos::from(4),
25557 );
25558
25559 if !modify_pending {
25560 assert!(
25561 rig.dispatch
25562 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
25563 );
25564 }
25565
25566 let (http_client, hits, server) = terminal_trades_server(&order, Vec::new()).await;
25567 let (client, _, _) = create_execution_client();
25568 let mut context = acked_probe_context(&client);
25569 context.http_client = http_client;
25570 context.dispatch = rig.dispatch.clone();
25571 context.registry = rig.registry.clone();
25572 context.emitter = rig.emitter.clone();
25573 for _ in 0..=ACKED_ORDER_PROBE_ATTEMPTS {
25574 recover_terminals(&context, true).await;
25575 }
25576
25577 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
25578 {
25579 let mut events = rig.dispatch.order_events.lock();
25580 release_terminal(rig.cloid, true, &rig.dispatch, &rig.emitter, &events);
25581 assert!(drain_events(&mut rig.rx).is_empty());
25582 assert!(
25583 rig.dispatch
25584 .terminals_pending
25585 .lock()
25586 .contains_key(&rig.cloid)
25587 );
25588 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
25589
25590 if modify_pending {
25591 assert!(
25592 rig.dispatch
25593 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
25594 );
25595 }
25596
25597 let deferred = events.remove(&rig.cloid).unwrap();
25598 flush_modify_events(
25599 rig.cloid,
25600 deferred,
25601 &rig.dispatch,
25602 &rig.emitter,
25603 &mut events,
25604 );
25605 }
25606
25607 let events = drain_events(&mut rig.rx);
25608 assert_eq!(events.len(), 2);
25609 assert!(
25610 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.trade_id == parse_lighter_trade_id(&trade).unwrap())
25611 );
25612 assert!(matches!(
25613 &events[1],
25614 ExecutionEvent::Order(OrderEventAny::Canceled(_))
25615 ));
25616 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25617 assert!(rig.dispatch.order_events.lock().is_empty());
25618 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
25619 server.abort();
25620 let _ = server.await;
25621 }
25622
25623 async fn terminal_trades_server(
25624 order: &LighterOrder,
25625 trades: Vec<LighterTrade>,
25626 ) -> (
25627 LighterHttpClient,
25628 Arc<AtomicUsize>,
25629 tokio::task::JoinHandle<()>,
25630 ) {
25631 let hits = Arc::new(AtomicUsize::new(0));
25632 let count = hits.clone();
25633 let order_index = order.order_index.to_string();
25634 let market_index = order.market_index.to_string();
25635
25636 let raw = order.clone();
25637
25638 let app = Router::new()
25639 .route(
25640 "/api/v1/accountActiveOrders",
25641 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
25642 )
25643 .route(
25644 "/api/v1/accountInactiveOrders",
25645 get(move || {
25646 let raw = raw.clone();
25647 async move { axum::Json(serde_json::json!({"code":200,"orders":[raw]})) }
25648 }),
25649 )
25650 .route(
25651 "/api/v1/trades",
25652 get(
25653 move |headers: HeaderMap,
25654 axum::extract::Query(query): axum::extract::Query<
25655 BTreeMap<String, String>,
25656 >| {
25657 let trades = trades.clone();
25658 let count = count.clone();
25659 let order_index = order_index.clone();
25660 let market_index = market_index.clone();
25661 async move {
25662 count.fetch_add(1, Ordering::Relaxed);
25663 assert_eq!(query.get("order_index"), Some(&order_index));
25664 assert_eq!(query.get("market_id"), Some(&market_index));
25665 assert_eq!(
25666 query.get("account_index"),
25667 Some(&TEST_ACCOUNT_INDEX_I64.to_string())
25668 );
25669 assert_eq!(query.get("sort_by").map(String::as_str), Some("timestamp"));
25670 assert_eq!(query.get("sort_dir").map(String::as_str), Some("desc"));
25671 assert_eq!(query.get("limit").map(String::as_str), Some("100"));
25672 assert!(!query.contains_key("from"));
25673 assert!(
25674 headers
25675 .get("authorization")
25676 .is_some_and(|auth| !auth.is_empty())
25677 );
25678 assert!(!query.contains_key("authorization"));
25679 assert!(!query.contains_key("auth"));
25680 axum::Json(serde_json::json!({"code":200,"trades":trades}))
25681 }
25682 },
25683 ),
25684 );
25685
25686 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
25687 let addr = listener.local_addr().unwrap();
25688 let server = tokio::spawn(async move {
25689 axum::serve(listener, app).await.unwrap();
25690 });
25691
25692 let http_client = LighterHttpClient::new(
25693 LighterEnvironment::Testnet,
25694 Some(format!("http://{addr}")),
25695 1,
25696 None,
25697 )
25698 .unwrap();
25699 (http_client, hits, server)
25700 }
25701
25702 #[rstest]
25703 #[case::recovered(true, false)]
25704 #[case::exhausted(false, false)]
25705 #[case::foreign_order(false, true)]
25706 #[tokio::test]
25707 async fn terminal_recovery_is_order_scoped_and_bounded(
25708 #[case] recovered: bool,
25709 #[case] foreign_order: bool,
25710 ) {
25711 let mut rig = dispatcher_rig("TERMINAL-RECOVERY");
25712 register_identity(&rig);
25713 rig.dispatch.mark_accepted_emitted(rig.cloid);
25714 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledNotEnoughLiquidity);
25715 let mut trade = dispatcher_test_trade(&rig, true);
25716 trade.bid_id = order.order_index;
25717 trade.bid_id_str = Some(order.order_id.clone());
25718 trade.size = Decimal::from_str("0.0025").unwrap();
25719 order.filled_base_amount = trade.size;
25720 dispatch_lighter_order(
25721 &order,
25722 &rig.dispatch,
25723 &rig.emitter,
25724 &rig.registry,
25725 account_id(),
25726 trader_id(),
25727 UnixNanos::from(1),
25728 );
25729
25730 let trades = if foreign_order {
25731 trade.bid_id += 1;
25732 trade.bid_id_str = Some(trade.bid_id.to_string());
25733 vec![trade]
25734 } else if recovered {
25735 vec![trade.clone(), trade]
25736 } else {
25737 Vec::new()
25738 };
25739
25740 let (http_client, hits, server) = terminal_trades_server(&order, trades).await;
25741 let (client, _, _) = create_execution_client();
25742 let mut context = acked_probe_context(&client);
25743 context.http_client = http_client;
25744 context.dispatch = rig.dispatch.clone();
25745 context.registry = rig.registry.clone();
25746 context.emitter = rig.emitter.clone();
25747 recover_terminals(&context, false).await;
25748 assert_eq!(hits.load(Ordering::Relaxed), 0, "deadline must be honored");
25749
25750 if foreign_order {
25751 let error = recover_terminal_trades(order.market_index, order.order_index, &context)
25752 .await
25753 .unwrap_err();
25754 assert_eq!(
25755 error.to_string(),
25756 "Lighter terminal recovery returned a trade outside the requested order"
25757 );
25758 hits.store(0, Ordering::Relaxed);
25759 }
25760
25761 let attempts = if recovered {
25762 1
25763 } else {
25764 ACKED_ORDER_PROBE_ATTEMPTS
25765 };
25766
25767 for attempt in 1..=attempts {
25768 recover_terminals(&context, true).await;
25769 assert_eq!(hits.load(Ordering::Relaxed), attempt);
25770
25771 if attempt < attempts {
25772 assert!(drain_events(&mut rig.rx).is_empty());
25773 assert_eq!(
25774 rig.dispatch.terminals_pending.lock()[&rig.cloid].attempts,
25775 attempt
25776 );
25777 }
25778 }
25779
25780 let events = drain_events(&mut rig.rx);
25781 assert_eq!(events.len(), if recovered { 2 } else { 1 });
25782
25783 if recovered {
25784 assert!(
25785 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
25786 if fill.last_qty == Quantity::from("0.0025"))
25787 );
25788 }
25789
25790 assert!(matches!(
25791 events.last(),
25792 Some(ExecutionEvent::Order(OrderEventAny::Canceled(_)))
25793 ));
25794 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25795 recover_terminals(&context, true).await;
25796 assert!(drain_events(&mut rig.rx).is_empty());
25797 assert_eq!(hits.load(Ordering::Relaxed), attempts);
25798 server.abort();
25799 let _ = server.await;
25800 }
25801
25802 #[rstest]
25803 #[case::reconnect(false)]
25804 #[case::stop(true)]
25805 #[tokio::test]
25806 async fn terminal_worker_recovers_immediately_on_lifecycle(#[case] stop: bool) {
25807 let mut rig = dispatcher_rig("TERMINAL-LIFECYCLE");
25808 register_identity(&rig);
25809 rig.dispatch.mark_accepted_emitted(rig.cloid);
25810 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
25811 let mut trade = dispatcher_test_trade(&rig, true);
25812 trade.bid_id = order.order_index;
25813 trade.bid_id_str = Some(order.order_id.clone());
25814 trade.size = Decimal::from_str("0.0025").unwrap();
25815 order.filled_base_amount = trade.size;
25816 dispatch_lighter_order(
25817 &order,
25818 &rig.dispatch,
25819 &rig.emitter,
25820 &rig.registry,
25821 account_id(),
25822 trader_id(),
25823 UnixNanos::from(1),
25824 );
25825 rig.dispatch
25826 .terminals_pending
25827 .lock()
25828 .get_mut(&rig.cloid)
25829 .unwrap()
25830 .retry_at = tokio::time::Instant::now() + Duration::from_secs(3600);
25831 let (http_client, hits, server) = terminal_trades_server(&order, vec![trade]).await;
25832 let (mut client, _, _) = create_execution_client();
25833 client.http_client = http_client;
25834 client.dispatch = rig.dispatch.clone();
25835 client.registry = rig.registry.clone();
25836 client.emitter = rig.emitter.clone();
25837 let context = acked_probe_context(&client);
25838 client
25839 .terminal_recovery_handle
25840 .spawn(run_terminal_recovery(
25841 context,
25842 client.terminal_producers_stopped.clone(),
25843 ))
25844 .unwrap();
25845
25846 if stop {
25847 client.stop().unwrap();
25848 } else {
25849 client.dispatch.terminals_recover.notify_one();
25850 }
25851
25852 if !stop {
25853 wait_until_async(
25854 || async { rig.dispatch.terminals_pending.lock().is_empty() },
25855 Duration::from_secs(2),
25856 )
25857 .await;
25858 client.stop().unwrap();
25859 }
25860
25861 client.finish_session_shutdown().await.unwrap();
25862 let events = drain_events(&mut rig.rx);
25863 assert_eq!(hits.load(Ordering::Relaxed), 1);
25864 assert_eq!(events.len(), 2);
25865 assert!(
25866 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
25867 if fill.last_qty == Quantity::from("0.0025"))
25868 );
25869 assert!(
25870 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Expired(event))
25871 if event.ts_event == UnixNanos::from(1_777_941_383_900_000_000))
25872 );
25873 assert!(client.terminal_recovery_handle.is_none());
25874 assert!(rig.dispatch.terminals_pending.lock().is_empty());
25875 server.abort();
25876 let _ = server.await;
25877 }
25878
25879 #[rstest]
25880 #[case::restored(true, false)]
25881 #[case::reconciled(false, false)]
25882 #[case::declined(false, true)]
25883 fn terminal_preserves_applied_fills_and_post_only_state(
25884 #[case] restored: bool,
25885 #[case] declined: bool,
25886 ) {
25887 let mut rig = dispatcher_rig("TERMINAL-BASELINE");
25888 register_identity(&rig);
25889 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
25890 let instrument = LIGHTER_INSTRUMENT_CACHE
25891 .get(&rig.instrument_id)
25892 .unwrap()
25893 .value()
25894 .clone();
25895 let mut order = OrderTestBuilder::new(OrderType::Limit)
25896 .trader_id(trader_id())
25897 .strategy_id(strategy_id())
25898 .instrument_id(rig.instrument_id)
25899 .client_order_id(rig.cloid)
25900 .side(OrderSide::Buy)
25901 .quantity(Quantity::from("0.0050"))
25902 .price(Price::from("2352.74"))
25903 .post_only(true)
25904 .build();
25905 dispatch_lighter_order(
25906 &frame,
25907 &rig.dispatch,
25908 &rig.emitter,
25909 &rig.registry,
25910 account_id(),
25911 trader_id(),
25912 UnixNanos::from(1),
25913 );
25914 frame.price = Decimal::from_str("2353.00").unwrap();
25915 dispatch_lighter_order(
25916 &frame,
25917 &rig.dispatch,
25918 &rig.emitter,
25919 &rig.registry,
25920 account_id(),
25921 trader_id(),
25922 UnixNanos::from(2),
25923 );
25924
25925 for event in drain_events(&mut rig.rx) {
25926 let ExecutionEvent::Order(event) = event else {
25927 panic!("expected order event");
25928 };
25929
25930 order.apply(event).unwrap();
25931 }
25932
25933 let mut first = dispatcher_test_trade(&rig, true);
25934 first.bid_id = frame.order_index;
25935 first.bid_id_str = Some(frame.order_id.clone());
25936 first.size = Decimal::from_str("0.0010").unwrap();
25937 let fill = parse_lighter_order_filled(
25938 &first,
25939 &instrument,
25940 &rig.dispatch.order_identity(&rig.cloid).unwrap(),
25941 rig.cloid,
25942 account_id(),
25943 trader_id(),
25944 TEST_ACCOUNT_INDEX_I64,
25945 UnixNanos::from(3),
25946 )
25947 .unwrap()
25948 .unwrap();
25949 let (mut client, cache, _) = create_execution_client();
25950 client.dispatch = rig.dispatch.clone();
25951 client.emitter = rig.emitter.clone();
25952 client.subscribe_applied_fills();
25953 cache
25954 .borrow_mut()
25955 .add_order(order.clone(), None, Some(client_id()), false)
25956 .unwrap();
25957
25958 if !restored {
25959 let mut report = parse_ws_fill_report(
25960 &first,
25961 TEST_ACCOUNT_INDEX_I64,
25962 &instrument,
25963 account_id(),
25964 UnixNanos::from(3),
25965 )
25966 .unwrap()
25967 .unwrap();
25968 report.client_order_id = Some(rig.cloid);
25969
25970 let sweep = finish_fill_reports(
25971 &client.core,
25972 &rig.dispatch,
25973 BTreeSet::new(),
25974 Ok(FillSweep {
25975 reports: vec![report.clone()],
25976 covers_window: true,
25977 withheld_instruments: AHashSet::new(),
25978 }),
25979 )
25980 .unwrap();
25981
25982 assert_eq!(
25983 rig.dispatch
25984 .order_identity(&rig.cloid)
25985 .unwrap()
25986 .filled_qty(),
25987 Decimal::ZERO
25988 );
25989 assert_eq!(sweep.reports, vec![report.clone()]);
25990 assert!(sweep.covers_window);
25991 assert!(sweep.withheld_instruments.is_empty());
25992 assert!(!order_shape_pending(
25993 &client.core,
25994 &rig.dispatch,
25995 &rig.cloid
25996 ));
25997 assert_eq!(rig.dispatch.trade_source(fill.trade_id), None);
25998
25999 let repeated = finish_fill_reports(
26000 &client.core,
26001 &rig.dispatch,
26002 BTreeSet::new(),
26003 Ok(FillSweep {
26004 reports: vec![report.clone()],
26005 covers_window: true,
26006 withheld_instruments: AHashSet::new(),
26007 }),
26008 )
26009 .unwrap();
26010
26011 assert_eq!(repeated.reports, vec![report]);
26012 }
26013
26014 msgbus::publish_order_event(
26015 "events.order.S-001".into(),
26016 &OrderEventAny::Filled(fill.clone()),
26017 );
26018 assert_eq!(
26019 rig.dispatch
26020 .order_identity(&rig.cloid)
26021 .unwrap()
26022 .filled_qty(),
26023 Decimal::ZERO
26024 );
26025
26026 if !declined {
26027 cache
26028 .borrow_mut()
26029 .update_order(&OrderEventAny::Filled(fill.clone()))
26030 .unwrap();
26031 msgbus::publish_order_event(
26032 "events.order.S-001".into(),
26033 &OrderEventAny::Filled(fill.clone()),
26034 );
26035 order.apply(OrderEventAny::Filled(fill.clone())).unwrap();
26036 }
26037
26038 if restored {
26039 rig.dispatch = WsDispatchState::new();
26040 rig.dispatch.seed_order_restorations(vec![order.clone()]);
26041 }
26042
26043 if !restored {
26044 assert_eq!(
26045 rig.dispatch
26046 .order_identity(&rig.cloid)
26047 .unwrap()
26048 .filled_qty(),
26049 if declined { Decimal::ZERO } else { first.size }
26050 );
26051 }
26052
26053 frame.status = LighterOrderStatus::CanceledPostOnly;
26054 frame.filled_base_amount = Decimal::from_str("0.0030").unwrap();
26055 dispatch_lighter_order(
26056 &frame,
26057 &rig.dispatch,
26058 &rig.emitter,
26059 &rig.registry,
26060 account_id(),
26061 trader_id(),
26062 UnixNanos::from(4),
26063 );
26064 let report =
26065 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(4))
26066 .unwrap()
26067 .with_client_order_id(rig.cloid);
26068 assert!(!order_report_ready(
26069 &client.core,
26070 &rig.dispatch,
26071 &report,
26072 &[]
26073 ));
26074 assert_eq!(
26075 execution_instruments_pending(&client.core, &rig.dispatch),
26076 AHashSet::from_iter([rig.instrument_id])
26077 );
26078 dispatch_lighter_trade(
26079 &first,
26080 &rig.dispatch,
26081 &rig.emitter,
26082 &rig.registry,
26083 account_id(),
26084 trader_id(),
26085 Some(TEST_ACCOUNT_INDEX_I64),
26086 UnixNanos::from(5),
26087 );
26088 let recovered = drain_events(&mut rig.rx);
26089
26090 if declined {
26091 assert_eq!(recovered.len(), 1);
26092
26093 let ExecutionEvent::Order(OrderEventAny::Filled(recovered_fill)) = &recovered[0] else {
26094 panic!("discarded HTTP report must remain recoverable as a real fill");
26095 };
26096
26097 assert_eq!(recovered_fill.trade_id, fill.trade_id);
26098 assert_eq!(recovered_fill.last_qty, fill.last_qty);
26099 cache
26100 .borrow_mut()
26101 .update_order(&OrderEventAny::Filled(recovered_fill.clone()))
26102 .unwrap();
26103 msgbus::publish_order_event(
26104 "events.order.S-001".into(),
26105 &OrderEventAny::Filled(recovered_fill.clone()),
26106 );
26107 order
26108 .apply(OrderEventAny::Filled(recovered_fill.clone()))
26109 .unwrap();
26110 } else {
26111 assert!(
26112 recovered.is_empty(),
26113 "historical fill must not repeat or release terminal"
26114 );
26115 }
26116
26117 let mut second = first;
26118 second.trade_id += 1;
26119 second.trade_id_str = Some(second.trade_id.to_string());
26120 second.size = Decimal::from_str("0.0020").unwrap();
26121 dispatch_lighter_trade(
26122 &second,
26123 &rig.dispatch,
26124 &rig.emitter,
26125 &rig.registry,
26126 account_id(),
26127 trader_id(),
26128 Some(TEST_ACCOUNT_INDEX_I64),
26129 UnixNanos::from(6),
26130 );
26131 let events = drain_events(&mut rig.rx);
26132 assert_eq!(events.len(), 2);
26133 assert!(matches!(
26134 &events[0],
26135 ExecutionEvent::Order(OrderEventAny::Filled(_))
26136 ));
26137 assert!(matches!(
26138 &events[1],
26139 ExecutionEvent::Order(OrderEventAny::Canceled(_))
26140 ));
26141
26142 for event in events {
26143 let ExecutionEvent::Order(event) = event else {
26144 panic!("expected order event");
26145 };
26146
26147 order.apply(event).unwrap();
26148 }
26149
26150 assert_eq!(order.status(), OrderStatus::Canceled);
26151 assert_eq!(order.filled_qty(), Quantity::from("0.0030"));
26152 assert_eq!(order.price(), Some(Price::from("2353.00")));
26153 assert_eq!(order.trade_ids().len(), 2);
26154 assert_eq!(
26155 rig.dispatch
26156 .order_identity(&rig.cloid)
26157 .unwrap()
26158 .filled_qty(),
26159 Decimal::from_str("0.0030").unwrap()
26160 );
26161 }
26162
26163 #[rstest]
26164 #[case::paged(false)]
26165 #[case::repeated_cursor(true)]
26166 #[tokio::test]
26167 async fn terminal_recovery_paginates_without_duplicate_fills(#[case] repeat_cursor: bool) {
26168 let mut rig = dispatcher_rig("TERMINAL-PAGES");
26169 register_identity(&rig);
26170 rig.dispatch.mark_accepted_emitted(rig.cloid);
26171 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
26172 order.filled_base_amount = Decimal::from_str("0.0030").unwrap();
26173 let mut first = dispatcher_test_trade(&rig, true);
26174 first.bid_id = order.order_index;
26175 first.bid_id_str = Some(order.order_id.clone());
26176 first.size = Decimal::from_str("0.0010").unwrap();
26177 let mut second = first.clone();
26178 second.trade_id += 1;
26179 second.trade_id_str = Some(second.trade_id.to_string());
26180 second.size = Decimal::from_str("0.0020").unwrap();
26181 let hits = Arc::new(AtomicUsize::new(0));
26182 let count = hits.clone();
26183
26184 let app = Router::new().route("/api/v1/trades", get(move |axum::extract::Query(query): axum::extract::Query<BTreeMap<String, String>>| {
26185 let first = first.clone();
26186 let second = second.clone();
26187 let count = count.clone();
26188 async move {
26189 count.fetch_add(1, Ordering::Relaxed);
26190
26191 if let Some(cursor) = query.get("cursor") {
26192 assert_eq!(cursor, "older");
26193 axum::Json(serde_json::json!({"code":200,"trades":[first],"next_cursor":if repeat_cursor { Some("older") } else { None }}))
26194 } else {
26195 axum::Json(serde_json::json!({"code":200,"trades":[second],"next_cursor":"older"}))
26196 }
26197 }
26198 }));
26199
26200 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
26201 let addr = listener.local_addr().unwrap();
26202 let server = tokio::spawn(async move {
26203 axum::serve(listener, app).await.unwrap();
26204 });
26205
26206 let (client, _, _) = create_execution_client();
26207 let mut context = acked_probe_context(&client);
26208 context.http_client = LighterHttpClient::new(
26209 LighterEnvironment::Testnet,
26210 Some(format!("http://{addr}")),
26211 1,
26212 None,
26213 )
26214 .unwrap();
26215 context.dispatch = rig.dispatch.clone();
26216 context.registry = rig.registry.clone();
26217 context.emitter = rig.emitter.clone();
26218 dispatch_lighter_order(
26219 &order,
26220 &rig.dispatch,
26221 &rig.emitter,
26222 &rig.registry,
26223 account_id(),
26224 trader_id(),
26225 UnixNanos::from(1),
26226 );
26227 let result = recover_terminal_trades(order.market_index, order.order_index, &context).await;
26228 assert_eq!(hits.load(Ordering::Relaxed), 2);
26229
26230 if repeat_cursor {
26231 assert_eq!(
26232 result.unwrap_err().to_string(),
26233 "Lighter terminal recovery repeated cursor"
26234 );
26235 assert!(drain_events(&mut rig.rx).is_empty());
26236 assert!(
26237 rig.dispatch
26238 .terminals_pending
26239 .lock()
26240 .contains_key(&rig.cloid)
26241 );
26242 } else {
26243 result.unwrap();
26244 let events = drain_events(&mut rig.rx);
26245 assert_eq!(events.len(), 3);
26246 assert!(
26247 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.last_qty == Quantity::from("0.0010"))
26248 );
26249 assert!(
26250 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.last_qty == Quantity::from("0.0020"))
26251 );
26252 assert!(matches!(
26253 &events[2],
26254 ExecutionEvent::Order(OrderEventAny::Canceled(_))
26255 ));
26256 assert!(rig.dispatch.terminals_pending.lock().is_empty());
26257 }
26258
26259 server.abort();
26260 let _ = server.await;
26261 }
26262
26263 #[rstest]
26264 #[case::normal(false, 0)]
26265 #[case::interrupted_shutdown(true, 0)]
26266 #[case::interrupted_lookup(false, 1)]
26267 #[case::repeated_reconnect(false, ACKED_ORDER_PROBE_ATTEMPTS)]
26268 #[tokio::test]
26269 async fn terminal_shutdown_bounds_unresponsive_http(
26270 #[case] interrupted: bool,
26271 #[case] reconnects: usize,
26272 ) {
26273 let mut rig = dispatcher_rig("TERMINAL-TIMEOUT");
26274 register_identity(&rig);
26275 rig.dispatch.mark_accepted_emitted(rig.cloid);
26276 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
26277 order.filled_base_amount = Decimal::from_str("0.0025").unwrap();
26278 dispatch_lighter_order(
26279 &order,
26280 &rig.dispatch,
26281 &rig.emitter,
26282 &rig.registry,
26283 account_id(),
26284 trader_id(),
26285 UnixNanos::from(1),
26286 );
26287 let hits = Arc::new(AtomicUsize::new(0));
26288 let count = hits.clone();
26289 let app = Router::new().route(
26290 "/api/v1/trades",
26291 get(move || {
26292 count.fetch_add(1, Ordering::Relaxed);
26293 std::future::pending::<String>()
26294 }),
26295 );
26296
26297 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
26298 let addr = listener.local_addr().unwrap();
26299 let server = tokio::spawn(async move {
26300 axum::serve(listener, app).await.unwrap();
26301 });
26302
26303 let (mut client, _, _) = create_execution_client();
26304 client.http_client = LighterHttpClient::new(
26305 LighterEnvironment::Testnet,
26306 Some(format!("http://{addr}")),
26307 60,
26308 None,
26309 )
26310 .unwrap();
26311 client.dispatch = rig.dispatch.clone();
26312 client.registry = rig.registry.clone();
26313 client.emitter = rig.emitter.clone();
26314 let context = acked_probe_context(&client);
26315 client
26316 .terminal_recovery_handle
26317 .spawn(run_terminal_recovery(
26318 context,
26319 client.terminal_producers_stopped.clone(),
26320 ))
26321 .unwrap();
26322
26323 for expected in 1..=reconnects {
26324 rig.dispatch.terminals_recover.notify_one();
26325 wait_until_async(
26326 || async { hits.load(Ordering::Relaxed) == expected },
26327 Duration::from_secs(2),
26328 )
26329 .await;
26330 }
26331
26332 client.stop().unwrap();
26333
26334 if interrupted {
26335 {
26336 let shutdown = client.finish_session_shutdown();
26337 tokio::pin!(shutdown);
26338 tokio::select! {
26339 result = &mut shutdown => panic!("shutdown completed before blocked recovery: {result:?}"),
26340 () = wait_until_async(|| async { hits.load(Ordering::Relaxed) == 1 }, Duration::from_secs(2)) => {},
26341 }
26342 }
26343
26344 assert!(client.terminal_recovery_handle.is_some());
26345 assert!(client.terminal_producers_stopped.is_cancelled());
26346 }
26347
26348 tokio::time::timeout(
26349 TERMINAL_SHUTDOWN_TIMEOUT + DISCONNECT_TIMEOUT,
26350 client.finish_session_shutdown(),
26351 )
26352 .await
26353 .unwrap()
26354 .unwrap();
26355 let events = drain_events(&mut rig.rx);
26356 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
26357 assert_eq!(events.len(), 1);
26358 assert!(
26359 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Expired(event))
26360 if event.client_order_id == rig.cloid)
26361 );
26362 assert!(rig.dispatch.terminals_pending.lock().is_empty());
26363 assert!(client.terminal_recovery_handle.is_none());
26364 server.abort();
26365 let _ = server.await;
26366 }
26367
26368 #[rstest]
26369 #[case::query_terminal(0, 0)]
26370 #[case::query_deferred(0, 1)]
26371 #[case::cancel_terminal(1, 0)]
26372 #[case::cancel_deferred(1, 1)]
26373 #[case::query_partial(0, 2)]
26374 #[case::cancel_partial(1, 2)]
26375 #[case::create_partial(2, 2)]
26376 #[case::query_queued_fill(0, 3)]
26377 #[case::cancel_queued_fill(1, 3)]
26378 #[case::create_queued_fill(2, 3)]
26379 #[tokio::test]
26380 async fn terminal_blocks_http_order_publication(#[case] route: u8, #[case] phase: u8) {
26381 let (mut client, cache, mut rx) = create_execution_client();
26382 let instrument = register_test_instrument(&client, &cache);
26383 let id = ClientOrderId::from("TERMINAL-HTTP-BARRIER");
26384 let venue = VenueOrderId::from("123");
26385 let mut factory = test_order_factory();
26386 let cached = test_limit_order(&mut factory, instrument, id.as_str());
26387 let index = client.dispatch.register_create_identity(&cached).unwrap();
26388 client
26389 .dispatch
26390 .mark_order_submission(&id, TEST_SUBMISSION_NONCE);
26391 cache_accepted_order(&cache, cached, venue, Some(client_id()));
26392 client.dispatch.mark_accepted_emitted(id);
26393 let mut order = reconciliation_raw_order(index, venue, LighterOrderStatus::Canceled);
26394 order.filled_base_amount = Decimal::from_str("0.0025").unwrap();
26395 if phase >= 2 {
26396 order.status = LighterOrderStatus::Open;
26397 order.remaining_base_amount = order.initial_base_amount - order.filled_base_amount;
26398
26399 if phase == 3 {
26400 let mut report =
26401 reconciliation_fill_report(instrument, index, venue, "QUEUED-REAL-FILL");
26402 report.client_order_id = Some(id);
26403 report.last_qty = Quantity::from("0.0025");
26404 let identity = client.dispatch.order_identity(&id).unwrap();
26405 let cached = cache.borrow().order(&id).unwrap().clone();
26406 let inst = cache.borrow().instrument(&instrument).unwrap().clone();
26407
26408 let OrderEventAny::Filled(fill) = TestOrderEventStubs::filled(
26409 &cached,
26410 &inst,
26411 Some(report.trade_id),
26412 None,
26413 Some(report.last_px),
26414 Some(report.last_qty),
26415 None,
26416 Some(report.commission),
26417 None,
26418 Some(account_id()),
26419 ) else {
26420 panic!("expected fill");
26421 };
26422
26423 assert!(!identity.has_publication());
26424 dispatch_lighter_fill(
26425 fill,
26426 &client.emitter,
26427 &client.dispatch,
26428 &client.dispatch.order_events.lock(),
26429 );
26430 }
26431 } else {
26432 dispatch_lighter_order(
26433 &order,
26434 &client.dispatch,
26435 &client.emitter,
26436 &client.registry,
26437 account_id(),
26438 trader_id(),
26439 UnixNanos::from(1),
26440 );
26441 assert!(client.dispatch.terminals_pending.lock().contains_key(&id));
26442
26443 if phase == 1 {
26444 client.dispatch.terminals_pending.lock().remove(&id);
26445 client
26446 .dispatch
26447 .order_events
26448 .lock()
26449 .entry(id)
26450 .or_default()
26451 .store_order(&order);
26452 }
26453 }
26454
26455 let app = Router::new()
26456 .route(
26457 "/api/v1/accountActiveOrders",
26458 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
26459 )
26460 .route(
26461 "/api/v1/accountInactiveOrders",
26462 get(move || {
26463 let order = order.clone();
26464 async move { axum::Json(serde_json::json!({"code":200,"orders":[order]})) }
26465 }),
26466 );
26467
26468 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
26469 let addr = listener.local_addr().unwrap();
26470 let server = tokio::spawn(async move {
26471 axum::serve(listener, app).await.unwrap();
26472 });
26473
26474 client.http_client = LighterHttpClient::new(
26475 LighterEnvironment::Testnet,
26476 Some(format!("http://{addr}")),
26477 1,
26478 None,
26479 )
26480 .unwrap();
26481
26482 if route == 1 {
26483 enqueue_cancel(&client, instrument, id, venue, 12);
26484 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
26485 assert!(
26486 !recover_acked_action_report(&pending, &acked_probe_context(&client))
26487 .await
26488 .unwrap()
26489 );
26490 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((0, 12)));
26491 } else if route == 2 {
26492 let cached = cache.borrow().order(&id).unwrap().clone();
26493
26494 let probe = AckedOrderProbe {
26495 order: Box::new(cached),
26496 client_order_index: index,
26497 connection_epoch: 0,
26498 nonce: TEST_SUBMISSION_NONCE,
26499 api_key_index: TEST_API_KEY_INDEX,
26500 tx_hash: "create-probe".to_string(),
26501 };
26502
26503 assert!(!probe_acked_create_report(&probe, &acked_probe_context(&client), 1).await);
26504 } else {
26505 client
26506 .query_order(QueryOrder::new(
26507 trader_id(),
26508 Some(client_id()),
26509 strategy_id(),
26510 instrument,
26511 id,
26512 Some(venue),
26513 UUID4::new(),
26514 UnixNanos::from(2),
26515 None,
26516 None,
26517 ))
26518 .unwrap();
26519 client.pending_tasks.begin_shutdown();
26520 client
26521 .pending_tasks
26522 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
26523 .await
26524 .unwrap();
26525 }
26526
26527 let events = drain_events(&mut rx);
26528 assert_eq!(events.len(), usize::from(phase == 3));
26529 assert_eq!(
26530 cache.borrow().order(&id).unwrap().filled_qty(),
26531 Quantity::from("0.0000")
26532 );
26533
26534 if phase == 3 {
26535 assert!(matches!(
26536 &events[0],
26537 ExecutionEvent::Order(OrderEventAny::Filled(_))
26538 ));
26539 }
26540
26541 assert!(client.dispatch.order_identities.contains_key(&id));
26542 server.abort();
26543 let _ = server.await;
26544 }
26545
26546 #[rstest]
26547 #[case::normal(false, false)]
26548 #[case::interrupted(true, false)]
26549 #[case::incomplete(false, true)]
26550 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
26551 async fn terminal_shutdown_drains_after_producer_drop(
26552 #[case] interrupted: bool,
26553 #[case] incomplete: bool,
26554 ) {
26555 let mut rig = dispatcher_rig("TERMINAL-LATE-PRODUCER");
26556 register_identity(&rig);
26557 rig.dispatch.mark_accepted_emitted(rig.cloid);
26558 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
26559 let mut trade = dispatcher_test_trade(&rig, true);
26560 trade.bid_id = order.order_index;
26561 trade.bid_id_str = Some(order.order_id.clone());
26562 order.filled_base_amount = trade.size;
26563 let (http_client, hits, server) = terminal_trades_server(&order, vec![trade]).await;
26564 let (mut client, _, _) = create_execution_client();
26565 client.http_client = http_client;
26566 client.dispatch = rig.dispatch.clone();
26567 client.registry = rig.registry.clone();
26568 client.emitter = rig.emitter.clone();
26569 client.subscribe_applied_fills();
26570 let context = acked_probe_context(&client);
26571 client
26572 .terminal_recovery_handle
26573 .spawn(run_terminal_recovery(
26574 context.clone(),
26575 client.terminal_producers_stopped.clone(),
26576 ))
26577 .unwrap();
26578 let (ready, started) = tokio::sync::oneshot::channel();
26579 let (release, released) = std::sync::mpsc::sync_channel::<()>(1);
26580
26581 client
26582 .pending_tasks
26583 .spawn(async move {
26584 let _guard = TaskGroupGuard::new(&[], move || {
26585 dispatch_lighter_order(
26586 &order,
26587 &context.dispatch,
26588 &context.emitter,
26589 &context.registry,
26590 account_id(),
26591 trader_id(),
26592 UnixNanos::from(2),
26593 );
26594 });
26595
26596 ready.send(()).unwrap();
26597
26598 if incomplete {
26599 released.recv().unwrap();
26600 } else {
26601 std::future::pending::<()>().await;
26602 }
26603 })
26604 .unwrap();
26605
26606 started.await.unwrap();
26607 client.stop().unwrap();
26608
26609 if interrupted {
26610 assert!(
26611 tokio::time::timeout(Duration::from_millis(20), client.finish_session_shutdown())
26612 .await
26613 .is_err()
26614 );
26615 assert!(!client.terminal_producers_stopped.is_cancelled());
26616 assert!(client.terminal_recovery_handle.is_some());
26617 assert!(!client.pending_tasks.is_empty());
26618 }
26619
26620 if incomplete {
26621 assert!(client.finish_session_shutdown().await.is_err());
26622 assert!(!client.terminal_producers_stopped.is_cancelled());
26623 assert!(client.terminal_recovery_handle.is_some());
26624 assert!(!client.pending_tasks.is_empty());
26625 assert!(client.fills_applied.is_some());
26626 release.send(()).unwrap();
26627 }
26628
26629 client.finish_session_shutdown().await.unwrap();
26630 let events = drain_events(&mut rig.rx);
26631 assert_eq!(hits.load(Ordering::Relaxed), 1);
26632 assert_eq!(events.len(), 2);
26633 assert!(matches!(
26634 &events[0],
26635 ExecutionEvent::Order(OrderEventAny::Filled(_))
26636 ));
26637 assert!(matches!(
26638 &events[1],
26639 ExecutionEvent::Order(OrderEventAny::Expired(_))
26640 ));
26641 assert!(rig.dispatch.terminals_pending.lock().is_empty());
26642 assert!(client.session_tasks_finished());
26643 assert!(client.fills_applied.is_none());
26644 server.abort();
26645 let _ = server.await;
26646 }
26647
26648 #[rstest]
26649 #[case::canceled(LighterOrderStatus::Canceled, true, false, false)]
26650 #[case::expired(LighterOrderStatus::CanceledExpired, true, false, false)]
26651 #[case::missing_trade(LighterOrderStatus::Canceled, false, false, false)]
26652 #[case::canceled_bundle_first(LighterOrderStatus::Canceled, true, true, true)]
26653 #[case::canceled_mass_first(LighterOrderStatus::Canceled, true, true, false)]
26654 #[case::expired_bundle_first(LighterOrderStatus::CanceledExpired, true, true, true)]
26655 #[case::expired_mass_first(LighterOrderStatus::CanceledExpired, true, true, false)]
26656 #[case::early_missing_trade(LighterOrderStatus::Canceled, false, true, false)]
26657 #[tokio::test]
26658 async fn terminal_mass_status_pairs_missing_fills(
26659 #[case] status: LighterOrderStatus,
26660 #[case] complete: bool,
26661 #[case] early_timestamp: bool,
26662 #[case] bundle_first: bool,
26663 ) {
26664 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
26665 cache
26666 .borrow_mut()
26667 .add_account(AccountAny::Margin(test_margin_account()))
26668 .unwrap();
26669 dispatch_reduce_frame(&rig, &frame, false);
26670 consume_reduce_events(&mut rig, &mut engine);
26671 let instrument = LIGHTER_INSTRUMENT_CACHE
26672 .get(&rig.instrument_id)
26673 .unwrap()
26674 .value()
26675 .clone();
26676 let mut first = dispatcher_test_trade(&rig, true);
26677 first.bid_id = frame.order_index;
26678 first.bid_id_str = Some(frame.order_id.clone());
26679 first.size = Decimal::from_str("0.0005").unwrap();
26680 first.usd_amount = first.size * first.price;
26681 let mut initial_fill = parse_ws_fill_report(
26682 &first,
26683 TEST_ACCOUNT_INDEX_I64,
26684 &instrument,
26685 account_id(),
26686 UnixNanos::from(2),
26687 )
26688 .unwrap()
26689 .unwrap();
26690 initial_fill.client_order_id = Some(rig.cloid);
26691 engine.reconcile_fill_report(&initial_fill);
26692 let mut second = first.clone();
26693 second.trade_id += 1;
26694 second.trade_id_str = Some(second.trade_id.to_string());
26695 second.timestamp += 1;
26696 second.size = Decimal::from_str("0.0010").unwrap();
26697 second.usd_amount = second.size * second.price;
26698 let second_id = parse_lighter_trade_id(&second).unwrap();
26699 frame.updated_at = if early_timestamp {
26700 second.timestamp - 2
26701 } else {
26702 second.timestamp + 1
26703 };
26704
26705 let terminal_timestamp = UnixNanos::from(frame.updated_at as u64 * 1_000_000);
26706 frame.status = status;
26707 frame.filled_base_amount =
26708 Decimal::from_str(if complete { "0.0015" } else { "0.0020" }).unwrap();
26709 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
26710 frame.remaining_base_amount = Decimal::ZERO;
26711
26712 let mut client = create_execution_client_with_cache(cache.clone());
26713 client.registry = rig.registry.clone();
26714 client.emitter = rig.emitter.clone();
26715 client
26716 .dispatch
26717 .seed_order_restorations(vec![cache.borrow().order(&rig.cloid).unwrap().clone()]);
26718 client.dispatch.note_active_market(frame.market_index);
26719
26720 let position = PositionStatusReport::new(
26721 account_id(),
26722 rig.instrument_id,
26723 PositionSide::Long,
26724 Quantity::from("0.0015"),
26725 UnixNanos::from(3),
26726 UnixNanos::from(3),
26727 None,
26728 None,
26729 None,
26730 );
26731 client
26732 .dispatch
26733 .replace_position_snapshot(&[position], &[], &[]);
26734 let trades = vec![first, second.clone(), second];
26735
26736 let app = Router::new()
26737 .route(
26738 "/api/v1/accountActiveOrders",
26739 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
26740 )
26741 .route(
26742 "/api/v1/accountInactiveOrders",
26743 get(move || {
26744 let frame = frame.clone();
26745 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
26746 }),
26747 )
26748 .route(
26749 "/api/v1/trades",
26750 get(move || {
26751 let trades = trades.clone();
26752 async move { axum::Json(serde_json::json!({"code":200,"trades":trades})) }
26753 }),
26754 );
26755
26756 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
26757 let addr = listener.local_addr().unwrap();
26758 let server = tokio::spawn(async move {
26759 axum::serve(listener, app).await.unwrap();
26760 });
26761
26762 client.http_client = LighterHttpClient::new(
26763 LighterEnvironment::Testnet,
26764 Some(format!("http://{addr}")),
26765 1,
26766 None,
26767 )
26768 .unwrap();
26769 let mass = client.generate_mass_status(None).await.unwrap().unwrap();
26770 assert_eq!(
26771 mass.order_reports().len(),
26772 usize::from(complete && !early_timestamp)
26773 );
26774 let bundles = drain_events(&mut rig.rx);
26775 assert_eq!(bundles.len(), usize::from(complete && early_timestamp));
26776 assert_eq!(mass.fill_reports().values().map(Vec::len).sum::<usize>(), 2);
26777 assert_eq!(mass.position_reports().len(), usize::from(complete));
26778 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
26779 let engine = Rc::new(RefCell::new(engine));
26780 engine
26781 .borrow_mut()
26782 .register_client(Box::new(client))
26783 .unwrap();
26784 let mut manager =
26785 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default()).unwrap();
26786 let applied = Rc::new(RefCell::new(Vec::new()));
26787 let captured = applied.clone();
26788 let handler = TypedHandler::from(move |event: &OrderEventAny| {
26789 captured.borrow_mut().push(event.clone());
26790 });
26791
26792 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
26793 let subscription = AppliedFillSubscription(handler);
26794
26795 if bundle_first {
26796 for event in &bundles {
26797 let ExecutionEvent::Report(report) = event else {
26798 panic!("expected bundled report")
26799 };
26800
26801 engine.borrow_mut().reconcile_execution_report(report);
26802 }
26803 }
26804
26805 manager.reconcile_execution_mass_status(&mass, &engine);
26806
26807 if !bundle_first {
26808 for event in &bundles {
26809 let ExecutionEvent::Report(report) = event else {
26810 panic!("expected bundled report")
26811 };
26812
26813 engine.borrow_mut().reconcile_execution_report(report);
26814 }
26815 }
26816
26817 drop(subscription);
26818 let applied = applied.take();
26819 assert_eq!(applied.len(), if complete { 2 } else { 1 });
26820
26821 let OrderEventAny::Filled(fill) = &applied[0] else {
26822 panic!("expected real fill first")
26823 };
26824
26825 assert_eq!(fill.trade_id, second_id);
26826 assert_eq!(fill.last_qty, Quantity::from("0.0010"));
26827
26828 if complete {
26829 assert!(matches!(
26830 applied.last(),
26831 Some(OrderEventAny::Canceled(_) | OrderEventAny::Expired(_))
26832 ));
26833 assert_eq!(applied.last().unwrap().ts_event(), terminal_timestamp);
26834 }
26835
26836 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
26837 assert_eq!(cached.filled_qty(), Quantity::from("0.0015"));
26838 assert_eq!(cached.trade_ids(), vec![&initial_fill.trade_id, &second_id]);
26839 assert_eq!(
26840 cached.status(),
26841 if !complete {
26842 OrderStatus::PartiallyFilled
26843 } else if status == LighterOrderStatus::CanceledExpired {
26844 OrderStatus::Expired
26845 } else {
26846 OrderStatus::Canceled
26847 }
26848 );
26849 assert!(
26850 nautilus_execution::reconciliation::inferred_reconciliation_trade_ids(&cached)
26851 .unwrap()
26852 .is_empty()
26853 );
26854 server.abort();
26855 let _ = server.await;
26856 }
26857
26858 #[rstest]
26859 #[case::observed(0)]
26860 #[case::synced(1)]
26861 #[case::restored(2)]
26862 #[case::restored_child(3)]
26863 #[tokio::test]
26864 async fn terminal_inferred_fills_exhaust_without_replay(#[case] mode: u8) {
26865 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
26866
26867 cache
26868 .borrow_mut()
26869 .add_account(AccountAny::Margin(test_margin_account()))
26870 .unwrap();
26871
26872 let mut client = create_execution_client_with_cache(cache.clone());
26873 client.dispatch = rig.dispatch.clone();
26874 client.emitter = rig.emitter.clone();
26875 client.registry = rig.registry.clone();
26876 if mode == 0 {
26877 client.subscribe_applied_fills();
26878 }
26879
26880 dispatch_reduce_frame(&rig, &frame, false);
26881 consume_reduce_events(&mut rig, &mut engine);
26882 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
26883 frame.remaining_base_amount -= frame.filled_base_amount;
26884 let instrument = LIGHTER_INSTRUMENT_CACHE
26885 .get(&rig.instrument_id)
26886 .unwrap()
26887 .value()
26888 .clone();
26889 let mut report =
26890 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(2))
26891 .unwrap()
26892 .with_client_order_id(rig.cloid);
26893 report.order_status = OrderStatus::PartiallyFilled;
26894 engine.reconcile_order_status_report(&report);
26895 let mut cached = cache.borrow().order(&rig.cloid).unwrap().clone();
26896 let inferred =
26897 nautilus_execution::reconciliation::inferred_reconciliation_trade_ids(&cached).unwrap();
26898 assert_eq!(inferred.len(), 1);
26899 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
26900
26901 if mode == 1 {
26902 assert!(order_shape_pending(&client.core, &rig.dispatch, &rig.cloid));
26903 } else if mode >= 2 {
26904 if mode == 3 {
26905 let mut history = cached.events().into_iter().cloned().collect::<Vec<_>>();
26906
26907 let OrderEventAny::Initialized(init) = &mut history[0] else {
26908 panic!("expected initialization");
26909 };
26910
26911 init.parent_order_id = Some(ClientOrderId::new("INFERRED-PARENT"));
26912 cached = OrderAny::from_events(history).unwrap();
26913 cache
26914 .borrow_mut()
26915 .add_order(cached.clone(), None, Some(client_id()), true)
26916 .unwrap();
26917 }
26918
26919 rig.dispatch = WsDispatchState::new();
26920 rig.dispatch.seed_order_restorations(vec![cached]);
26921 client.dispatch = rig.dispatch.clone();
26922 dispatch_reduce_frame(&rig, &frame, false);
26923 assert!(drain_events(&mut rig.rx).is_empty());
26924 }
26925
26926 let positions_before = cache
26927 .borrow()
26928 .positions_open(None, None, None, None, None)
26929 .iter()
26930 .map(|position| position.quantity)
26931 .collect::<Vec<_>>();
26932 assert_eq!(positions_before, vec![Quantity::from("0.0010")]);
26933 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
26934 assert!(identity.has_inferred_fills());
26935 assert_eq!(identity.filled_qty(), Decimal::ZERO);
26936 assert_eq!(rig.dispatch.trade_source(inferred[0]), None);
26937 assert_eq!(
26938 execution_instruments_pending(&client.core, &client.dispatch),
26939 AHashSet::from_iter([rig.instrument_id])
26940 );
26941 assert!(!order_report_ready(
26942 &client.core,
26943 &rig.dispatch,
26944 &report,
26945 &[]
26946 ));
26947 let mut trade = dispatcher_test_trade(&rig, true);
26948 trade.bid_id = frame.order_index;
26949 trade.bid_id_str = Some(frame.order_id.clone());
26950 trade.size = frame.filled_base_amount;
26951 let mut fill = parse_ws_fill_report(
26952 &trade,
26953 TEST_ACCOUNT_INDEX_I64,
26954 &instrument,
26955 account_id(),
26956 UnixNanos::from(3),
26957 )
26958 .unwrap()
26959 .unwrap();
26960 fill.client_order_id = Some(rig.cloid);
26961 frame.status = LighterOrderStatus::Canceled;
26962 frame.remaining_base_amount = Decimal::ZERO;
26963 dispatch_reduce_frame(&rig, &frame, false);
26964 assert!(drain_events(&mut rig.rx).is_empty());
26965 let (http, hits, server) = terminal_trades_server(&frame, vec![trade]).await;
26966 client.http_client = http;
26967 let context = acked_probe_context(&client);
26968
26969 if mode == 3 {
26970 let command = GenerateOrderStatusReport::new(
26971 UUID4::new(),
26972 UnixNanos::from(4),
26973 Some(rig.instrument_id),
26974 Some(rig.cloid),
26975 Some(VenueOrderId::new(&frame.order_id)),
26976 None,
26977 None,
26978 );
26979 assert!(client.generate_order_status_report(&command).await.is_err());
26980 wait_until_async(
26981 || async {
26982 client
26983 .dispatch
26984 .order_events
26985 .lock()
26986 .get(&rig.cloid)
26987 .is_none_or(|entry| entry.recovery_epoch.is_none())
26988 },
26989 Duration::from_secs(6),
26990 )
26991 .await;
26992 }
26993
26994 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
26995 recover_terminals(&context, true).await;
26996 let events = consume_reduce_events(&mut rig, &mut engine);
26997
26998 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
26999 assert!(events.is_empty());
27000 } else {
27001 assert_eq!(events.len(), 1);
27002 assert!(
27003 matches!(&events[0], OrderEventAny::Canceled(event) if event.ts_event == report.ts_last)
27004 );
27005 }
27006 }
27007
27008 let sweep = finish_fill_reports(
27009 &client.core,
27010 &rig.dispatch,
27011 BTreeSet::new(),
27012 Ok(FillSweep {
27013 reports: vec![fill],
27014 covers_window: true,
27015 withheld_instruments: AHashSet::new(),
27016 }),
27017 )
27018 .unwrap();
27019
27020 assert!(sweep.reports.is_empty());
27021 assert!(!sweep.covers_window);
27022 assert_eq!(
27023 sweep.withheld_instruments,
27024 AHashSet::from_iter([rig.instrument_id])
27025 );
27026 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
27027 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
27028 assert_eq!(cached.trade_ids(), inferred.iter().collect::<Vec<_>>());
27029 assert_eq!(cached.status(), OrderStatus::Canceled);
27030 assert_eq!(
27031 cache
27032 .borrow()
27033 .positions_open(None, None, None, None, None)
27034 .iter()
27035 .map(|position| position.quantity)
27036 .collect::<Vec<_>>(),
27037 positions_before
27038 );
27039 assert_eq!(identity.filled_qty(), Decimal::ZERO);
27040 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
27041 assert!(rig.dispatch.terminals_pending.lock().is_empty());
27042 inferred_reconciliation_trade_ids(&cached).unwrap();
27043 assert_eq!(
27044 client
27045 .dispatch
27046 .order_events
27047 .lock()
27048 .get(&rig.cloid)
27049 .map_or(0, |entry| entry.fills.len()),
27050 0
27051 );
27052 client.dispatch.replace_position_snapshot(&[], &[], &[]);
27053 assert!(execution_instruments_pending(&client.core, &client.dispatch).is_empty());
27054
27055 let command = GeneratePositionStatusReports::new(
27056 UUID4::new(),
27057 UnixNanos::from(5),
27058 Some(rig.instrument_id),
27059 None,
27060 None,
27061 None,
27062 None,
27063 );
27064 assert!(
27065 client
27066 .generate_position_status_reports(&command)
27067 .await
27068 .unwrap()
27069 .is_empty()
27070 );
27071 let task = client
27072 .generate_position_status_reports_task(&command)
27073 .unwrap();
27074 task.collection.await;
27075 assert!((task.finish)().unwrap().is_empty());
27076 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
27077
27078 let config = ExecutionManagerConfig {
27079 position_check_threshold_ns: DurationNanos::from_secs(0),
27080 ..ExecutionManagerConfig::default()
27081 };
27082
27083 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
27084 for event in manager.check_positions_consistency(&[&client]).await {
27085 engine.process(&event);
27086 }
27087
27088 assert_eq!(
27089 cache
27090 .borrow()
27091 .positions_open(None, None, None, None, None)
27092 .iter()
27093 .map(|position| position.signed_decimal_qty())
27094 .sum::<Decimal>(),
27095 Decimal::ZERO
27096 );
27097 assert_eq!(
27098 cache.borrow().order(&rig.cloid).unwrap().trade_ids(),
27099 inferred.iter().collect::<Vec<_>>()
27100 );
27101 assert_eq!(identity.filled_qty(), Decimal::ZERO);
27102 cache
27103 .borrow_mut()
27104 .purge_closed_orders(UnixNanos::from(u64::MAX), 0, false)
27105 .unwrap();
27106 assert!(cache.borrow().order(&rig.cloid).is_none());
27107 assert!(execution_instruments_pending(&client.core, &client.dispatch).is_empty());
27108 server.abort();
27109 let _ = server.await;
27110 }
27111
27112 #[rstest]
27113 #[case::applied(true, false)]
27114 #[case::discarded(false, false)]
27115 #[case::venue_uuid(true, true)]
27116 fn terminal_tracks_engine_applied_reconciliation(
27117 #[case] applied: bool,
27118 #[case] venue_uuid: bool,
27119 ) {
27120 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
27121
27122 let mut client = create_execution_client_with_cache(cache.clone());
27123 client.dispatch = rig.dispatch.clone();
27124 client.emitter = rig.emitter.clone();
27125 client.subscribe_applied_fills();
27126 client.subscribe_applied_fills();
27127 dispatch_reduce_frame(&rig, &frame, false);
27128 consume_reduce_events(&mut rig, &mut engine);
27129 let mut trade = dispatcher_test_trade(&rig, true);
27130 trade.bid_id = frame.order_index;
27131 trade.bid_id_str = Some(frame.order_id.clone());
27132 trade.size = Decimal::from_str("0.0010").unwrap();
27133 if venue_uuid {
27134 trade.trade_id_str = Some("4b1da1f0-0d43-5a14-99b0-7245e76569af".to_string());
27135 }
27136
27137 let instrument = LIGHTER_INSTRUMENT_CACHE
27138 .get(&rig.instrument_id)
27139 .unwrap()
27140 .value()
27141 .clone();
27142 let mut report = parse_ws_fill_report(
27143 &trade,
27144 TEST_ACCOUNT_INDEX_I64,
27145 &instrument,
27146 account_id(),
27147 UnixNanos::from(2),
27148 )
27149 .unwrap()
27150 .unwrap();
27151 report.client_order_id = Some(rig.cloid);
27152
27153 for _ in 0..2 {
27154 let reports = finish_fill_reports(
27155 &client.core,
27156 &rig.dispatch,
27157 BTreeSet::new(),
27158 Ok(FillSweep {
27159 reports: vec![report.clone()],
27160 covers_window: true,
27161 withheld_instruments: AHashSet::new(),
27162 }),
27163 )
27164 .unwrap();
27165
27166 assert_eq!(reports.reports, vec![report.clone()]);
27167 assert!(!order_shape_pending(
27168 &client.core,
27169 &rig.dispatch,
27170 &rig.cloid
27171 ));
27172 }
27173
27174 if applied {
27175 engine.reconcile_fill_report(&report);
27176 }
27177
27178 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
27179 assert_eq!(
27180 identity.filled_qty(),
27181 if applied { trade.size } else { Decimal::ZERO }
27182 );
27183 assert_eq!(
27184 rig.dispatch.trade_source(report.trade_id),
27185 if applied {
27186 Some(TradeDedupSource::Reconciliation)
27187 } else {
27188 None
27189 }
27190 );
27191 assert!(!identity.has_publication());
27192 frame.status = LighterOrderStatus::Canceled;
27193 frame.filled_base_amount = trade.size;
27194 frame.remaining_base_amount = Decimal::ZERO;
27195 dispatch_reduce_frame(&rig, &frame, false);
27196
27197 if !applied {
27198 assert!(drain_events(&mut rig.rx).is_empty());
27199 dispatch_lighter_trade(
27200 &trade,
27201 &rig.dispatch,
27202 &rig.emitter,
27203 &rig.registry,
27204 account_id(),
27205 trader_id(),
27206 Some(TEST_ACCOUNT_INDEX_I64),
27207 UnixNanos::from(3),
27208 );
27209 }
27210
27211 let events = consume_reduce_events(&mut rig, &mut engine);
27212 assert_eq!(events.len(), if applied { 1 } else { 2 });
27213 assert!(matches!(events.last(), Some(OrderEventAny::Canceled(_))));
27214
27215 if !applied {
27216 assert!(
27217 matches!(&events[0], OrderEventAny::Filled(fill) if fill.trade_id == report.trade_id)
27218 );
27219 }
27220
27221 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
27222 assert_eq!(order.status(), OrderStatus::Canceled);
27223 assert_eq!(order.filled_qty(), Quantity::from("0.0010"));
27224 assert_eq!(order.trade_ids(), vec![&report.trade_id]);
27225 assert!(rig.dispatch.terminals_pending.lock().is_empty());
27226 frame.filled_base_amount += Decimal::from_str("0.0001").unwrap();
27227 dispatch_reduce_frame(&rig, &frame, false);
27228 assert!(drain_events(&mut rig.rx).is_empty());
27229 }
27230
27231 #[rstest]
27232 #[case::drop(true)]
27233 #[case::shutdown(false)]
27234 #[tokio::test]
27235 async fn terminal_fill_subscription_detaches(#[case] drop_client: bool) {
27236 let (mut rig, cache, mut engine, frame) = reduce_order_replay();
27237
27238 let mut client = create_execution_client_with_cache(cache.clone());
27239 client.dispatch = rig.dispatch.clone();
27240 client.subscribe_applied_fills();
27241 client.subscribe_applied_fills();
27242 dispatch_reduce_frame(&rig, &frame, false);
27243 consume_reduce_events(&mut rig, &mut engine);
27244 let mut report = reconciliation_fill_report(
27245 rig.instrument_id,
27246 frame.client_order_index,
27247 VenueOrderId::new(&frame.order_id),
27248 "APPLIED-FIRST",
27249 );
27250 report.client_order_id = Some(rig.cloid);
27251 report.last_qty = Quantity::from("0.0010");
27252 engine.reconcile_fill_report(&report);
27253 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
27254 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
27255
27256 if !drop_client {
27257 client.stop().unwrap();
27258 client.finish_session_shutdown().await.unwrap();
27259 assert!(client.fills_applied.is_none());
27260 }
27261
27262 drop(client);
27263 report.trade_id = TradeId::new("APPLIED-SECOND");
27264 engine.reconcile_fill_report(&report);
27265 assert_eq!(
27266 cache.borrow().order(&rig.cloid).unwrap().filled_qty(),
27267 Quantity::from("0.0020")
27268 );
27269 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
27270 assert_eq!(rig.dispatch.trade_source(report.trade_id), None);
27271 }
27272
27273 fn test_margin_account() -> MarginAccount {
27274 MarginAccount::new(
27275 AccountState::new(
27276 account_id(),
27277 AccountType::Margin,
27278 vec![AccountBalance::new(
27279 Money::from("10000.00000000 USDC"),
27280 Money::from("0.00000000 USDC"),
27281 Money::from("10000.00000000 USDC"),
27282 )],
27283 Vec::new(),
27284 true,
27285 UUID4::new(),
27286 UnixNanos::from(1),
27287 UnixNanos::from(1),
27288 Some(Currency::from("USDC")),
27289 ),
27290 true,
27291 )
27292 }
27293
27294 #[rstest]
27295 #[case::inline_cancel(0, LighterOrderStatus::Canceled)]
27296 #[case::worker_cancel(1, LighterOrderStatus::Canceled)]
27297 #[case::query_cancel(2, LighterOrderStatus::Canceled)]
27298 #[case::inline_expire(0, LighterOrderStatus::CanceledExpired)]
27299 #[case::worker_expire(1, LighterOrderStatus::CanceledExpired)]
27300 #[case::query_expire(2, LighterOrderStatus::CanceledExpired)]
27301 #[case::query_filled(2, LighterOrderStatus::Filled)]
27302 #[tokio::test]
27303 async fn terminal_query_recovers_missed_frames(
27304 #[case] route: u8,
27305 #[case] status: LighterOrderStatus,
27306 ) {
27307 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
27308 cache
27309 .borrow_mut()
27310 .add_account(AccountAny::Margin(test_margin_account()))
27311 .unwrap();
27312 dispatch_reduce_frame(&rig, &frame, false);
27313 consume_reduce_events(&mut rig, &mut engine);
27314
27315 let mut client = create_execution_client_with_cache(cache.clone());
27316 client.dispatch = rig.dispatch.clone();
27317 client.registry = rig.registry.clone();
27318 client.emitter = rig.emitter.clone();
27319 frame.status = status;
27320 let filled = status == LighterOrderStatus::Filled;
27321 let qty = Quantity::from(if filled { "0.0027" } else { "0.0010" });
27322 frame.filled_base_amount = qty.as_decimal();
27323 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27324 frame.remaining_base_amount = Decimal::ZERO;
27325 let mut trade = dispatcher_test_trade(&rig, true);
27326 trade.bid_id = frame.order_index;
27327 trade.bid_id_str = Some(frame.order_id.clone());
27328 trade.size = frame.filled_base_amount;
27329 trade.usd_amount = trade.size * trade.price;
27330 let trade_id = parse_lighter_trade_id(&trade).unwrap();
27331 let terminal_timestamp = UnixNanos::from(frame.updated_at as u64 * 1_000_000);
27332 let venue = VenueOrderId::from(frame.order_id.as_str());
27333 let raw = frame.clone();
27334 let hits = Arc::new(AtomicUsize::new(0));
27335 let counted = hits.clone();
27336
27337 let app = Router::new()
27338 .route(
27339 "/api/v1/accountActiveOrders",
27340 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
27341 )
27342 .route(
27343 "/api/v1/accountInactiveOrders",
27344 get(move || {
27345 let raw = raw.clone();
27346 async move { axum::Json(serde_json::json!({"code":200,"orders":[raw]})) }
27347 }),
27348 )
27349 .route(
27350 "/api/v1/trades",
27351 get(move || {
27352 let trade = trade.clone();
27353 let counted = counted.clone();
27354 async move {
27355 counted.fetch_add(1, Ordering::Relaxed);
27356 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
27357 }
27358 }),
27359 );
27360
27361 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
27362 let addr = listener.local_addr().unwrap();
27363 let server = tokio::spawn(async move {
27364 axum::serve(listener, app).await.unwrap();
27365 });
27366
27367 client.http_client = LighterHttpClient::new(
27368 LighterEnvironment::Testnet,
27369 Some(format!("http://{addr}")),
27370 1,
27371 None,
27372 )
27373 .unwrap();
27374
27375 let command = GenerateOrderStatusReport::new(
27376 UUID4::new(),
27377 UnixNanos::from(4),
27378 Some(rig.instrument_id),
27379 Some(rig.cloid),
27380 Some(venue),
27381 None,
27382 None,
27383 );
27384
27385 if filled && route == 2 {
27386 dispatch_reduce_frame(&rig, &frame, false);
27387 assert!(!client.dispatch.order_identities.contains_key(&rig.cloid));
27388 }
27389
27390 let context = acked_probe_context(&client);
27391 let report = match route {
27392 0 => client.generate_order_status_report(&command).await.unwrap(),
27393 1 => {
27394 let task = client.generate_order_status_report_task(&command).unwrap();
27395 task.collection.await;
27396 (task.finish)().unwrap()
27397 }
27398 _ => {
27399 client
27400 .query_order(QueryOrder::new(
27401 trader_id(),
27402 Some(client_id()),
27403 strategy_id(),
27404 rig.instrument_id,
27405 rig.cloid,
27406 Some(venue),
27407 UUID4::new(),
27408 UnixNanos::from(4),
27409 None,
27410 None,
27411 ))
27412 .unwrap();
27413 client.pending_tasks.begin_shutdown();
27414 client
27415 .pending_tasks
27416 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
27417 .await
27418 .unwrap();
27419 None
27420 }
27421 };
27422
27423 if !filled || route != 2 {
27424 assert!(drain_events(&mut rig.rx).is_empty());
27425 }
27426
27427 assert_eq!(
27428 client
27429 .dispatch
27430 .terminals_pending
27431 .lock()
27432 .contains_key(&rig.cloid),
27433 route == 2 && !filled
27434 );
27435 let applied = Rc::new(RefCell::new(Vec::new()));
27436 let captured = applied.clone();
27437 let handler = TypedHandler::from(move |event: &OrderEventAny| {
27438 captured.borrow_mut().push(event.clone());
27439 });
27440
27441 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
27442 let subscription = AppliedFillSubscription(handler);
27443
27444 if let Some(report) = report {
27445 assert_eq!(report.filled_qty, qty);
27446
27447 let command = GenerateFillReports::new(
27448 UUID4::new(),
27449 UnixNanos::from(5),
27450 Some(rig.instrument_id),
27451 Some(venue),
27452 None,
27453 None,
27454 None,
27455 None,
27456 );
27457 let fills = client.generate_fill_reports(command).await.unwrap();
27458 let bundle = EngineExecutionReport::OrderWithFills(Box::new(report), fills);
27459 engine.reconcile_execution_report(&bundle);
27460 } else {
27461 if !filled {
27462 recover_terminals(&context, true).await;
27463 }
27464
27465 consume_reduce_events(&mut rig, &mut engine);
27466 }
27467
27468 drop(subscription);
27469 let events = applied.take();
27470 assert_eq!(events.len(), if filled { 1 } else { 2 });
27471
27472 let OrderEventAny::Filled(fill) = &events[0] else {
27473 panic!("expected real fill first");
27474 };
27475
27476 assert_eq!(fill.trade_id, trade_id);
27477 assert_eq!(fill.last_qty, qty);
27478
27479 if !filled {
27480 assert!(matches!(
27481 events[1],
27482 OrderEventAny::Canceled(_) | OrderEventAny::Expired(_)
27483 ));
27484 assert_eq!(events[1].ts_event(), terminal_timestamp);
27485 }
27486
27487 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
27488 assert_eq!(cached.trade_ids(), vec![&trade_id]);
27489 assert_eq!(cached.filled_qty(), qty);
27490 assert_eq!(hits.load(Ordering::Relaxed), 1);
27491 server.abort();
27492 let _ = server.await;
27493 }
27494
27495 #[rstest]
27496 #[case::bulk_complete(true, 2)]
27497 #[case::targeted_complete(false, 2)]
27498 #[case::bulk_partial(true, 1)]
27499 #[case::targeted_partial(false, 1)]
27500 #[case::bulk_empty(true, 0)]
27501 #[case::targeted_empty(false, 0)]
27502 #[tokio::test]
27503 async fn terminal_targeted_reconciliation_recovers_offsetting_fills(
27504 #[case] bulk_report: bool,
27505 #[case] coverage: u8,
27506 ) {
27507 let (mut rig, cache, mut engine, mut buy) = reduce_order_replay();
27508 cache
27509 .borrow_mut()
27510 .add_account(AccountAny::Margin(test_margin_account()))
27511 .unwrap();
27512 dispatch_reduce_frame(&rig, &buy, false);
27513 consume_reduce_events(&mut rig, &mut engine);
27514 let sell_id = ClientOrderId::from("MISSED-SELL");
27515 let sell_order = OrderTestBuilder::new(OrderType::Limit)
27516 .trader_id(trader_id())
27517 .strategy_id(strategy_id())
27518 .instrument_id(rig.instrument_id)
27519 .client_order_id(sell_id)
27520 .side(OrderSide::Sell)
27521 .quantity(Quantity::from("0.0027"))
27522 .price(Price::from("2352.74"))
27523 .build();
27524 let sell_index = rig.dispatch.register_create_identity(&sell_order).unwrap();
27525 rig.dispatch
27526 .mark_order_submission(&sell_id, TEST_SUBMISSION_NONCE + 1);
27527 cache
27528 .borrow_mut()
27529 .add_order(sell_order, None, Some(client_id()), false)
27530 .unwrap();
27531 engine.process(&OrderEventAny::Submitted(OrderSubmitted::new(
27532 trader_id(),
27533 strategy_id(),
27534 rig.instrument_id,
27535 sell_id,
27536 account_id(),
27537 UUID4::new(),
27538 UnixNanos::from(1),
27539 UnixNanos::from(1),
27540 )));
27541 let mut sell = buy.clone();
27542 sell.order_index += 1;
27543 sell.order_id = sell.order_index.to_string();
27544 sell.client_order_index = sell_index;
27545 sell.client_order_id = sell_index.to_string();
27546 sell.is_ask = true;
27547 sell.side = Some(LighterOrderSide::Sell);
27548 dispatch_reduce_frame(&rig, &sell, false);
27549 consume_reduce_events(&mut rig, &mut engine);
27550 buy.status = LighterOrderStatus::Canceled;
27551 sell.status = LighterOrderStatus::CanceledExpired;
27552 for frame in [&mut buy, &mut sell] {
27553 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
27554 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27555 frame.remaining_base_amount = Decimal::ZERO;
27556 }
27557
27558 let mut buy_trade = dispatcher_test_trade(&rig, true);
27559 buy_trade.bid_id = buy.order_index;
27560 buy_trade.bid_id_str = Some(buy.order_id.clone());
27561
27562 let qty = Quantity::from(match coverage {
27563 0 => "0.0000",
27564 1 => "0.0005",
27565 _ => "0.0010",
27566 });
27567
27568 buy_trade.size = qty.as_decimal();
27569 buy_trade.usd_amount = buy_trade.size * buy_trade.price;
27570 let mut sell_trade = dispatcher_test_trade(&rig, false);
27571 sell_trade.trade_id += 1;
27572 sell_trade.trade_id_str = Some(sell_trade.trade_id.to_string());
27573 sell_trade.ask_id = sell.order_index;
27574 sell_trade.ask_id_str = Some(sell.order_id.clone());
27575 sell_trade.ask_client_id = sell_index;
27576 sell_trade.ask_client_id_str = Some(sell_index.to_string());
27577 sell_trade.size = buy_trade.size;
27578 sell_trade.usd_amount = sell_trade.size * sell_trade.price;
27579 buy.updated_at = buy_trade.timestamp + 1;
27580 sell.updated_at = sell_trade.timestamp + 1;
27581 let expected = [
27582 (
27583 rig.cloid,
27584 parse_lighter_trade_id(&buy_trade).unwrap(),
27585 OrderStatus::Canceled,
27586 ),
27587 (
27588 sell_id,
27589 parse_lighter_trade_id(&sell_trade).unwrap(),
27590 OrderStatus::Expired,
27591 ),
27592 ];
27593 let orders = vec![buy, sell];
27594
27595 let trades = if coverage == 0 {
27596 Vec::new()
27597 } else {
27598 vec![buy_trade, sell_trade]
27599 };
27600
27601 let history_hits = Arc::new(AtomicUsize::new(0));
27602 let hits = Arc::new(AtomicUsize::new(0));
27603 let counted = hits.clone();
27604
27605 let app = Router::new()
27606 .route(
27607 "/api/v1/accountActiveOrders",
27608 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
27609 )
27610 .route(
27611 "/api/v1/accountInactiveOrders",
27612 get(move || {
27613 let orders = orders.clone();
27614 let history_hits = history_hits.clone();
27615 async move {
27616 let first = history_hits.fetch_add(1, Ordering::Relaxed) == 0;
27617
27618 let orders = if !bulk_report && first {
27619 Vec::new()
27620 } else {
27621 orders
27622 };
27623
27624 axum::Json(serde_json::json!({"code":200,"orders":orders}))
27625 }
27626 }),
27627 )
27628 .route(
27629 "/api/v1/trades",
27630 get(move || {
27631 let trades = trades.clone();
27632 let counted = counted.clone();
27633 async move {
27634 counted.fetch_add(1, Ordering::Relaxed);
27635 axum::Json(serde_json::json!({"code":200,"trades":trades}))
27636 }
27637 }),
27638 );
27639
27640 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
27641 let addr = listener.local_addr().unwrap();
27642 let server = tokio::spawn(async move {
27643 axum::serve(listener, app).await.unwrap();
27644 });
27645
27646 let mut client = create_execution_client_with_cache(cache.clone());
27647 client.dispatch = rig.dispatch.clone();
27648 client.registry = rig.registry.clone();
27649 client.emitter = rig.emitter.clone();
27650 client.dispatch.note_active_market(TEST_MARKET_INDEX);
27651 client.dispatch.replace_position_snapshot(&[], &[], &[]);
27652 client.http_client = LighterHttpClient::new(
27653 LighterEnvironment::Testnet,
27654 Some(format!("http://{addr}")),
27655 1,
27656 None,
27657 )
27658 .unwrap();
27659 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
27660
27661 let config = ExecutionManagerConfig {
27662 open_check_lookback_mins: None,
27663 open_check_missing_retries: 0,
27664 open_check_open_only: false,
27665 open_check_threshold_ns: DurationNanos::from_secs(0),
27666 single_order_query_delay_ms: 0,
27667 ..ExecutionManagerConfig::default()
27668 };
27669
27670 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
27671 assert!(
27672 cache
27673 .borrow()
27674 .positions_open(None, None, None, None, None)
27675 .is_empty()
27676 );
27677 let events = manager.check_open_orders(&[&client]).await;
27678 for event in &events {
27679 engine.process(event);
27680 }
27681
27682 assert_eq!(hits.load(Ordering::Relaxed), 2);
27683 assert_eq!(events.len(), usize::from(coverage) * 2);
27684
27685 for (id, trade_id, status) in expected {
27686 let order_events = events
27687 .iter()
27688 .filter(|event| event.client_order_id() == id)
27689 .collect::<Vec<_>>();
27690
27691 if coverage > 0 {
27692 assert!(
27693 matches!(order_events[0], OrderEventAny::Filled(fill) if fill.trade_id == trade_id && fill.last_qty == qty)
27694 );
27695 }
27696
27697 if coverage == 2 {
27698 assert!(matches!(
27699 order_events[1],
27700 OrderEventAny::Canceled(_) | OrderEventAny::Expired(_)
27701 ));
27702 }
27703
27704 let order = cache.borrow().order(&id).unwrap().clone();
27705 assert_eq!(
27706 order.status(),
27707 match coverage {
27708 0 => OrderStatus::Accepted,
27709 1 => OrderStatus::PartiallyFilled,
27710 _ => status,
27711 }
27712 );
27713 assert_eq!(order.filled_qty(), qty);
27714 let trade_ids: Vec<_> = std::iter::once(&trade_id)
27715 .take(usize::from(coverage > 0))
27716 .collect();
27717 assert_eq!(order.trade_ids(), trade_ids);
27718 assert!(
27719 inferred_reconciliation_trade_ids(&order)
27720 .unwrap()
27721 .is_empty()
27722 );
27723 }
27724
27725 assert_eq!(
27726 cache
27727 .borrow()
27728 .positions_open(None, None, None, None, None)
27729 .iter()
27730 .map(|position| position.signed_decimal_qty())
27731 .sum::<Decimal>(),
27732 Decimal::ZERO
27733 );
27734 assert!(drain_events(&mut rig.rx).is_empty());
27735 server.abort();
27736 let _ = server.await;
27737 }
27738
27739 #[rstest]
27740 #[case::bulk_partial(true, false)]
27741 #[case::targeted_partial(false, false)]
27742 #[case::bulk_filled(true, true)]
27743 #[case::targeted_filled(false, true)]
27744 #[tokio::test]
27745 async fn terminal_reconciliation_preserves_real_fill_gate(
27746 #[case] bulk: bool,
27747 #[case] filled: bool,
27748 ) {
27749 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
27750 cache
27751 .borrow_mut()
27752 .add_account(AccountAny::Margin(test_margin_account()))
27753 .unwrap();
27754 dispatch_reduce_frame(&rig, &frame, false);
27755 consume_reduce_events(&mut rig, &mut engine);
27756
27757 frame.status = if filled {
27758 LighterOrderStatus::Filled
27759 } else {
27760 LighterOrderStatus::Open
27761 };
27762
27763 let qty = Quantity::from(if filled { "0.0027" } else { "0.0010" });
27764 frame.filled_base_amount = qty.as_decimal();
27765 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27766 frame.remaining_base_amount = frame.initial_base_amount - frame.filled_base_amount;
27767 let mut trade = dispatcher_test_trade(&rig, true);
27768 trade.bid_id = frame.order_index;
27769 trade.bid_id_str = Some(frame.order_id.clone());
27770 trade.size = frame.filled_base_amount;
27771 trade.usd_amount = trade.size * trade.price;
27772 let trade_id = parse_lighter_trade_id(&trade).unwrap();
27773 let response_trade = trade.clone();
27774 let trade_hits = Arc::new(AtomicUsize::new(0));
27775 let hits = trade_hits.clone();
27776 let active = frame.clone();
27777 let inactive = frame.clone();
27778 let history_hits = Arc::new(AtomicUsize::new(0));
27779 let active_hits = history_hits.clone();
27780
27781 let app = Router::new()
27782 .route(
27783 "/api/v1/trades",
27784 get(move || {
27785 let trade = response_trade.clone();
27786 let hits = hits.clone();
27787 async move {
27788 hits.fetch_add(1, Ordering::Relaxed);
27789 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
27790 }
27791 }),
27792 )
27793 .route(
27794 "/api/v1/accountActiveOrders",
27795 get(move || {
27796 let active = active.clone();
27797 let hits = active_hits.clone();
27798 async move {
27799 let orders =
27800 if !filled && (bulk || hits.fetch_add(1, Ordering::Relaxed) > 0) {
27801 vec![active]
27802 } else {
27803 Vec::new()
27804 };
27805
27806 axum::Json(serde_json::json!({"code":200,"orders":orders}))
27807 }
27808 }),
27809 )
27810 .route(
27811 "/api/v1/accountInactiveOrders",
27812 get(move || {
27813 let inactive = inactive.clone();
27814 let hits = history_hits.clone();
27815 async move {
27816 let orders = if filled && (bulk || hits.fetch_add(1, Ordering::Relaxed) > 0)
27817 {
27818 vec![inactive]
27819 } else {
27820 Vec::new()
27821 };
27822
27823 axum::Json(serde_json::json!({"code":200,"orders":orders}))
27824 }
27825 }),
27826 );
27827
27828 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
27829 let addr = listener.local_addr().unwrap();
27830 let server = tokio::spawn(async move {
27831 axum::serve(listener, app).await.unwrap();
27832 });
27833
27834 let mut client = create_execution_client_with_cache(cache.clone());
27835 client.dispatch = rig.dispatch.clone();
27836 client.registry = rig.registry.clone();
27837 client.emitter = rig.emitter.clone();
27838 client.dispatch.note_active_market(TEST_MARKET_INDEX);
27839 client.http_client = LighterHttpClient::new(
27840 LighterEnvironment::Testnet,
27841 Some(format!("http://{addr}")),
27842 1,
27843 None,
27844 )
27845 .unwrap();
27846 client.subscribe_applied_fills();
27847 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
27848
27849 let config = ExecutionManagerConfig {
27850 open_check_lookback_mins: None,
27851 open_check_missing_retries: 0,
27852 open_check_open_only: false,
27853 open_check_threshold_ns: DurationNanos::from_secs(0),
27854 single_order_query_delay_ms: 0,
27855 ..ExecutionManagerConfig::default()
27856 };
27857
27858 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
27859 let events = manager.check_open_orders(&[&client]).await;
27860 assert!(
27861 events.is_empty(),
27862 "uncovered reports must not create inferred fills"
27863 );
27864
27865 let command = GenerateOrderStatusReport::new(
27866 UUID4::new(),
27867 UnixNanos::from(4),
27868 Some(rig.instrument_id),
27869 Some(rig.cloid),
27870 Some(VenueOrderId::new(&frame.order_id)),
27871 None,
27872 None,
27873 );
27874 assert!(client.generate_order_status_report(&command).await.is_err());
27875 let task = client.generate_order_status_report_task(&command).unwrap();
27876 task.collection.await;
27877 assert!((task.finish)().is_err());
27878 assert_eq!(rig.rx.len(), usize::from(filled));
27879 assert_eq!(trade_hits.load(Ordering::Relaxed), usize::from(filled));
27880 dispatch_lighter_trade(
27881 &trade,
27882 &client.dispatch,
27883 &client.emitter,
27884 &client.registry,
27885 account_id(),
27886 trader_id(),
27887 Some(TEST_ACCOUNT_INDEX_I64),
27888 UnixNanos::from(5),
27889 );
27890 consume_reduce_events(&mut rig, &mut engine);
27891 let report = client
27892 .generate_order_status_report(&command)
27893 .await
27894 .unwrap()
27895 .unwrap();
27896 assert_eq!(report.filled_qty, qty);
27897 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
27898 assert_eq!(cached.trade_ids(), vec![&trade_id]);
27899 assert_eq!(cached.filled_qty(), qty);
27900 assert_eq!(
27901 cached.status(),
27902 if filled {
27903 OrderStatus::Filled
27904 } else {
27905 OrderStatus::PartiallyFilled
27906 }
27907 );
27908 assert!(
27909 inferred_reconciliation_trade_ids(&cached)
27910 .unwrap()
27911 .is_empty()
27912 );
27913 server.abort();
27914 let _ = server.await;
27915 }
27916
27917 #[rstest]
27918 #[case::canceled(LighterOrderStatus::Canceled)]
27919 #[case::filled(LighterOrderStatus::Filled)]
27920 #[tokio::test]
27921 async fn terminal_mass_status_recovers_child_without_sweep_fills(
27922 #[case] status: LighterOrderStatus,
27923 ) {
27924 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
27925 cache
27926 .borrow_mut()
27927 .add_account(AccountAny::Margin(test_margin_account()))
27928 .unwrap();
27929 dispatch_reduce_frame(&rig, &frame, false);
27930 consume_reduce_events(&mut rig, &mut engine);
27931 let mut history = cache
27932 .borrow()
27933 .order(&rig.cloid)
27934 .unwrap()
27935 .events()
27936 .into_iter()
27937 .cloned()
27938 .collect::<Vec<_>>();
27939
27940 let OrderEventAny::Initialized(init) = &mut history[0] else {
27941 panic!("expected initialization");
27942 };
27943
27944 init.parent_order_id = Some(ClientOrderId::new("OFFLINE-PARENT"));
27945 let cached = OrderAny::from_events(history).unwrap();
27946 cache
27947 .borrow_mut()
27948 .add_order(cached.clone(), None, Some(client_id()), true)
27949 .unwrap();
27950
27951 let mut client = create_execution_client_with_cache(cache.clone());
27952 client.registry = rig.registry.clone();
27953 client.emitter = rig.emitter.clone();
27954 client.dispatch.seed_order_restorations(vec![cached]);
27955 client.dispatch.note_active_market(TEST_MARKET_INDEX);
27956 client.dispatch.replace_position_snapshot(&[], &[], &[]);
27957 frame.status = status;
27958
27959 let qty = Quantity::from(if status == LighterOrderStatus::Filled {
27960 "0.0027"
27961 } else {
27962 "0.0010"
27963 });
27964
27965 frame.filled_base_amount = qty.as_decimal();
27966 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
27967 frame.remaining_base_amount = Decimal::ZERO;
27968 let mut trade = dispatcher_test_trade(&rig, true);
27969 trade.bid_id = frame.order_index;
27970 trade.bid_id_str = Some(frame.order_id.clone());
27971 trade.size = qty.as_decimal();
27972 trade.usd_amount = trade.size * trade.price;
27973 let trade_id = parse_lighter_trade_id(&trade).unwrap();
27974 let hits = Arc::new(AtomicUsize::new(0));
27975 let counted = hits.clone();
27976
27977 let venue = VenueOrderId::new(&frame.order_id);
27978
27979 let app = Router::new()
27980 .route(
27981 "/api/v1/accountActiveOrders",
27982 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
27983 )
27984 .route(
27985 "/api/v1/accountInactiveOrders",
27986 get(move || {
27987 let frame = frame.clone();
27988 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
27989 }),
27990 )
27991 .route(
27992 "/api/v1/trades",
27993 get(move |Query(query): Query<AHashMap<String, String>>| {
27994 let trade = trade.clone();
27995 let hits = counted.clone();
27996 async move {
27997 let trades = if query.contains_key("order_index") {
27998 hits.fetch_add(1, Ordering::Relaxed);
27999 vec![trade]
28000 } else {
28001 Vec::new()
28002 };
28003
28004 axum::Json(serde_json::json!({"code":200,"trades":trades}))
28005 }
28006 }),
28007 );
28008
28009 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
28010 let addr = listener.local_addr().unwrap();
28011 let server = tokio::spawn(async move {
28012 axum::serve(listener, app).await.unwrap();
28013 });
28014
28015 client.http_client = LighterHttpClient::new(
28016 LighterEnvironment::Testnet,
28017 Some(format!("http://{addr}")),
28018 1,
28019 None,
28020 )
28021 .unwrap();
28022 assert_eq!(
28023 cache.borrow().client_order_id(&venue).copied(),
28024 Some(rig.cloid)
28025 );
28026 let mass = client.generate_mass_status(None).await.unwrap().unwrap();
28027 assert!(
28028 client
28029 .dispatch
28030 .order_identity(&rig.cloid)
28031 .unwrap()
28032 .child_fill_baseline()
28033 .is_some()
28034 );
28035 wait_until_async(
28036 || async {
28037 client
28038 .dispatch
28039 .order_identity(&rig.cloid)
28040 .is_some_and(|identity| identity.has_child_publication())
28041 },
28042 Duration::from_secs(3),
28043 )
28044 .await;
28045
28046 assert!(mass.fill_reports().is_empty());
28047 client.pending_tasks.begin_shutdown();
28048 client
28049 .pending_tasks
28050 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
28051 .await
28052 .unwrap();
28053
28054 for event in drain_events(&mut rig.rx) {
28055 match event {
28056 ExecutionEvent::Order(event) => engine.process(&event),
28057 ExecutionEvent::Report(report) => {
28058 engine.reconcile_execution_report(&report);
28059 }
28060 _ => panic!("unexpected child recovery event"),
28061 }
28062 }
28063
28064 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
28065 assert_eq!(hits.load(Ordering::Relaxed), 1);
28066 assert_eq!(
28067 cached.status(),
28068 if status == LighterOrderStatus::Filled {
28069 OrderStatus::Filled
28070 } else {
28071 OrderStatus::Canceled
28072 }
28073 );
28074 assert_eq!(cached.filled_qty(), qty);
28075 assert_eq!(cached.trade_ids(), vec![&trade_id]);
28076 assert!(
28077 inferred_reconciliation_trade_ids(&cached)
28078 .unwrap()
28079 .is_empty()
28080 );
28081 server.abort();
28082 let _ = server.await;
28083 }
28084
28085 #[tokio::test]
28086 async fn terminal_child_recovery_rechecks_inferred_overlap_after_http() {
28087 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
28088 cache
28089 .borrow_mut()
28090 .add_account(AccountAny::Margin(test_margin_account()))
28091 .unwrap();
28092 dispatch_reduce_frame(&rig, &frame, false);
28093 consume_reduce_events(&mut rig, &mut engine);
28094 let mut history = cache
28095 .borrow()
28096 .order(&rig.cloid)
28097 .unwrap()
28098 .events()
28099 .into_iter()
28100 .cloned()
28101 .collect::<Vec<_>>();
28102
28103 let OrderEventAny::Initialized(init) = &mut history[0] else {
28104 panic!("expected initialization");
28105 };
28106
28107 init.parent_order_id = Some(ClientOrderId::new("RACING-PARENT"));
28108 let cached = OrderAny::from_events(history).unwrap();
28109 cache
28110 .borrow_mut()
28111 .add_order(cached.clone(), None, Some(client_id()), true)
28112 .unwrap();
28113 rig.dispatch = WsDispatchState::new();
28114 rig.dispatch
28115 .restore_reconciled_order(
28116 &cached,
28117 frame.client_order_index,
28118 VenueOrderId::new(&frame.order_id),
28119 false,
28120 )
28121 .unwrap();
28122 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
28123
28124 let mut client = create_execution_client_with_cache(cache.clone());
28125 client.dispatch = rig.dispatch.clone();
28126 client.registry = rig.registry.clone();
28127 client.emitter = rig.emitter.clone();
28128 client.subscribe_applied_fills();
28129 frame.status = LighterOrderStatus::Canceled;
28130 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
28131 frame.remaining_base_amount = Decimal::ZERO;
28132 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
28133 let instrument = LIGHTER_INSTRUMENT_CACHE
28134 .get(&rig.instrument_id)
28135 .unwrap()
28136 .value()
28137 .clone();
28138 let mut partial =
28139 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(3))
28140 .unwrap()
28141 .with_client_order_id(rig.cloid);
28142 partial.order_status = OrderStatus::PartiallyFilled;
28143 let mut trade = dispatcher_test_trade(&rig, true);
28144 trade.bid_id = frame.order_index;
28145 trade.bid_id_str = Some(frame.order_id.clone());
28146 trade.size = frame.filled_base_amount;
28147 trade.usd_amount = trade.size * trade.price;
28148 let requested = Arc::new(tokio::sync::Notify::new());
28149 let resume = Arc::new(tokio::sync::Notify::new());
28150 let signal = requested.clone();
28151 let gate = resume.clone();
28152
28153 let app = Router::new()
28154 .route(
28155 "/api/v1/accountActiveOrders",
28156 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
28157 )
28158 .route(
28159 "/api/v1/accountInactiveOrders",
28160 get(move || {
28161 let frame = frame.clone();
28162 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
28163 }),
28164 )
28165 .route(
28166 "/api/v1/trades",
28167 get(move || {
28168 let trade = trade.clone();
28169 let signal = signal.clone();
28170 let gate = gate.clone();
28171 async move {
28172 signal.notify_one();
28173 gate.notified().await;
28174 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
28175 }
28176 }),
28177 );
28178
28179 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
28180 let addr = listener.local_addr().unwrap();
28181 let server = tokio::spawn(async move {
28182 axum::serve(listener, app).await.unwrap();
28183 });
28184
28185 client.http_client = LighterHttpClient::new(
28186 LighterEnvironment::Testnet,
28187 Some(format!("http://{addr}")),
28188 1,
28189 None,
28190 )
28191 .unwrap();
28192 let context = acked_probe_context(&client);
28193 let (result, ()) = tokio::join!(recover_child_order(rig.cloid, 0, &context), async {
28194 tokio::time::timeout(Duration::from_secs(2), requested.notified())
28195 .await
28196 .unwrap();
28197 engine.reconcile_order_status_report(&partial);
28198 assert!(identity.has_inferred_fills());
28199 resume.notify_one();
28200 });
28201 assert!(result.unwrap());
28202 assert_eq!(
28203 client
28204 .dispatch
28205 .order_events
28206 .lock()
28207 .get(&rig.cloid)
28208 .map_or(0, |entry| entry.fills.len()),
28209 0
28210 );
28211 assert!(drain_events(&mut rig.rx).is_empty());
28212 assert_eq!(identity.filled_qty(), Decimal::ZERO);
28213 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
28214 let inferred = inferred_reconciliation_trade_ids(&cached).unwrap();
28215 assert_eq!(inferred.len(), 1);
28216 assert_eq!(cached.trade_ids(), inferred.iter().collect::<Vec<_>>());
28217 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
28218 server.abort();
28219 let _ = server.await;
28220 }
28221
28222 fn create_execution_client_with_cache(cache: Rc<RefCell<Cache>>) -> LighterExecutionClient {
28223 let core = ExecutionClientCore::new(
28224 trader_id(),
28225 client_id(),
28226 test_config().resolved_venue(),
28227 OmsType::Netting,
28228 account_id(),
28229 AccountType::Margin,
28230 None,
28231 cache,
28232 );
28233 LighterExecutionClient::new(core, test_config()).unwrap()
28234 }
28235}